Slopper
Commit: bb21041
Parent: d34d51b

Use subtitles as context for clip description

Mårten Åsberg committed on 2026-05-10 at 22:00
Browse files at this commit
Directory.Packages.props +4 -0
diff --git a/Directory.Packages.props b/Directory.Packages.props
index f33d812..c562d5d 100644
@@ -23,6 +23,10 @@
Include="Microsoft.Extensions.Hosting.Abstractions"
Version="11.0.0-preview.3.26207.106"
/>
<PackageVersion
Include="Microsoft.Extensions.Caching.Memory"
Version="11.0.0-preview.3.26207.106"
/>
<PackageVersion Include="Microsoft.Extensions.Http" Version="11.0.0-preview.3.26207.106" />
<PackageVersion
Include="Microsoft.Extensions.Logging.Abstractions"
src/Api/packages.lock.json +22 -0
diff --git a/src/Api/packages.lock.json b/src/Api/packages.lock.json
index 66c4ab6..b2966a7 100644
@@ -147,6 +147,14 @@
"SQLitePCLRaw.core": "2.1.11"
}
},
"Microsoft.Extensions.Caching.Abstractions": {
"type": "Transitive",
"resolved": "11.0.0-preview.3.26207.106",
"contentHash": "rIYCe4u6m5u9s0QM+30K9CynYrk7IKl7MmDhxAN1rn0m7zjFni0kGvLJ1UiAClbxj8OwQpdeOoD0XrBHiYZ4Dw==",
"dependencies": {
"Microsoft.Extensions.Primitives": "11.0.0-preview.3.26207.106"
}
},
"Microsoft.Extensions.Configuration": {
"type": "Transitive",
"resolved": "11.0.0-preview.3.26207.106",
@@ -477,6 +485,7 @@
"type": "Project",
"dependencies": {
"FFMpegCore": "[5.4.0, )",
"Microsoft.Extensions.Caching.Memory": "[11.0.0-preview.3.26207.106, )",
"Microsoft.Extensions.Logging.Abstractions": "[11.0.0-preview.3.26207.106, )",
"OpenTelemetry": "[1.15.3, )",
"Slopper.Domain": "[1.0.0, )",
@@ -530,6 +539,19 @@
"resolved": "10.5.2",
"contentHash": "Ei+YWV9Ybnps7pR1dgjlG29gelXEwZkhLVAcWmKe6HvXS6LNBYgSdWiY3Hk9OZXYtK34rv/NtLWBQYQGOBQYPQ=="
},
"Microsoft.Extensions.Caching.Memory": {
"type": "CentralTransitive",
"requested": "[11.0.0-preview.3.26207.106, )",
"resolved": "11.0.0-preview.3.26207.106",
"contentHash": "aF0hfOFPT/4e1s4vI1opaVJRXPzRe8sHQydVDnnV2R8KDBAbCnPj7yDyQA7fc/nOjcUss/kfiWVzB+LznQTh5w==",
"dependencies": {
"Microsoft.Extensions.Caching.Abstractions": "11.0.0-preview.3.26207.106",
"Microsoft.Extensions.DependencyInjection.Abstractions": "11.0.0-preview.3.26207.106",
"Microsoft.Extensions.Logging.Abstractions": "11.0.0-preview.3.26207.106",
"Microsoft.Extensions.Options": "11.0.0-preview.3.26207.106",
"Microsoft.Extensions.Primitives": "11.0.0-preview.3.26207.106"
}
},
"Microsoft.Extensions.Hosting": {
"type": "CentralTransitive",
"requested": "[11.0.0-preview.3.26207.106, )",
src/Cli/Program.cs +3 -3
diff --git a/src/Cli/Program.cs b/src/Cli/Program.cs
index 2567fd0..f7c6dc4 100644
@@ -24,9 +24,9 @@ var logger = app.Services.GetRequiredService<ILogger<Program>>();
using var scope = app.Services.CreateScope();
var clipDescriber = scope.ServiceProvider.GetRequiredService<ClipDescriber>();
var description = await clipDescriber.DescribeClip(
new(Guid.NewGuid(), args[0], new Subtitles.Embedded(3)),
TimeSpan.FromSeconds(11),
TimeSpan.FromSeconds(16),
new(Guid.NewGuid(), args[0], new Subtitles.Embedded(2)),
TimeSpan.FromSeconds(239),
TimeSpan.FromSeconds(40),
CancellationToken.None
);
logger.LogInformation("{Description}", description);
src/Cli/Properties/launchSettings.json +1 -1
diff --git a/src/Cli/Properties/launchSettings.json b/src/Cli/Properties/launchSettings.json
index 2c6eba0..c2230de 100644
@@ -8,7 +8,7 @@
"environmentVariables": {
"DOTNET_ENVIRONMENT": "Development"
},
"commandLineArgs": "D:/slopper/media/S02E06.mkv"
"commandLineArgs": "D:/slopper/media/S01E01.mkv"
}
}
}
src/Cli/packages.lock.json +17 -15
diff --git a/src/Cli/packages.lock.json b/src/Cli/packages.lock.json
index e6d940a..d1a0770 100644
@@ -149,22 +149,10 @@
},
"Microsoft.Extensions.Caching.Abstractions": {
"type": "Transitive",
"resolved": "10.0.7",
"contentHash": "pUDgQKEqNUFlerDIFRg7zzoDVRPEWIG7nR40h8Gzg8RXza4Ry0lWZ7u91bmwu3iUDCxw3Dv6TLHVFoAgY0gy7Q==",
"dependencies": {
"Microsoft.Extensions.Primitives": "10.0.7"
}
},
"Microsoft.Extensions.Caching.Memory": {
"type": "Transitive",
"resolved": "10.0.7",
"contentHash": "6eULH/sc97yfCEV31g7AgUzHc7dIm0DGBcofoE8GgBaXbdAPPhathN8rYcgi1TSiG1QucCdqKiVNaDEPAEXL5Q==",
"resolved": "11.0.0-preview.3.26207.106",
"contentHash": "rIYCe4u6m5u9s0QM+30K9CynYrk7IKl7MmDhxAN1rn0m7zjFni0kGvLJ1UiAClbxj8OwQpdeOoD0XrBHiYZ4Dw==",
"dependencies": {
"Microsoft.Extensions.Caching.Abstractions": "10.0.7",
"Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.7",
"Microsoft.Extensions.Logging.Abstractions": "10.0.7",
"Microsoft.Extensions.Options": "10.0.7",
"Microsoft.Extensions.Primitives": "10.0.7"
"Microsoft.Extensions.Primitives": "11.0.0-preview.3.26207.106"
}
},
"Microsoft.Extensions.Configuration": {
@@ -477,6 +465,7 @@
"type": "Project",
"dependencies": {
"FFMpegCore": "[5.4.0, )",
"Microsoft.Extensions.Caching.Memory": "[11.0.0-preview.3.26207.106, )",
"Microsoft.Extensions.Logging.Abstractions": "[11.0.0-preview.3.26207.106, )",
"OpenTelemetry": "[1.15.3, )",
"Slopper.Domain": "[1.0.0, )",
@@ -536,6 +525,19 @@
"resolved": "10.5.2",
"contentHash": "Ei+YWV9Ybnps7pR1dgjlG29gelXEwZkhLVAcWmKe6HvXS6LNBYgSdWiY3Hk9OZXYtK34rv/NtLWBQYQGOBQYPQ=="
},
"Microsoft.Extensions.Caching.Memory": {
"type": "CentralTransitive",
"requested": "[11.0.0-preview.3.26207.106, )",
"resolved": "11.0.0-preview.3.26207.106",
"contentHash": "aF0hfOFPT/4e1s4vI1opaVJRXPzRe8sHQydVDnnV2R8KDBAbCnPj7yDyQA7fc/nOjcUss/kfiWVzB+LznQTh5w==",
"dependencies": {
"Microsoft.Extensions.Caching.Abstractions": "11.0.0-preview.3.26207.106",
"Microsoft.Extensions.DependencyInjection.Abstractions": "11.0.0-preview.3.26207.106",
"Microsoft.Extensions.Logging.Abstractions": "11.0.0-preview.3.26207.106",
"Microsoft.Extensions.Options": "11.0.0-preview.3.26207.106",
"Microsoft.Extensions.Primitives": "11.0.0-preview.3.26207.106"
}
},
"Microsoft.Extensions.Hosting.Abstractions": {
"type": "CentralTransitive",
"requested": "[11.0.0-preview.3.26207.106, )",
src/Domain/ClipDescriber.cs +15 -2
diff --git a/src/Domain/ClipDescriber.cs b/src/Domain/ClipDescriber.cs
index 81fe83d..933cd9a 100644
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
@@ -11,6 +12,7 @@ namespace Slopper.Domain;
public sealed class ClipDescriber(
IFrameExtractor frameExtractor,
ISubtitleReader subtitleReader,
IChatClient chatClient,
IOptionsMonitor<ClipDescriberOptions> options
)
@@ -18,16 +20,27 @@ public sealed class ClipDescriber(
public async Task<string> DescribeClip(
MediaItem media,
TimeSpan start,
TimeSpan end,
TimeSpan duration,
CancellationToken cancellationToken
)
{
var frames = await frameExtractor.ExtractFrames(media, [start, end], cancellationToken);
var frames = await frameExtractor.ExtractFrames(media, [start, start + duration], cancellationToken);
var subtitles = await subtitleReader.ReadSubtitles(media, start, duration, cancellationToken);
List<AIContent> contents = [];
foreach (var frame in frames)
{
contents.Add(new DataContent(frame, "image/png"));
}
contents.Add(new TextContent(options.CurrentValue.Prompt));
if (subtitles.Count > 0)
{
contents.Add(
new TextContent(
$"Subtitles from this clip:\n{string.Join("\n", subtitles.Select(s => string.Join(" / ", s.Lines)))}"
)
);
}
var response = await chatClient.GetResponseAsync(
[new ChatMessage(ChatRole.User, contents)],
src/Domain/ClipGenerator.cs +1 -1
diff --git a/src/Domain/ClipGenerator.cs b/src/Domain/ClipGenerator.cs
index dff7747..a25b18e 100644
@@ -29,7 +29,7 @@ public sealed class ClipGenerator(
var clipId = Guid.CreateVersion7();
var clipPath = Path.Join(clipDirectory, $"{clipId}.mp4");
var description = await clipDescriber.DescribeClip(media, start, start + duration, cancellationToken);
var description = await clipDescriber.DescribeClip(media, start, duration, cancellationToken);
await clipExtractor.ExtractClip(media, start, duration, clipPath, cancellationToken);
src/Domain/ISubtitleLinesReader.cs +10 -1
diff --git a/src/Domain/ISubtitleLinesReader.cs b/src/Domain/ISubtitleLinesReader.cs
index 2d938dd..0814c0a 100644
@@ -1,3 +1,5 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
@@ -5,5 +7,12 @@ namespace Slopper.Domain;
public interface ISubtitleReader
{
Task<SubtitleEntry[]> ReadSubtitles(MediaItem media, CancellationToken cancellationToken);
Task<IReadOnlyList<SubtitleEntry>> ReadSubtitles(MediaItem media, CancellationToken cancellationToken);
Task<IReadOnlyList<SubtitleEntry>> ReadSubtitles(
MediaItem media,
TimeSpan start,
TimeSpan duration,
CancellationToken cancellationToken
);
}
src/Infrastructure/Database/packages.lock.json +16 -15
diff --git a/src/Infrastructure/Database/packages.lock.json b/src/Infrastructure/Database/packages.lock.json
index bf3436e..6f75e42 100644
@@ -197,22 +197,10 @@
},
"Microsoft.Extensions.Caching.Abstractions": {
"type": "Transitive",
"resolved": "10.0.7",
"contentHash": "pUDgQKEqNUFlerDIFRg7zzoDVRPEWIG7nR40h8Gzg8RXza4Ry0lWZ7u91bmwu3iUDCxw3Dv6TLHVFoAgY0gy7Q==",
"dependencies": {
"Microsoft.Extensions.Primitives": "10.0.7"
}
},
"Microsoft.Extensions.Caching.Memory": {
"type": "Transitive",
"resolved": "10.0.7",
"contentHash": "6eULH/sc97yfCEV31g7AgUzHc7dIm0DGBcofoE8GgBaXbdAPPhathN8rYcgi1TSiG1QucCdqKiVNaDEPAEXL5Q==",
"resolved": "11.0.0-preview.3.26207.106",
"contentHash": "rIYCe4u6m5u9s0QM+30K9CynYrk7IKl7MmDhxAN1rn0m7zjFni0kGvLJ1UiAClbxj8OwQpdeOoD0XrBHiYZ4Dw==",
"dependencies": {
"Microsoft.Extensions.Caching.Abstractions": "10.0.7",
"Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.7",
"Microsoft.Extensions.Logging.Abstractions": "10.0.7",
"Microsoft.Extensions.Options": "10.0.7",
"Microsoft.Extensions.Primitives": "10.0.7"
"Microsoft.Extensions.Primitives": "11.0.0-preview.3.26207.106"
}
},
"Microsoft.Extensions.Configuration": {
@@ -417,6 +405,19 @@
"resolved": "10.5.2",
"contentHash": "Ei+YWV9Ybnps7pR1dgjlG29gelXEwZkhLVAcWmKe6HvXS6LNBYgSdWiY3Hk9OZXYtK34rv/NtLWBQYQGOBQYPQ=="
},
"Microsoft.Extensions.Caching.Memory": {
"type": "CentralTransitive",
"requested": "[11.0.0-preview.3.26207.106, )",
"resolved": "11.0.0-preview.3.26207.106",
"contentHash": "aF0hfOFPT/4e1s4vI1opaVJRXPzRe8sHQydVDnnV2R8KDBAbCnPj7yDyQA7fc/nOjcUss/kfiWVzB+LznQTh5w==",
"dependencies": {
"Microsoft.Extensions.Caching.Abstractions": "11.0.0-preview.3.26207.106",
"Microsoft.Extensions.DependencyInjection.Abstractions": "11.0.0-preview.3.26207.106",
"Microsoft.Extensions.Logging.Abstractions": "11.0.0-preview.3.26207.106",
"Microsoft.Extensions.Options": "11.0.0-preview.3.26207.106",
"Microsoft.Extensions.Primitives": "11.0.0-preview.3.26207.106"
}
},
"Microsoft.Extensions.Logging.Abstractions": {
"type": "CentralTransitive",
"requested": "[11.0.0-preview.3.26207.106, )",
src/Infrastructure/Ffmpeg/Ffmpeg.csproj +1 -0
diff --git a/src/Infrastructure/Ffmpeg/Ffmpeg.csproj b/src/Infrastructure/Ffmpeg/Ffmpeg.csproj
index 12b84e3..0838fcf 100644
@@ -8,6 +8,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="FFMpegCore" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
<PackageReference Include="OpenTelemetry" />
<PackageReference Include="SubtitlesParserV2" />
src/Infrastructure/Ffmpeg/ServiceCollectionExtensions.cs +1 -0
diff --git a/src/Infrastructure/Ffmpeg/ServiceCollectionExtensions.cs b/src/Infrastructure/Ffmpeg/ServiceCollectionExtensions.cs
index fe08e30..db21053 100644
@@ -7,6 +7,7 @@ public static class ServiceCollectionExtensions
{
public static IServiceCollection AddFfmpegServices(this IServiceCollection services) =>
services
.AddMemoryCache()
.AddTransient<IClipExtractor, ClipExtractor>()
.AddTransient<ISubtitleReader, SubtitleReader>()
.AddTransient<IFrameExtractor, FrameExtractor>();
src/Infrastructure/Ffmpeg/SubtitleReader.cs +41 -7
diff --git a/src/Infrastructure/Ffmpeg/SubtitleReader.cs b/src/Infrastructure/Ffmpeg/SubtitleReader.cs
index c117e2f..d91a4e3 100644
@@ -1,25 +1,59 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FFMpegCore;
using FFMpegCore.Pipes;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Slopper.Domain;
using SubtitlesParserV2;
namespace Slopper.Infrastructure.Ffmpeg;
internal sealed class SubtitleReader(ILogger<SubtitleReader> logger) : ISubtitleReader
internal sealed class SubtitleReader(ILogger<SubtitleReader> logger, IMemoryCache cache) : ISubtitleReader
{
public async Task<SubtitleEntry[]> ReadSubtitles(MediaItem media, CancellationToken cancellationToken) =>
media.Subtitles switch
public async Task<IReadOnlyList<SubtitleEntry>> ReadSubtitles(
MediaItem media,
CancellationToken cancellationToken
) => await ReadSubtitlesInternal(media);
public async Task<IReadOnlyList<SubtitleEntry>> ReadSubtitles(
MediaItem media,
TimeSpan start,
TimeSpan duration,
CancellationToken cancellationToken
)
{
var all = await ReadSubtitlesInternal(media);
var first = Array.FindIndex(all, s => start <= s.Start);
var end = start + duration;
var last = Array.FindIndex(all, s => end <= s.Start + s.Duration);
if (last < first)
{
Subtitles.External(var path) => ReadSubtitles(path),
Subtitles.Embedded(var index) => await ReadSubtitles(media.Path, index),
_ => throw new ArgumentException("Unknown subtitle type"),
};
return [];
}
return new ArraySegment<SubtitleEntry>(all, first, last - first + 1);
}
private async Task<SubtitleEntry[]> ReadSubtitlesInternal(MediaItem media) =>
(
await cache.GetOrCreateAsync(
media,
async entry =>
{
entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(15);
return media.Subtitles switch
{
Subtitles.External(var path) => ReadSubtitles(path),
Subtitles.Embedded(var index) => await ReadSubtitles(media.Path, index),
_ => throw new ArgumentException("Unknown subtitle type"),
};
}
)
)!;
private static SubtitleEntry[] ReadSubtitles(string path)
{
src/Infrastructure/Ffmpeg/packages.lock.json +21 -0
diff --git a/src/Infrastructure/Ffmpeg/packages.lock.json b/src/Infrastructure/Ffmpeg/packages.lock.json
index 2ea4d17..c4ec2ef 100644
@@ -17,6 +17,19 @@
"Instances": "3.0.2"
}
},
"Microsoft.Extensions.Caching.Memory": {
"type": "Direct",
"requested": "[11.0.0-preview.3.26207.106, )",
"resolved": "11.0.0-preview.3.26207.106",
"contentHash": "aF0hfOFPT/4e1s4vI1opaVJRXPzRe8sHQydVDnnV2R8KDBAbCnPj7yDyQA7fc/nOjcUss/kfiWVzB+LznQTh5w==",
"dependencies": {
"Microsoft.Extensions.Caching.Abstractions": "11.0.0-preview.3.26207.106",
"Microsoft.Extensions.DependencyInjection.Abstractions": "11.0.0-preview.3.26207.106",
"Microsoft.Extensions.Logging.Abstractions": "11.0.0-preview.3.26207.106",
"Microsoft.Extensions.Options": "11.0.0-preview.3.26207.106",
"Microsoft.Extensions.Primitives": "11.0.0-preview.3.26207.106"
}
},
"Microsoft.Extensions.Logging.Abstractions": {
"type": "Direct",
"requested": "[11.0.0-preview.3.26207.106, )",
@@ -51,6 +64,14 @@
"resolved": "3.0.2",
"contentHash": "LfhegDpmA8PuHW58RmgVvCDG/mfVCTU+Vhy4ppmXLJfAer33Xl0NocDy92OwSL6CnkVdx41O/I0+BjNhU1JtMQ=="
},
"Microsoft.Extensions.Caching.Abstractions": {
"type": "Transitive",
"resolved": "11.0.0-preview.3.26207.106",
"contentHash": "rIYCe4u6m5u9s0QM+30K9CynYrk7IKl7MmDhxAN1rn0m7zjFni0kGvLJ1UiAClbxj8OwQpdeOoD0XrBHiYZ4Dw==",
"dependencies": {
"Microsoft.Extensions.Primitives": "11.0.0-preview.3.26207.106"
}
},
"Microsoft.Extensions.Configuration": {
"type": "Transitive",
"resolved": "11.0.0-preview.3.26207.106",