📄
src/Domain/ClipSelector.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
using System; using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.AI; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; namespace Slopper.Domain; public sealed class ClipSelector( ISubtitleReader subtitleReader, IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator, IOptions<ClipSelectorOptions> options ) { private readonly Lazy<Task<IReadOnlyList<Embedding<float>>>> clippableQuotesEmbeddings = new(async () => await embeddingGenerator.GenerateAsync(options.Value.ClippableQuotes) ); public async Task<(TimeSpan start, TimeSpan duration)> PickClip( MediaItem media, CancellationToken cancellationToken ) { var subtitles = await subtitleReader.ReadSubtitles(media, cancellationToken); var subtitleLines = subtitles.SelectMany(s => s.Lines); var subtitleEmbeddings = await embeddingGenerator.GenerateAsync( subtitleLines, cancellationToken: cancellationToken ); return FindBestMatch( subtitleEmbeddings.Zip(subtitles.SelectMany(s => s.Lines.Select(_ => s))), await clippableQuotesEmbeddings.Value ); } private static (TimeSpan, TimeSpan) FindBestMatch( IEnumerable<(Embedding<float> embedding, SubtitleEntry subtitle)> subtitleEmbeddings, IReadOnlyList<Embedding<float>> clippableQuotesEmbeddings ) { var se = subtitleEmbeddings.ToArray(); (float, int)? best = null; for (var i = 0; i < se.Length; i++) { var score = CalculateSimilarity(se[i].embedding, clippableQuotesEmbeddings); if (best is not (var previousBestScore, _) || previousBestScore < score) { best = (score, i); } } if (best is not (_, var bestIndex)) { throw new Exception("No subtitle entries found, no best match possible."); } var start = se[int.Max(0, bestIndex - 2)].subtitle.Start; var end = se[int.Min(bestIndex + 1, se.Length - 1)].subtitle; var duration = end.Start + end.Duration - start; return (start, duration); } private static float CalculateSimilarity( Embedding<float> subtitleEmbedding, IReadOnlyList<Embedding<float>> clippableQuotesEmbeddings ) => clippableQuotesEmbeddings.Max(qe => CalculateEmbeddingSimilarity(subtitleEmbedding, qe)); private static float CalculateEmbeddingSimilarity(Embedding<float> a, Embedding<float> b) { var vectorA = a.Vector.Span; var vectorB = b.Vector.Span; float dotProduct = 0.0f; float magnitudeA = 0.0f; float magnitudeB = 0.0f; int count = int.Min(vectorA.Length, vectorB.Length); for (int i = 0; i < count; i++) { dotProduct += vectorA[i] * vectorB[i]; magnitudeA += vectorA[i] * vectorA[i]; magnitudeB += vectorB[i] * vectorB[i]; } if (magnitudeA <= 0.0f || magnitudeB <= 0.0f) { return 0.0f; } return dotProduct / (float.Sqrt(magnitudeA) * float.Sqrt(magnitudeB)); } } public sealed class ClipSelectorOptions { [Required, MinLength(1)] public required IReadOnlyList<string> ClippableQuotes { get; set; } } [OptionsValidator] internal sealed partial class ClipSelectorOptionsValidator : IValidateOptions<ClipSelectorOptions>; public static class ClipSelectorServiceCollectionExtensions { extension(IServiceCollection services) { public IServiceCollection AddClipSelector() { services.AddOptions<ClipSelectorOptions>().BindConfiguration("ClipSelector").ValidateOnStart(); services.AddTransient<IValidateOptions<ClipSelectorOptions>, ClipSelectorOptionsValidator>(); services.AddSingleton<ClipSelector>(); return services; } } }