Descriptions for clips

public sealed record Clip(Guid Id, TimeSpan Duration, DateTimeOffset CreatedAt, string? Description)

public static Clip FromDomain(Domain.Clip clip) => new(clip.Id, clip.Duration, clip.CreatedAt, clip.Description);

"ClipDescriber": {

"Prompt": "Give one short sentence caption of these frames from a video clip, optimized for engagement on TikTok and Instagram Reels. Only ever give on sentence, no alternatives, not formatting, just a plain text caption, optimized for engagement."

},

var clipDescriber = scope.ServiceProvider.GetRequiredService<ClipDescriber>();

var description = await clipDescriber.DescribeClip(

TimeSpan.FromSeconds(16),

logger.LogInformation("{Description}", description);

"ClipDescriber": {

"Prompt": "Give one short sentence caption of these frames from a video clip, optimized for engagement on TikTok and Instagram Reels. Only ever give on sentence, no alternatives, not formatting, just a plain text caption, optimized for engagement."

},

DateTimeOffset CreatedAt,

string? Description

using System;

using System.Collections.Generic;

using System.ComponentModel.DataAnnotations;

using System.Threading;

using System.Threading.Tasks;

using Microsoft.Extensions.AI;

using Microsoft.Extensions.DependencyInjection;

using Microsoft.Extensions.Options;

namespace Slopper.Domain;

public sealed class ClipDescriber(

IFrameExtractor frameExtractor,

IChatClient chatClient,

IOptionsMonitor<ClipDescriberOptions> options

)

{

public async Task<string> DescribeClip(

MediaItem media,

TimeSpan start,

TimeSpan end,

CancellationToken cancellationToken

)

{

var frames = await frameExtractor.ExtractFrames(media, [start, end], cancellationToken);

List<AIContent> contents = [];

foreach (var frame in frames)

contents.Add(new DataContent(frame, "image/png"));

contents.Add(new TextContent(options.CurrentValue.Prompt));

var response = await chatClient.GetResponseAsync(

[new ChatMessage(ChatRole.User, contents)],

cancellationToken: cancellationToken

);

return response.Text;

}

}

public sealed class ClipDescriberOptions

{

[Required]

public required string Prompt { get; set; }

}

[OptionsValidator]

internal sealed partial class ClipDescriberOptionsValidator : IValidateOptions<ClipDescriberOptions>;

public static class ClipDescriberServiceCollectionExtensions

{

extension(IServiceCollection services)

{

public IServiceCollection AddClipDescriber()

{

services.AddOptions<ClipDescriberOptions>().BindConfiguration("ClipDescriber").ValidateOnStart();

services.AddTransient<IValidateOptions<ClipDescriberOptions>, ClipDescriberOptionsValidator>();

services.AddTransient<ClipDescriber>();

return services;

}

}

}

IClipRepository clipRepository,

ClipDescriber clipDescriber

var description = await clipDescriber.DescribeClip(media, start, start + duration, cancellationToken);

var clip = new Clip(clipId, media.Id, clipPath, start, duration, utcNow, description);

services.AddClipDescriber();

using System;

using System.Collections.Generic;

using System.Threading;

using System.Threading.Tasks;

namespace Slopper.Domain;

public interface IFrameExtractor

{

Task<IReadOnlyList<byte[]>> ExtractFrames(

MediaItem media,

IReadOnlyList<TimeSpan> timestamps,

CancellationToken cancellationToken

);

}

description: string | null;

<div v-if="clip.description" class="description">

<p>{{ clip.description }}</p>

</div>

position: relative;

.description {

position: absolute;

bottom: 0;

left: 0;

right: 0;

padding: 1rem;

background: linear-gradient(to bottom, rgba(0, 0, 0, 0) 0%, rgba(0, 0, 0, 0.75) 1.2rem);

color: #ffffff;

line-height: 1.4;

}

internal sealed class AiOptions

public required string EmbeddingModel { get; set; }

[Required]

public required string DescriptionModel { get; set; }

internal sealed partial class AiOptionsValidator : IValidateOptions<AiOptions>;

using System;

using System.Net.Http;

services.AddOptions<AiOptions>().BindConfiguration("Ai").ValidateOnStart();

services.AddTransient<IValidateOptions<AiOptions>, AiOptionsValidator>();

services

.AddHttpClient("Ollama")

.ConfigureHttpClient(

(sp, client) =>

var options = sp.GetRequiredService<IOptions<AiOptions>>().Value;

client.BaseAddress = options.Endpoint;

client.Timeout = TimeSpan.FromMinutes(10);

if (options.BasicAuth is not null)

{

client.DefaultRequestHeaders.Authorization = new("Basic", options.BasicAuth);

}

);

services

.AddEmbeddingGenerator(sp =>

{

var factory = sp.GetRequiredService<IHttpClientFactory>();

var options = sp.GetRequiredService<IOptions<AiOptions>>().Value;

return new OllamaApiClient(factory.CreateClient("Ollama"), options.EmbeddingModel);

})

.UseOpenTelemetry();

services

.AddChatClient(sp =>

{

var factory = sp.GetRequiredService<IHttpClientFactory>();

var options = sp.GetRequiredService<IOptions<AiOptions>>().Value;

return new OllamaApiClient(factory.CreateClient("Ollama"), options.DescriptionModel);

})

.UseOpenTelemetry();

// <auto-generated />

using System;

using Microsoft.EntityFrameworkCore;

using Microsoft.EntityFrameworkCore.Infrastructure;

using Microsoft.EntityFrameworkCore.Migrations;

using Microsoft.EntityFrameworkCore.Storage.ValueConversion;

using Slopper.Infrastructure.Database.Slopper;

#nullable disable

namespace Slopper.Infrastructure.Database.Slopper.Migrations

{

[DbContext(typeof(SlopperDbContext))]

[Migration("20260509175844_AddClipDescription")]

partial class AddClipDescription

{

/// <inheritdoc />

protected override void BuildTargetModel(ModelBuilder modelBuilder)

{

#pragma warning disable 612, 618

modelBuilder.HasAnnotation("ProductVersion", "10.0.7");

modelBuilder.Entity("Slopper.Domain.Clip", b =>

{

b.Property<Guid>("Id")

.ValueGeneratedOnAdd()

.HasColumnType("TEXT");

b.Property<DateTimeOffset>("CreatedAt")

.HasColumnType("TEXT");

b.Property<string>("Description")

.HasColumnType("TEXT");

b.Property<TimeSpan>("Duration")

.HasColumnType("TEXT");

b.Property<Guid>("MediaItemId")

.HasColumnType("TEXT");

b.Property<string>("Path")

.IsRequired()

.HasColumnType("TEXT");

b.Property<TimeSpan>("Start")

.HasColumnType("TEXT");

b.HasKey("Id");

b.ToTable("Clips");

});

#pragma warning restore 612, 618

}

}

}

using Microsoft.EntityFrameworkCore.Migrations;

#nullable disable

namespace Slopper.Infrastructure.Database.Slopper.Migrations

{

/// <inheritdoc />

public partial class AddClipDescription : Migration

{

/// <inheritdoc />

protected override void Up(MigrationBuilder migrationBuilder)

{

migrationBuilder.AddColumn<string>(name: "Description", table: "Clips", type: "TEXT", nullable: true);

}

/// <inheritdoc />

protected override void Down(MigrationBuilder migrationBuilder)

{

migrationBuilder.DropColumn(name: "Description", table: "Clips");

}

}

}

b.Property<string>("Description")

.HasColumnType("TEXT");

clipBuilder.Property(c => c.Description);

using FFMpegCore;

using FFMpegCore.Arguments;

namespace Slopper.Infrastructure.Ffmpeg;

internal static class FfmpegExtensions

{

extension(FFMpegArgumentOptions options)

{

public FFMpegArgumentOptions WithArgument(string argument) =>

options.WithArgument(new CustomArgument(argument));

}

extension(VideoFilterOptions filter)

{

public VideoFilterOptions Add(string key, string value) =>

filter.Add(new CustomVideoFilterArgument(key, value));

public VideoFilterOptions Add(IVideoFilterArgument argument)

{

filter.Arguments.Add(argument);

return filter;

}

}

}

internal sealed record CustomVideoFilterArgument(string Key, string Value) : IVideoFilterArgument;

using System;

using System.Collections.Generic;

using System.IO;

using System.Threading;

using System.Threading.Tasks;

using FFMpegCore;

using FFMpegCore.Pipes;

using Microsoft.Extensions.Logging;

using Slopper.Domain;

namespace Slopper.Infrastructure.Ffmpeg;

internal sealed class FrameExtractor(ILogger<FrameExtractor> logger) : IFrameExtractor

{

public async Task<IReadOnlyList<byte[]>> ExtractFrames(

MediaItem media,

IReadOnlyList<TimeSpan> timestamps,

CancellationToken cancellationToken

)

{

var frames = new List<byte[]>(timestamps.Count);

foreach (var timestamp in timestamps)

{

using var stream = new MemoryStream();

var args = FFMpegArguments

.FromFileInput(media.Path, addArguments: options => options.Seek(timestamp))

.OutputToPipe(

new StreamPipeSink(stream),

addArguments: options =>

options

.WithArgument("-frames:v 1")

.WithVideoFilters(filter => filter.Scale(-1, 720))

.ForceFormat("image2pipe")

.WithArgument("-c:v png")

)

.CancellableThrough(cancellationToken);

using var activity = Tracing.StartExtractFrame(media.Path, timestamp);

logger.LogInformation("Running ffmpeg {FfmpegArguments}", args.Arguments);

await args.ProcessAsynchronously();

frames.Add(stream.ToArray());

}

return frames;

}

}

services

.AddTransient<IClipExtractor, ClipExtractor>()

.AddTransient<ISubtitleReader, SubtitleReader>()

.AddTransient<IFrameExtractor, FrameExtractor>();

public static Activity? StartExtractFrame(string path, TimeSpan timestamp) =>

FfmpegActivity

.StartActivity("ExtractFrame", ActivityKind.Internal)

?.SetTag("Path", path)

?.SetTag("Timestamp", timestamp);