Slopper
Commit: ac32f4b
Parent: 0eca1b8

Add tags to clips

Mårten Åsberg committed on 2026-05-11 at 15:08
Browse files at this commit
src/Api/Clip.cs +4 -2
diff --git a/src/Api/Clip.cs b/src/Api/Clip.cs
index 3b7008c..b458aa7 100644
@@ -1,8 +1,10 @@
using System;
using System.Linq;
namespace Slopper.Api;
public sealed record Clip(Guid Id, TimeSpan Duration, DateTimeOffset CreatedAt, string? Description)
public sealed record Clip(Guid Id, TimeSpan Duration, DateTimeOffset CreatedAt, string? Caption, string[] Tags)
{
public static Clip FromDomain(Domain.Clip clip) => new(clip.Id, clip.Duration, clip.CreatedAt, clip.Description);
public static Clip FromDomain(Domain.Clip clip) =>
new(clip.Id, clip.Duration, clip.CreatedAt, clip.Caption, [.. clip.Tags.Select(t => t.Value)]);
}
src/Api/appsettings.Development.json +1 -1
diff --git a/src/Api/appsettings.Development.json b/src/Api/appsettings.Development.json
index 4d5f6e2..5d2cecc 100644
@@ -10,7 +10,7 @@
"Interval": "00:00:10.000"
},
"ClipDescriber": {
"Prompt": "Give one short sentence caption of these frames from a video clip, optimized for engagement on TikTok and Instagram Reels. Only ever give on sentence, no alternatives, not formatting, just a plain text caption, optimized for engagement."
"Prompt": "Give one short sentence caption and tags for the attached frames from a video clip, optimized for engagement on TikTok and Instagram Reels. Answer with JSON, one string field for `caption` and an array of strings for `tags`. Make the caption one sentence only optimized for engagement, and the tags one word or camel case without the hashtag symbol."
},
"ClipSelector": {
"ClippableQuotes": [
src/Cli/Program.cs +29 -27
diff --git a/src/Cli/Program.cs b/src/Cli/Program.cs
index 412d1ae..3e5d32b 100644
@@ -1,53 +1,55 @@
using System;
using System.Threading.Tasks;
using System.IO;
using System.Threading;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Slopper.Cli;
using Slopper.Domain;
using Slopper.Domain.Describer;
using Slopper.Infrastructure.Ai;
using Slopper.Infrastructure.Database;
using Slopper.Infrastructure.Ffmpeg;
using Winton.Extensions.Configuration.Consul;
var builder = Host.CreateApplicationBuilder();
builder.Configuration.AddConsul(
"slopper",
options =>
{
options.ConsulConfigurationOptions = options => options.Address = new(builder.Configuration["Consul:Address"]!);
options.ConsulHttpClientOptions = client =>
client.DefaultRequestHeaders.Authorization = new("Basic", builder.Configuration["Consul:BasicAuth"]);
options.ReloadOnChange = true;
options.PollWaitTime = TimeSpan.FromSeconds(5);
}
);
builder.ConfigureOpenTelemetry();
builder.Services.AddClipSelector().AddClipGenerator();
builder.Services.AddJellyfinDatabase().AddSlopperDatabase().AddFfmpegServices().AddAi();
builder.Services.AddOptions<TestOption>().BindConfiguration("");
using var app = builder.Build();
await app.StartAsync();
var options = app.Services.GetRequiredService<IOptionsMonitor<TestOption>>();
var logger = app.Services.GetRequiredService<ILogger<Program>>();
var lifetime = app.Services.GetRequiredService<IHostApplicationLifetime>();
using var scope = app.Services.CreateScope();
var timeProvider = scope.ServiceProvider.GetRequiredService<TimeProvider>();
var clipSelector = scope.ServiceProvider.GetRequiredService<ClipSelector>();
var clipDescriber = scope.ServiceProvider.GetRequiredService<ClipDescriber>();
var clipExtractor = scope.ServiceProvider.GetRequiredService<IClipExtractor>();
var clipRepository = scope.ServiceProvider.GetRequiredService<IClipRepository>();
do
{
logger.LogInformation("Consul: {Description}", options.CurrentValue.ConsulOptionForSlopper);
await Task.Delay(TimeSpan.FromSeconds(5));
} while (!lifetime.ApplicationStopping.IsCancellationRequested);
var utcNow = timeProvider.GetUtcNow();
var media = new MediaItem(Guid.CreateVersion7(utcNow), args[0], new Subtitles.Embedded(2));
var (start, duration) = await clipSelector.PickClip(media, CancellationToken.None);
class TestOption
var clipId = Guid.CreateVersion7(utcNow);
var clipPath = Path.Join(args[1], $"{clipId}.mp4");
var descriptionTask = clipDescriber.DescribeClip(media, start, duration, CancellationToken.None);
await clipExtractor.ExtractClip(media, start, duration, clipPath, CancellationToken.None);
var description = await descriptionTask;
var clip = new Clip(clipId, media.Id, clipPath, start, duration, utcNow, description.Caption)
{
public required string ConsulOptionForSlopper { get; set; }
}
Tags = description.Tags,
};
await clipRepository.Save(clip, CancellationToken.None);
await app.StopAsync();
src/Cli/Properties/launchSettings.json +1 -1
diff --git a/src/Cli/Properties/launchSettings.json b/src/Cli/Properties/launchSettings.json
index c2230de..e91d42b 100644
@@ -8,7 +8,7 @@
"environmentVariables": {
"DOTNET_ENVIRONMENT": "Development"
},
"commandLineArgs": "D:/slopper/media/S01E01.mkv"
"commandLineArgs": "D:/slopper/media/S01E01.mkv D:/slopper/media/"
}
}
}
src/Cli/appsettings.Development.json +1 -1
diff --git a/src/Cli/appsettings.Development.json b/src/Cli/appsettings.Development.json
index 75d9278..204e8d2 100644
@@ -6,7 +6,7 @@
}
},
"ClipDescriber": {
"Prompt": "Give one short sentence caption of these frames from a video clip, optimized for engagement on TikTok and Instagram Reels. Only ever give on sentence, no alternatives, not formatting, just a plain text caption, optimized for engagement."
"Prompt": "Give one short sentence caption and tags for the attached frames from a video clip, optimized for engagement on TikTok and Instagram Reels. Answer with JSON, one string field for `caption` and an array of plain strings for `tags` with at least 2 values. Make the caption one sentence only optimized for engagement, and the tags one word or camel case without the hashtag symbol."
},
"ClipSelector": {
"ClippableQuotes": [
src/Domain/Clip.cs +6 -2
diff --git a/src/Domain/Clip.cs b/src/Domain/Clip.cs
index ed02efb..d648c0e 100644
@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
namespace Slopper.Domain;
@@ -9,5 +10,8 @@ public sealed record Clip(
TimeSpan Start,
TimeSpan Duration,
DateTimeOffset CreatedAt,
string? Description
);
string? Caption
)
{
public required IReadOnlySet<Tag> Tags { get; init; }
}
src/Domain/ClipGenerator.cs +8 -2
diff --git a/src/Domain/ClipGenerator.cs b/src/Domain/ClipGenerator.cs
index 19ae6ba..09915d8 100644
@@ -5,6 +5,7 @@ using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using Slopper.Domain.Describer;
namespace Slopper.Domain;
@@ -26,14 +27,19 @@ public sealed class ClipGenerator(
var (start, duration) = await clipSelector.PickClip(media, cancellationToken);
var utcNow = timeProvider.GetUtcNow();
var clipId = Guid.CreateVersion7();
var clipId = Guid.CreateVersion7(utcNow);
var clipPath = Path.Join(clipDirectory, $"{clipId}.mp4");
var descriptionTask = clipDescriber.DescribeClip(media, start, duration, cancellationToken);
await clipExtractor.ExtractClip(media, start, duration, clipPath, cancellationToken);
var clip = new Clip(clipId, media.Id, clipPath, start, duration, utcNow, await descriptionTask);
var description = await descriptionTask;
var clip = new Clip(clipId, media.Id, clipPath, start, duration, utcNow, description.Caption)
{
Tags = description.Tags,
};
await clipRepository.Save(clip, cancellationToken);
return clip;
src/Domain/Describer/AiDtos.cs +10 -0
diff --git a/src/Domain/Describer/AiDtos.cs b/src/Domain/Describer/AiDtos.cs
new file mode 100644
index 0000000..fce2817
@@ -0,0 +1,10 @@
using System.Collections.Immutable;
using System.Text.Json.Serialization;
namespace Slopper.Domain.Describer;
[JsonSerializable(typeof(ClipDescriptionAiDto))]
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)]
internal sealed partial class AiDtoSerializerContext : JsonSerializerContext;
internal sealed record ClipDescriptionAiDto(string Caption, IImmutableSet<string> Tags);
src/Domain/Describer/ClipDescriber.cs +14 -29
diff --git a/src/Domain/ClipDescriber.cs b/src/Domain/Describer/ClipDescriber.cs
similarity index 68%
rename from src/Domain/ClipDescriber.cs
rename to src/Domain/Describer/ClipDescriber.cs
index 933cd9a..2c7e6e0 100644
@@ -1,14 +1,13 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
namespace Slopper.Domain;
namespace Slopper.Domain.Describer;
public sealed class ClipDescriber(
IFrameExtractor frameExtractor,
@@ -17,7 +16,7 @@ public sealed class ClipDescriber(
IOptionsMonitor<ClipDescriberOptions> options
)
{
public async Task<string> DescribeClip(
public async Task<ClipDescription> DescribeClip(
MediaItem media,
TimeSpan start,
TimeSpan duration,
@@ -44,33 +43,19 @@ public sealed class ClipDescriber(
var response = await chatClient.GetResponseAsync(
[new ChatMessage(ChatRole.User, contents)],
cancellationToken: cancellationToken
new()
{
ResponseFormat = ChatResponseFormat.ForJsonSchema<ClipDescriptionAiDto>(
AiDtoSerializerContext.Default.Options
),
},
cancellationToken
);
return response.Text;
}
}
public sealed class ClipDescriberOptions
{
[Required]
public required string Prompt { get; set; }
}
[OptionsValidator]
internal sealed partial class ClipDescriberOptionsValidator : IValidateOptions<ClipDescriberOptions>;
var result =
JsonSerializer.Deserialize(response.Text, AiDtoSerializerContext.Default.ClipDescriptionAiDto)
?? throw new Exception("Literal null response from description agent.");
public static class ClipDescriberServiceCollectionExtensions
{
extension(IServiceCollection services)
{
public IServiceCollection AddClipDescriber()
{
services.AddOptions<ClipDescriberOptions>().BindConfiguration("ClipDescriber").ValidateOnStart();
services.AddTransient<IValidateOptions<ClipDescriberOptions>, ClipDescriberOptionsValidator>();
services.AddTransient<ClipDescriber>();
return services;
}
return new(result.Caption, result.Tags.Select(t => new Tag(t)).ToHashSet());
}
}
src/Domain/Describer/ClipDescriberOptions.cs +13 -0
diff --git a/src/Domain/Describer/ClipDescriberOptions.cs b/src/Domain/Describer/ClipDescriberOptions.cs
new file mode 100644
index 0000000..37529ee
@@ -0,0 +1,13 @@
using System.ComponentModel.DataAnnotations;
using Microsoft.Extensions.Options;
namespace Slopper.Domain.Describer;
public sealed class ClipDescriberOptions
{
[Required]
public required string Prompt { get; set; }
}
[OptionsValidator]
internal sealed partial class ClipDescriberOptionsValidator : IValidateOptions<ClipDescriberOptions>;
src/Domain/Describer/ClipDescriberServiceCollectionExtensions.cs +19 -0
diff --git a/src/Domain/Describer/ClipDescriberServiceCollectionExtensions.cs b/src/Domain/Describer/ClipDescriberServiceCollectionExtensions.cs
new file mode 100644
index 0000000..5bec347
@@ -0,0 +1,19 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
namespace Slopper.Domain.Describer;
public static class ClipDescriberServiceCollectionExtensions
{
extension(IServiceCollection services)
{
public IServiceCollection AddClipDescriber()
{
services.AddOptions<ClipDescriberOptions>().BindConfiguration("ClipDescriber").ValidateOnStart();
services.AddTransient<IValidateOptions<ClipDescriberOptions>, ClipDescriberOptionsValidator>();
services.AddTransient<ClipDescriber>();
return services;
}
}
}
src/Domain/Describer/ClipDescription.cs +5 -0
diff --git a/src/Domain/Describer/ClipDescription.cs b/src/Domain/Describer/ClipDescription.cs
new file mode 100644
index 0000000..13efbfd
@@ -0,0 +1,5 @@
using System.Collections.Generic;
namespace Slopper.Domain;
public sealed record ClipDescription(string Caption, IReadOnlySet<Tag> Tags);
src/Domain/Tag.cs +3 -0
diff --git a/src/Domain/Tag.cs b/src/Domain/Tag.cs
new file mode 100644
index 0000000..5fdd632
@@ -0,0 +1,3 @@
namespace Slopper.Domain;
public sealed record Tag(string Value);
src/Frontend/src/Clip.ts +2 -1
diff --git a/src/Frontend/src/Clip.ts b/src/Frontend/src/Clip.ts
index 3763d97..0e87b16 100644
@@ -2,5 +2,6 @@ export interface Clip {
id: string;
duration: string;
createdAt: Date;
description: string | null;
caption: string | null;
tags: Array<string>;
}
src/Frontend/src/components/ClipFeed.vue +0 -1
diff --git a/src/Frontend/src/components/ClipFeed.vue b/src/Frontend/src/components/ClipFeed.vue
index 3538f88..5754ccf 100644
@@ -26,7 +26,6 @@ async function fetchClips(after?: string) {
if (newClips.length === 0) {
exhausted.value = true;
} else {
console.log(newClips);
clips.value.push(...newClips);
}
} finally {
src/Frontend/src/components/ClipItem.vue +10 -3
diff --git a/src/Frontend/src/components/ClipItem.vue b/src/Frontend/src/components/ClipItem.vue
index a0c94ce..de1986d 100644
@@ -36,8 +36,11 @@ onUnmounted(() => observer?.disconnect());
<template>
<div class="clip">
<video ref="videoEl" :src="`/api/clips/${clip.id}/stream`" loop playsinline muted></video>
<div v-if="clip.description" class="description">
<p>{{ clip.description }}</p>
<div v-if="clip.caption" class="caption">
<p>
<span>{{ clip.caption }}</span>
<span v-for="tag in clip.tags" class="tag">#{{ tag }}</span>
</p>
</div>
</div>
</template>
@@ -56,7 +59,7 @@ video {
object-fit: cover;
}
.description {
.caption {
position: absolute;
bottom: 0;
left: 0;
@@ -66,4 +69,8 @@ video {
color: #ffffff;
line-height: 1.4;
}
.tag {
margin-inline-start: 1ch;
}
</style>
src/Infrastructure/Database/Slopper/ClipRepository.cs +2 -2
diff --git a/src/Infrastructure/Database/Slopper/ClipRepository.cs b/src/Infrastructure/Database/Slopper/ClipRepository.cs
index 2d06b99..b6d0c5a 100644
@@ -12,7 +12,7 @@ internal sealed class ClipRepository(SlopperDbContext slopperContext) : IClipRep
{
public IAsyncEnumerable<Clip> GetLatest(Guid? after, int limit, CancellationToken cancellationToken)
{
IQueryable<Clip> query = slopperContext.Clips.OrderByDescending(c => c.Id);
IQueryable<Clip> query = slopperContext.Clips.Include(c => c.Tags).OrderByDescending(c => c.Id);
if (after is Guid afterKey)
{
query = query.Where(q => q.Id < afterKey);
@@ -21,7 +21,7 @@ internal sealed class ClipRepository(SlopperDbContext slopperContext) : IClipRep
}
public async Task<Clip?> Get(Guid id, CancellationToken cancellationToken) =>
await slopperContext.Clips.AsNoTracking().SingleOrDefaultAsync(c => c.Id == id, cancellationToken);
await slopperContext.Clips.Include(c => c.Tags).SingleOrDefaultAsync(c => c.Id == id, cancellationToken);
public async Task Save(Clip clip, CancellationToken cancellationToken)
{
src/Infrastructure/Database/Slopper/Migrations/20260511111909_AddTags.Designer.cs +76 -0
diff --git a/src/Infrastructure/Database/Slopper/Migrations/20260511111909_AddTags.Designer.cs b/src/Infrastructure/Database/Slopper/Migrations/20260511111909_AddTags.Designer.cs
new file mode 100644
index 0000000..b5ff2aa
@@ -0,0 +1,76 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Slopper.Infrastructure.Database.Slopper;
#nullable disable
namespace Slopper.Infrastructure.Database.Slopper.Migrations
{
[DbContext(typeof(SlopperDbContext))]
[Migration("20260511111909_AddTags")]
partial class AddTags
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder.HasAnnotation("ProductVersion", "10.0.7");
modelBuilder.Entity("Slopper.Domain.Clip", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("TEXT");
b.Property<string>("Caption")
.HasColumnType("TEXT");
b.Property<DateTimeOffset>("CreatedAt")
.HasColumnType("TEXT");
b.Property<TimeSpan>("Duration")
.HasColumnType("TEXT");
b.Property<Guid>("MediaItemId")
.HasColumnType("TEXT");
b.Property<string>("Path")
.IsRequired()
.HasColumnType("TEXT");
b.Property<TimeSpan>("Start")
.HasColumnType("TEXT");
b.HasKey("Id");
b.ToTable("Clips");
});
modelBuilder.Entity("Slopper.Domain.Clip", b =>
{
b.OwnsMany("Slopper.Domain.Tag", "Tags", b1 =>
{
b1.Property<Guid>("ClipId")
.HasColumnType("TEXT");
b1.Property<string>("Value")
.HasColumnType("TEXT");
b1.HasKey("ClipId", "Value");
b1.ToTable("Tag");
b1.WithOwner()
.HasForeignKey("ClipId");
});
b.Navigation("Tags");
});
#pragma warning restore 612, 618
}
}
}
src/Infrastructure/Database/Slopper/Migrations/20260511111909_AddTags.cs +45 -0
diff --git a/src/Infrastructure/Database/Slopper/Migrations/20260511111909_AddTags.cs b/src/Infrastructure/Database/Slopper/Migrations/20260511111909_AddTags.cs
new file mode 100644
index 0000000..108c95b
@@ -0,0 +1,45 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Slopper.Infrastructure.Database.Slopper.Migrations
{
/// <inheritdoc />
public partial class AddTags : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.RenameColumn(name: "Description", table: "Clips", newName: "Caption");
migrationBuilder.CreateTable(
name: "Tag",
columns: table => new
{
Value = table.Column<string>(type: "TEXT", nullable: false),
ClipId = table.Column<Guid>(type: "TEXT", nullable: false),
},
constraints: table =>
{
table.PrimaryKey("PK_Tag", x => new { x.ClipId, x.Value });
table.ForeignKey(
name: "FK_Tag_Clips_ClipId",
column: x => x.ClipId,
principalTable: "Clips",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade
);
}
);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(name: "Tag");
migrationBuilder.RenameColumn(name: "Caption", table: "Clips", newName: "Description");
}
}
}
src/Infrastructure/Database/Slopper/Migrations/SlopperDbContextModelSnapshot.cs +23 -2
diff --git a/src/Infrastructure/Database/Slopper/Migrations/SlopperDbContextModelSnapshot.cs b/src/Infrastructure/Database/Slopper/Migrations/SlopperDbContextModelSnapshot.cs
index c7822b7..7d416b2 100644
@@ -23,10 +23,10 @@ namespace Slopper.Infrastructure.Database.Slopper.Migrations
.ValueGeneratedOnAdd()
.HasColumnType("TEXT");
b.Property<DateTimeOffset>("CreatedAt")
b.Property<string>("Caption")
.HasColumnType("TEXT");
b.Property<string>("Description")
b.Property<DateTimeOffset>("CreatedAt")
.HasColumnType("TEXT");
b.Property<TimeSpan>("Duration")
@@ -46,6 +46,27 @@ namespace Slopper.Infrastructure.Database.Slopper.Migrations
b.ToTable("Clips");
});
modelBuilder.Entity("Slopper.Domain.Clip", b =>
{
b.OwnsMany("Slopper.Domain.Tag", "Tags", b1 =>
{
b1.Property<Guid>("ClipId")
.HasColumnType("TEXT");
b1.Property<string>("Value")
.HasColumnType("TEXT");
b1.HasKey("ClipId", "Value");
b1.ToTable("Tag");
b1.WithOwner()
.HasForeignKey("ClipId");
});
b.Navigation("Tags");
});
#pragma warning restore 612, 618
}
}
src/Infrastructure/Database/Slopper/SlopperDbContext.cs +2 -1
diff --git a/src/Infrastructure/Database/Slopper/SlopperDbContext.cs b/src/Infrastructure/Database/Slopper/SlopperDbContext.cs
index a3af510..0ea8f8d 100644
@@ -14,6 +14,7 @@ internal sealed class SlopperDbContext(DbContextOptions options) : DbContext(opt
clipBuilder.Property(c => c.MediaItemId);
clipBuilder.Property(c => c.Path);
clipBuilder.Property(c => c.CreatedAt);
clipBuilder.Property(c => c.Description);
clipBuilder.Property(c => c.Caption);
clipBuilder.OwnsMany(c => c.Tags).HasKey("ClipId", "Value");
}
}