From d025031a2fb5a3e534ae794e214b5b8740a7fc32 Mon Sep 17 00:00:00 2001 From: Samuele Lorefice Date: Tue, 23 Dec 2025 22:45:30 +0100 Subject: [PATCH] Extensive testing methodology rework --- EncodingSampleTest/Program.cs | 232 +++++++++++++++++++++++++--------- 1 file changed, 174 insertions(+), 58 deletions(-) diff --git a/EncodingSampleTest/Program.cs b/EncodingSampleTest/Program.cs index 75ac9d5..4282e07 100644 --- a/EncodingSampleTest/Program.cs +++ b/EncodingSampleTest/Program.cs @@ -1,84 +1,139 @@ -using System; +using System.Diagnostics; +using System.Text; using EncodingSampleTest; using FFMpegCore; using FFMpegCore.Enums; +const int MAX_JOBS = 8; string ffmpegPath = Path.Combine(Environment.CurrentDirectory, "vendor"); string tempPath = Path.Combine(Environment.CurrentDirectory, "tmp"); string outputDir = Path.Combine(Environment.CurrentDirectory, "output"); +const string inputFile = "testVid.mp4"; Directory.CreateDirectory(outputDir); Directory.CreateDirectory(tempPath); -const string inputFile = "testVid.mp4"; -const int MAX_JOBS = 3; +const string tableHeader = "Width\tHeight\tPreset\tCRF\tSize Ratio\tEncode Time\tSpeed\tVMAF\n"; +StringBuilder AV1GPUStats = new(tableHeader); +StringBuilder HEVCGPUStats = new(tableHeader); +StringBuilder H264GPUStats = new(tableHeader); +StringBuilder AV1CPUStats = new(tableHeader); +StringBuilder HEVCCPUStats = new(tableHeader); +StringBuilder H264CPUStats = new(tableHeader); GlobalFFOptions.Configure(options => { options.BinaryFolder = ffmpegPath; options.TemporaryFilesFolder = tempPath; + options.UseCache = true; }); if(!File.Exists(inputFile)) return; +var originalFile = new FileInfo(inputFile); var mediaInfo = FFProbe.Analyse(inputFile); if (mediaInfo.PrimaryVideoStream == null) { Console.WriteLine("No video stream found."); return; } -var W = mediaInfo.PrimaryVideoStream.Width / 2; -var H = mediaInfo.PrimaryVideoStream.Height / 2; +var W = mediaInfo.PrimaryVideoStream.Width; +var H = mediaInfo.PrimaryVideoStream.Height; -Queue GpuEncodingTasks = new(); +List GpuTasks = new(); List CpuTasks = new(); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); CancellationToken cancellationToken = cancellationTokenSource.Token; -for (int cq = 24; cq <= 34; cq += 2) { - GpuEncodingTasks.Enqueue(Encode(Encoder.AV1_NVENC, 1920, 1080, cq, cancellationToken)); - GpuEncodingTasks.Enqueue(Encode(Encoder.HEVC_NVENC, 1920, 1080, cq, cancellationToken)); - GpuEncodingTasks.Enqueue(Encode(Encoder.H264_NVENC, 1920, 1080, cq, cancellationToken)); - CpuTasks.Add(Encode(Encoder.AV1_CPU, 1920, 1080, cq, cancellationToken)); - CpuTasks.Add(Encode(Encoder.HEVC_CPU, 1920, 1080, cq, cancellationToken)); - CpuTasks.Add(Encode(Encoder.H264_CPU, 1920, 1080, cq, cancellationToken)); -} -//Run all GPU tasks sequentially -while (GpuEncodingTasks.Count > 0) { - var task = GpuEncodingTasks.Dequeue(); - try { - task.RunSynchronously(); - } catch (Exception ex) { - Console.WriteLine($"Error during GPU encoding: {ex.Message}"); +Console.WriteLine("Generating Tasks:"); +for (int cq = 24; cq <= 34; cq += 3) { + for (int preset = 12; preset <= 18; preset+=3) { + GpuTasks.Add(Encode(Encoder.AV1_NVENC, 1280, 720, cq, preset, cancellationToken)); + GpuTasks.Add(Encode(Encoder.AV1_NVENC, 1920, 1080, cq, preset, cancellationToken)); + GpuTasks.Add(Encode(Encoder.AV1_NVENC, W, H, cq, preset, cancellationToken)); + + GpuTasks.Add(Encode(Encoder.HEVC_NVENC, 1280, 720, cq, preset, cancellationToken)); + GpuTasks.Add(Encode(Encoder.HEVC_NVENC, 1920, 1080, cq, preset, cancellationToken)); + GpuTasks.Add(Encode(Encoder.HEVC_NVENC, W, H, cq, preset, cancellationToken)); + + GpuTasks.Add(Encode(Encoder.H264_NVENC, 1280, 720, cq, preset, cancellationToken)); + GpuTasks.Add(Encode(Encoder.H264_NVENC, 1920, 1080, cq, preset, cancellationToken)); + GpuTasks.Add(Encode(Encoder.H264_NVENC, W, H, cq, preset, cancellationToken)); + } + for (int preset = 4; preset <= 7; preset++) { + CpuTasks.Add(Encode(Encoder.AV1_CPU, 1280, 720, cq, preset, cancellationToken)); + CpuTasks.Add(Encode(Encoder.AV1_CPU, 1920, 1080, cq, preset, cancellationToken)); + CpuTasks.Add(Encode(Encoder.AV1_CPU, W, H, cq, preset, cancellationToken)); + } + for (int preset = 3; preset <= 5; preset++) { + CpuTasks.Add(Encode(Encoder.HEVC_CPU, 1280, 720, cq, preset, cancellationToken)); + CpuTasks.Add(Encode(Encoder.HEVC_CPU, 1920, 1080, cq, preset, cancellationToken)); + CpuTasks.Add(Encode(Encoder.HEVC_CPU, W, H, cq, preset, cancellationToken)); + + CpuTasks.Add(Encode(Encoder.H264_CPU, 1280, 720, cq, preset, cancellationToken)); + CpuTasks.Add(Encode(Encoder.H264_CPU, 1920, 1080, cq, preset, cancellationToken)); + CpuTasks.Add(Encode(Encoder.H264_CPU, W, H, cq, preset, cancellationToken)); } } -//Run all CPU tasks in parallel -try { - while (CpuTasks.Count > 0) { - var runningTasks = CpuTasks.Take(MAX_JOBS).ToList(); - CpuTasks = CpuTasks.Skip(MAX_JOBS).ToList(); - await Task.WhenAll(runningTasks); +//Run all GPU tasks sequentially +while (GpuTasks.Count(x => x.IsCompleted) < GpuTasks.Count) { + GpuTasks.First(x => !x.IsCompleted).RunSynchronously(); + Console.WriteLine(GpuTasks.Count(x => !x.IsCompleted) + " GPU tasks remaining..."); +} + +//Run all CPU tasks sequentially +while (CpuTasks.Count(x => x.IsCompleted) < CpuTasks.Count) { + CpuTasks.First(x => !x.IsCompleted).RunSynchronously(); + Console.WriteLine(CpuTasks.Count(x => !x.IsCompleted) + " CPU tasks remaining..."); +} + +//Calculate VMAF score in parallel +Dictionary> vmafResults = new(); +var tasks = GpuTasks.Concat(CpuTasks).OfType>().Select(x => x.Result).ToList(); +while (tasks.Count > 0) { + var vmafTasks = tasks.Take(MAX_JOBS); + tasks = tasks.Skip(MAX_JOBS).ToList(); + + foreach (var task in vmafTasks) + vmafResults.Add(task, ComputeVMAFScore(inputFile, task.Filename)); + + Task.WhenAll(vmafResults.Values).Wait(); + Console.WriteLine(tasks.Count + " VMAF tasks remaining..."); +} + +Console.WriteLine("Computing Results:"); +foreach (var result in vmafResults) { + //"Width\tHeight\tPreset\tCRF\tSize Ratio\tEncode Time\tSpeed\n"; + var es = result.Key; + string line = $"{es.Width}\t{es.Height}\t{es.preset}\t{es.CRF}\t{es.SizeRatio:F3}\t{es.EncodeTime:g}\t"+ + $"{(es.EncodeTime.TotalSeconds / mediaInfo.Duration.TotalSeconds):F3}x\t{result.Value.Result:F2}\n"; + Console.WriteLine(line); + switch (es.EncoderType) { + case Encoder.AV1_NVENC: AV1GPUStats.Append(line); break; + case Encoder.HEVC_NVENC: HEVCGPUStats.Append(line); break; + case Encoder.H264_NVENC: H264GPUStats.Append(line); break; + case Encoder.AV1_CPU: AV1CPUStats.Append(line); break; + case Encoder.HEVC_CPU: HEVCCPUStats.Append(line); break; + case Encoder.H264_CPU: H264CPUStats.Append(line); break; } - await Task.WhenAll(CpuTasks); -} -catch (Exception ex) { - Console.WriteLine($"Error during CPU encoding: {ex.Message}"); } +File.WriteAllText(Path.Combine(outputDir, "AV1_NVENC_Stats.tsv"), AV1GPUStats.ToString()); +File.WriteAllText(Path.Combine(outputDir, "HEVC_NVENC_Stats.tsv"), HEVCGPUStats.ToString()); +File.WriteAllText(Path.Combine(outputDir, "H264_NVENC_Stats.tsv"), H264GPUStats.ToString()); +File.WriteAllText(Path.Combine(outputDir, "AV1_CPU_Stats.tsv"), AV1CPUStats.ToString()); +File.WriteAllText(Path.Combine(outputDir, "HEVC_CPU_Stats.tsv"), HEVCCPUStats.ToString()); +File.WriteAllText(Path.Combine(outputDir, "H264_CPU_Stats.tsv"), H264CPUStats.ToString()); + cancellationTokenSource.Cancel(); return; - -Task Encode(Encoder encoder, int W = -1, int H = -1, int CQ = 23, CancellationToken cancellationToken = default) { - var outputFile = Path.Combine(outputDir, $"output_{encoder}-{W}x{H}_cq{CQ}.mp4"); - if (File.Exists(outputFile) && new FileInfo(outputFile).Length > 0) { - Console.WriteLine($"Skipping existing file {outputFile}"); - return Task.CompletedTask; - } +Task Encode(Encoder encoder, int W = -1, int H = -1, int CQ = 23, int preset = 0, CancellationToken cancellationToken = default) { + var outputFile = Path.Combine(outputDir, $"output_{encoder}-{W}x{H}_p{preset}_cq{CQ}.mp4"); FFMpegArgumentProcessor ffmpegArgs = encoder switch { - Encoder.AV1_NVENC => AV1Encode( outputFile, W, H, CQ), - Encoder.HEVC_NVENC => HEVCEncode( outputFile, W, H, CQ), - Encoder.H264_NVENC => H264Encode( outputFile, W, H, CQ), - Encoder.AV1_CPU => AV1EncodeCPU( outputFile, W, H, CQ), - Encoder.HEVC_CPU => HEVCEncodeCPU( outputFile, W, H, CQ), - Encoder.H264_CPU => H264EncodeCPU( outputFile, W, H, CQ), + Encoder.AV1_NVENC => AV1Encode( outputFile, W, H, CQ, preset), + Encoder.HEVC_NVENC => HEVCEncode( outputFile, W, H, CQ, preset), + Encoder.H264_NVENC => H264Encode( outputFile, W, H, CQ, preset), + Encoder.AV1_CPU => AV1EncodeCPU( outputFile, W, H, CQ, preset), + Encoder.HEVC_CPU => HEVCEncodeCPU( outputFile, W, H, CQ, preset), + Encoder.H264_CPU => H264EncodeCPU( outputFile, W, H, CQ, preset), _ => throw new ArgumentOutOfRangeException(nameof(encoder), encoder, null) }; @@ -87,20 +142,33 @@ Task Encode(Encoder encoder, int W = -1, int H = -1, int CQ = 23, CancellationTo .CancellableThrough(cancellationToken); Console.WriteLine(ffmpegArgs.Arguments); - return new Task(_ => { + return new Task(_ => { Console.WriteLine("Starting encoding: " + outputFile); - return ffmpegArgs.ProcessAsynchronously().Result; + var sw = Stopwatch.StartNew(); + ffmpegArgs.ProcessSynchronously(); + sw.Stop(); + var fileSize = new FileInfo(outputFile).Length; + return new EncodeStats() { + Filename = outputFile, + EncoderType = encoder, + Width = W, + Height = H, + CRF = CQ, + preset = preset, + SizeRatio = originalFile.Length / (float)fileSize, + EncodeTime = sw.Elapsed + }; }, cancellationToken); } -FFMpegArgumentProcessor AV1Encode(string filePath, int W = -1, int H = -1, int CQ = 23) { // AV1 is visually lossless at CQ 23 +FFMpegArgumentProcessor AV1Encode(string filePath, int W = -1, int H = -1, int CQ = 23, int preset = 0) { // AV1 is visually lossless at CQ 23 return FFMpegArguments .FromFileInput(inputFile, false, options => options.WithHardwareAcceleration()) .OutputToFile(filePath, true, options => options .CopyChannel(Channel.Audio) .CopyChannel(Channel.Subtitle) .WithVideoCodec("av1_nvenc") - .WithArgument(new NvencSpeedPreset(NvencSpeed.p7)) + .WithArgument(new NvencSpeedPreset((NvencSpeed)preset)) .WithArgument(new NvencTuneArgument(NvencTune.hq)) .WithArgument(new NvencHighBitDepthArgument(true)) .WithCustomArgument("-rc vbr") @@ -110,14 +178,14 @@ FFMpegArgumentProcessor AV1Encode(string filePath, int W = -1, int H = -1, int C ); } -FFMpegArgumentProcessor HEVCEncode(string filePath, int W = -1, int H = -1, int CQ = 23) { +FFMpegArgumentProcessor HEVCEncode(string filePath, int W = -1, int H = -1, int CQ = 23, int preset = 0) { return FFMpegArguments .FromFileInput(inputFile, true, options => options.WithHardwareAcceleration()) .OutputToFile(filePath, true, options => options .CopyChannel(Channel.Audio) .CopyChannel(Channel.Subtitle) .WithVideoCodec("hevc_nvenc") - .WithArgument(new NvencSpeedPreset(NvencSpeed.p7)) + .WithArgument(new NvencSpeedPreset((NvencSpeed)preset)) .WithArgument(new NvencTuneArgument(NvencTune.hq)) .WithCustomArgument("-rc vbr_hq") .WithCustomArgument($"-cq {(byte)CQ}") @@ -126,14 +194,14 @@ FFMpegArgumentProcessor HEVCEncode(string filePath, int W = -1, int H = -1, int ); } -FFMpegArgumentProcessor H264Encode(string filePath, int W = -1, int H = -1, int CQ = 23) { +FFMpegArgumentProcessor H264Encode(string filePath, int W = -1, int H = -1, int CQ = 23, int preset = 0) { return FFMpegArguments .FromFileInput(inputFile, true, options => options.WithHardwareAcceleration()) .OutputToFile(filePath, true, options => options .CopyChannel(Channel.Audio) .CopyChannel(Channel.Subtitle) .WithVideoCodec("h264_nvenc") - .WithArgument(new NvencSpeedPreset(NvencSpeed.p7)) + .WithArgument(new NvencSpeedPreset((NvencSpeed)preset)) .WithArgument(new NvencTuneArgument(NvencTune.hq)) .WithCustomArgument("-rc vbr_hq") .WithCustomArgument($"-cq {(byte)CQ}") // approximate CRF to CQ @@ -142,14 +210,14 @@ FFMpegArgumentProcessor H264Encode(string filePath, int W = -1, int H = -1, int ); } -FFMpegArgumentProcessor AV1EncodeCPU(string filePath, int W = -1, int H = -1, int CQ = 23) { +FFMpegArgumentProcessor AV1EncodeCPU(string filePath, int W = -1, int H = -1, int CQ = 23, int preset = 0) { return FFMpegArguments .FromFileInput(inputFile) .OutputToFile(filePath, true, options => options .CopyChannel(Channel.Audio) .CopyChannel(Channel.Subtitle) .WithVideoCodec("libsvtav1") - .WithCustomArgument($"-preset 3") + .WithCustomArgument($"-preset {preset}") .WithConstantRateFactor(CQ) .WithCustomArgument("-svtav1-params tune=0") .WithVideoFilters(filterOptions => {if (W > 0 && H > 0) filterOptions.Scale(W, H);}) @@ -157,14 +225,14 @@ FFMpegArgumentProcessor AV1EncodeCPU(string filePath, int W = -1, int H = -1, in ); } -FFMpegArgumentProcessor HEVCEncodeCPU(string filePath, int W = -1, int H = -1, int CQ = 23) { +FFMpegArgumentProcessor HEVCEncodeCPU(string filePath, int W = -1, int H = -1, int CQ = 23, int preset = 0) { return FFMpegArguments .FromFileInput(inputFile) .OutputToFile(filePath, true, options => options .CopyChannel(Channel.Audio) .CopyChannel(Channel.Subtitle) .WithVideoCodec(VideoCodec.LibX265) - .WithSpeedPreset(Speed.Slow) + .WithSpeedPreset(ToX264Speed(preset)) .WithConstantRateFactor(CQ) .WithCustomArgument("-x265-params profile=main10:high-tier=1:aq-mode=3:psy-rd=1:rc-lookahead=60:bframes=6") .WithVideoFilters(filterOptions => {if (W > 0 && H > 0) filterOptions.Scale(W, H);}) @@ -172,14 +240,14 @@ FFMpegArgumentProcessor HEVCEncodeCPU(string filePath, int W = -1, int H = -1, i ); } -FFMpegArgumentProcessor H264EncodeCPU(string filePath, int W = -1, int H = -1, int CQ = 23) { +FFMpegArgumentProcessor H264EncodeCPU(string filePath, int W = -1, int H = -1, int CQ = 23, int preset = 0) { return FFMpegArguments .FromFileInput(inputFile) .OutputToFile(filePath, true, options => options .CopyChannel(Channel.Audio) .CopyChannel(Channel.Subtitle) .WithVideoCodec(VideoCodec.LibX264) - .WithSpeedPreset(Speed.Slow) + .WithSpeedPreset(ToX264Speed(preset)) .WithConstantRateFactor(CQ) .WithCustomArgument("-x265-params profile=main10:high-tier=1:aq-mode=3:psy-rd=1:rc-lookahead=60:bframes=6") .WithVideoFilters(filterOptions => {if (W > 0 && H > 0) filterOptions.Scale(W, H);}) @@ -187,6 +255,43 @@ FFMpegArgumentProcessor H264EncodeCPU(string filePath, int W = -1, int H = -1, i ); } +async Task ComputeVMAFScore(string referenceFile, string distortedFile) { + float VMAF = -1; + ProcessStartInfo startInfo = new ProcessStartInfo { + FileName = Path.Combine(ffmpegPath, "ffmpeg"), + Arguments = $"-i \"{distortedFile}\" -i \"{referenceFile}\" -lavfi libvmaf -f null -", + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true + }; + using Process ffmpeg = Process.Start(startInfo)!; + string output = ffmpeg.StandardOutput.ReadToEnd(); + await ffmpeg.WaitForExitAsync(); + // Parse VMAF score from output + string marker = "VMAF score: "; + int index = output.LastIndexOf(marker, StringComparison.Ordinal); + if (index != -1) { + int start = index + marker.Length; + int end = output.IndexOf("\n", start, StringComparison.Ordinal); + string scoreStr = output.Substring(start, end - start).Trim(); + if (float.TryParse(scoreStr, out float score)) { + VMAF = score; + } + } + return VMAF; +} + +Speed ToX264Speed(int p) => p switch { + 1 => Speed.VerySlow, + 2 => Speed.Slower, + 3 => Speed.Slow, + 4 => Speed.Medium, + 5 => Speed.Fast, + 6 => Speed.Faster, + 7 => Speed.VeryFast, + _ => Speed.Medium +}; + public enum Encoder { AV1_NVENC, HEVC_NVENC, @@ -194,4 +299,15 @@ public enum Encoder { AV1_CPU, HEVC_CPU, H264_CPU +} + +record struct EncodeStats { + public string Filename; + public Encoder EncoderType; + public int Width; + public int Height; + public int CRF; + public int preset; + public float SizeRatio; + public TimeSpan EncodeTime; } \ No newline at end of file