Extensive testing methodology rework
This commit is contained in:
Samuele Lorefice
@@ -1,84 +1,139 @@
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.Text;
|
||||
using EncodingSampleTest;
|
||||
using FFMpegCore;
|
||||
using FFMpegCore.Enums;
|
||||
|
||||
const int MAX_JOBS = 8;
|
||||
string ffmpegPath = Path.Combine(Environment.CurrentDirectory, "vendor");
|
||||
string tempPath = Path.Combine(Environment.CurrentDirectory, "tmp");
|
||||
string outputDir = Path.Combine(Environment.CurrentDirectory, "output");
|
||||
const string inputFile = "testVid.mp4";
|
||||
Directory.CreateDirectory(outputDir);
|
||||
Directory.CreateDirectory(tempPath);
|
||||
const string inputFile = "testVid.mp4";
|
||||
const int MAX_JOBS = 3;
|
||||
const string tableHeader = "Width\tHeight\tPreset\tCRF\tSize Ratio\tEncode Time\tSpeed\tVMAF\n";
|
||||
StringBuilder AV1GPUStats = new(tableHeader);
|
||||
StringBuilder HEVCGPUStats = new(tableHeader);
|
||||
StringBuilder H264GPUStats = new(tableHeader);
|
||||
StringBuilder AV1CPUStats = new(tableHeader);
|
||||
StringBuilder HEVCCPUStats = new(tableHeader);
|
||||
StringBuilder H264CPUStats = new(tableHeader);
|
||||
|
||||
GlobalFFOptions.Configure(options => {
|
||||
options.BinaryFolder = ffmpegPath;
|
||||
options.TemporaryFilesFolder = tempPath;
|
||||
options.UseCache = true;
|
||||
});
|
||||
|
||||
if(!File.Exists(inputFile)) return;
|
||||
|
||||
var originalFile = new FileInfo(inputFile);
|
||||
var mediaInfo = FFProbe.Analyse(inputFile);
|
||||
if (mediaInfo.PrimaryVideoStream == null) {
|
||||
Console.WriteLine("No video stream found.");
|
||||
return;
|
||||
}
|
||||
var W = mediaInfo.PrimaryVideoStream.Width / 2;
|
||||
var H = mediaInfo.PrimaryVideoStream.Height / 2;
|
||||
var W = mediaInfo.PrimaryVideoStream.Width;
|
||||
var H = mediaInfo.PrimaryVideoStream.Height;
|
||||
|
||||
Queue<Task> GpuEncodingTasks = new();
|
||||
List<Task> GpuTasks = new();
|
||||
List<Task> CpuTasks = new();
|
||||
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
|
||||
|
||||
CancellationToken cancellationToken = cancellationTokenSource.Token;
|
||||
for (int cq = 24; cq <= 34; cq += 2) {
|
||||
GpuEncodingTasks.Enqueue(Encode(Encoder.AV1_NVENC, 1920, 1080, cq, cancellationToken));
|
||||
GpuEncodingTasks.Enqueue(Encode(Encoder.HEVC_NVENC, 1920, 1080, cq, cancellationToken));
|
||||
GpuEncodingTasks.Enqueue(Encode(Encoder.H264_NVENC, 1920, 1080, cq, cancellationToken));
|
||||
CpuTasks.Add(Encode(Encoder.AV1_CPU, 1920, 1080, cq, cancellationToken));
|
||||
CpuTasks.Add(Encode(Encoder.HEVC_CPU, 1920, 1080, cq, cancellationToken));
|
||||
CpuTasks.Add(Encode(Encoder.H264_CPU, 1920, 1080, cq, cancellationToken));
|
||||
}
|
||||
//Run all GPU tasks sequentially
|
||||
while (GpuEncodingTasks.Count > 0) {
|
||||
var task = GpuEncodingTasks.Dequeue();
|
||||
try {
|
||||
task.RunSynchronously();
|
||||
} catch (Exception ex) {
|
||||
Console.WriteLine($"Error during GPU encoding: {ex.Message}");
|
||||
Console.WriteLine("Generating Tasks:");
|
||||
for (int cq = 24; cq <= 34; cq += 3) {
|
||||
for (int preset = 12; preset <= 18; preset+=3) {
|
||||
GpuTasks.Add(Encode(Encoder.AV1_NVENC, 1280, 720, cq, preset, cancellationToken));
|
||||
GpuTasks.Add(Encode(Encoder.AV1_NVENC, 1920, 1080, cq, preset, cancellationToken));
|
||||
GpuTasks.Add(Encode(Encoder.AV1_NVENC, W, H, cq, preset, cancellationToken));
|
||||
|
||||
GpuTasks.Add(Encode(Encoder.HEVC_NVENC, 1280, 720, cq, preset, cancellationToken));
|
||||
GpuTasks.Add(Encode(Encoder.HEVC_NVENC, 1920, 1080, cq, preset, cancellationToken));
|
||||
GpuTasks.Add(Encode(Encoder.HEVC_NVENC, W, H, cq, preset, cancellationToken));
|
||||
|
||||
GpuTasks.Add(Encode(Encoder.H264_NVENC, 1280, 720, cq, preset, cancellationToken));
|
||||
GpuTasks.Add(Encode(Encoder.H264_NVENC, 1920, 1080, cq, preset, cancellationToken));
|
||||
GpuTasks.Add(Encode(Encoder.H264_NVENC, W, H, cq, preset, cancellationToken));
|
||||
}
|
||||
for (int preset = 4; preset <= 7; preset++) {
|
||||
CpuTasks.Add(Encode(Encoder.AV1_CPU, 1280, 720, cq, preset, cancellationToken));
|
||||
CpuTasks.Add(Encode(Encoder.AV1_CPU, 1920, 1080, cq, preset, cancellationToken));
|
||||
CpuTasks.Add(Encode(Encoder.AV1_CPU, W, H, cq, preset, cancellationToken));
|
||||
}
|
||||
for (int preset = 3; preset <= 5; preset++) {
|
||||
CpuTasks.Add(Encode(Encoder.HEVC_CPU, 1280, 720, cq, preset, cancellationToken));
|
||||
CpuTasks.Add(Encode(Encoder.HEVC_CPU, 1920, 1080, cq, preset, cancellationToken));
|
||||
CpuTasks.Add(Encode(Encoder.HEVC_CPU, W, H, cq, preset, cancellationToken));
|
||||
|
||||
CpuTasks.Add(Encode(Encoder.H264_CPU, 1280, 720, cq, preset, cancellationToken));
|
||||
CpuTasks.Add(Encode(Encoder.H264_CPU, 1920, 1080, cq, preset, cancellationToken));
|
||||
CpuTasks.Add(Encode(Encoder.H264_CPU, W, H, cq, preset, cancellationToken));
|
||||
}
|
||||
}
|
||||
|
||||
//Run all CPU tasks in parallel
|
||||
try {
|
||||
while (CpuTasks.Count > 0) {
|
||||
var runningTasks = CpuTasks.Take(MAX_JOBS).ToList();
|
||||
CpuTasks = CpuTasks.Skip(MAX_JOBS).ToList();
|
||||
await Task.WhenAll(runningTasks);
|
||||
//Run all GPU tasks sequentially
|
||||
while (GpuTasks.Count(x => x.IsCompleted) < GpuTasks.Count) {
|
||||
GpuTasks.First(x => !x.IsCompleted).RunSynchronously();
|
||||
Console.WriteLine(GpuTasks.Count(x => !x.IsCompleted) + " GPU tasks remaining...");
|
||||
}
|
||||
|
||||
//Run all CPU tasks sequentially
|
||||
while (CpuTasks.Count(x => x.IsCompleted) < CpuTasks.Count) {
|
||||
CpuTasks.First(x => !x.IsCompleted).RunSynchronously();
|
||||
Console.WriteLine(CpuTasks.Count(x => !x.IsCompleted) + " CPU tasks remaining...");
|
||||
}
|
||||
|
||||
//Calculate VMAF score in parallel
|
||||
Dictionary<EncodeStats,Task<float>> vmafResults = new();
|
||||
var tasks = GpuTasks.Concat(CpuTasks).OfType<Task<EncodeStats>>().Select(x => x.Result).ToList();
|
||||
while (tasks.Count > 0) {
|
||||
var vmafTasks = tasks.Take(MAX_JOBS);
|
||||
tasks = tasks.Skip(MAX_JOBS).ToList();
|
||||
|
||||
foreach (var task in vmafTasks)
|
||||
vmafResults.Add(task, ComputeVMAFScore(inputFile, task.Filename));
|
||||
|
||||
Task.WhenAll(vmafResults.Values).Wait();
|
||||
Console.WriteLine(tasks.Count + " VMAF tasks remaining...");
|
||||
}
|
||||
|
||||
Console.WriteLine("Computing Results:");
|
||||
foreach (var result in vmafResults) {
|
||||
//"Width\tHeight\tPreset\tCRF\tSize Ratio\tEncode Time\tSpeed\n";
|
||||
var es = result.Key;
|
||||
string line = $"{es.Width}\t{es.Height}\t{es.preset}\t{es.CRF}\t{es.SizeRatio:F3}\t{es.EncodeTime:g}\t"+
|
||||
$"{(es.EncodeTime.TotalSeconds / mediaInfo.Duration.TotalSeconds):F3}x\t{result.Value.Result:F2}\n";
|
||||
Console.WriteLine(line);
|
||||
switch (es.EncoderType) {
|
||||
case Encoder.AV1_NVENC: AV1GPUStats.Append(line); break;
|
||||
case Encoder.HEVC_NVENC: HEVCGPUStats.Append(line); break;
|
||||
case Encoder.H264_NVENC: H264GPUStats.Append(line); break;
|
||||
case Encoder.AV1_CPU: AV1CPUStats.Append(line); break;
|
||||
case Encoder.HEVC_CPU: HEVCCPUStats.Append(line); break;
|
||||
case Encoder.H264_CPU: H264CPUStats.Append(line); break;
|
||||
}
|
||||
await Task.WhenAll(CpuTasks);
|
||||
}
|
||||
catch (Exception ex) {
|
||||
Console.WriteLine($"Error during CPU encoding: {ex.Message}");
|
||||
}
|
||||
File.WriteAllText(Path.Combine(outputDir, "AV1_NVENC_Stats.tsv"), AV1GPUStats.ToString());
|
||||
File.WriteAllText(Path.Combine(outputDir, "HEVC_NVENC_Stats.tsv"), HEVCGPUStats.ToString());
|
||||
File.WriteAllText(Path.Combine(outputDir, "H264_NVENC_Stats.tsv"), H264GPUStats.ToString());
|
||||
File.WriteAllText(Path.Combine(outputDir, "AV1_CPU_Stats.tsv"), AV1CPUStats.ToString());
|
||||
File.WriteAllText(Path.Combine(outputDir, "HEVC_CPU_Stats.tsv"), HEVCCPUStats.ToString());
|
||||
File.WriteAllText(Path.Combine(outputDir, "H264_CPU_Stats.tsv"), H264CPUStats.ToString());
|
||||
|
||||
cancellationTokenSource.Cancel();
|
||||
return;
|
||||
|
||||
|
||||
Task Encode(Encoder encoder, int W = -1, int H = -1, int CQ = 23, CancellationToken cancellationToken = default) {
|
||||
var outputFile = Path.Combine(outputDir, $"output_{encoder}-{W}x{H}_cq{CQ}.mp4");
|
||||
if (File.Exists(outputFile) && new FileInfo(outputFile).Length > 0) {
|
||||
Console.WriteLine($"Skipping existing file {outputFile}");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
Task<EncodeStats> Encode(Encoder encoder, int W = -1, int H = -1, int CQ = 23, int preset = 0, CancellationToken cancellationToken = default) {
|
||||
var outputFile = Path.Combine(outputDir, $"output_{encoder}-{W}x{H}_p{preset}_cq{CQ}.mp4");
|
||||
|
||||
FFMpegArgumentProcessor ffmpegArgs = encoder switch {
|
||||
Encoder.AV1_NVENC => AV1Encode( outputFile, W, H, CQ),
|
||||
Encoder.HEVC_NVENC => HEVCEncode( outputFile, W, H, CQ),
|
||||
Encoder.H264_NVENC => H264Encode( outputFile, W, H, CQ),
|
||||
Encoder.AV1_CPU => AV1EncodeCPU( outputFile, W, H, CQ),
|
||||
Encoder.HEVC_CPU => HEVCEncodeCPU( outputFile, W, H, CQ),
|
||||
Encoder.H264_CPU => H264EncodeCPU( outputFile, W, H, CQ),
|
||||
Encoder.AV1_NVENC => AV1Encode( outputFile, W, H, CQ, preset),
|
||||
Encoder.HEVC_NVENC => HEVCEncode( outputFile, W, H, CQ, preset),
|
||||
Encoder.H264_NVENC => H264Encode( outputFile, W, H, CQ, preset),
|
||||
Encoder.AV1_CPU => AV1EncodeCPU( outputFile, W, H, CQ, preset),
|
||||
Encoder.HEVC_CPU => HEVCEncodeCPU( outputFile, W, H, CQ, preset),
|
||||
Encoder.H264_CPU => H264EncodeCPU( outputFile, W, H, CQ, preset),
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(encoder), encoder, null)
|
||||
};
|
||||
|
||||
@@ -87,20 +142,33 @@ Task Encode(Encoder encoder, int W = -1, int H = -1, int CQ = 23, CancellationTo
|
||||
.CancellableThrough(cancellationToken);
|
||||
|
||||
Console.WriteLine(ffmpegArgs.Arguments);
|
||||
return new Task<bool>(_ => {
|
||||
return new Task<EncodeStats>(_ => {
|
||||
Console.WriteLine("Starting encoding: " + outputFile);
|
||||
return ffmpegArgs.ProcessAsynchronously().Result;
|
||||
var sw = Stopwatch.StartNew();
|
||||
ffmpegArgs.ProcessSynchronously();
|
||||
sw.Stop();
|
||||
var fileSize = new FileInfo(outputFile).Length;
|
||||
return new EncodeStats() {
|
||||
Filename = outputFile,
|
||||
EncoderType = encoder,
|
||||
Width = W,
|
||||
Height = H,
|
||||
CRF = CQ,
|
||||
preset = preset,
|
||||
SizeRatio = originalFile.Length / (float)fileSize,
|
||||
EncodeTime = sw.Elapsed
|
||||
};
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
FFMpegArgumentProcessor AV1Encode(string filePath, int W = -1, int H = -1, int CQ = 23) { // AV1 is visually lossless at CQ 23
|
||||
FFMpegArgumentProcessor AV1Encode(string filePath, int W = -1, int H = -1, int CQ = 23, int preset = 0) { // AV1 is visually lossless at CQ 23
|
||||
return FFMpegArguments
|
||||
.FromFileInput(inputFile, false, options => options.WithHardwareAcceleration())
|
||||
.OutputToFile(filePath, true, options => options
|
||||
.CopyChannel(Channel.Audio)
|
||||
.CopyChannel(Channel.Subtitle)
|
||||
.WithVideoCodec("av1_nvenc")
|
||||
.WithArgument(new NvencSpeedPreset(NvencSpeed.p7))
|
||||
.WithArgument(new NvencSpeedPreset((NvencSpeed)preset))
|
||||
.WithArgument(new NvencTuneArgument(NvencTune.hq))
|
||||
.WithArgument(new NvencHighBitDepthArgument(true))
|
||||
.WithCustomArgument("-rc vbr")
|
||||
@@ -110,14 +178,14 @@ FFMpegArgumentProcessor AV1Encode(string filePath, int W = -1, int H = -1, int C
|
||||
);
|
||||
}
|
||||
|
||||
FFMpegArgumentProcessor HEVCEncode(string filePath, int W = -1, int H = -1, int CQ = 23) {
|
||||
FFMpegArgumentProcessor HEVCEncode(string filePath, int W = -1, int H = -1, int CQ = 23, int preset = 0) {
|
||||
return FFMpegArguments
|
||||
.FromFileInput(inputFile, true, options => options.WithHardwareAcceleration())
|
||||
.OutputToFile(filePath, true, options => options
|
||||
.CopyChannel(Channel.Audio)
|
||||
.CopyChannel(Channel.Subtitle)
|
||||
.WithVideoCodec("hevc_nvenc")
|
||||
.WithArgument(new NvencSpeedPreset(NvencSpeed.p7))
|
||||
.WithArgument(new NvencSpeedPreset((NvencSpeed)preset))
|
||||
.WithArgument(new NvencTuneArgument(NvencTune.hq))
|
||||
.WithCustomArgument("-rc vbr_hq")
|
||||
.WithCustomArgument($"-cq {(byte)CQ}")
|
||||
@@ -126,14 +194,14 @@ FFMpegArgumentProcessor HEVCEncode(string filePath, int W = -1, int H = -1, int
|
||||
);
|
||||
}
|
||||
|
||||
FFMpegArgumentProcessor H264Encode(string filePath, int W = -1, int H = -1, int CQ = 23) {
|
||||
FFMpegArgumentProcessor H264Encode(string filePath, int W = -1, int H = -1, int CQ = 23, int preset = 0) {
|
||||
return FFMpegArguments
|
||||
.FromFileInput(inputFile, true, options => options.WithHardwareAcceleration())
|
||||
.OutputToFile(filePath, true, options => options
|
||||
.CopyChannel(Channel.Audio)
|
||||
.CopyChannel(Channel.Subtitle)
|
||||
.WithVideoCodec("h264_nvenc")
|
||||
.WithArgument(new NvencSpeedPreset(NvencSpeed.p7))
|
||||
.WithArgument(new NvencSpeedPreset((NvencSpeed)preset))
|
||||
.WithArgument(new NvencTuneArgument(NvencTune.hq))
|
||||
.WithCustomArgument("-rc vbr_hq")
|
||||
.WithCustomArgument($"-cq {(byte)CQ}") // approximate CRF to CQ
|
||||
@@ -142,14 +210,14 @@ FFMpegArgumentProcessor H264Encode(string filePath, int W = -1, int H = -1, int
|
||||
);
|
||||
}
|
||||
|
||||
FFMpegArgumentProcessor AV1EncodeCPU(string filePath, int W = -1, int H = -1, int CQ = 23) {
|
||||
FFMpegArgumentProcessor AV1EncodeCPU(string filePath, int W = -1, int H = -1, int CQ = 23, int preset = 0) {
|
||||
return FFMpegArguments
|
||||
.FromFileInput(inputFile)
|
||||
.OutputToFile(filePath, true, options => options
|
||||
.CopyChannel(Channel.Audio)
|
||||
.CopyChannel(Channel.Subtitle)
|
||||
.WithVideoCodec("libsvtav1")
|
||||
.WithCustomArgument($"-preset 3")
|
||||
.WithCustomArgument($"-preset {preset}")
|
||||
.WithConstantRateFactor(CQ)
|
||||
.WithCustomArgument("-svtav1-params tune=0")
|
||||
.WithVideoFilters(filterOptions => {if (W > 0 && H > 0) filterOptions.Scale(W, H);})
|
||||
@@ -157,14 +225,14 @@ FFMpegArgumentProcessor AV1EncodeCPU(string filePath, int W = -1, int H = -1, in
|
||||
);
|
||||
}
|
||||
|
||||
FFMpegArgumentProcessor HEVCEncodeCPU(string filePath, int W = -1, int H = -1, int CQ = 23) {
|
||||
FFMpegArgumentProcessor HEVCEncodeCPU(string filePath, int W = -1, int H = -1, int CQ = 23, int preset = 0) {
|
||||
return FFMpegArguments
|
||||
.FromFileInput(inputFile)
|
||||
.OutputToFile(filePath, true, options => options
|
||||
.CopyChannel(Channel.Audio)
|
||||
.CopyChannel(Channel.Subtitle)
|
||||
.WithVideoCodec(VideoCodec.LibX265)
|
||||
.WithSpeedPreset(Speed.Slow)
|
||||
.WithSpeedPreset(ToX264Speed(preset))
|
||||
.WithConstantRateFactor(CQ)
|
||||
.WithCustomArgument("-x265-params profile=main10:high-tier=1:aq-mode=3:psy-rd=1:rc-lookahead=60:bframes=6")
|
||||
.WithVideoFilters(filterOptions => {if (W > 0 && H > 0) filterOptions.Scale(W, H);})
|
||||
@@ -172,14 +240,14 @@ FFMpegArgumentProcessor HEVCEncodeCPU(string filePath, int W = -1, int H = -1, i
|
||||
);
|
||||
}
|
||||
|
||||
FFMpegArgumentProcessor H264EncodeCPU(string filePath, int W = -1, int H = -1, int CQ = 23) {
|
||||
FFMpegArgumentProcessor H264EncodeCPU(string filePath, int W = -1, int H = -1, int CQ = 23, int preset = 0) {
|
||||
return FFMpegArguments
|
||||
.FromFileInput(inputFile)
|
||||
.OutputToFile(filePath, true, options => options
|
||||
.CopyChannel(Channel.Audio)
|
||||
.CopyChannel(Channel.Subtitle)
|
||||
.WithVideoCodec(VideoCodec.LibX264)
|
||||
.WithSpeedPreset(Speed.Slow)
|
||||
.WithSpeedPreset(ToX264Speed(preset))
|
||||
.WithConstantRateFactor(CQ)
|
||||
.WithCustomArgument("-x265-params profile=main10:high-tier=1:aq-mode=3:psy-rd=1:rc-lookahead=60:bframes=6")
|
||||
.WithVideoFilters(filterOptions => {if (W > 0 && H > 0) filterOptions.Scale(W, H);})
|
||||
@@ -187,6 +255,43 @@ FFMpegArgumentProcessor H264EncodeCPU(string filePath, int W = -1, int H = -1, i
|
||||
);
|
||||
}
|
||||
|
||||
async Task<float> ComputeVMAFScore(string referenceFile, string distortedFile) {
|
||||
float VMAF = -1;
|
||||
ProcessStartInfo startInfo = new ProcessStartInfo {
|
||||
FileName = Path.Combine(ffmpegPath, "ffmpeg"),
|
||||
Arguments = $"-i \"{distortedFile}\" -i \"{referenceFile}\" -lavfi libvmaf -f null -",
|
||||
RedirectStandardOutput = true,
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true
|
||||
};
|
||||
using Process ffmpeg = Process.Start(startInfo)!;
|
||||
string output = ffmpeg.StandardOutput.ReadToEnd();
|
||||
await ffmpeg.WaitForExitAsync();
|
||||
// Parse VMAF score from output
|
||||
string marker = "VMAF score: ";
|
||||
int index = output.LastIndexOf(marker, StringComparison.Ordinal);
|
||||
if (index != -1) {
|
||||
int start = index + marker.Length;
|
||||
int end = output.IndexOf("\n", start, StringComparison.Ordinal);
|
||||
string scoreStr = output.Substring(start, end - start).Trim();
|
||||
if (float.TryParse(scoreStr, out float score)) {
|
||||
VMAF = score;
|
||||
}
|
||||
}
|
||||
return VMAF;
|
||||
}
|
||||
|
||||
Speed ToX264Speed(int p) => p switch {
|
||||
1 => Speed.VerySlow,
|
||||
2 => Speed.Slower,
|
||||
3 => Speed.Slow,
|
||||
4 => Speed.Medium,
|
||||
5 => Speed.Fast,
|
||||
6 => Speed.Faster,
|
||||
7 => Speed.VeryFast,
|
||||
_ => Speed.Medium
|
||||
};
|
||||
|
||||
public enum Encoder {
|
||||
AV1_NVENC,
|
||||
HEVC_NVENC,
|
||||
@@ -194,4 +299,15 @@ public enum Encoder {
|
||||
AV1_CPU,
|
||||
HEVC_CPU,
|
||||
H264_CPU
|
||||
}
|
||||
|
||||
record struct EncodeStats {
|
||||
public string Filename;
|
||||
public Encoder EncoderType;
|
||||
public int Width;
|
||||
public int Height;
|
||||
public int CRF;
|
||||
public int preset;
|
||||
public float SizeRatio;
|
||||
public TimeSpan EncodeTime;
|
||||
}
|
||||
Reference in New Issue
Block a user