Fixed scoring calculation, added step by step score calculation to avoid losing everything everytime.

EncodingSampleTest/Program.cs

@@ -36,14 +36,14 @@ if (mediaInfo.PrimaryVideoStream == null) {
 var W = mediaInfo.PrimaryVideoStream.Width;
 var H = mediaInfo.PrimaryVideoStream.Height;
 
-List<Task> GpuTasks = new();
+List<Task<EncodeStats>> GpuTasks = new();
-List<Task> CpuTasks = new();
+List<Task<EncodeStats>> CpuTasks = new();
 CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
 
 CancellationToken cancellationToken = cancellationTokenSource.Token;
 Console.WriteLine("Generating Tasks:");
-for (int cq = 24; cq <= 34; cq += 3) {
+for (int cq = 24; cq <= 34; cq++) {
-    for (int preset = 12; preset <= 18; preset+=2) {
+    for (int preset = 12; preset <= 18; preset++) {
         //GpuTasks.Add(Encode(Encoder.AV1_NVENC, 1280, 720, cq, preset, cancellationToken));
         //GpuTasks.Add(Encode(Encoder.AV1_NVENC, 1920, 1080, cq, preset, cancellationToken));
         GpuTasks.Add(Encode(Encoder.AV1_NVENC, W, H, cq, preset, cancellationToken));
@@ -56,12 +56,12 @@ for (int cq = 24; cq <= 34; cq += 3) {
         //GpuTasks.Add(Encode(Encoder.H264_NVENC, 1920, 1080, cq, preset, cancellationToken));
         //GpuTasks.Add(Encode(Encoder.H264_NVENC, W, H, cq, preset, cancellationToken));
     }
-    for (int preset = 5; preset <= 7; preset++) {
+    for (int preset = 3; preset <= 10; preset++) {
         //CpuTasks.Add(Encode(Encoder.AV1_CPU,  1280, 720, cq, preset, cancellationToken));
         //CpuTasks.Add(Encode(Encoder.AV1_CPU,  1920, 1080, cq, preset, cancellationToken));
         CpuTasks.Add(Encode(Encoder.AV1_CPU,  W, H, cq, preset, cancellationToken));
     }
-    for (int preset = 3; preset <= 5; preset++) {
+    for (int preset = 2; preset <= 5; preset++) {
         //CpuTasks.Add(Encode(Encoder.HEVC_CPU, 1280, 720, cq, preset, cancellationToken));
         //CpuTasks.Add(Encode(Encoder.HEVC_CPU, 1920, 1080, cq, preset, cancellationToken));
         CpuTasks.Add(Encode(Encoder.HEVC_CPU, W, H, cq, preset, cancellationToken));
@@ -74,39 +74,42 @@ for (int cq = 24; cq <= 34; cq += 3) {
 
 //Run all GPU tasks sequentially
 while (GpuTasks.Count(x => x.IsCompleted) < GpuTasks.Count) {
-    GpuTasks.First(x => !x.IsCompleted).RunSynchronously();
+    var task = GpuTasks.First(x => !x.IsCompleted);
+    task.Start();
+    await task;
+    Console.WriteLine("Computing Scores...");
+    await ComputeScores(task.Result);
     Console.WriteLine(GpuTasks.Count(x => !x.IsCompleted) + " GPU tasks remaining...");
 }
 
 //Run all CPU tasks sequentially
 while (CpuTasks.Count(x => x.IsCompleted) < CpuTasks.Count) {
-    CpuTasks.First(x => !x.IsCompleted).RunSynchronously();
+    var task = CpuTasks.First(x => !x.IsCompleted);
+    task.Start();
+    await task;
+    Console.WriteLine("Computing Scores...");
+    await ComputeScores(task.Result);
     Console.WriteLine(CpuTasks.Count(x => !x.IsCompleted) + " CPU tasks remaining...");
 }
 
-//Calculate VMAF, PSNR and SSIM scores
+cancellationTokenSource.Cancel();
-Console.WriteLine("Computing VMAF Scores:");
+return;
-Dictionary<EncodeStats,Task<float>> vmafResults = new();
-Dictionary<EncodeStats,Task<Tuple<float,float>>> psnrResults = new();
-Dictionary<EncodeStats,Task<float>> ssimResults = new();
-var taskList = GpuTasks.Concat(CpuTasks).OfType<Task<EncodeStats>>().Select(x => x.Result).ToList();
-foreach (var task in taskList) {
-    var encodeStats = task;
-    vmafResults[encodeStats] = ComputeVMAFScore(inputFile, encodeStats.Filename);
-    psnrResults[encodeStats] = ComputePSNRScore(inputFile, encodeStats.Filename);
-    ssimResults[encodeStats] = ComputeSSIMScore(inputFile, encodeStats.Filename);
-}
 
-Console.WriteLine("Computing Results:");
+async Task ComputeScores(EncodeStats es) {
-foreach (var result in taskList) {
+    Console.WriteLine("Computing VMAF...");
-    //"Width\tHeight\tPreset\tCRF\tSize Ratio\tEncode Time\tSpeed\tVMAF\tSSIM\tPSNR_avg\tPSNR_MSE\n"
+    var vmaf = ComputeVMAFScore(inputFile, es.Filename);//Task.FromResult(-1f);
-    var es = result;
+    Console.WriteLine($"VMAF= {vmaf.Result}");
+    var psnr = ComputePSNRScore(inputFile, es.Filename);
+    Console.WriteLine($"PSNR= {psnr.Result.Item1} - {psnr.Result.Item2}");
+    var ssim = ComputeSSIMScore(inputFile, es.Filename);
+    Console.WriteLine($"SSIM= {ssim.Result}");
+    await Task.WhenAll([vmaf, psnr, ssim]);
     string line = $"{es.Width}\t{es.Height}\t{es.preset}\t{es.CRF}\t{es.SizeRatio:F3}\t{es.EncodeTime:g}\t"+
-    $"{(mediaInfo.Duration.TotalSeconds / es.EncodeTime.TotalSeconds):F3}x"+
+                  $"{(mediaInfo.Duration.TotalSeconds / es.EncodeTime.TotalSeconds):F3}x\t"+
-    $"\t{vmafResults[result]:F2}\n" + //VMAF
+                  $"{vmaf.Result:F2}\t" + //VMAF
-    $"\t{ssimResults[result]:F4}\t" + //SSIM
+                  $"{ssim.Result:F4}\t" + //SSIM
-    $"\t{psnrResults[result].Result.Item1:F2}\t" + //PSNR_avg
+                  $"{psnr.Result.Item1:F2}\t" + //PSNR_avg
-    $"{psnrResults[result].Result.Item2:F2}";      //PSNR_MSE
+                  $"{psnr.Result.Item2:F2}\n";    //PSNR_MSE
     
     Console.WriteLine(line);
     switch (es.EncoderType) {
@@ -117,16 +120,17 @@ foreach (var result in taskList) {
         case Encoder.HEVC_CPU: HEVCCPUStats.Append(line); break;
         case Encoder.H264_CPU: H264CPUStats.Append(line); break;
     }
+    WriteAllStats();
 }
+
+void WriteAllStats() {
     File.WriteAllText(Path.Combine(outputDir, "AV1_NVENC_Stats.tsv"), AV1GPUStats.ToString());
     File.WriteAllText(Path.Combine(outputDir, "HEVC_NVENC_Stats.tsv"), HEVCGPUStats.ToString());
     //File.WriteAllText(Path.Combine(outputDir, "H264_NVENC_Stats.tsv"), H264GPUStats.ToString());
     File.WriteAllText(Path.Combine(outputDir, "AV1_CPU_Stats.tsv"), AV1CPUStats.ToString());
     File.WriteAllText(Path.Combine(outputDir, "HEVC_CPU_Stats.tsv"), HEVCCPUStats.ToString());
     //File.WriteAllText(Path.Combine(outputDir, "H264_CPU_Stats.tsv"), H264CPUStats.ToString());
-
+}
-cancellationTokenSource.Cancel();
-return;
 
 Task<EncodeStats> Encode(Encoder encoder, int W = -1, int H = -1, int CQ = 23, int preset = 0, CancellationToken cancellationToken = default) {
     var outputFile = Path.Combine(outputDir, $"output_{encoder}-{W}x{H}_p{preset}_cq{CQ}.mp4");
@@ -264,12 +268,14 @@ async Task<float> ComputeVMAFScore(string referenceFile, string distortedFile) {
     ProcessStartInfo startInfo = new ProcessStartInfo {
         FileName = Path.Combine(ffmpegPath, "ffmpeg"),
         Arguments = $"-i \"{distortedFile}\" -i \"{referenceFile}\"  -lavfi libvmaf=n_threads=32 -f null -",
-        RedirectStandardOutput = true,
+        CreateNoWindow = true,
         UseShellExecute = false,
-        CreateNoWindow = true
+        RedirectStandardOutput = false,
+        RedirectStandardError = true
     };
     using Process ffmpeg = Process.Start(startInfo)!;
-    string output = ffmpeg.StandardOutput.ReadToEnd();
+    //string output = await ffmpeg.StandardOutput.ReadToEndAsync();
+    string output = await ffmpeg.StandardError.ReadToEndAsync();
     await ffmpeg.WaitForExitAsync();
     // Parse VMAF score from output
     string marker = "VMAF score: ";
@@ -292,20 +298,23 @@ async Task<float> ComputeVMAFScore(string referenceFile, string distortedFile) {
 /// <param name="distortedFile">The path to the distorted video file.</param>
 /// <returns>A tuple containing the average PSNR and average MSE values.</returns>
 async Task<Tuple<float,float>> ComputePSNRScore(string referenceFile, string distortedFile) {
+    string dir = Environment.CurrentDirectory;
+    Environment.CurrentDirectory = Path.GetDirectoryName(distortedFile)!;
+    string psnrLogFile = Path.Combine(Path.GetDirectoryName(distortedFile)!, $"{Path.GetFileNameWithoutExtension(distortedFile)}.psnr");
+    
     float PSNR = -1;
     float MSE = -1;
     ProcessStartInfo startInfo = new ProcessStartInfo {
         FileName = Path.Combine(ffmpegPath, "ffmpeg"),
-        Arguments = $"-i \"{distortedFile}\" -i \"{referenceFile}\" -lavfi psnr=stats_file={Path.GetFileNameWithoutExtension(distortedFile)}.psnr:stats_version=2 -f null -",
+        //TODO: This is a damn hack.
-        RedirectStandardOutput = true,
+        Arguments = $"-i \"{distortedFile}\" -i \"..\\{referenceFile}\" -lavfi psnr=stats_file={Path.GetFileName(psnrLogFile)}:stats_version=2 -f null -",
         UseShellExecute = false,
         CreateNoWindow = true
     };
     using Process ffmpeg = Process.Start(startInfo)!;
-    string output = ffmpeg.StandardOutput.ReadToEnd();
     await ffmpeg.WaitForExitAsync();
     // Parse PSNR score from stats file
-    var PsnrLog = File.ReadAllLines(Path.Combine(Path.GetDirectoryName(distortedFile)!, $"{Path.GetFileNameWithoutExtension(distortedFile)}.psnr"));
+    var PsnrLog = File.ReadAllLines(psnrLogFile);
     
     List<float> mseVals = new();
     List<float> psnrVals = new();
@@ -319,6 +328,7 @@ async Task<Tuple<float,float>> ComputePSNRScore(string referenceFile, string dis
     }
     PSNR = psnrVals.Count > 0 ? psnrVals.Average() : -1;
     MSE = mseVals.Count > 0 ? mseVals.Average() : -1;
+    Environment.CurrentDirectory = dir;
     return new (PSNR, MSE);
 }
 
@@ -327,12 +337,13 @@ async Task<float> ComputeSSIMScore(string referenceFile, string distortedFile) {
     ProcessStartInfo startInfo = new ProcessStartInfo {
         FileName = Path.Combine(ffmpegPath, "ffmpeg"),
         Arguments = $"-i \"{distortedFile}\" -i \"{referenceFile}\" -lavfi ssim -f null -",
-        RedirectStandardOutput = true,
+        RedirectStandardOutput = false,
+        RedirectStandardError = true,
         UseShellExecute = false,
         CreateNoWindow = true
     };
     using Process ffmpeg = Process.Start(startInfo)!;
-    string output = ffmpeg.StandardOutput.ReadToEnd();
+    string output = await ffmpeg.StandardError.ReadToEndAsync();
     await ffmpeg.WaitForExitAsync();
     
     // Parse SSIM score from output