MilkyShots/Lactose/Jobs/FileSystemCrawlJob.cs

using Butter;
using Butter.Types;
using Lactose.Models;
using Lactose.Repositories;
using System.Collections;
using System.Data;
using static System.String;

namespace Lactose.Jobs;

public sealed class FileSystemCrawlJob : Job {
    public override string      Name   { get; } = "Directory Scan: ";
    public override JobStatus   JobStatus { get; }
    IAssetRepository            assetRepository;
    string                      workingPath;
    Guid                        folderId;
    ILogger<FileSystemCrawlJob> logger;
    JobManager                  jobManager;
    List<Job>                   childJobs = [];

    public FileSystemCrawlJob(
        Guid folderId,
        string workingPath,
        ILogger<FileSystemCrawlJob> logger,
        IAssetRepository assetRepository,
        JobManager jobManager
    ) {
        JobStatus               = new(this);
        this.logger          = logger;
        this.assetRepository = assetRepository;
        this.workingPath     = workingPath;
        this.folderId        = folderId;
        this.jobManager      = jobManager;
        Name         += $"{workingPath}";
    }

    ///<inheritdoc />
    protected override async Task TaskJob(CancellationToken token) {
        logger.LogInformation($"Started crawling directory {workingPath}");

        DirectoryInfo dir = new(workingPath);

        if (!dir.Exists) {
            logger.LogWarning("Directory {Dir} does not exist. Skipping crawl.", workingPath);
            return;
        }

        int steps,
            totalSteps;

        var files = dir.GetFiles();
        logger.LogDebug($"Found {files.Length} files.");

        var folders = dir.GetDirectories();
        logger.LogDebug($"Found {folders.Length} folders.");

        totalSteps = files.Length + folders.Length;
        steps      = 0;

        foreach (var folder in folders) {
            if (token.IsCancellationRequested) {
                // Cancel all child jobs
                lock (childJobs) { childJobs.ForEach(j => j.Cancel()); }
                // Report this job as canceled
                JobStatus.Cancel();
                return;
            }

            steps++;
            var job = jobManager.CreateJob<FileSystemCrawlJob>(folderId, folder.FullName);
            job.ParentJobId = Id;
            // When the job is done, remove it from the child jobs list
            job.Done += (o, _) => { lock (childJobs) { childJobs.Remove((o as Job)!); } };
            lock (childJobs) { childJobs.Add(job); }
            jobManager.EnqueueJob(job);
            logger.LogDebug($"Enqueued crawl job for folder {folder.FullName}");
            JobStatus.UpdateProgress(Math.Clamp(steps/(float)totalSteps, 0, 1), $"Processing folder {folder.FullName}");
        }

        foreach (var file in files) {
            if (token.IsCancellationRequested) {
                // Cancel all child jobs
                lock (childJobs) { childJobs.ForEach(j => j.Cancel()); }
                // Report this job as canceled
                JobStatus.Cancel();
                return;
            }

            steps++;
            JobStatus.UpdateProgress(Math.Clamp(steps/(float)totalSteps, 0, 1), $"Processing file {file.FullName}");

            if (assetRepository.FindByPath(file.FullName) != null) continue;

            var asset = AssetFromPath(file.FullName);
            if (asset == null) continue;

            try {
                assetRepository.Insert(asset);
                assetRepository.Save();
                logger.LogInformation("Inserted asset {Asset} into the database.", asset.OriginalPath);
            } catch (DuplicateNameException) {
                logger.LogWarning("Asset {Asset} already exists in the database. Skipping.", asset.OriginalPath);
            } catch (Exception e) {
                logger.LogError(e, "Error inserting asset {Asset} into the database. {e}", asset.OriginalPath, e);
            }
        }

        // Wait for all child jobs to complete
        int pendingCount;
        lock (childJobs) { pendingCount = childJobs.Count; }
        JobStatus.UpdateProgress(Math.Clamp(steps/(float)totalSteps, 0, 1), $"Waiting for {pendingCount} child jobs to complete.");
        JobStatus.Wait();

        while (true) {
            lock (childJobs) { if (childJobs.Count == 0) break; }
            if (token.IsCancellationRequested) {
                // Cancel all child jobs
                lock (childJobs) { childJobs.ForEach(j => j.Cancel()); }
                // Cancel this job as well
                JobStatus.Cancel();
                return;
            }
            try {
                await Task.Delay(50, token).ConfigureAwait(false);
            } catch (OperationCanceledException) {
                lock (childJobs) { childJobs.ForEach(j => j.Cancel()); }
                JobStatus.Cancel();
                return;
            }
        }
        JobStatus.Complete("Crawl completed successfully.");
    }

    Asset? AssetFromPath(string filePath) {
        logger.LogTrace("Loading asset from file path {filePath}", filePath);
        // Get the file info
        var finfo = new FileInfo(filePath);

        // Determine the type of the file
        string      ext = finfo.Extension.ToLower();
        EAssetType? type;

        // ReSharper disable ArrangeObjectCreationWhenTypeNotEvident
        var mimeImg = MimeTypes.Image.FirstOrDefault(mime => mime.Extensions.Contains(ext), new(Empty, []));
        var mimeVid = MimeTypes.Video.FirstOrDefault(mime => mime.Extensions.Contains(ext), new(Empty, []));

        if (mimeImg.MimeType != Empty) type      = EAssetType.Image;
        else if (mimeVid.MimeType != Empty) type = EAssetType.Video;
        else return null;

        // Create a new asset
        var asset = new Asset() {
            FolderId         = folderId,
            OriginalPath     = finfo.FullName,
            OriginalFilename = finfo.Name,
            Type             = type.Value,
            IsPubliclyShared = false,
            CreatedAt        = finfo.CreationTime,
            UpdatedAt        = finfo.LastWriteTime,
            MimeType = type.Value switch {
                EAssetType.Image => mimeImg.MimeType,
                EAssetType.Video => mimeVid.MimeType,
                _                => throw new ArgumentOutOfRangeException()
            },
            FileSize = finfo.Length,
            Hash     = new BitArray(64)
        };
        
        logger.LogDebug(
            $"""
             Asset created from path: {asset.OriginalPath}
             Data:
                 - Filename: {asset.OriginalFilename}
                 - Type: {asset.Type}
                 - MimeType: {asset.MimeType}
                 - CreatedAt: {asset.CreatedAt}
                 - UpdatedAt: {asset.UpdatedAt}
                 - FileSize: {asset.FileSize} bytes   
             """);
        return asset;
    }
}