From c8056aade49a50a8d3954dfb7973729111ae029e Mon Sep 17 00:00:00 2001 From: George Bates <61670316+Pear-231@users.noreply.github.com> Date: Mon, 29 Dec 2025 19:37:07 +0000 Subject: [PATCH] Change Notes: - Added better compression logging to help with spotting bugs with files that shouldn't be compressed. --- .../SharedCore/ErrorHandling/PackFileLog.cs | 148 ++++++++++-------- .../PackFiles/PackFileContainerLoader.cs | 6 +- 2 files changed, 83 insertions(+), 71 deletions(-) diff --git a/Shared/SharedCore/ErrorHandling/PackFileLog.cs b/Shared/SharedCore/ErrorHandling/PackFileLog.cs index d813d869a..3a51adf93 100644 --- a/Shared/SharedCore/ErrorHandling/PackFileLog.cs +++ b/Shared/SharedCore/ErrorHandling/PackFileLog.cs @@ -3,21 +3,15 @@ namespace Shared.Core.ErrorHandling { - public class CompressionStats + public class CompressionInformation(long diskSize = 0, long uncompressedSize = 0) { - public long DiskSize { get; set; } - public long UncompressedSize { get; set; } + public long DiskSize { get; set; } = diskSize; + public long UncompressedSize { get; set; } = uncompressedSize; - public CompressionStats(long diskSize = 0, long uncompressedSize = 0) + public void Add(CompressionInformation compressionInformation) { - DiskSize = diskSize; - UncompressedSize = uncompressedSize; - } - - public void Add(CompressionStats stat) - { - DiskSize += stat.DiskSize; - UncompressedSize += stat.UncompressedSize; + DiskSize += compressionInformation.DiskSize; + UncompressedSize += compressionInformation.UncompressedSize; } } @@ -25,22 +19,19 @@ public static class PackFileLog { private static readonly ILogger s_logger = Logging.CreateStatic(typeof(PackFileLog)); - public static Dictionary GetCompressionStats(PackFileContainer container) + public static Dictionary GetCompressionInformation(PackFileContainer container) { - var stats = new Dictionary(); + var compressionInformation = new Dictionary(); foreach (var packFile in container.FileList.Values) { if (packFile.DataSource is PackedFileSource source) { - var format = source.IsCompressed - ? source.CompressionFormat - : CompressionFormat.None; - - if (!stats.TryGetValue(format, out var totals)) + var compressionFormat = source.IsCompressed ? source.CompressionFormat : CompressionFormat.None; + if (!compressionInformation.TryGetValue(compressionFormat, out var totals)) { - totals = new CompressionStats(); - stats[format] = totals; + totals = new CompressionInformation(); + compressionInformation[compressionFormat] = totals; } totals.DiskSize += source.Size; @@ -48,77 +39,98 @@ public static Dictionary GetCompressionStat } } - return stats; + return compressionInformation; } public static void LogPackCompression(PackFileContainer container) { - var stats = GetCompressionStats(container); + var compressionInformation = GetCompressionInformation(container); var totalFiles = container.FileList.Count; - var packSizeFmt = FormatSize(container.OriginalLoadByteSize); + var packSize = FormatSize(container.OriginalLoadByteSize); + + var loadingPart = $"Loading {container.Name}.pack ({totalFiles} files, {packSize})"; - var loadingPart = $"Loading {container.Name}.pack ({totalFiles} files, {packSizeFmt})"; + var fileCountsByCompressionFormat = new Dictionary(); + var fileTypeCountsByCompressionFormat = new Dictionary>(); - var fileCounts = new Dictionary(); - foreach (var pf in container.FileList.Values) + foreach (var packFile in container.FileList.Values) { - if (pf.DataSource is PackedFileSource src) + if (packFile.DataSource is not PackedFileSource packedFileSource) + continue; + + var compressionFormat = packedFileSource.IsCompressed ? packedFileSource.CompressionFormat : CompressionFormat.None; + + if (!fileCountsByCompressionFormat.TryGetValue(compressionFormat, out var fileCount)) + fileCountsByCompressionFormat[compressionFormat] = 1; + else + fileCountsByCompressionFormat[compressionFormat] = fileCount + 1; + + var fileType = string.IsNullOrWhiteSpace(packFile.Extension) ? "no_extension" : packFile.Extension; + + if (!fileTypeCountsByCompressionFormat.TryGetValue(compressionFormat, out var fileTypeCounts)) { - var fmt = src.IsCompressed - ? src.CompressionFormat - : CompressionFormat.None; - - if (!fileCounts.TryGetValue(fmt, out var cnt)) - fileCounts[fmt] = 1; - else - fileCounts[fmt] = cnt + 1; + fileTypeCounts = new Dictionary(StringComparer.OrdinalIgnoreCase); + fileTypeCountsByCompressionFormat[compressionFormat] = fileTypeCounts; } + + if (!fileTypeCounts.TryGetValue(fileType, out var fileTypeCount)) + fileTypeCounts[fileType] = 1; + else + fileTypeCounts[fileType] = fileTypeCount + 1; } - var segments = stats - .OrderBy(kvp => kvp.Key) - .Select(kvp => + var segments = new List(); + + foreach (var compressionEntry in compressionInformation.OrderBy(compressionEntry => compressionEntry.Key)) + { + var compressionFormat = compressionEntry.Key; + var count = fileCountsByCompressionFormat.TryGetValue(compressionFormat, out var fileCount)? fileCount : 0; + var diskSize = FormatSize(compressionEntry.Value.DiskSize); + + var fileSizes = compressionFormat == CompressionFormat.None + ? $"Disk Size: {diskSize}" + : $"Disk Size: {diskSize}, Uncompressed Size: {FormatSize(compressionEntry.Value.UncompressedSize)}"; + + var fileTypes = string.Empty; + if (fileTypeCountsByCompressionFormat.TryGetValue(compressionFormat, out var fileTypeCounts) && fileTypeCounts.Count > 0) { - var fmt = kvp.Key; - var count = fileCounts.TryGetValue(fmt, out var c) ? c : 0; - var disk = FormatSize(kvp.Value.DiskSize); + var fileTypeSegments = new List(); - if (fmt == CompressionFormat.None) - return $"{fmt}: {count} files, {disk} (Disk Size)"; + foreach (var fileTypeEntry in fileTypeCounts.OrderBy(fileTypeEntry => fileTypeEntry.Key, StringComparer.OrdinalIgnoreCase)) + fileTypeSegments.Add($"{fileTypeEntry.Key} ({fileTypeEntry.Value})"); - var unc = FormatSize(kvp.Value.UncompressedSize); - return $"{fmt}: {count} files, {disk} (Disk Size), {unc} (Uncompressed Size)"; - }) - .ToList(); + fileTypes = $": {string.Join(", ", fileTypeSegments)}"; + } + + segments.Add($"{compressionFormat} ({count} files, {fileSizes}){fileTypes}"); + } - var compressionPart = $"File Compression – {string.Join(" | ", segments)}"; - s_logger.Here().Information($"{loadingPart} | {compressionPart}"); + s_logger.Here().Information($"{loadingPart} | {string.Join(" | ", segments)}"); } - public static void LogPacksCompression(IDictionary globalStats) + public static void LogPacksCompression(IDictionary allCompressionInformation) { - var segments = globalStats - .OrderBy(kvp => kvp.Key) - .Select(kvp => - { - var format = kvp.Key; - var diskFormatted = FormatSize(kvp.Value.DiskSize); + var segments = new List(); - if (format == CompressionFormat.None) - return $"{format}: {diskFormatted} (Disk Size)"; + foreach (var compressionEntry in allCompressionInformation.OrderBy(compressionEntry => compressionEntry.Key)) + { + var compressionFormat = compressionEntry.Key; + var diskSize = FormatSize(compressionEntry.Value.DiskSize); - var uncompressedFormatted = FormatSize(kvp.Value.UncompressedSize); - return $"{format}: {diskFormatted} (Disk Size), {uncompressedFormatted} (Uncompressed Size)"; - }) - .ToList(); + if (compressionFormat == CompressionFormat.None) + { + segments.Add($"{compressionFormat}: {diskSize} (Disk Size)"); + continue; + } - var totalDisk = globalStats.Values.Sum(stat => stat.DiskSize); - var totalUncompressed = globalStats.Values.Sum(stat => stat.UncompressedSize); + var uncompressedSize = FormatSize(compressionEntry.Value.UncompressedSize); + segments.Add($"{compressionFormat}: {diskSize} (Disk Size), {uncompressedSize} (Uncompressed Size)"); + } - var totalDiskFormatted = FormatSize(totalDisk); - var totalUncompressedFormatted = FormatSize(totalUncompressed); + var totalDiskSize = FormatSize(allCompressionInformation.Values.Sum(compressionInformation => compressionInformation.DiskSize)); + var totalUncompressedSize = FormatSize(allCompressionInformation.Values.Sum(compressionInformation => compressionInformation.UncompressedSize)); - var totalSegment = $"Total: {totalDiskFormatted} (Disk Size), {totalUncompressedFormatted} (Uncompressed Size)"; + var totalSegment = $"Total: {totalDiskSize} (Disk Size), {totalUncompressedSize} (Uncompressed Size)"; var summary = string.Join(" | ", segments.Append(totalSegment)); s_logger.Here().Information($"Size of compressed files in all packs by format - {summary}"); diff --git a/Shared/SharedCore/PackFiles/PackFileContainerLoader.cs b/Shared/SharedCore/PackFiles/PackFileContainerLoader.cs index 23b95ddd7..e47a746e2 100644 --- a/Shared/SharedCore/PackFiles/PackFileContainerLoader.cs +++ b/Shared/SharedCore/PackFiles/PackFileContainerLoader.cs @@ -110,7 +110,7 @@ private static void AddFolderContentToPackFile(PackFileContainer container, stri } var packList = new List(); - var packsCompressionStats = new ConcurrentDictionary(); + var packsCompressionStats = new ConcurrentDictionary(); Parallel.ForEach(allCaPackFiles, packFilePath => { @@ -124,11 +124,11 @@ private static void AddFolderContentToPackFile(PackFileContainer container, stri packList.Add(pack); PackFileLog.LogPackCompression(pack); - var packCompressionStats = PackFileLog.GetCompressionStats(pack); + var packCompressionStats = PackFileLog.GetCompressionInformation(pack); foreach (var kvp in packCompressionStats) { if (!packsCompressionStats.TryGetValue(kvp.Key, out var existingStats)) - packsCompressionStats[kvp.Key] = new CompressionStats(kvp.Value.DiskSize, kvp.Value.UncompressedSize); + packsCompressionStats[kvp.Key] = new CompressionInformation(kvp.Value.DiskSize, kvp.Value.UncompressedSize); else existingStats.Add(kvp.Value); }