refactor(compiler): utf8 module compression

.github/workflows/build.yaml

@@ -27,7 +27,7 @@ jobs:
         run: dotnet restore
 
       - name: Build
-        run: dotnet publish ./src/Compiler/Compiler.csproj -c Release -r win-x64 -f net10.0-windows
+        run: dotnet publish ./src/Compiler/Compiler.csproj -c Release -r win-x64 -f net10.0
 
       - uses: actions/upload-artifact@v4
         with:

src/Compiler/CompilerSettings.cs

@@ -0,0 +1,33 @@
+// Copyright (c) 2026 James Draycott <me@racci.dev>. All Rights Reserved.
+// Licensed under the AGPL-3.0-or-later License, See LICENSE in the project root
+// for license information.
+
+using System.IO.Compression;
+
+namespace Compiler;
+
+internal enum EmbeddedLocalTextCompression {
+    None,
+    GZip
+}
+
+internal static class CompilerSettings {
+    internal static EmbeddedLocalTextCompression EmbeddedLocalTextCompression { get; set; } = EmbeddedLocalTextCompression.GZip;
+
+    internal static CompressionLevel EmbeddedLocalTextCompressionLevel { get; set; } = CompressionLevel.Optimal;
+
+    internal static void ConfigureEmbeddedLocalTextCompression(string mode) {
+        switch (mode.ToLowerInvariant()) {
+            case "none":
+                EmbeddedLocalTextCompression = EmbeddedLocalTextCompression.None;
+                EmbeddedLocalTextCompressionLevel = CompressionLevel.NoCompression;
+                break;
+            case "gzip":
+                EmbeddedLocalTextCompression = EmbeddedLocalTextCompression.GZip;
+                EmbeddedLocalTextCompressionLevel = CompressionLevel.Optimal;
+                break;
+            default:
+                throw new ArgumentException($"Unsupported embedded compression mode '{mode}'. Use none or gzip.");
+        }
+    }
+}

src/Compiler/Module/Compiled/Compiled.cs

@@ -16,10 +16,14 @@ namespace Compiler.Module.Compiled;
 
 public enum ContentType {
     UTF8String,
-    Base64Utf8,
     Zip
 }
 
+public enum ContentCompression {
+    None,
+    GZip
+}
+
 [method: Pure]
 public abstract class Compiled(ModuleSpec moduleSpec, RequirementGroup requirements) {
     public Compiled(
@@ -87,6 +91,11 @@ Fin<string> ComputeHash() {
     /// </summary>
     public abstract ContentType Type { get; }
 
+    /// <summary>
+    /// Determines how the content bytes of this module should be decompressed.
+    /// </summary>
+    public abstract ContentCompression Compression { get; }
+
     public virtual Fin<string> GetIdentityHash() => this.ComputedHash();
 
     public Fin<string> GetNameHash() => this.GetIdentityHash().Map(hash => $"{this.ModuleSpec.Name}-{hash[..6]}");
@@ -110,6 +119,7 @@ from hash in this.GetIdentityHash()
             Version = '{{this.Version}}';
             Hash = '{{hash[..6]}}';
             Type = '{{this.Type}}';
+            Compression = '{{this.Compression}}';
             Content = {{content}}
         }
         """;

src/Compiler/Module/Compiled/Local.cs

@@ -5,6 +5,7 @@
 using System.Collections;
 using System.Diagnostics.CodeAnalysis;
 using System.Diagnostics.Contracts;
+using System.IO.Compression;
 using System.Text;
 using Compiler.Requirements;
 using Compiler.Text;
@@ -13,31 +14,39 @@
 namespace Compiler.Module.Compiled;
 
 public class CompiledLocalModule : Compiled {
-    public override ContentType Type { get; } = ContentType.Base64Utf8;
+    public override ContentType Type { get; } = ContentType.UTF8String;
+
+    public override ContentCompression Compression { get; }
 
     // Local modules are always version 0.0.1, as they are not versioned.
     public override Version Version { get; } = new Version(0, 0, 1);
 
     public virtual CompiledDocument Document { get; }
 
-    [Pure]
+    private readonly EmbeddedLocalTextCompression CompressionMode;
+
+    private readonly CompressionLevel CompressionLevel;
+
     public CompiledLocalModule(
         PathedModuleSpec moduleSpec,
         CompiledDocument document,
         RequirementGroup requirements
     ) : base(moduleSpec, requirements) {
         this.Document = document;
-        this.SetContentBytes(new(() => this.GetRawContentText().Map(text => Encoding.UTF8.GetBytes(text))));
+        this.CompressionMode = CompilerSettings.EmbeddedLocalTextCompression;
+        this.CompressionLevel = CompilerSettings.EmbeddedLocalTextCompressionLevel;
+        this.Compression = this.CompressionMode == EmbeddedLocalTextCompression.None ? ContentCompression.None : ContentCompression.GZip;
+        this.SetContentBytes(new(this.GetRawContentBytes));
     }
 
     [Pure]
-    protected virtual Fin<string> GetRawContentText() {
+    protected virtual Fin<byte[]> GetRawContentBytes() {
         var content = new StringBuilder();
 
         foreach (var requirement in this.Requirements.GetRequirements()) {
             var hashResult = requirement switch {
                 ModuleSpec req => this.FindSibling(req) is { } sibling
-                    ? sibling.GetIdentityHash().Map(hash => hash[..6])
+                    ? sibling.GetNameHash().Map(hash => hash[(sibling.ModuleSpec.Name.Length + 1)..])
                     : Fin.Fail<string>(Error.New($"Missing compiled sibling for module requirement {requirement} in {this.ModuleSpec.Name}.")),
                 _ => Pure(requirement.HashString[..6])
             };
@@ -53,11 +62,28 @@ protected virtual Fin<string> GetRawContentText() {
         content.AppendLine()
             .Append(this.Document.GetContent());
 
-        return content.ToString();
+        return Encoding.UTF8.GetBytes(content.ToString());
     }
 
     public override Fin<string> StringifyContent() =>
-        this.GetRawContentText().Map(text => $"'{Convert.ToBase64String(Encoding.UTF8.GetBytes(text))}'");
+        this.GetRawContentBytes().Map(bytes => this.CompressionMode == EmbeddedLocalTextCompression.None
+            ? $"'{Encoding.UTF8.GetString(bytes).Replace("'", "''")}'"
+            : $"'{Convert.ToBase64String(Compress(bytes, this.CompressionLevel))}'");
+
+    /// <summary>Returns the raw payload bytes as they appear in the embedded output (compressed or raw).</summary>
+    internal Fin<byte[]> GetEmbeddedPayloadBytes() =>
+        this.GetRawContentBytes().Map(bytes => this.CompressionMode == EmbeddedLocalTextCompression.None
+            ? bytes
+            : Compress(bytes, this.CompressionLevel));
+
+    protected static byte[] Compress(byte[] bytes, CompressionLevel compressionLevel) {
+        using var output = new MemoryStream();
+        using (var gzip = new GZipStream(output, compressionLevel, true)) {
+            gzip.Write(bytes, 0, bytes.Length);
+        }
+
+        return output.ToArray();
+    }
 
     public Fin<Unit> ValidateRequirementsResolved() {
         foreach (var requirement in this.Requirements.GetRequirements<ModuleSpec>()) {

src/Compiler/Module/Compiled/Remote.cs

@@ -4,10 +4,10 @@
 
 using System.Collections;
 using System.IO.Compression;
-using System.Security.Cryptography;
 using System.Management.Automation;
 using System.Management.Automation.Runspaces;
 using System.Reflection;
+using System.Security.Cryptography;
 using System.Text;
 using System.Text.Json;
 using CommandLine;
@@ -39,11 +39,12 @@ private sealed record ExtraModuleInfo(
 
     private Lock UpdatingArchiveLock { get; } = new();
     private Option<byte[]> UpdatedContentBytes;
-
     private readonly Fin<string> IdentityHash;
 
     public override ContentType Type => ContentType.Zip;
 
+    public override ContentCompression Compression => ContentCompression.None;
+
     public override Version Version { get; }
 
     public CompiledRemoteModule(

src/Compiler/Program.cs

@@ -16,6 +16,7 @@
 using System.Text;
 using CommandLine;
 using Compiler.Analyser;
+using Compiler.Module.Compiled;
 using Compiler.Module.Resolvable;
 using Compiler.Requirements;
 using Extended.Collections.Generic;
@@ -65,6 +66,9 @@ public class Options {
 
         [Option('f', "force", Required = false, HelpText = "Force overwrite of output file.")]
         public bool Force { get; set; }
+
+        [Option("embedded-compression", Required = false, Default = "gzip", HelpText = "Embedded local text compression mode: none|gzip.")]
+        public string EmbeddedCompression { get; set; } = "gzip";
     }
 
     private static async Task<int> Main(string[] args) {
@@ -78,7 +82,12 @@ private static async Task<int> Main(string[] args) {
             async opts => {
                 CleanInput(opts);
                 IsDebugging = SetupLogger(opts) <= LogLevel.Debug;
-
+                try {
+                    CompilerSettings.ConfigureEmbeddedLocalTextCompression(opts.EmbeddedCompression);
+                } catch (ArgumentException ex) {
+                    Errors.Add(ex);
+                    return;
+                }
                 if (GetFilesToCompile(opts.Input!).IsErr(out var error, out var filesToCompile)) {
                     Errors.Add(error);
                     return;
@@ -92,8 +101,8 @@ private static async Task<int> Main(string[] args) {
                 var scriptCreationTasks = filesToCompile.Select(async scriptPath => {
                     var pathedModuleSpec = new PathedModuleSpec(sourceRoot, Path.GetFullPath(scriptPath));
                     var maybeScript = await Resolvable.TryCreateScript(pathedModuleSpec, superParent);
-                    if (maybeScript.IsErr(out var error, out var resolvableScript)) {
-                        Errors.Add(error.Enrich(pathedModuleSpec));
+                    if (maybeScript.IsErr(out var err, out var resolvableScript)) {
+                        Errors.Add(err.Enrich(pathedModuleSpec));
                         return;
                     }
 
@@ -109,6 +118,8 @@ await Output(
                             scriptPath,
                             output,
                             opts.Force);
+
+                        LogCompressionSummary(compiled);
                     });
                 }).ToArray();
 
@@ -125,10 +136,7 @@ await Output(
         Option<string> sourceDirectory = None;
         Option<string> outputDirectory = None;
         if (result.Value.AsOption().IsSome(out var opts)) {
-            sourceDirectory = opts.Input.AsOption().Map(input => {
-                return File.Exists(opts.Input) ? Path.GetDirectoryName(opts.Input)! : opts.Input;
-            })!;
-
+            sourceDirectory = opts.Input.AsOption().Map(_ => File.Exists(opts.Input) ? Path.GetDirectoryName(opts.Input)! : opts.Input)!;
             outputDirectory = opts.Output.AsOption().Map(Path.GetFullPath);
         }
         await OutputErrors(Errors, sourceDirectory, outputDirectory);
@@ -142,6 +150,16 @@ await Output(
         return Errors.IsEmpty ? 0 : Errors.All(e => !e.IsExceptional) ? 0 : 1;
     }
 
+    private static void LogCompressionSummary(CompiledScript compiled) {
+        var localModules = compiled.Graph.Vertices.OfType<CompiledLocalModule>().ToArray();
+        var rawBytes = localModules.Sum(module => module.GetContentBytes().Match(bytes => bytes.Length, _ => 0));
+        var payloadBytes = localModules.Sum(module => module.GetEmbeddedPayloadBytes().Match(bytes => bytes.Length, _ => 0));
+        var serializedBytes = compiled.GetPowerShellObject().Match(Encoding.UTF8.GetByteCount, _ => 0);
+        var savings = rawBytes == 0 ? 0d : 1d - ((double)payloadBytes / rawBytes);
+
+        Logger.Info($"Compression summary: mode={CompilerSettings.EmbeddedLocalTextCompression}, local_modules={localModules.Length}, raw_bytes={rawBytes}, payload_bytes={payloadBytes}, serialized_bytes={serializedBytes}, savings={savings:P1}");
+    }
+
     public static void CleanInput(Options opts) {
         ArgumentException.ThrowIfNullOrWhiteSpace(opts.Input, nameof(opts.Input));
 
@@ -336,7 +354,6 @@ bool forceOverwrite
         var outputPath = GetOutputLocation(sourceDirectory, outputDirectory, fileName);
         Logger.Debug($"Preparing output for {fileName} -> {outputPath}");
         if (File.Exists(outputPath)) {
-
             var hashEngine = System.Security.Cryptography.SHA256.Create();
             var existingFileStream = File.OpenRead(outputPath);
             var hash = hashEngine.ComputeHash(existingFileStream);