diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e71300d9..5daeee1a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -27,7 +27,7 @@ jobs: run: dotnet restore - name: Build - run: dotnet publish ./src/Compiler/Compiler.csproj -c Release -r win-x64 -f net10.0-windows + run: dotnet publish ./src/Compiler/Compiler.csproj -c Release -r win-x64 -f net10.0 - uses: actions/upload-artifact@v4 with: diff --git a/src/Compiler/CompilerSettings.cs b/src/Compiler/CompilerSettings.cs new file mode 100644 index 00000000..1552979e --- /dev/null +++ b/src/Compiler/CompilerSettings.cs @@ -0,0 +1,33 @@ +// Copyright (c) 2026 James Draycott . All Rights Reserved. +// Licensed under the AGPL-3.0-or-later License, See LICENSE in the project root +// for license information. + +using System.IO.Compression; + +namespace Compiler; + +internal enum EmbeddedLocalTextCompression { + None, + GZip +} + +internal static class CompilerSettings { + internal static EmbeddedLocalTextCompression EmbeddedLocalTextCompression { get; set; } = EmbeddedLocalTextCompression.GZip; + + internal static CompressionLevel EmbeddedLocalTextCompressionLevel { get; set; } = CompressionLevel.Optimal; + + internal static void ConfigureEmbeddedLocalTextCompression(string mode) { + switch (mode.ToLowerInvariant()) { + case "none": + EmbeddedLocalTextCompression = EmbeddedLocalTextCompression.None; + EmbeddedLocalTextCompressionLevel = CompressionLevel.NoCompression; + break; + case "gzip": + EmbeddedLocalTextCompression = EmbeddedLocalTextCompression.GZip; + EmbeddedLocalTextCompressionLevel = CompressionLevel.Optimal; + break; + default: + throw new ArgumentException($"Unsupported embedded compression mode '{mode}'. Use none or gzip."); + } + } +} diff --git a/src/Compiler/Module/Compiled/Compiled.cs b/src/Compiler/Module/Compiled/Compiled.cs index 4c126d1f..f15a9187 100644 --- a/src/Compiler/Module/Compiled/Compiled.cs +++ b/src/Compiler/Module/Compiled/Compiled.cs @@ -16,10 +16,14 @@ namespace Compiler.Module.Compiled; public enum ContentType { UTF8String, - Base64Utf8, Zip } +public enum ContentCompression { + None, + GZip +} + [method: Pure] public abstract class Compiled(ModuleSpec moduleSpec, RequirementGroup requirements) { public Compiled( @@ -87,6 +91,11 @@ Fin ComputeHash() { /// public abstract ContentType Type { get; } + /// + /// Determines how the content bytes of this module should be decompressed. + /// + public abstract ContentCompression Compression { get; } + public virtual Fin GetIdentityHash() => this.ComputedHash(); public Fin GetNameHash() => this.GetIdentityHash().Map(hash => $"{this.ModuleSpec.Name}-{hash[..6]}"); @@ -110,6 +119,7 @@ from hash in this.GetIdentityHash() Version = '{{this.Version}}'; Hash = '{{hash[..6]}}'; Type = '{{this.Type}}'; + Compression = '{{this.Compression}}'; Content = {{content}} } """; diff --git a/src/Compiler/Module/Compiled/Local.cs b/src/Compiler/Module/Compiled/Local.cs index c80c81f2..c3976596 100644 --- a/src/Compiler/Module/Compiled/Local.cs +++ b/src/Compiler/Module/Compiled/Local.cs @@ -5,6 +5,7 @@ using System.Collections; using System.Diagnostics.CodeAnalysis; using System.Diagnostics.Contracts; +using System.IO.Compression; using System.Text; using Compiler.Requirements; using Compiler.Text; @@ -13,31 +14,39 @@ namespace Compiler.Module.Compiled; public class CompiledLocalModule : Compiled { - public override ContentType Type { get; } = ContentType.Base64Utf8; + public override ContentType Type { get; } = ContentType.UTF8String; + + public override ContentCompression Compression { get; } // Local modules are always version 0.0.1, as they are not versioned. public override Version Version { get; } = new Version(0, 0, 1); public virtual CompiledDocument Document { get; } - [Pure] + private readonly EmbeddedLocalTextCompression CompressionMode; + + private readonly CompressionLevel CompressionLevel; + public CompiledLocalModule( PathedModuleSpec moduleSpec, CompiledDocument document, RequirementGroup requirements ) : base(moduleSpec, requirements) { this.Document = document; - this.SetContentBytes(new(() => this.GetRawContentText().Map(text => Encoding.UTF8.GetBytes(text)))); + this.CompressionMode = CompilerSettings.EmbeddedLocalTextCompression; + this.CompressionLevel = CompilerSettings.EmbeddedLocalTextCompressionLevel; + this.Compression = this.CompressionMode == EmbeddedLocalTextCompression.None ? ContentCompression.None : ContentCompression.GZip; + this.SetContentBytes(new(this.GetRawContentBytes)); } [Pure] - protected virtual Fin GetRawContentText() { + protected virtual Fin GetRawContentBytes() { var content = new StringBuilder(); foreach (var requirement in this.Requirements.GetRequirements()) { var hashResult = requirement switch { ModuleSpec req => this.FindSibling(req) is { } sibling - ? sibling.GetIdentityHash().Map(hash => hash[..6]) + ? sibling.GetNameHash().Map(hash => hash[(sibling.ModuleSpec.Name.Length + 1)..]) : Fin.Fail(Error.New($"Missing compiled sibling for module requirement {requirement} in {this.ModuleSpec.Name}.")), _ => Pure(requirement.HashString[..6]) }; @@ -53,11 +62,28 @@ protected virtual Fin GetRawContentText() { content.AppendLine() .Append(this.Document.GetContent()); - return content.ToString(); + return Encoding.UTF8.GetBytes(content.ToString()); } public override Fin StringifyContent() => - this.GetRawContentText().Map(text => $"'{Convert.ToBase64String(Encoding.UTF8.GetBytes(text))}'"); + this.GetRawContentBytes().Map(bytes => this.CompressionMode == EmbeddedLocalTextCompression.None + ? $"'{Encoding.UTF8.GetString(bytes).Replace("'", "''")}'" + : $"'{Convert.ToBase64String(Compress(bytes, this.CompressionLevel))}'"); + + /// Returns the raw payload bytes as they appear in the embedded output (compressed or raw). + internal Fin GetEmbeddedPayloadBytes() => + this.GetRawContentBytes().Map(bytes => this.CompressionMode == EmbeddedLocalTextCompression.None + ? bytes + : Compress(bytes, this.CompressionLevel)); + + protected static byte[] Compress(byte[] bytes, CompressionLevel compressionLevel) { + using var output = new MemoryStream(); + using (var gzip = new GZipStream(output, compressionLevel, true)) { + gzip.Write(bytes, 0, bytes.Length); + } + + return output.ToArray(); + } public Fin ValidateRequirementsResolved() { foreach (var requirement in this.Requirements.GetRequirements()) { diff --git a/src/Compiler/Module/Compiled/Remote.cs b/src/Compiler/Module/Compiled/Remote.cs index 3685d777..02400f52 100644 --- a/src/Compiler/Module/Compiled/Remote.cs +++ b/src/Compiler/Module/Compiled/Remote.cs @@ -4,10 +4,10 @@ using System.Collections; using System.IO.Compression; -using System.Security.Cryptography; using System.Management.Automation; using System.Management.Automation.Runspaces; using System.Reflection; +using System.Security.Cryptography; using System.Text; using System.Text.Json; using CommandLine; @@ -39,11 +39,12 @@ private sealed record ExtraModuleInfo( private Lock UpdatingArchiveLock { get; } = new(); private Option UpdatedContentBytes; - private readonly Fin IdentityHash; public override ContentType Type => ContentType.Zip; + public override ContentCompression Compression => ContentCompression.None; + public override Version Version { get; } public CompiledRemoteModule( diff --git a/src/Compiler/Program.cs b/src/Compiler/Program.cs index fa87edd3..e31237f0 100644 --- a/src/Compiler/Program.cs +++ b/src/Compiler/Program.cs @@ -16,6 +16,7 @@ using System.Text; using CommandLine; using Compiler.Analyser; +using Compiler.Module.Compiled; using Compiler.Module.Resolvable; using Compiler.Requirements; using Extended.Collections.Generic; @@ -65,6 +66,9 @@ public class Options { [Option('f', "force", Required = false, HelpText = "Force overwrite of output file.")] public bool Force { get; set; } + + [Option("embedded-compression", Required = false, Default = "gzip", HelpText = "Embedded local text compression mode: none|gzip.")] + public string EmbeddedCompression { get; set; } = "gzip"; } private static async Task Main(string[] args) { @@ -78,7 +82,12 @@ private static async Task Main(string[] args) { async opts => { CleanInput(opts); IsDebugging = SetupLogger(opts) <= LogLevel.Debug; - + try { + CompilerSettings.ConfigureEmbeddedLocalTextCompression(opts.EmbeddedCompression); + } catch (ArgumentException ex) { + Errors.Add(ex); + return; + } if (GetFilesToCompile(opts.Input!).IsErr(out var error, out var filesToCompile)) { Errors.Add(error); return; @@ -92,8 +101,8 @@ private static async Task Main(string[] args) { var scriptCreationTasks = filesToCompile.Select(async scriptPath => { var pathedModuleSpec = new PathedModuleSpec(sourceRoot, Path.GetFullPath(scriptPath)); var maybeScript = await Resolvable.TryCreateScript(pathedModuleSpec, superParent); - if (maybeScript.IsErr(out var error, out var resolvableScript)) { - Errors.Add(error.Enrich(pathedModuleSpec)); + if (maybeScript.IsErr(out var err, out var resolvableScript)) { + Errors.Add(err.Enrich(pathedModuleSpec)); return; } @@ -109,6 +118,8 @@ await Output( scriptPath, output, opts.Force); + + LogCompressionSummary(compiled); }); }).ToArray(); @@ -125,10 +136,7 @@ await Output( Option sourceDirectory = None; Option outputDirectory = None; if (result.Value.AsOption().IsSome(out var opts)) { - sourceDirectory = opts.Input.AsOption().Map(input => { - return File.Exists(opts.Input) ? Path.GetDirectoryName(opts.Input)! : opts.Input; - })!; - + sourceDirectory = opts.Input.AsOption().Map(_ => File.Exists(opts.Input) ? Path.GetDirectoryName(opts.Input)! : opts.Input)!; outputDirectory = opts.Output.AsOption().Map(Path.GetFullPath); } await OutputErrors(Errors, sourceDirectory, outputDirectory); @@ -142,6 +150,16 @@ await Output( return Errors.IsEmpty ? 0 : Errors.All(e => !e.IsExceptional) ? 0 : 1; } + private static void LogCompressionSummary(CompiledScript compiled) { + var localModules = compiled.Graph.Vertices.OfType().ToArray(); + var rawBytes = localModules.Sum(module => module.GetContentBytes().Match(bytes => bytes.Length, _ => 0)); + var payloadBytes = localModules.Sum(module => module.GetEmbeddedPayloadBytes().Match(bytes => bytes.Length, _ => 0)); + var serializedBytes = compiled.GetPowerShellObject().Match(Encoding.UTF8.GetByteCount, _ => 0); + var savings = rawBytes == 0 ? 0d : 1d - ((double)payloadBytes / rawBytes); + + Logger.Info($"Compression summary: mode={CompilerSettings.EmbeddedLocalTextCompression}, local_modules={localModules.Length}, raw_bytes={rawBytes}, payload_bytes={payloadBytes}, serialized_bytes={serializedBytes}, savings={savings:P1}"); + } + public static void CleanInput(Options opts) { ArgumentException.ThrowIfNullOrWhiteSpace(opts.Input, nameof(opts.Input)); @@ -336,7 +354,6 @@ bool forceOverwrite var outputPath = GetOutputLocation(sourceDirectory, outputDirectory, fileName); Logger.Debug($"Preparing output for {fileName} -> {outputPath}"); if (File.Exists(outputPath)) { - var hashEngine = System.Security.Cryptography.SHA256.Create(); var existingFileStream = File.OpenRead(outputPath); var hash = hashEngine.ComputeHash(existingFileStream); diff --git a/src/Compiler/Resources/ScriptTemplate.ps1 b/src/Compiler/Resources/ScriptTemplate.ps1 index 1a6de2ed..7b669169 100644 --- a/src/Compiler/Resources/ScriptTemplate.ps1 +++ b/src/Compiler/Resources/ScriptTemplate.ps1 @@ -29,39 +29,33 @@ begin { [Int]$Script:ModuleLockTimeoutSeconds = 180; [Int]$Script:ModuleLockRetryMilliseconds = 200; - function Convert-Base64Utf8ToBytes { + function Convert-Base64GZipUtf8ToString { [CmdletBinding()] - [OutputType([Byte[]])] + [OutputType([String])] param( [Parameter(Mandatory)] - [String]$Base64, - - [Parameter(Mandatory)] - [Boolean]$PSBelow6, - - [Parameter(Mandatory)] - [Boolean]$IncludeBom + [String]$Base64 ) - $Local:Bytes = [System.Convert]::FromBase64String($Base64); - if (-not ($IncludeBom -and $PSBelow6)) { - return $Local:Bytes; - } - - return [Byte[]]($Bom + $Local:Bytes); - } - - function Write-ModuleBytes { - [CmdletBinding()] - param( - [Parameter(Mandatory)] - [String]$Path, - - [Parameter(Mandatory)] - [Byte[]]$Bytes - ) + $Local:CompressedBytes = [System.Convert]::FromBase64String($Base64); + $Local:InputStream = [System.IO.MemoryStream]::new($Local:CompressedBytes); + try { + $Local:OutputStream = [System.IO.MemoryStream]::new(); + try { + $Local:GZipStream = [System.IO.Compression.GZipStream]::new($Local:InputStream, [System.IO.Compression.CompressionMode]::Decompress); + try { + $Local:GZipStream.CopyTo($Local:OutputStream); + } finally { + $Local:GZipStream.Dispose(); + } - [System.IO.File]::WriteAllBytes($Path, $Bytes); + return [System.Text.Encoding]::UTF8.GetString($Local:OutputStream.ToArray()); + } finally { + $Local:OutputStream.Dispose(); + } + } finally { + $Local:InputStream.Dispose(); + } } function Test-UTF8ModuleReady { @@ -125,7 +119,7 @@ begin { [String]$ModuleHash, [Parameter(Mandatory)] - [ValidateSet('Base64Utf8', 'Zip')] + [ValidateSet('UTF8String', 'Zip')] [String]$ModuleType, [String]$ModulePath, @@ -167,7 +161,7 @@ begin { return $Local:LockHandle; } catch [System.IO.IOException] { $Local:IsReady = switch ($ModuleType) { - 'Base64Utf8' { Test-UTF8ModuleReady -ModulePath $ModulePath -ReadyPath $ReadyPath -Bom $Bom -PSBelow6:$PSBelow6; break; } + 'UTF8String' { Test-UTF8ModuleReady -ModulePath $ModulePath -ReadyPath $ReadyPath -Bom $Bom -PSBelow6:$PSBelow6; break; } 'Zip' { Test-ZipModuleReady -ReadyPath $ReadyPath -ModuleFolderPath $ModuleFolderPath; break; } } @@ -194,7 +188,7 @@ begin { [String]$LockPath, [Parameter(Mandatory)] - [ValidateSet('Base64Utf8', 'Zip')] + [ValidateSet('UTF8String', 'Zip')] [String]$ModuleType, [String]$ModulePath, @@ -215,7 +209,7 @@ begin { $Local:IsReady = $false; switch ($ModuleType) { - 'Base64Utf8' { + 'UTF8String' { if ($Local:OwnerSucceeded -and $ReadyPath -and -not (Test-Path -Path $ReadyPath -PathType Leaf)) { $Local:HasValidContent = $false; if (Test-Path -Path $ModulePath -PathType Leaf) { @@ -275,6 +269,7 @@ begin { $Local:Type = $_.Type; $Local:Hash = $_.Hash; $Local:Content = $_.Content; + $Local:Compression = if ($null -ne $_.Compression -and -not [String]::IsNullOrWhiteSpace([String]$_.Compression)) { [String]$_.Compression } else { 'None' }; $Local:NameHash = "$Local:Name-$Local:Hash"; if (-not $Local:Name -or -not $Local:Type -or -not $Local:Hash -or -not $Local:Content) { Write-Warning "Invalid module definition: $($_), skipping..."; @@ -291,7 +286,7 @@ begin { $Local:ModuleReadyPath = Join-Path -Path $Local:ModuleFolderPath -ChildPath '.ready'; switch ($_.Type) { - 'Base64Utf8' { + 'UTF8String' { $Local:IsRootScript = $null -eq $Script:ScriptPath; $Local:FileSuffix = if ($Local:IsRootScript) { 'ps1' } else { 'psm1' }; $Local:InnerModulePath = Join-Path -Path $Local:ModuleFolderPath -ChildPath "$Local:NameHash.$Local:FileSuffix"; @@ -299,8 +294,12 @@ begin { if ($Local:IsRootScript) { $Local:RootScriptPath = [System.IO.Path]::ChangeExtension([System.IO.Path]::GetTempFileName(), '.ps1'); Write-Verbose "Writing root script content to temp file: $Local:RootScriptPath" - $Local:RootBytes = Convert-Base64Utf8ToBytes -Base64 $Content -PSBelow6:$Local:PSBelow6 -IncludeBom:$true; - Write-ModuleBytes -Path $Local:RootScriptPath -Bytes $Local:RootBytes; + if ($Local:Compression -eq 'GZip') { + $Local:RootContent = Convert-Base64GZipUtf8ToString -Base64 $Local:Content; + } else { + $Local:RootContent = $Local:Content; + } + Set-Content -Path $Local:RootScriptPath -Value $Local:RootContent -Encoding $Local:Encoding -Force -WhatIf:$False; $Script:ScriptPath = $Local:RootScriptPath; $Script:TransientScriptPath = $Local:RootScriptPath; return; @@ -308,18 +307,22 @@ begin { if (-not (Test-UTF8ModuleReady -ModulePath $Local:InnerModulePath -ReadyPath $Local:ModuleReadyPath -Bom $Local:Bom -PSBelow6:$Local:PSBelow6)) { [Boolean]$Local:Utf8Succeeded = $false; - $Local:LockHandle = Wait-ModuleLock -LockPath $Local:ModuleLockPath -ModuleName $Local:Name -ModuleHash $Local:Hash -ModuleType 'Base64Utf8' -ModulePath $Local:InnerModulePath -ReadyPath $Local:ModuleReadyPath -Bom $Local:Bom -PSBelow6:$Local:PSBelow6; + $Local:LockHandle = Wait-ModuleLock -LockPath $Local:ModuleLockPath -ModuleName $Local:Name -ModuleHash $Local:Hash -ModuleType 'UTF8String' -ModulePath $Local:InnerModulePath -ReadyPath $Local:ModuleReadyPath -Bom $Local:Bom -PSBelow6:$Local:PSBelow6; try { if (-not (Test-UTF8ModuleReady -ModulePath $Local:InnerModulePath -ReadyPath $Local:ModuleReadyPath -Bom $Local:Bom -PSBelow6:$Local:PSBelow6)) { Write-Verbose "Writing content to module file: $Local:InnerModulePath" - $Local:ModuleBytes = Convert-Base64Utf8ToBytes -Base64 $Content -PSBelow6:$Local:PSBelow6 -IncludeBom:$true; - Write-ModuleBytes -Path $Local:InnerModulePath -Bytes $Local:ModuleBytes; + if ($Local:Compression -eq 'GZip') { + $Local:ModuleContent = Convert-Base64GZipUtf8ToString -Base64 $Local:Content; + } else { + $Local:ModuleContent = $Local:Content; + } + Set-Content -Path $Local:InnerModulePath -Value $Local:ModuleContent -Encoding $Local:Encoding -Force -WhatIf:$False; $Local:Utf8Succeeded = $true; } else { $Local:Utf8Succeeded = $true; } } finally { - Complete-ModuleLock -LockHandle $Local:LockHandle -LockPath $Local:ModuleLockPath -ModuleType 'Base64Utf8' -ModulePath $Local:InnerModulePath -ReadyPath $Local:ModuleReadyPath -Bom $Local:Bom -PSBelow6:$Local:PSBelow6 -OperationSucceeded $Local:Utf8Succeeded; + Complete-ModuleLock -LockHandle $Local:LockHandle -LockPath $Local:ModuleLockPath -ModuleType 'UTF8String' -ModulePath $Local:InnerModulePath -ReadyPath $Local:ModuleReadyPath -Bom $Local:Bom -PSBelow6:$Local:PSBelow6 -OperationSucceeded $Local:Utf8Succeeded; } } } diff --git a/tests/Compiler/Integration/ScriptTemplateRuntimeTests.cs b/tests/Compiler/Integration/ScriptTemplateRuntimeTests.cs index a2f57f50..55465355 100644 --- a/tests/Compiler/Integration/ScriptTemplateRuntimeTests.cs +++ b/tests/Compiler/Integration/ScriptTemplateRuntimeTests.cs @@ -553,6 +553,51 @@ public async Task GeneratedScript_WithWrapper_CapturesTerminatingErrorDetail() = }); }); + [Test] + public async Task GeneratedScript_UnicodeLocalModuleUsesGzipAndPreservesExtractedBytes() => await InvokeWithInjectedModuleOptOut(async () => { + var sourceRoot = TestUtils.GenerateUniqueDirectory(); + var outputRoot = TestUtils.GenerateUniqueDirectory(); + var programDataRoot = TestUtils.GenerateUniqueDirectory(); + var tempRoot = TestUtils.GenerateUniqueDirectory(); + + var moduleDir = Path.Combine(sourceRoot, "Unicode"); + Directory.CreateDirectory(moduleDir); + var modulePath = Path.Combine(moduleDir, "Unicode.psm1"); + var scriptPath = Path.Combine(sourceRoot, "Root.ps1"); + + var moduleContent = @" +function Get-UnicodePayload { + [CmdletBinding()] + param() + '📦-🗑️-🔄-Ω' +} +Export-ModuleMember -Function Get-UnicodePayload +".TrimStart(); + await File.WriteAllTextAsync(modulePath, moduleContent); + await File.WriteAllTextAsync(scriptPath, "using module ./Unicode/Unicode.psm1\nGet-UnicodePayload"); + + var compiledScriptPath = await CompileScriptToOutput(sourceRoot, outputRoot, scriptPath); + var generatedScript = await File.ReadAllTextAsync(compiledScriptPath); + var result = await RunPwsh(compiledScriptPath, programDataRoot, tempRoot); + var modulesRoot = GetModulesRoot(programDataRoot, [result]); + var moduleDirectory = FindSingleModuleDirectory(modulesRoot, "Unicode-"); + var moduleFile = Directory.GetFiles(moduleDirectory, "Unicode-*.psm1", SearchOption.TopDirectoryOnly).Single(); + var extractedBytes = await File.ReadAllBytesAsync(moduleFile); + var extractedText = Encoding.UTF8.GetString(extractedBytes); + + Assert.Multiple(() => { + Assert.That(result.ExitCode, Is.EqualTo(0), FormatResult(result)); + Assert.That(result.StandardOutput, Does.Contain("📦-🗑️-🔄-Ω")); + Assert.That(generatedScript, Does.Not.Contain("📦")); + Assert.That(generatedScript, Does.Not.Contain("🗑️")); + Assert.That(generatedScript, Does.Not.Contain("🔄")); + Assert.That(generatedScript, Does.Not.Contain("Ω")); + Assert.That(generatedScript, Is.EqualTo(Encoding.ASCII.GetString(Encoding.ASCII.GetBytes(generatedScript)))); + Assert.That(generatedScript, Does.Match("['\"]?[A-Za-z0-9+/=]+['\"]?")); + Assert.That(extractedText, Does.Contain("📦-🗑️-🔄-Ω")); + }); + }); + [Test] public async Task GeneratedScript_UnicodeLocalModulePreservesOutputAndExtractedBytes() => await InvokeWithInjectedModuleOptOut(async () => { var sourceRoot = TestUtils.GenerateUniqueDirectory(); @@ -592,12 +637,99 @@ function Get-UnicodePayload { Assert.That(generatedScript, Does.Not.Contain("🗑️")); Assert.That(generatedScript, Does.Not.Contain("🔄")); Assert.That(generatedScript, Does.Not.Contain("Ω")); + Assert.That(generatedScript, Does.Contain("Compression = 'GZip'")); + Assert.That(generatedScript, Does.Contain("Type = 'UTF8String'")); Assert.That(generatedScript, Is.EqualTo(Encoding.ASCII.GetString(Encoding.ASCII.GetBytes(generatedScript)))); Assert.That(generatedScript, Does.Match("['\"]?[A-Za-z0-9+/=]+['\"]?")); Assert.That(extractedText, Does.Contain("📦-🗑️-🔄-Ω")); }); }); + [Test] + public async Task GeneratedScript_LocalTextPayloadUsesGzipAndRunsEndToEnd() => await InvokeWithInjectedModuleOptOut(async () => { + var sourceRoot = TestUtils.GenerateUniqueDirectory(); + var outputRoot = TestUtils.GenerateUniqueDirectory(); + var programDataRoot = TestUtils.GenerateUniqueDirectory(); + var tempRoot = TestUtils.GenerateUniqueDirectory(); + + var moduleDir = Path.Combine(sourceRoot, "Text"); + Directory.CreateDirectory(moduleDir); + var modulePath = Path.Combine(moduleDir, "Text.psm1"); + var scriptPath = Path.Combine(sourceRoot, "Root.ps1"); + + var moduleContent = @" +function Get-LocalTextPayload { + [CmdletBinding()] + param() + 'gzip local text payload' +} +Export-ModuleMember -Function Get-LocalTextPayload +".TrimStart(); + await File.WriteAllTextAsync(modulePath, moduleContent); + await File.WriteAllTextAsync(scriptPath, "using module ./Text/Text.psm1\nGet-LocalTextPayload"); + + var compiledScriptPath = await CompileScriptToOutput(sourceRoot, outputRoot, scriptPath); + var generatedScript = await File.ReadAllTextAsync(compiledScriptPath); + var result = await RunPwsh(compiledScriptPath, programDataRoot, tempRoot); + var modulesRoot = GetModulesRoot(programDataRoot, [result]); + var moduleDirectory = FindSingleModuleDirectory(modulesRoot, "Text-"); + var moduleFile = Directory.GetFiles(moduleDirectory, "Text-*.psm1", SearchOption.TopDirectoryOnly).Single(); + var extractedText = await File.ReadAllTextAsync(moduleFile); + + Assert.Multiple(() => { + Assert.That(result.ExitCode, Is.EqualTo(0), FormatResult(result)); + Assert.That(result.StandardOutput, Does.Contain("gzip local text payload")); + Assert.That(generatedScript, Does.Contain("Compression = 'GZip'")); + Assert.That(generatedScript, Does.Contain("Type = 'UTF8String'")); + Assert.That(generatedScript, Does.Not.Contain("Compression = 'None'")); + Assert.That(extractedText, Does.Contain("gzip local text payload")); + }); + }); + + [Test, NonParallelizable] + public async Task GeneratedScript_LocalTextPayloadUsesNoneModeEndToEnd() => await InvokeWithInjectedModuleOptOut(async () => { + var previousCompression = CompilerSettings.EmbeddedLocalTextCompression; + var previousLevel = CompilerSettings.EmbeddedLocalTextCompressionLevel; + CompilerSettings.ConfigureEmbeddedLocalTextCompression("none"); + try { + var sourceRoot = TestUtils.GenerateUniqueDirectory(); + var outputRoot = TestUtils.GenerateUniqueDirectory(); + var programDataRoot = TestUtils.GenerateUniqueDirectory(); + var tempRoot = TestUtils.GenerateUniqueDirectory(); + + var moduleDir = Path.Combine(sourceRoot, "Text"); + Directory.CreateDirectory(moduleDir); + var modulePath = Path.Combine(moduleDir, "Text.psm1"); + var scriptPath = Path.Combine(sourceRoot, "Root.ps1"); + + var moduleContent = @" +function Get-LocalTextPayload { + [CmdletBinding()] + param() + 'none local text payload' +} +Export-ModuleMember -Function Get-LocalTextPayload +".TrimStart(); + await File.WriteAllTextAsync(modulePath, moduleContent); + await File.WriteAllTextAsync(scriptPath, "using module ./Text/Text.psm1\nGet-LocalTextPayload"); + + var compiledScriptPath = await CompileScriptToOutput(sourceRoot, outputRoot, scriptPath); + var generatedScript = await File.ReadAllTextAsync(compiledScriptPath); + var result = await RunPwsh(compiledScriptPath, programDataRoot, tempRoot); + + Assert.Multiple(() => { + Assert.That(result.ExitCode, Is.EqualTo(0), FormatResult(result)); + Assert.That(result.StandardOutput, Does.Contain("none local text payload")); + Assert.That(generatedScript, Does.Contain("Compression = 'None'")); + Assert.That(generatedScript, Does.Contain("Type = 'UTF8String'")); + Assert.That(generatedScript, Does.Not.Contain("Compression = 'GZip'")); + }); + } finally { + CompilerSettings.EmbeddedLocalTextCompression = previousCompression; + CompilerSettings.EmbeddedLocalTextCompressionLevel = previousLevel; + } + }); + private static async Task InvokeWithInjectedModuleOptOut(Func action) { var previous = Environment.GetEnvironmentVariable("COMPILER_SKIP_INJECTED_MODULES"); Environment.SetEnvironmentVariable("COMPILER_SKIP_INJECTED_MODULES", bool.TrueString); diff --git a/tests/Compiler/Module/Compiled/Local.cs b/tests/Compiler/Module/Compiled/Local.cs index daaded32..1e7dd2f7 100644 --- a/tests/Compiler/Module/Compiled/Local.cs +++ b/tests/Compiler/Module/Compiled/Local.cs @@ -2,6 +2,7 @@ // Licensed under the AGPL-3.0-or-later License, See LICENSE in the project root // for license information. +using System.IO.Compression; using System.Management.Automation.Language; using System.Text; using System.Text.RegularExpressions; @@ -14,7 +15,7 @@ namespace Compiler.Test.Module.Compiled; [TestFixture] -public class CompiledLocalModuleTests { +public partial class CompiledLocalModuleTests { [Test, Repeat(10), Parallelizable] public async Task StringifyContent_ReturnsValidAstContent() { var module = await TestData.GetRandomCompiledModule(); @@ -82,7 +83,8 @@ public void StringifyContent_EmbeddedHashedImportUsesPlainModuleReference() { CompiledUtils.AddDependency(module, remoteDependency); var output = module.StringifyContent().Unwrap(); - var decodedOutput = DecodeQuotedBase64Payload(output); + var bytes = Convert.FromBase64String(StripQuotedBase64(output)); + var decodedOutput = DecompressGzip(bytes); var remoteHash = remoteDependency.GetNameHash().Unwrap(); Assert.Multiple(() => { @@ -111,10 +113,119 @@ public void StringifyContent_UnicodeLocalModuleUsesAsciiSafePayloadContract() { }); } - private static string DecodeQuotedBase64Payload(string payload) { - var match = Regex.Match(payload, "^[\"'](?[A-Za-z0-9+/=]+)[\"']$", RegexOptions.Singleline); - var encoded = match.Success ? match.Groups["content"].Value : payload; - return Encoding.UTF8.GetString(Convert.FromBase64String(encoded)); + [Test] + public void StringifyContent_LocalTextPayloadUsesGzipRoundtrip() { + var moduleContent = "function Invoke-GzipLocal { 'local gzip payload' }"; + var module = TestData.CreateModule(moduleContent, "GzipLocalModule"); + var output = module.StringifyContent().Unwrap(); + var bytes = Convert.FromBase64String(StripQuotedBase64(output)); + + Assert.Multiple(() => { + Assert.That(bytes, Is.Not.Empty); + Assert.That(DecompressGzip(bytes), Does.Contain("Invoke-GzipLocal")); + Assert.That(DecompressGzip(bytes), Does.Contain("local gzip payload")); + }); + } + + [Test] + public void StringifyContent_LocalTextPayloadMetadataUsesPowerShellObject() { + var moduleContent = "function Invoke-GzipLocal { 'local gzip payload' }"; + var root = TestData.CreateModule("Write-Host 'Root';"); + var module = TestData.CreateModule(moduleContent, "GzipLocalModule"); + CompiledUtils.AddDependency(root, module); + var output = module.GetPowerShellObject().Unwrap().ToString(); + + Assert.Multiple(() => { + Assert.That(output, Does.Contain("Compression = 'GZip'")); + Assert.That(output, Does.Contain("Type = 'UTF8String'")); + }); + } + + [Test, NonParallelizable] + public void StringifyContent_LocalTextPayloadNoneModeEmitsPlainPowerShellText() { + try { + CompilerSettings.ConfigureEmbeddedLocalTextCompression("none"); + var root = TestData.CreateModule("Write-Host 'Root';"); + var moduleContent = "function Invoke-PlainLocal { 'local plain payload' }"; + var module = TestData.CreateModule(moduleContent, "PlainLocalModule"); + CompiledUtils.AddDependency(root, module); + var output = module.StringifyContent().Unwrap(); + var metadata = module.GetPowerShellObject().Unwrap().ToString(); + + Assert.Multiple(() => { + Assert.That(metadata, Does.Contain("Compression = 'None'")); + Assert.That(metadata, Does.Contain("Type = 'UTF8String'")); + Assert.That(output, Does.Contain("Invoke-PlainLocal")); + Assert.That(output, Does.Contain("local plain payload")); + Assert.That(output, Does.Not.Match("^[\"'][A-Za-z0-9+/=]+[\"']$")); + }); + } finally { + CompilerSettings.ConfigureEmbeddedLocalTextCompression("gzip"); + } + } + + [Test, NonParallelizable] + public void StringifyContent_BenchmarkSummaryReportsSavingsForGzipAndNone() { + try { + var gzipModule = TestData.CreateModule($"function Invoke-GzipSummary {{ '{new string('a', 2048)}' }}", "GzipSummaryModule"); + var gzipRaw = gzipModule.GetContentBytes().Unwrap(); + var gzipPayload = gzipModule.GetEmbeddedPayloadBytes().Unwrap(); + + CompilerSettings.ConfigureEmbeddedLocalTextCompression("none"); + var noneModule = TestData.CreateModule("function Invoke-NoneSummary { 'none summary payload' }", "NoneSummaryModule"); + var noneRaw = noneModule.GetContentBytes().Unwrap(); + var nonePayload = noneModule.GetEmbeddedPayloadBytes().Unwrap(); + + Assert.Multiple(() => { + Assert.That(gzipPayload, Has.Length.LessThan(gzipRaw.Length)); + Assert.That(gzipRaw.Length - gzipPayload.Length, Is.GreaterThan(0)); + Assert.That((gzipRaw.Length - gzipPayload.Length) * 100.0 / gzipRaw.Length, Is.GreaterThan(0)); + Assert.That(nonePayload, Has.Length.EqualTo(noneRaw.Length)); + Assert.That(noneRaw.Length - nonePayload.Length, Is.EqualTo(0)); + Assert.That((noneRaw.Length - nonePayload.Length) * 100.0 / noneRaw.Length, Is.EqualTo(0)); + }); + } finally { + CompilerSettings.ConfigureEmbeddedLocalTextCompression("gzip"); + } + } + + [Test] + public async Task StringifyContent_RemotePayloadKeepsNoCompressionMetadata() { + var module = await CompiledRemoteModuleTests.TestData.GetTestRemoteModule(); + var output = module.StringifyContent().Unwrap(); + var bytes = Convert.FromBase64String(StripQuotedBase64(output)); + + Assert.Multiple(() => { + Assert.That(bytes, Is.Not.Empty); + using var zipArchive = new ZipArchive(new MemoryStream(bytes), ZipArchiveMode.Read, false); + Assert.That(zipArchive.Entries, Is.Not.Empty); + }); + } + + [Test] + public async Task GetPowerShellObject_RemotePayloadUsesNoneCompressionMetadata() { + var module = await CompiledRemoteModuleTests.TestData.GetTestRemoteModule(); + var output = module.GetPowerShellObject().Unwrap().ToString(); + + Assert.Multiple(() => { + Assert.That(output, Does.Contain("Compression = 'None'")); + Assert.That(output, Does.Contain("Type = 'Zip'")); + }); + } + + private static string DecompressGzip(byte[] bytes) { + using var input = new MemoryStream(bytes); + using var gzip = new GZipStream(input, CompressionMode.Decompress); + using var reader = new StreamReader(gzip, Encoding.UTF8, true); + return reader.ReadToEnd(); + } + + [GeneratedRegex("^[\"'](?[A-Za-z0-9+/=]+)[\"']$", RegexOptions.Singleline)] + private static partial Regex Base64ContentRegex(); + + private static string StripQuotedBase64(string payload) { + var match = Base64ContentRegex().Match(payload); + return match.Success ? match.Groups["content"].Value : payload; } public static class TestData { diff --git a/tests/Compiler/Module/Compiled/Remote.cs b/tests/Compiler/Module/Compiled/Remote.cs index dfcc37fd..2a853e99 100644 --- a/tests/Compiler/Module/Compiled/Remote.cs +++ b/tests/Compiler/Module/Compiled/Remote.cs @@ -70,6 +70,32 @@ public async Task StringifyContent_RenamesEmbeddedArchiveManifestToHash() { }); } + [Test] + public async Task GetPowerShellObject_UsesNoneCompressionMetadata() { + var module = await TestData.GetTestRemoteModule(); + var output = module.GetPowerShellObject().Unwrap().ToString(); + + Assert.Multiple(() => { + Assert.That(output, Does.Contain("Compression = 'None'")); + Assert.That(output, Does.Contain("Type = 'Zip'")); + }); + } + + [Test] + public async Task GetPowerShellObject_UsesStableIdentityHash() { + var module = await TestData.GetTestRemoteModule(); + var expectedNameHash = module.GetNameHash().Unwrap(); + var expectedHash = expectedNameHash[(module.ModuleSpec.Name.Length + 1)..]; + + _ = module.StringifyContent().Unwrap(); + var output = module.GetPowerShellObject().Unwrap().ToString(); + + Assert.Multiple(() => { + Assert.That(module.GetNameHash().Unwrap(), Is.EqualTo(expectedNameHash)); + Assert.That(output, Does.Contain($"Hash = '{expectedHash}'")); + }); + } + public static class TestData { private static readonly Dictionary TestableRemoteModules = new() { ["Microsoft.PowerShell.PSResourceGet"] = "1.0.5", diff --git a/tests/Compiler/Module/Compiled/Script.cs b/tests/Compiler/Module/Compiled/Script.cs index 37c9ff00..9f607f00 100644 --- a/tests/Compiler/Module/Compiled/Script.cs +++ b/tests/Compiler/Module/Compiled/Script.cs @@ -3,6 +3,7 @@ // for license information. using System.Reflection; +using System.IO.Compression; using System.Text; using System.Text.RegularExpressions; using Compiler.Module.Compiled; @@ -55,7 +56,7 @@ public void GetPowerShellObject_AddsErrorWhenDefineMissing() { var output = module.GetPowerShellObject().Unwrap(); var embeddedPayload = GetEmbeddedModulePayload(output); - var decodedOutput = DecodeBase64Payload(embeddedPayload); + var decodedOutput = DecodeBase64Payload(embeddedPayload, output.Contains("Compression = 'GZip'", StringComparison.Ordinal)); Assert.That(decodedOutput, Does.Contain("!DEFINE UNKNOWN_TOKEN")); } @@ -66,8 +67,16 @@ private static string GetEmbeddedModulePayload(string output) { return match.Groups["content"].Value; } - private static string DecodeBase64Payload(string payload) { - return Encoding.UTF8.GetString(Convert.FromBase64String(payload)); + private static string DecodeBase64Payload(string payload, bool gzip) { + var bytes = Convert.FromBase64String(payload); + if (!gzip) { + return Encoding.UTF8.GetString(bytes); + } + + using var input = new MemoryStream(bytes); + using var gzipStream = new GZipStream(input, CompressionMode.Decompress); + using var reader = new StreamReader(gzipStream, Encoding.UTF8, true); + return reader.ReadToEnd(); } [Test] diff --git a/tests/Compiler/Program.cs b/tests/Compiler/Program.cs index 7ad3b8ce..6f5de7cc 100644 --- a/tests/Compiler/Program.cs +++ b/tests/Compiler/Program.cs @@ -72,6 +72,17 @@ public void EnsureDirectoryStructure( } } + [Test, NonParallelizable] + public void ContentCompressionMode_DefaultsToGZipAndAcceptsNone() { + Assert.That(CompilerSettings.EmbeddedLocalTextCompression, Is.EqualTo(EmbeddedLocalTextCompression.GZip)); + try { + CompilerSettings.ConfigureEmbeddedLocalTextCompression("none"); + Assert.That(CompilerSettings.EmbeddedLocalTextCompression, Is.EqualTo(EmbeddedLocalTextCompression.None)); + } finally { + CompilerSettings.ConfigureEmbeddedLocalTextCompression("gzip"); + } + } + [Test] public async Task Output_ToFile( [Values(false, true)] bool overwrite,