From e30788c39c44ccd8eed100b963baf4fb0c07d7ff Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Mon, 19 Oct 2026 23:39:31 +0000 Subject: [PATCH 1/8] Prototype Linux Helix result cache and shadow validation Add a NativeAOT content-addressed cache and an opt-in Azure Pipelines shadow lane that always executes fresh Helix work items. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ee547d49-5419-4268-b01f-6b2757f99953 --- .../libraries/helix-cache-shadow.yml | 59 +++ eng/pipelines/runtime.yml | 9 + src/tools/helix-cache/Cache.cs | 178 +++++++ src/tools/helix-cache/HelixCache.csproj | 12 + src/tools/helix-cache/JsonFile.cs | 114 +++++ src/tools/helix-cache/LinuxEnvironment.cs | 101 ++++ src/tools/helix-cache/Models.cs | 120 +++++ src/tools/helix-cache/Program.cs | 82 ++++ src/tools/helix-cache/README.md | 198 ++++++++ src/tools/helix-cache/Results.cs | 94 ++++ src/tools/helix-cache/SR.cs | 16 + src/tools/helix-cache/Shadow.proj | 47 ++ src/tools/helix-cache/Snapshot.cs | 159 +++++++ src/tools/helix-cache/Strings.resx | 25 + src/tools/helix-cache/shadow.py | 204 ++++++++ src/tools/helix-cache/tests/test_cache.py | 438 ++++++++++++++++++ 16 files changed, 1856 insertions(+) create mode 100644 eng/pipelines/libraries/helix-cache-shadow.yml create mode 100644 src/tools/helix-cache/Cache.cs create mode 100644 src/tools/helix-cache/HelixCache.csproj create mode 100644 src/tools/helix-cache/JsonFile.cs create mode 100644 src/tools/helix-cache/LinuxEnvironment.cs create mode 100644 src/tools/helix-cache/Models.cs create mode 100644 src/tools/helix-cache/Program.cs create mode 100644 src/tools/helix-cache/README.md create mode 100644 src/tools/helix-cache/Results.cs create mode 100644 src/tools/helix-cache/SR.cs create mode 100644 src/tools/helix-cache/Shadow.proj create mode 100644 src/tools/helix-cache/Snapshot.cs create mode 100644 src/tools/helix-cache/Strings.resx create mode 100644 src/tools/helix-cache/shadow.py create mode 100644 src/tools/helix-cache/tests/test_cache.py diff --git a/eng/pipelines/libraries/helix-cache-shadow.yml b/eng/pipelines/libraries/helix-cache-shadow.yml new file mode 100644 index 00000000000000..73115a9357a198 --- /dev/null +++ b/eng/pipelines/libraries/helix-cache-shadow.yml @@ -0,0 +1,59 @@ +parameters: + enabled: false + archType: '' + osGroup: '' + osSubgroup: '' + buildConfig: '' + runtimeFlavor: '' + runtimeVariant: '' + helixQueues: '' + targetRid: '' + nameSuffix: '' + platform: '' + shouldContinueOnError: false + +steps: +- ${{ if and(parameters.enabled, eq(parameters.platform, 'linux_x64'), eq(variables['System.TeamProject'], 'public')) }}: + # Persist completed entries only. Input snapshots are recreated in each job. + - task: Cache@2 + displayName: Helix shadow result cache + continueOnError: true + inputs: + key: '"helix-results" | "v1" | "linux-x64" | "System.Memory.Data.Tests" | "${{ parameters.buildConfig }}" | "$(Build.BuildId)"' + path: $(Pipeline.Workspace)/.helix-cache/entries + restoreKeys: | + "helix-results" | "v1" | "linux-x64" | "System.Memory.Data.Tests" | "${{ parameters.buildConfig }}" + + - script: | + set -euo pipefail + dotnet="$(Build.SourcesDirectory)/.dotnet/dotnet" + project="$(Build.SourcesDirectory)/src/tools/helix-cache" + output="$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/helix-cache-shadow" + cache="$(Pipeline.Workspace)/.helix-cache" + mkdir -p "$cache/entries" + + "$dotnet" publish "$project/HelixCache.csproj" -c Release -r linux-x64 \ + -o "$(Build.SourcesDirectory)/artifacts/tools/helix-cache" + executable="$(Build.SourcesDirectory)/artifacts/tools/helix-cache/helix-cache" + python3 "$project/tests/test_cache.py" "$executable" + + "$dotnet" msbuild "$(Build.SourcesDirectory)/src/libraries/sendtohelix.proj" \ + -t:PrepareCorrelationPayloadDirectory -p:Configuration=$(_BuildConfig) -p:RuntimeFlavor=CoreCLR + settings="$("$dotnet" msbuild "$(Build.SourcesDirectory)/src/libraries/sendtohelix.proj" \ + -nologo -p:Configuration=$(_BuildConfig) -getProperty:NetCoreAppCurrentBuildSettings)" + archives="$("$dotnet" msbuild "$(Build.SourcesDirectory)/src/libraries/System.Memory.Data/tests/System.Memory.Data.Tests.csproj" \ + -nologo -p:Configuration=$(_BuildConfig) -getProperty:TestArchiveTestsDir)" + python3 "$project/shadow.py" prepare \ + --executable "$executable" --cache "$cache" --output "$output" \ + --payload "$archives/System.Memory.Data.Tests.zip" \ + --runtime "$(Build.SourcesDirectory)/artifacts/helix/runtime/test-runtime-$settings.zip" \ + --queue AzureLinux.3.Amd64.Open + + # A cache hit never bypasses this submission. + "$dotnet" msbuild "$project/Shadow.proj" -t:Submit \ + -p:HelixCacheShadowDirectory="$output" -p:ShadowCreator=dotnet-bot + python3 "$project/shadow.py" collect \ + --executable "$executable" --cache "$cache" --output "$output" + displayName: Validate Helix cache in shadow mode (always execute) + continueOnError: true + timeoutInMinutes: 75 diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index d97952180cb74f..032aae21615fb2 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -1,3 +1,9 @@ +parameters: +- name: helixCacheShadow + displayName: Validate Linux Helix result caching (shadow mode) + type: boolean + default: false + # Setting batch to true, triggers one build at a time. # if there is a push while a build in progress, it will wait, # until the running build finishes, and produce a build with all the changes @@ -385,6 +391,9 @@ extends: eq(variables['archType'], 'arm64'), eq(variables['coreclrContainsChange'], true)), eq(variables['isRollingBuild'], true)) + - template: /eng/pipelines/libraries/helix-cache-shadow.yml + parameters: + enabled: ${{ parameters.helixCacheShadow }} condition: >- or( eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), diff --git a/src/tools/helix-cache/Cache.cs b/src/tools/helix-cache/Cache.cs new file mode 100644 index 00000000000000..6f1553620cb06d --- /dev/null +++ b/src/tools/helix-cache/Cache.cs @@ -0,0 +1,178 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; + +namespace HelixCache; + +internal sealed class Cache +{ + internal const int ProtocolVersion = 1; + private readonly string _root; + private readonly Dictionary _snapshots = new(StringComparer.Ordinal); + + internal Cache(string directory) => _root = Path.GetFullPath(directory); + + internal LookupResult Lookup(Submission submission, string baseDirectory, LinuxEnvironment environment) + { + JsonFile.ValidateVersion(submission.SchemaVersion); + JsonFile.Require(submission.WorkItems.Count > 0, nameof(submission.WorkItems)); + LookupResult result = new(); + HashSet names = new(StringComparer.Ordinal); + foreach (WorkItem item in submission.WorkItems) + { + JsonFile.Require(!string.IsNullOrWhiteSpace(item.Name) && names.Add(item.Name), nameof(item.Name)); + JsonFile.Require(!string.IsNullOrWhiteSpace(item.Command), nameof(item.Command)); + JsonFile.Require(item.TimeoutSeconds > 0, nameof(item.TimeoutSeconds)); + JsonFile.Require(item.Queues.Count > 0 && item.Queues.Distinct(StringComparer.Ordinal).Count() == item.Queues.Count, nameof(item.Queues)); + JsonFile.Require(item.PreCommands.All(c => c is not null) && item.PostCommands.All(c => c is not null) && + item.Environment.All(p => !string.IsNullOrEmpty(p.Key) && p.Value is not null), nameof(item.Environment)); + Snapshot payload = Capture(item.Payload, baseDirectory); + List correlation = item.CorrelationPayloads.Select(p => Capture(p, baseDirectory)).ToList(); + + foreach (string target in item.Queues) + { + JsonFile.Require(!string.IsNullOrWhiteSpace(target), nameof(item.Queues)); + ExecutionInputs inputs = new() + { + Name = item.Name, + TargetQueue = target, + EnvironmentFingerprint = environment.Resolve(target), + Payload = Identity(item.Payload, payload), + CorrelationPayloads = item.CorrelationPayloads.Select((p, i) => Identity(p, correlation[i])).ToList(), + Command = item.Command, + PreCommands = item.PreCommands, + PostCommands = item.PostCommands, + Environment = item.Environment, + TimeoutSeconds = item.TimeoutSeconds, + ExpectedExitCode = item.ExpectedExitCode, + }; + ExecutionRecord execution = new() + { + SchemaVersion = ProtocolVersion, + Key = Key(inputs), + Inputs = inputs, + PayloadPath = payload.Path, + CorrelationPayloadPaths = correlation.Select(p => p.Path).ToList(), + }; + string entry = EntryPath(execution.Key); + bool hit = Directory.Exists(entry); + if (hit) + { + ValidateEntry(entry, execution.Key); + } + result.Entries.Add(new LookupEntry + { + Status = hit ? "hit" : "miss", + Execution = execution, + ResultsPath = hit ? Path.Combine(entry, "results") : null, + }); + } + } + return result; + } + + internal void Store(ExecutionRecord execution, string results) + { + ValidateRecord(execution); + VerifySnapshot(execution.PayloadPath, execution.Inputs.Payload.Digest); + for (int i = 0; i < execution.CorrelationPayloadPaths.Count; i++) + { + VerifySnapshot(execution.CorrelationPayloadPaths[i], execution.Inputs.CorrelationPayloads[i].Digest); + } + results = Path.GetFullPath(results); + Snapshot.EnsureOutside(results, _root); + using TemporaryDirectory temporary = new(Path.Combine(_root, "tmp")); + Snapshot.EnsureOutside(results, temporary.Path); + string stagedResults = Path.Combine(temporary.Path, "results"); + Snapshot.Copy(results, stagedResults); + TestSummary summary = Results.Validate(stagedResults, execution); + StoredEntry entry = new() + { + SchemaVersion = ProtocolVersion, + Execution = execution, + CreatedUtc = DateTimeOffset.UtcNow, + ResultsDigest = Snapshot.Hash(stagedResults), + Summary = summary, + }; + JsonFile.Write(Path.Combine(temporary.Path, "entry.json"), entry, CacheJsonContext.Default.StoredEntry); + string final = EntryPath(execution.Key); + Directory.CreateDirectory(Path.GetDirectoryName(final)!); + try + { + Directory.Move(temporary.Path, final); + } + catch (IOException) when (Directory.Exists(final)) + { + ValidateEntry(final, execution.Key); + } + } + + private Snapshot Capture(Payload payload, string baseDirectory) + { + JsonFile.Require(!string.IsNullOrWhiteSpace(payload.Path), nameof(payload.Path)); + JsonFile.RelativePath(payload.Destination, allowEmpty: true); + string path = Path.GetFullPath(payload.Path, baseDirectory); + if (!_snapshots.TryGetValue(path, out Snapshot? snapshot)) + { + Snapshot.EnsureOutside(path, _root); + snapshot = Snapshot.Capture(path, _root); + _snapshots.Add(path, snapshot); + } + return snapshot; + } + + private static PayloadIdentity Identity(Payload payload, Snapshot snapshot) => new() + { + Digest = snapshot.Digest, + Destination = payload.Destination, + Kind = snapshot.Kind, + }; + + private void ValidateEntry(string directory, string key) + { + StoredEntry entry = JsonFile.Read(Path.Combine(directory, "entry.json"), CacheJsonContext.Default.StoredEntry); + JsonFile.ValidateVersion(entry.SchemaVersion); + ValidateRecord(entry.Execution); + string results = Path.Combine(directory, "results"); + if (entry.Execution.Key != key || Snapshot.Hash(results) != entry.ResultsDigest) + { + throw new InvalidDataException(SR.Get("Corrupt", directory)); + } + TestSummary summary = Results.Validate(results, entry.Execution); + if (summary.Total != entry.Summary.Total || summary.Passed != entry.Summary.Passed || summary.Skipped != entry.Summary.Skipped) + { + throw new InvalidDataException(SR.Get("Corrupt", directory)); + } + } + + private static void ValidateRecord(ExecutionRecord execution) + { + JsonFile.ValidateVersion(execution.SchemaVersion); + if (!IsDigest(execution.Key) || Key(execution.Inputs) != execution.Key || + execution.CorrelationPayloadPaths.Count != execution.Inputs.CorrelationPayloads.Count) + { + throw new InvalidDataException(SR.Get("Corrupt", execution.Key)); + } + } + + private static void VerifySnapshot(string path, string digest) + { + if (Snapshot.Hash(path) != digest) + { + throw new InvalidDataException(SR.Get("Corrupt", path)); + } + } + + private string EntryPath(string key) => Path.Combine(_root, "entries", "v1", key[..2], key); + + private static string Key(ExecutionInputs inputs) => + JsonFile.Digest(JsonSerializer.SerializeToElement(inputs, CacheJsonContext.Default.ExecutionInputs)); + + internal static bool IsDigest(string value) => + value.Length == 64 && value.All(c => c is >= '0' and <= '9' or >= 'a' and <= 'f'); +} diff --git a/src/tools/helix-cache/HelixCache.csproj b/src/tools/helix-cache/HelixCache.csproj new file mode 100644 index 00000000000000..8fbae8b8951256 --- /dev/null +++ b/src/tools/helix-cache/HelixCache.csproj @@ -0,0 +1,12 @@ + + + helix-cache + $(NetCoreAppToolCurrent) + Exe + false + true + enable + false + true + + diff --git a/src/tools/helix-cache/JsonFile.cs b/src/tools/helix-cache/JsonFile.cs new file mode 100644 index 00000000000000..e07132a81c5e02 --- /dev/null +++ b/src/tools/helix-cache/JsonFile.cs @@ -0,0 +1,114 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization.Metadata; + +namespace HelixCache; + +internal static class JsonFile +{ + internal static T Read(string path, JsonTypeInfo type) + { + using FileStream stream = File.OpenRead(path); + using JsonDocument document = JsonDocument.Parse(stream); + using (Utf8JsonWriter validator = new(Stream.Null)) + { + WriteCanonical(validator, document.RootElement); + } + return JsonSerializer.Deserialize(document.RootElement, type) ?? throw new InvalidDataException(SR.Get("InvalidField", path)); + } + + internal static void Write(string path, T value, JsonTypeInfo type) + { + path = Path.GetFullPath(path); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + string temporary = path + "." + Guid.NewGuid().ToString("N") + ".tmp"; + try + { + using (FileStream stream = new(temporary, FileMode.CreateNew, FileAccess.Write)) + { + JsonSerializer.Serialize(stream, value, type); + stream.Flush(flushToDisk: true); + } + File.Move(temporary, path, overwrite: true); + } + finally + { + File.Delete(temporary); + } + } + + internal static string Digest(JsonElement value) + { + using MemoryStream stream = new(); + using (Utf8JsonWriter writer = new(stream)) + { + WriteCanonical(writer, value); + } + return Convert.ToHexStringLower(SHA256.HashData(stream.GetBuffer().AsSpan(0, checked((int)stream.Length)))); + } + + internal static void WriteCanonical(Utf8JsonWriter writer, JsonElement value) + { + switch (value.ValueKind) + { + case JsonValueKind.Object: + writer.WriteStartObject(); + HashSet names = new(StringComparer.Ordinal); + foreach (JsonProperty property in value.EnumerateObject().OrderBy(p => p.Name, StringComparer.Ordinal)) + { + if (!names.Add(property.Name)) + { + throw new InvalidDataException(SR.Get("Duplicate", property.Name)); + } + writer.WritePropertyName(property.Name); + WriteCanonical(writer, property.Value); + } + writer.WriteEndObject(); + break; + case JsonValueKind.Array: + writer.WriteStartArray(); + foreach (JsonElement item in value.EnumerateArray()) + { + Require(item.ValueKind != JsonValueKind.Null, nameof(item)); + WriteCanonical(writer, item); + } + writer.WriteEndArray(); + break; + default: + value.WriteTo(writer); + break; + } + } + + internal static void ValidateVersion(int version) + { + if (version != Cache.ProtocolVersion) + { + throw new InvalidDataException(SR.Get("Version", version, Cache.ProtocolVersion)); + } + } + + internal static void Require(bool condition, string field) + { + if (!condition) + { + throw new InvalidDataException(SR.Get("InvalidField", field)); + } + } + + internal static void RelativePath(string path, bool allowEmpty = false) + { + Require(allowEmpty || !string.IsNullOrWhiteSpace(path), nameof(path)); + if (Path.IsPathRooted(path) || path.Split('/').Any(p => p is "." or "..") || path.Contains('\\')) + { + throw new InvalidDataException(SR.Get("RelativePath", path)); + } + } +} diff --git a/src/tools/helix-cache/LinuxEnvironment.cs b/src/tools/helix-cache/LinuxEnvironment.cs new file mode 100644 index 00000000000000..fc231da1df16f6 --- /dev/null +++ b/src/tools/helix-cache/LinuxEnvironment.cs @@ -0,0 +1,101 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net.Http; +using System.Text.Json; +using System.Threading.Tasks; + +namespace HelixCache; + +internal sealed class LinuxEnvironment : IDisposable +{ + private readonly JsonDocument _queues; + private readonly Dictionary _fingerprints = new(StringComparer.Ordinal); + + private LinuxEnvironment(JsonDocument queues) => _queues = queues; + + internal static async Task LoadAsync(string? queueInfo) + { + if (queueInfo is not null) + { + using FileStream file = File.OpenRead(queueInfo); + return new LinuxEnvironment(await JsonDocument.ParseAsync(file)); + } + + using HttpClient client = new() { Timeout = TimeSpan.FromSeconds(60) }; + using Stream stream = await client.GetStreamAsync("https://helix.dot.net/api/2019-06-17/info/queues"); + return new LinuxEnvironment(await JsonDocument.ParseAsync(stream)); + } + + internal JsonElement Resolve(string target) + { + if (_fingerprints.TryGetValue(target, out JsonElement fingerprint)) + { + return fingerprint; + } + + // Matches Arcade JobSender's (alias)host@container syntax; retain a digest's second '@'. + // https://github.com/dotnet/arcade/blob/7103cc604358625941ee1d105e662f0d697085c6/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs#L374-L399 + int separator = target.IndexOf('@'); + string host = separator < 0 ? target : target[..separator]; + string? container = separator < 0 ? null : target[(separator + 1)..]; + if (host.StartsWith('(')) + { + int close = host.IndexOf(')'); + JsonFile.Require(separator >= 0 && close > 1 && close < host.Length - 1, nameof(target)); + host = host[(close + 1)..]; + } + + if (container is not null) + { + int digestStart = container.LastIndexOf("@sha256:", StringComparison.Ordinal); + if (digestStart <= 0 || !Cache.IsDigest(container[(digestStart + "@sha256:".Length)..])) + { + throw new InvalidDataException(SR.Get("Container", container)); + } + } + + JsonElement? queue = null; + foreach (JsonElement candidate in _queues.RootElement.EnumerateArray()) + { + if (string.Equals(candidate.GetProperty("QueueId").GetString(), host, StringComparison.OrdinalIgnoreCase)) + { + JsonFile.Require(queue is null, nameof(target)); + queue = candidate; + } + } + + if (queue is not JsonElement info || + !string.Equals(info.GetProperty("OperatingSystemGroup").GetString(), "linux", StringComparison.OrdinalIgnoreCase) || + !string.Equals(info.GetProperty("Architecture").GetString(), "AMD64", StringComparison.OrdinalIgnoreCase) || + !info.GetProperty("IsAvailable").GetBoolean()) + { + throw new InvalidDataException(SR.Get("Queue", host)); + } + + using MemoryStream buffer = new(); + using (Utf8JsonWriter writer = new(buffer)) + { + writer.WriteStartObject(); + foreach (string property in new[] { "QueueId", "Architecture", "OperatingSystemGroup", "GalleryImage", "PreInstalledImage", "PreparedImage", "Artifacts", "WorkspacePath" }) + { + if (info.TryGetProperty(property, out JsonElement value)) + { + writer.WritePropertyName(property); + JsonFile.WriteCanonical(writer, value); + } + } + writer.WriteString("Container", container); + writer.WriteEndObject(); + } + using JsonDocument document = JsonDocument.Parse(buffer.ToArray()); + fingerprint = document.RootElement.Clone(); + _fingerprints.Add(target, fingerprint); + return fingerprint; + } + + public void Dispose() => _queues.Dispose(); +} diff --git a/src/tools/helix-cache/Models.cs b/src/tools/helix-cache/Models.cs new file mode 100644 index 00000000000000..baa49be7963030 --- /dev/null +++ b/src/tools/helix-cache/Models.cs @@ -0,0 +1,120 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace HelixCache; + +internal sealed class Submission +{ + public required int SchemaVersion { get; init; } + public required List WorkItems { get; init; } +} + +internal sealed class WorkItem +{ + public required string Name { get; init; } + public required List Queues { get; init; } + public required Payload Payload { get; init; } + public List CorrelationPayloads { get; init; } = []; + public required string Command { get; init; } + public List PreCommands { get; init; } = []; + public List PostCommands { get; init; } = []; + public Dictionary Environment { get; init; } = []; + public required int TimeoutSeconds { get; init; } + public required int ExpectedExitCode { get; init; } +} + +internal sealed class Payload +{ + public required string Path { get; init; } + public string Destination { get; init; } = ""; +} + +internal sealed class PayloadIdentity +{ + public required string Digest { get; init; } + public required string Destination { get; init; } + public required string Kind { get; init; } +} + +internal sealed class ExecutionInputs +{ + public required string Name { get; init; } + public required string TargetQueue { get; init; } + public required JsonElement EnvironmentFingerprint { get; init; } + public required PayloadIdentity Payload { get; init; } + public required List CorrelationPayloads { get; init; } + public required string Command { get; init; } + public required List PreCommands { get; init; } + public required List PostCommands { get; init; } + public required Dictionary Environment { get; init; } + public required int TimeoutSeconds { get; init; } + public required int ExpectedExitCode { get; init; } +} + +internal sealed class ExecutionRecord +{ + public required int SchemaVersion { get; init; } + public required string Key { get; init; } + public required ExecutionInputs Inputs { get; init; } + public required string PayloadPath { get; init; } + public required List CorrelationPayloadPaths { get; init; } +} + +internal sealed class LookupResult +{ + public int SchemaVersion { get; init; } = Cache.ProtocolVersion; + public List Entries { get; init; } = []; +} + +internal sealed class LookupEntry +{ + public required string Status { get; init; } + public required ExecutionRecord Execution { get; init; } + public string? ResultsPath { get; init; } +} + +internal sealed class ResultReceipt +{ + public required int SchemaVersion { get; init; } + public required bool Completed { get; init; } + public required bool InfrastructureSucceeded { get; init; } + public required int ExitCode { get; init; } + public required string JobId { get; init; } + public required string WorkItemName { get; init; } + public required string TargetQueue { get; init; } + public required List ResultFiles { get; init; } +} + +internal sealed class TestSummary +{ + public int Total { get; set; } + public int Passed { get; set; } + public int Skipped { get; set; } +} + +internal sealed class StoredEntry +{ + public required int SchemaVersion { get; init; } + public required ExecutionRecord Execution { get; init; } + public required DateTimeOffset CreatedUtc { get; init; } + public required string ResultsDigest { get; init; } + public required TestSummary Summary { get; init; } +} + +[JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + WriteIndented = true, + UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow, + RespectNullableAnnotations = true)] +[JsonSerializable(typeof(Submission))] +[JsonSerializable(typeof(ExecutionRecord))] +[JsonSerializable(typeof(ExecutionInputs))] +[JsonSerializable(typeof(LookupResult))] +[JsonSerializable(typeof(ResultReceipt))] +[JsonSerializable(typeof(StoredEntry))] +internal sealed partial class CacheJsonContext : JsonSerializerContext; diff --git a/src/tools/helix-cache/Program.cs b/src/tools/helix-cache/Program.cs new file mode 100644 index 00000000000000..9d2b772274468d --- /dev/null +++ b/src/tools/helix-cache/Program.cs @@ -0,0 +1,82 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net.Http; +using System.Text.Json; +using System.Threading.Tasks; +using System.Xml; + +namespace HelixCache; + +internal static class Program +{ + private static async Task Main(string[] args) + { + try + { + if (args is ["--help"] or ["-h"]) + { + Console.WriteLine(SR.Get("Usage")); + return 0; + } + if (!OperatingSystem.IsLinux()) + { + throw new PlatformNotSupportedException(SR.Get("UnsupportedPlatform")); + } + if (args.Length < 2 || args[0] is not ("lookup" or "store") || args.Length % 2 != 0) + { + throw new InvalidDataException(SR.Get("Usage")); + } + + Dictionary options = new(StringComparer.Ordinal); + for (int i = 2; i < args.Length; i += 2) + { + if (!options.TryAdd(args[i], args[i + 1]) || string.IsNullOrWhiteSpace(args[i + 1])) + { + throw new InvalidDataException(SR.Get("Usage")); + } + } + + Cache cache = new(Take(options, "--cache")); + string manifest = Path.GetFullPath(args[1]); + if (args[0] == "lookup") + { + string output = Take(options, "--output"); + options.Remove("--queue-info", out string? queueInfo); + JsonFile.Require(options.Count == 0, nameof(options)); + Submission submission = JsonFile.Read(manifest, CacheJsonContext.Default.Submission); + using LinuxEnvironment environment = await LinuxEnvironment.LoadAsync(queueInfo); + LookupResult result = cache.Lookup(submission, Path.GetDirectoryName(manifest)!, environment); + JsonFile.Write(output, result, CacheJsonContext.Default.LookupResult); + } + else + { + string results = Take(options, "--results"); + JsonFile.Require(options.Count == 0, nameof(options)); + ExecutionRecord execution = JsonFile.Read(manifest, CacheJsonContext.Default.ExecutionRecord); + cache.Store(execution, results); + Console.Error.WriteLine(SR.Get("Stored", execution.Key)); + } + return 0; + } + catch (Exception ex) when (ex is InvalidDataException or IOException or UnauthorizedAccessException or JsonException or + HttpRequestException or TaskCanceledException or XmlException or ArgumentException or + InvalidOperationException or KeyNotFoundException or OverflowException or PlatformNotSupportedException) + { + Console.Error.WriteLine(SR.Get("Error", ex.Message)); + return 1; + } + } + + private static string Take(Dictionary options, string name) + { + if (!options.Remove(name, out string? value)) + { + throw new InvalidDataException(SR.Get("Usage")); + } + return value; + } +} diff --git a/src/tools/helix-cache/README.md b/src/tools/helix-cache/README.md new file mode 100644 index 00000000000000..3affdccb1a5092 --- /dev/null +++ b/src/tools/helix-cache/README.md @@ -0,0 +1,198 @@ +# Helix result cache prototype + +A C# NativeAOT CLI for a local, trusted filesystem cache of Linux x64 Helix +work-item results. An opt-in Azure Pipelines shadow lane exercises actual +submission and collection without replacing any test executions or results. + +## Build and exercise + +From the repository root, using its configured SDK and NuGet feeds: + +```sh +dotnet publish src/tools/helix-cache/HelixCache.csproj -c Release -r linux-x64 +python3 src/tools/helix-cache/tests/test_cache.py \ + artifacts/bin/HelixCache/Release/net11.0/linux-x64/publish/helix-cache +``` + +NativeAOT publishing requires a supported native compiler/linker and development +libraries. For managed development, build with `-p:PublishAot=false` and run the +generated apphost with the SDK runtime available. + +## Lookup + +```sh +helix-cache lookup submission.json --cache /path/to/cache --output lookup.json +``` + +Example `submission.json`: + +```json +{ + "schemaVersion": 1, + "workItems": [ + { + "name": "Example.Tests", + "queues": ["AzureLinux.3.Amd64.Open"], + "payload": { "path": "Example.Tests.zip" }, + "correlationPayloads": [ + { "path": "test-runtime.zip", "destination": "" } + ], + "command": "./RunTests.sh", + "preCommands": [], + "postCommands": [], + "environment": { "DOTNET_TieredCompilation": "1" }, + "timeoutSeconds": 900, + "expectedExitCode": 0 + } + ] +} +``` + +Input paths are relative to the submission manifest. A payload is either a ZIP +archive (hashed as opaque bytes) or a directory. Directory snapshots preserve +Unix modes and empty directories and normalize filesystem timestamps. Symlinks +in directory inputs/results are rejected; put linked payloads in an archive. +Shared inputs are captured once per invocation. + +The tool resolves the actual host queue, requires an available Linux AMD64 queue, +and fingerprints its declared image, ordered provisioning recipe and workspace +path. Docker targets use Arcade's `(alias)host@image` syntax and must include a +lowercase `@sha256:` digest. Operational queue state is excluded. The full target +string remains part of the execution identity, including its alias. + +For offline use or deterministic tests, supply `--queue-info queues.json`, where +the file is a trusted snapshot of the array returned by +`https://helix.dot.net/api/2019-06-17/info/queues`. Without that option the tool +fetches queue information once from that endpoint. + +Each `lookup.json` entry contains `status` (`hit` or `miss`), an `execution` +record, and a `resultsPath` for hits. Hits and misses both exit zero; malformed +inputs, HTTP failures, unreadable files and corrupt entries exit nonzero. +Diagnostics go to stderr. The caller must retain every requested item when +combining cached and fresh results, including an all-hit submission. + +**Submit the snapshot paths in the execution record, not the original input +paths.** Treat snapshots as immutable. If the sender needs writable directories +(the Helix SDK can write `.payload` markers), make separate submission copies +from these snapshots. Do not resolve inputs again after computing the key. + +## Store completed results + +Save one entry's `execution` object from `lookup.json` as `execution.json`, and +collect the original work item's results into a directory: + +```sh +helix-cache store execution.json --results downloaded-results --cache /path/to/cache +``` + +The directory must include a `result.json` receipt constructed by the trusted +result collector from Helix's completed-work-item status: + +```json +{ + "schemaVersion": 1, + "completed": true, + "infrastructureSucceeded": true, + "exitCode": 0, + "jobId": "00000000-0000-0000-0000-000000000001", + "workItemName": "Example.Tests", + "targetQueue": "AzureLinux.3.Amd64.Open", + "resultFiles": ["testResults.xml"] +} +``` + +`infrastructureSucceeded` must be false for Helix infrastructure errors, +cancellations or timeouts. Submission success is not completion. This tool does +not independently authenticate the receipt with Helix. + +V1 accepts xUnit XML (`assemblies` / `assembly` / `test`) only. Each assembly must +contain tests whose results agree with its total/passed/failed/skipped counts; +failures, errors, empty or incomplete reports are rejected. Skips are retained. +All files in the results directory are copied and checksummed. The execution key +and input snapshots are checked before publishing an entry. Atomic directory +renames make concurrent stores safe; an existing valid entry wins. + +Store uses the environment captured at lookup, never today's queue metadata. +Lookup checks artifact integrity and reports corruption as an error rather than +returning a cached pass. Original job provenance and reports are retained. +Cached results must be labeled as reused when published; no new Helix job is +invented. TRX/JUnit, cache eviction and replacing normal MSBuild test submissions +with cached results are outside this prototype. + +## Azure Pipelines shadow validation + +Queue `eng/pipelines/runtime.yml` with its `helixCacheShadow` boolean parameter +enabled. The default is **false**. The additional steps run only in the public +project's Linux x64 `CoreCLR_AllSubsets` job, after its normal submissions. +They use the existing build's `System.Memory.Data.Tests.zip` and runtime archive +and submit one additional work item to `AzureLinux.3.Amd64.Open`. + +`Cache@2` restores `$(Pipeline.Workspace)/.helix-cache/entries`, using a rolling +per-build snapshot key and a stable restore prefix, as the compiler caches do. +The cache task's build ID is **not** part of an execution digest. Only completed +entries are persisted; frozen input snapshots are recreated in each job. +Existing Azure Pipelines branch/PR cache scoping applies. + +`shadow.py` is a thin Python standard-library orchestration/transport layer; all +input capture, digest computation, cache validation and admission use the C# +NativeAOT executable. Its `prepare` command records the full command, upload +post-command, timeout and queue, runs native lookup, and exports frozen paths +to `submission.props`. `Shadow.proj` submits those exact archives through the +pinned Helix SDK without injecting extra reporters. Its default target only +validates inputs; submission requires **`-t:Submit`**, which waits for completion +and cancels the submitted job on task cancellation. + +`collect` checks the public Helix API's job, host queue, work-item identity, terminal state, +infrastructure errors, failure reason and exit code, then downloads the uploaded +xUnit XML and console log. It constructs the receipt from that completion, +passes the report through native admission, and compares any hit with the fresh +test identities/outcomes/skip reasons. Ordering, worker directory prefixes and +execution timings are ignored. The original cached job provenance is retained. +Every hit still executes on Helix; no cached result is published as a new run. + +Evidence is included in the existing build-log artifact under +`artifacts/log//helix-cache-shadow/`, including `lookup.json`, +`execution.json`, `work-item.json`, `summary.json`, and fresh results. Shadow +errors are visible but use `continueOnError`, so they do not replace or weaken +the normal test decision. Internal queues and authentication are not supported +by this first shadow lane. + +For local preparation without submitting, after building the archives: + +```sh +python3 src/tools/helix-cache/shadow.py prepare \ + --executable artifacts/bin/HelixCache/Release/net11.0/linux-x64/publish/helix-cache \ + --cache artifacts/tmp/helix-cache \ + --output artifacts/log/Release/helix-cache-shadow \ + --payload artifacts/helix/tests/linux.AnyCPU.Release/System.Memory.Data.Tests.zip \ + --runtime artifacts/helix/runtime/test-runtime-net11.0-linux-Release-x64.zip \ + --queue AzureLinux.3.Amd64.Open +dotnet msbuild src/tools/helix-cache/Shadow.proj -t:Validate \ + -p:HelixCacheShadowDirectory="$PWD/artifacts/log/Release/helix-cache-shadow" +``` + +The integration suite mocks public completion/file endpoints; live submission, +cross-build restoration and unchanged-rebuild hit rates require the opt-in CI +run. ZIPs are currently hashed byte-for-byte, so archive timestamps or other +nondeterministic build outputs can prevent reuse despite unchanged sources. + +## Cache contract + +Keys include the schema namespace, work-item name, target queue, declared Linux +environment, payload identities and destinations, ordered correlation overlays, +commands, environment overrides, timeout and expected exit code. Object +properties are sorted ordinally; command and overlay order is preserved. Local +absolute paths and result timestamps are not key material. + +This is a fingerprint of the **declared Linux environment**, not an attestation of +the worker's installed packages, kernel or CPU features. Native queues can drift +without changing their published recipes; a pinned container identifies its +userspace but not its host. External services and randomized/stress trials may +also make result reuse inappropriate. Enable caching only where that equivalence +contract is acceptable. Build-specific command arguments remain in the key; +there is no heuristic removal of build IDs or reporting arguments. + +The cache and manifests are trusted local inputs, not a shared-cache security +boundary. Input snapshots live under `inputs/v1`; completed results are under +`entries/v1//`. Do not mutate entries or delete snapshots while +their executions are outstanding. diff --git a/src/tools/helix-cache/Results.cs b/src/tools/helix-cache/Results.cs new file mode 100644 index 00000000000000..dfccbef2179b07 --- /dev/null +++ b/src/tools/helix-cache/Results.cs @@ -0,0 +1,94 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Xml; +using System.Xml.Linq; + +namespace HelixCache; + +internal static class Results +{ + internal static TestSummary Validate(string directory, ExecutionRecord execution) + { + ResultReceipt receipt = JsonFile.Read(Path.Combine(directory, "result.json"), CacheJsonContext.Default.ResultReceipt); + JsonFile.ValidateVersion(receipt.SchemaVersion); + if (!receipt.Completed || !receipt.InfrastructureSucceeded || + receipt.ExitCode != execution.Inputs.ExpectedExitCode || + receipt.WorkItemName != execution.Inputs.Name || + receipt.TargetQueue != execution.Inputs.TargetQueue || + !Guid.TryParse(receipt.JobId, out _) || receipt.ResultFiles.Count == 0) + { + throw new InvalidDataException(SR.Get("Result", directory)); + } + + TestSummary summary = new(); + HashSet reports = new(StringComparer.Ordinal); + foreach (string report in receipt.ResultFiles) + { + JsonFile.RelativePath(report); + JsonFile.Require(reports.Add(report), nameof(receipt.ResultFiles)); + string path = Path.Combine(directory, report); + using XmlReader reader = XmlReader.Create(path, new XmlReaderSettings { DtdProcessing = DtdProcessing.Prohibit, XmlResolver = null }); + XDocument document = XDocument.Load(reader); + if (document.Root is not XElement root || root.Name != "assemblies" || !root.Elements("assembly").Any()) + { + throw new InvalidDataException(SR.Get("Report", report)); + } + + foreach (XElement assembly in root.Elements("assembly")) + { + TestSummary actual = new(); + foreach (XElement test in assembly.Descendants("test")) + { + checked + { + actual.Total++; + switch ((string?)test.Attribute("result")) + { + case "Pass": + actual.Passed++; + break; + case "Skip": + actual.Skipped++; + break; + default: + throw new InvalidDataException(SR.Get("Report", report)); + } + } + } + + if (actual.Total == 0 || Count(assembly, "total") != actual.Total || + Count(assembly, "passed") != actual.Passed || Count(assembly, "skipped") != actual.Skipped || + Count(assembly, "failed") != 0 || + assembly.Descendants("error").Any() || assembly.Descendants("failure").Any() || + (assembly.Attribute("errors") is not null && Count(assembly, "errors") != 0)) + { + throw new InvalidDataException(SR.Get("Report", report)); + } + + checked + { + summary.Total += actual.Total; + summary.Passed += actual.Passed; + summary.Skipped += actual.Skipped; + } + } + } + + return summary; + } + + private static int Count(XElement assembly, string name) + { + if (!int.TryParse((string?)assembly.Attribute(name), NumberStyles.None, CultureInfo.InvariantCulture, out int count)) + { + throw new InvalidDataException(SR.Get("Report", name)); + } + return count; + } +} diff --git a/src/tools/helix-cache/SR.cs b/src/tools/helix-cache/SR.cs new file mode 100644 index 00000000000000..ddee04d4ff2a1e --- /dev/null +++ b/src/tools/helix-cache/SR.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Globalization; +using System.Resources; + +namespace HelixCache; + +internal static class SR +{ + private static readonly ResourceManager s_resources = new("HelixCache.Strings", typeof(SR).Assembly); + + internal static string Get(string name, params object?[] arguments) => + string.Format(CultureInfo.CurrentCulture, + s_resources.GetString(name) ?? throw new MissingManifestResourceException(name), arguments); +} diff --git a/src/tools/helix-cache/Shadow.proj b/src/tools/helix-cache/Shadow.proj new file mode 100644 index 00000000000000..313e7857c79b48 --- /dev/null +++ b/src/tools/helix-cache/Shadow.proj @@ -0,0 +1,47 @@ + + + + + $(ShadowQueue) + + + + + + $(ShadowPayload) + $(ShadowCommand) + $(ShadowPostCommand) + $(ShadowTimeout) + + + + + + + + + + + + + + + + + + 1 + $(ShadowCancellationToken) + + + + + diff --git a/src/tools/helix-cache/Snapshot.cs b/src/tools/helix-cache/Snapshot.cs new file mode 100644 index 00000000000000..310a06cf2b8697 --- /dev/null +++ b/src/tools/helix-cache/Snapshot.cs @@ -0,0 +1,159 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text.Json; + +namespace HelixCache; + +internal sealed class Snapshot +{ + private static readonly DateTime s_timestamp = new(2000, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + internal required string Path { get; init; } + internal required string Digest { get; init; } + internal required string Kind { get; init; } + + internal static Snapshot Capture(string source, string cache) + { + source = System.IO.Path.GetFullPath(source); + FileAttributes attributes = File.GetAttributes(source); + RejectLink(source, attributes); + bool directory = (attributes & FileAttributes.Directory) != 0; + if (!directory && !source.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidDataException(SR.Get("Payload", source)); + } + + using TemporaryDirectory temporary = new(System.IO.Path.Combine(cache, "tmp")); + EnsureOutside(source, temporary.Path); + string name = directory ? "payload" : System.IO.Path.GetFileName(source); + string staged = System.IO.Path.Combine(temporary.Path, name); + Copy(source, staged); + string digest = Hash(staged); + string parent = System.IO.Path.Combine(cache, "inputs", "v1"); + Directory.CreateDirectory(parent); + string final = System.IO.Path.Combine(parent, digest); + try + { + Directory.Move(temporary.Path, final); + } + catch (IOException) when (Directory.Exists(final)) + { + string existing = System.IO.Path.Combine(final, name); + if (Hash(existing) != digest) + { + throw new InvalidDataException(SR.Get("Corrupt", existing)); + } + } + + return new Snapshot { Path = System.IO.Path.Combine(final, name), Digest = digest, Kind = directory ? "directory" : "archive" }; + } + + internal static void EnsureOutside(string source, string destination) + { + if (Directory.Exists(source) && + (destination == source || destination.StartsWith(System.IO.Path.TrimEndingDirectorySeparator(source) + "/", StringComparison.Ordinal))) + { + throw new InvalidDataException(SR.Get("NestedCache", destination, source)); + } + } + + internal static void Copy(string source, string destination) + { + FileAttributes attributes = File.GetAttributes(source); + RejectLink(source, attributes); + UnixFileMode mode = File.GetUnixFileMode(source); + if ((attributes & FileAttributes.Directory) != 0) + { + Directory.CreateDirectory(destination); + foreach (string child in Directory.EnumerateFileSystemEntries(source).Order(StringComparer.Ordinal)) + { + Copy(child, System.IO.Path.Combine(destination, System.IO.Path.GetFileName(child))); + } + Directory.SetLastWriteTimeUtc(destination, s_timestamp); + } + else + { + using (FileStream input = File.OpenRead(source)) + using (FileStream output = new(destination, FileMode.CreateNew, FileAccess.Write)) + { + input.CopyTo(output); + } + File.SetLastWriteTimeUtc(destination, s_timestamp); + } + File.SetUnixFileMode(destination, mode); + } + + internal static string Hash(string path) + { + using MemoryStream manifest = new(); + using (Utf8JsonWriter writer = new(manifest)) + { + writer.WriteStartArray(); + WriteTree(writer, path, Directory.Exists(path) ? "" : System.IO.Path.GetFileName(path)); + writer.WriteEndArray(); + } + return Convert.ToHexStringLower(SHA256.HashData(manifest.GetBuffer().AsSpan(0, checked((int)manifest.Length)))); + } + + private static void WriteTree(Utf8JsonWriter writer, string path, string relative) + { + FileAttributes attributes = File.GetAttributes(path); + RejectLink(path, attributes); + bool directory = (attributes & FileAttributes.Directory) != 0; + writer.WriteStartArray(); + writer.WriteStringValue(relative); + writer.WriteStringValue(directory ? "directory" : "file"); + writer.WriteNumberValue((int)File.GetUnixFileMode(path)); + if (!directory) + { + using FileStream file = File.OpenRead(path); + writer.WriteStringValue(Convert.ToHexStringLower(SHA256.HashData(file))); + } + writer.WriteEndArray(); + if (directory) + { + foreach (string child in Directory.EnumerateFileSystemEntries(path).Order(StringComparer.Ordinal)) + { + string name = System.IO.Path.GetFileName(child); + WriteTree(writer, child, relative.Length == 0 ? name : relative + "/" + name); + } + } + } + + private static void RejectLink(string path, FileAttributes attributes) + { + if ((attributes & FileAttributes.ReparsePoint) != 0) + { + throw new InvalidDataException(SR.Get("Symlink", path)); + } + } +} + +internal sealed class TemporaryDirectory : IDisposable +{ + internal string Path { get; } + + internal TemporaryDirectory(string parent) + { + Path = System.IO.Path.Combine(parent, Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public void Dispose() + { + if (Directory.Exists(Path)) + { + // Copied inputs can contain read-only directories. + foreach (string directory in Directory.EnumerateDirectories(Path, "*", SearchOption.AllDirectories).Prepend(Path)) + { + File.SetUnixFileMode(directory, File.GetUnixFileMode(directory) | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + Directory.Delete(Path, recursive: true); + } + } +} diff --git a/src/tools/helix-cache/Strings.resx b/src/tools/helix-cache/Strings.resx new file mode 100644 index 00000000000000..740f83f520b6f4 --- /dev/null +++ b/src/tools/helix-cache/Strings.resx @@ -0,0 +1,25 @@ + + + text/microsoft-resx + 2.0 + System.Resources.ResXResourceReader, System.Windows.Forms + System.Resources.ResXResourceWriter, System.Windows.Forms + Usage: helix-cache lookup submission.json --cache DIRECTORY --output lookup.json [--queue-info queues.json] + helix-cache store execution.json --results DIRECTORY --cache DIRECTORY +Only Linux x64 queues and digest-pinned container references are supported. + helix-cache: {0} + Invalid or missing manifest field: {0}. + Unsupported schema version {0}; expected {1}. + This prototype requires a Linux host. + Queue '{0}' is missing, unavailable, or not a Linux AMD64 queue. + Container reference '{0}' must end in an explicit @sha256: digest. + Duplicate name or property: {0}. + Payload '{0}' must be a directory or a ZIP archive. + Symbolic links are not supported in directory payloads or results: {0}. Use an archive for payloads containing links. + Cache or staging directory '{0}' must not be inside input directory '{1}'. + Cache entry or snapshot failed validation: {0}. + Results are incomplete, unsuccessful, or do not match the execution record: {0}. + Expected a completed, nonempty, passing xUnit report: {0}. + Expected a relative path without traversal: {0}. + Stored {0}. + diff --git a/src/tools/helix-cache/shadow.py b/src/tools/helix-cache/shadow.py new file mode 100644 index 00000000000000..34a94dab8809d0 --- /dev/null +++ b/src/tools/helix-cache/shadow.py @@ -0,0 +1,204 @@ +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. + +"""Prepare frozen Helix submissions and collect public results; caching stays in the native CLI.""" + +import argparse +from collections import Counter +import json +from pathlib import Path +import subprocess +import time +from urllib.parse import quote +from urllib.request import urlopen +import uuid +import xml.etree.ElementTree as ET + + +API = "https://helix.dot.net/api/2019-06-17" +WORK_ITEM = "System.Memory.Data.Tests" +COMMAND = 'bash ./RunTests.sh --runtime-path "$HELIX_CORRELATION_PAYLOAD"' +POST_COMMAND = 'cp testResults.xml "$HELIX_WORKITEM_UPLOAD_ROOT/testResults.xml"' + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2) + "\n") + + +def read_json(path): + return json.loads(path.read_text()) + + +def invoke(executable, *args): + subprocess.run([str(executable), *map(str, args)], check=True) + + +def msbuild_literal(value): + for char in "%$@();'*?": + value = value.replace(char, f"%{ord(char):02X}") + return value + + +def prepare(args): + for archive in (args.payload, args.runtime): + if not archive.is_file() or archive.suffix.lower() != ".zip": + raise ValueError(f"The shadow submitter requires an existing ZIP archive: {archive}") + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + submission = { + "schemaVersion": 1, + "workItems": [{ + "name": WORK_ITEM, + "queues": [args.queue], + "payload": {"path": str(args.payload.resolve())}, + "correlationPayloads": [{"path": str(args.runtime.resolve())}], + "command": COMMAND, + "postCommands": [POST_COMMAND], + "timeoutSeconds": 900, + "expectedExitCode": 0, + }], + } + write_json(output / "submission.json", submission) + options = ["--queue-info", args.queue_info] if args.queue_info else [] + invoke(args.executable, "lookup", output / "submission.json", + "--cache", args.cache, "--output", output / "lookup.json", *options) + entry, = read_json(output / "lookup.json")["entries"] + execution = entry["execution"] + write_json(output / "execution.json", execution) + project = ET.Element("Project") + properties = ET.SubElement(project, "PropertyGroup") + for name, value in { + "ShadowName": execution["inputs"]["name"], + "ShadowQueue": execution["inputs"]["targetQueue"], + "ShadowPayload": execution["payloadPath"], + "ShadowRuntime": execution["correlationPayloadPaths"][0], + "ShadowCommand": execution["inputs"]["command"], + "ShadowPostCommand": execution["inputs"]["postCommands"][0], + "ShadowTimeout": "00:15:00", + }.items(): + ET.SubElement(properties, name).text = msbuild_literal(value) + ET.indent(project, space=" ") + ET.ElementTree(project).write(output / "submission.props", encoding="utf-8", xml_declaration=True) + print(f"Helix cache shadow: {entry['status']} {execution['key']}; execution will NOT be skipped.") + + +def get_json(url): + with urlopen(url, timeout=60) as response: + return json.load(response) + + +def download(url, destination): + with urlopen(url, timeout=60) as response: + destination.write_bytes(response.read()) + + +def completed_work_item(job_id, name, wait_seconds): + deadline = time.monotonic() + wait_seconds + url = f"{API}/jobs/{job_id}/workitems/{quote(name, safe='')}" + while True: + item = get_json(url) + if item["Job"] != job_id or item["Name"] != name: + raise ValueError("Helix returned a different job or work item.") + if item["State"] in ("Finished", "Passed", "Failed"): + return item + if item["State"] not in ("Unscheduled", "Waiting", "Running"): + raise ValueError(f"Unexpected Helix work item state: {item['State']}") + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError(f"Helix job {job_id} did not complete within {wait_seconds} seconds.") + time.sleep(min(15, remaining)) + + +def outcomes(path): + root = ET.parse(path).getroot() + return Counter( + (Path(assembly.attrib["name"]).name, test.attrib["name"], test.attrib["result"], + test.findtext("reason/message", default="") if test.attrib["result"] == "Skip" else "") + for assembly in root.findall("assembly") + for test in assembly.findall("collection/test") + ) + + +def collect(args): + output = args.output.resolve() + execution = read_json(output / "execution.json") + lookup, = read_json(output / "lookup.json")["entries"] + job_id = str(uuid.UUID((output / "job-id.txt").read_text().strip())) + summary = {"key": execution["key"], "lookup": lookup["status"], "jobId": job_id, "mode": "shadow"} + write_json(output / "summary.json", summary) + job = get_json(f"{API}/jobs/{job_id}") + expected_queue = execution["inputs"]["environmentFingerprint"]["QueueId"] + if job["Name"] != job_id or job["QueueId"].casefold() != expected_queue.casefold(): + raise ValueError("Helix returned a different job or target queue.") + item = completed_work_item(job_id, execution["inputs"]["name"], args.wait_seconds) + write_json(output / "work-item.json", item) + if (item["State"] not in ("Finished", "Passed") or item.get("FailureReason") is not None + or item.get("Errors") or type(item.get("ExitCode")) is not int + or item["ExitCode"] != execution["inputs"]["expectedExitCode"]): + raise ValueError(f"Helix work item did not complete successfully; see {output / 'work-item.json'}.") + files = get_json(f"{API}/jobs/{job_id}/workitems/{quote(item['Name'], safe='')}/files?latestOnly=true") + reports = [file for file in files if file["Name"] == "testResults.xml"] + if len(reports) != 1: + raise ValueError(f"Expected exactly one uploaded testResults.xml; found {len(reports)}.") + report, = reports + results = output / "results" + results.mkdir() + download(report["Link"], results / "testResults.xml") + if item.get("ConsoleOutputUri"): + download(item["ConsoleOutputUri"], results / "console.log") + write_json(results / "result.json", { + "schemaVersion": 1, + "completed": True, + "infrastructureSucceeded": True, + "exitCode": item["ExitCode"], + "jobId": job_id, + "workItemName": item["Name"], + "targetQueue": execution["inputs"]["targetQueue"], + "resultFiles": ["testResults.xml"], + }) + write_json(results / "provenance.json", { + "kind": "helix-execution", + "jobId": job_id, + "machineName": item["MachineName"], + "state": item["State"], + }) + invoke(args.executable, "store", output / "execution.json", "--results", results, "--cache", args.cache) + fresh = outcomes(results / "testResults.xml") + summary["tests"] = sum(fresh.values()) + summary["comparison"] = "not-applicable" + if lookup["status"] == "hit": + cached_results = Path(lookup["resultsPath"]) + cached = outcomes(cached_results / "testResults.xml") + summary["cachedJobId"] = read_json(cached_results / "result.json")["jobId"] + summary["comparison"] = "match" if cached == fresh else "mismatch" + write_json(output / "summary.json", summary) + if cached != fresh: + raise ValueError(f"Cached and fresh test outcomes differ; see {output / 'summary.json'}.") + summary["stored"] = True + write_json(output / "summary.json", summary) + print(json.dumps(summary, indent=2)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + for name in ("prepare", "collect"): + command = commands.add_parser(name) + command.add_argument("--executable", type=Path, required=True) + command.add_argument("--cache", type=Path, required=True) + command.add_argument("--output", type=Path, required=True) + if name == "prepare": + command.add_argument("--payload", type=Path, required=True) + command.add_argument("--runtime", type=Path, required=True) + command.add_argument("--queue", required=True) + command.add_argument("--queue-info", type=Path) + else: + command.add_argument("--wait-seconds", type=int, default=3600) + args = parser.parse_args() + if args.command == "collect" and args.wait_seconds <= 0: + parser.error("--wait-seconds must be positive") + (prepare if args.command == "prepare" else collect)(args) + + +if __name__ == "__main__": + main() diff --git a/src/tools/helix-cache/tests/test_cache.py b/src/tools/helix-cache/tests/test_cache.py new file mode 100644 index 00000000000000..1c3fcca56a6637 --- /dev/null +++ b/src/tools/helix-cache/tests/test_cache.py @@ -0,0 +1,438 @@ +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. + +"""Run against the published native binary: python3 test_cache.py /path/to/helix-cache.""" + +import argparse +from contextlib import redirect_stdout +import copy +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from concurrent.futures import ThreadPoolExecutor +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +import shadow + + +EXECUTABLE = None + + +class CacheTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="helix-cache-test-") + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.cache = self.root / "cache" + self.payload = self.root / "payload" + self.payload.mkdir() + (self.payload / "run.sh").write_text("#!/bin/sh\nexit 0\n") + (self.payload / "run.sh").chmod(0o755) + self.shared = self.root / "shared.zip" + self.shared.write_bytes(b"opaque archive bytes") + self.queues = [{ + "QueueId": "linux.test.open", + "Architecture": "AMD64", + "OperatingSystemGroup": "linux", + "IsAvailable": True, + "GalleryImage": {"Version": "1"}, + "Artifacts": [{"Id": "prepare", "Parameters": {"version": "1"}}], + "QueueDepth": 0, + }] + self.submission = { + "schemaVersion": 1, + "workItems": [{ + "name": "suite", + "queues": ["linux.test.open"], + "payload": {"path": "payload"}, + "correlationPayloads": [{"path": "shared.zip", "destination": "runtime"}], + "command": "./run.sh", + "environment": {"SECOND": "2", "FIRST": "1"}, + "timeoutSeconds": 60, + "expectedExitCode": 0, + }], + } + + def write_json(self, name, value): + path = self.root / name + path.write_text(json.dumps(value)) + return path + + def invoke(self, *args, success=True): + result = subprocess.run( + [EXECUTABLE, *map(str, args)], capture_output=True, text=True, timeout=30 + ) + self.assertEqual(0 if success else 1, result.returncode, result.stderr) + self.assertEqual("", result.stdout) + if not success: + self.assertIn("helix-cache:", result.stderr) + return result + + def lookup(self, success=True): + self.invoke( + "lookup", self.write_json("submission.json", self.submission), + "--cache", self.cache, "--output", self.root / "lookup.json", + "--queue-info", self.write_json("queues.json", self.queues), + success=success, + ) + if success: + return json.loads((self.root / "lookup.json").read_text())["entries"] + return None + + def create_results(self, execution): + results = self.root / "results" + results.mkdir(exist_ok=True) + receipt = { + "schemaVersion": 1, + "completed": True, + "infrastructureSucceeded": True, + "exitCode": 0, + "jobId": "00000000-0000-0000-0000-000000000001", + "workItemName": execution["inputs"]["name"], + "targetQueue": execution["inputs"]["targetQueue"], + "resultFiles": ["tests.xml"], + } + self.write_json("results/result.json", receipt) + (results / "tests.xml").write_text( + '' + '' + '' + ) + return results + + def store(self, execution, success=True): + return self.invoke( + "store", self.write_json("execution.json", execution), + "--results", self.root / "results", "--cache", self.cache, + success=success, + ) + + def test_round_trip_uses_frozen_inputs_and_preserves_results(self): + first = self.lookup()[0] + self.assertEqual("miss", first["status"]) + execution = first["execution"] + snapshot = Path(execution["payloadPath"]) + self.assertNotEqual(self.payload, snapshot) + self.assertEqual(0o755, (snapshot / "run.sh").stat().st_mode & 0o777) + self.create_results(execution) + self.store(execution) + hit = self.lookup()[0] + self.assertEqual("hit", hit["status"]) + self.assertEqual(execution["key"], hit["execution"]["key"]) + self.assertEqual( + (self.root / "results/tests.xml").read_bytes(), + (Path(hit["resultsPath"]) / "tests.xml").read_bytes(), + ) + (self.payload / "run.sh").write_text("changed") + self.assertEqual("#!/bin/sh\nexit 0\n", (snapshot / "run.sh").read_text()) + self.assertEqual("miss", self.lookup()[0]["status"]) + + def test_execution_and_environment_changes_invalidate_keys(self): + baseline = self.lookup()[0]["execution"]["key"] + original = copy.deepcopy(self.submission) + for field, value in [ + ("command", "./run.sh other"), + ("preCommands", ["export STRESS=1"]), + ("postCommands", ["echo done"]), + ("timeoutSeconds", 61), + ("expectedExitCode", 100), + ("environment", {"FIRST": "changed"}), + ]: + with self.subTest(field=field): + self.submission = copy.deepcopy(original) + self.submission["workItems"][0][field] = value + self.assertNotEqual(baseline, self.lookup()[0]["execution"]["key"]) + self.submission = original + self.queues[0]["GalleryImage"]["Version"] = "2" + self.assertNotEqual(baseline, self.lookup()[0]["execution"]["key"]) + self.queues[0]["GalleryImage"]["Version"] = "1" + self.queues[0]["Artifacts"][0]["Parameters"]["version"] = "2" + self.assertNotEqual(baseline, self.lookup()[0]["execution"]["key"]) + + def test_irrelevant_order_and_live_queue_state_do_not_change_key(self): + baseline = self.lookup()[0]["execution"]["key"] + self.queues[0]["QueueDepth"] = 300 + self.queues[0]["IsInProtectedMode"] = True + self.submission["workItems"][0]["environment"] = {"FIRST": "1", "SECOND": "2"} + os.utime(self.payload / "run.sh", (1000000000, 1000000000)) + self.assertEqual(baseline, self.lookup()[0]["execution"]["key"]) + + def test_paths_do_not_affect_keys_but_modes_and_empty_directories_do(self): + baseline = self.lookup()[0]["execution"]["key"] + relocated = self.root / "relocated" + self.payload.rename(relocated) + self.submission["workItems"][0]["payload"]["path"] = "relocated" + self.assertEqual(baseline, self.lookup()[0]["execution"]["key"]) + (relocated / "run.sh").chmod(0o644) + self.assertNotEqual(baseline, self.lookup()[0]["execution"]["key"]) + (relocated / "run.sh").chmod(0o755) + (relocated / "empty").mkdir() + self.assertNotEqual(baseline, self.lookup()[0]["execution"]["key"]) + + def test_queue_partitioning_and_digest_pinned_containers(self): + target = "(Debian.Amd64)linux.test.open@registry/image:tag@sha256:" + "a" * 64 + self.submission["workItems"][0]["queues"].append(target) + entries = self.lookup() + self.assertEqual(2, len(entries)) + self.assertNotEqual(entries[0]["execution"]["key"], entries[1]["execution"]["key"]) + self.assertEqual( + "registry/image:tag@sha256:" + "a" * 64, + entries[1]["execution"]["inputs"]["environmentFingerprint"]["Container"], + ) + self.create_results(entries[0]["execution"]) + self.store(entries[0]["execution"]) + self.assertEqual(["hit", "miss"], [e["status"] for e in self.lookup()]) + self.submission["workItems"][0]["queues"] = ["linux.test.open@registry/image:latest"] + self.lookup(success=False) + self.submission["workItems"][0]["queues"] = ["(alias)linux.test.open"] + self.lookup(success=False) + + def test_shared_payload_bytes_destination_and_overlay_order_matter(self): + baseline = self.lookup()[0]["execution"]["key"] + self.shared.write_bytes(b"new archive") + self.assertNotEqual(baseline, self.lookup()[0]["execution"]["key"]) + self.shared.write_bytes(b"opaque archive bytes") + self.submission["workItems"][0]["correlationPayloads"][0]["destination"] = "other" + self.assertNotEqual(baseline, self.lookup()[0]["execution"]["key"]) + correlations = self.submission["workItems"][0]["correlationPayloads"] + correlations.append({"path": "shared.zip", "destination": "second"}) + before = self.lookup()[0]["execution"]["key"] + correlations.reverse() + self.assertNotEqual(before, self.lookup()[0]["execution"]["key"]) + + def test_unsuccessful_receipts_are_not_stored(self): + execution = self.lookup()[0]["execution"] + self.create_results(execution) + original = json.loads((self.root / "results/result.json").read_text()) + for field, value in [ + ("completed", False), ("infrastructureSucceeded", False), + ("exitCode", 1), ("workItemName", "other"), + ("targetQueue", "other"), ("jobId", "not-a-guid"), + ("resultFiles", []), ("resultFiles", ["../outside.xml"]), + ("resultFiles", ["tests.xml", "tests.xml"]), + ]: + with self.subTest(field=field, value=value): + receipt = dict(original, **{field: value}) + self.write_json("results/result.json", receipt) + self.store(execution, success=False) + self.assertEqual("miss", self.lookup()[0]["status"]) + + def test_reports_must_be_complete_nonempty_and_passing(self): + execution = self.lookup()[0]["execution"] + self.create_results(execution) + for report in [ + "", + '', + '', + '' + '', + '' + '', + ']>' + '&x;', + ]: + with self.subTest(report=report): + (self.root / "results/tests.xml").write_text(report) + self.store(execution, success=False) + self.assertEqual("miss", self.lookup()[0]["status"]) + + def test_corruption_is_an_error_not_a_hit_or_silent_miss(self): + execution = self.lookup()[0]["execution"] + self.create_results(execution) + self.store(execution) + hit = self.lookup()[0] + (Path(hit["resultsPath"]) / "tests.xml").write_text("corrupted") + self.lookup(success=False) + + def test_modified_snapshot_or_forged_key_cannot_be_stored(self): + execution = self.lookup()[0]["execution"] + self.create_results(execution) + forged = dict(execution, key="0" * 64) + self.store(forged, success=False) + (Path(execution["payloadPath"]) / "run.sh").write_text("modified after lookup") + self.store(execution, success=False) + + def test_store_rejects_results_containing_its_staging_directory(self): + execution = self.lookup()[0]["execution"] + result = self.invoke( + "store", self.write_json("execution.json", execution), + "--results", self.cache / "tmp", "--cache", self.cache, + success=False, + ) + self.assertIn("must not be inside input directory", result.stderr) + self.assertEqual([], list((self.cache / "tmp").iterdir())) + self.assertEqual("miss", self.lookup()[0]["status"]) + + def test_duplicate_stores_are_atomic_and_idempotent(self): + execution = self.lookup()[0]["execution"] + self.create_results(execution) + record = self.write_json("execution.json", execution) + with ThreadPoolExecutor(max_workers=2) as pool: + operations = [ + pool.submit( + self.invoke, "store", record, "--results", self.root / "results", + "--cache", self.cache, + ) for _ in range(2) + ] + for operation in operations: + operation.result() + self.assertEqual("hit", self.lookup()[0]["status"]) + self.assertEqual([], list((self.cache / "tmp").iterdir())) + + def test_rejects_unsupported_queues_links_and_nested_cache(self): + for field, value in [ + ("Architecture", "ARM64"), ("OperatingSystemGroup", "windows"), + ("IsAvailable", False), + ]: + with self.subTest(field=field): + original = self.queues[0][field] + self.queues[0][field] = value + self.lookup(success=False) + self.queues[0][field] = original + (self.payload / "link").symlink_to(self.shared) + self.lookup(success=False) + (self.payload / "link").unlink() + self.cache = self.payload / "cache" + self.lookup(success=False) + + def test_strict_manifest_and_cli_validation(self): + original = copy.deepcopy(self.submission) + for transform in [ + lambda s: s.update(schemaVersion=2), + lambda s: s.update(workItems=[]), + lambda s: s.update(workItems=[None]), + lambda s: s.update(unrecognized=True), + lambda s: s["workItems"][0].pop("expectedExitCode"), + lambda s: s["workItems"][0].update(command=None), + lambda s: s["workItems"][0].update(timeoutSeconds=0), + ]: + with self.subTest(transform=transform): + self.submission = copy.deepcopy(original) + transform(self.submission) + self.lookup(success=False) + self.invoke("store", "missing", "--cache", self.cache, success=False) + self.invoke("lookup", "missing", "--cache", self.cache, "--cache", self.cache, success=False) + + def prepare_shadow(self, name="shadow"): + args = argparse.Namespace( + executable=Path(EXECUTABLE), cache=self.cache, output=self.root / name, + payload=self.shared, runtime=self.shared, queue=self.queues[0]["QueueId"], + queue_info=self.write_json("queues.json", self.queues), wait_seconds=60, + ) + with redirect_stdout(io.StringIO()): + shadow.prepare(args) + return args + + def collect_shadow(self, args, job_id, report=None, item_changes=None, files=None, job_changes=None): + (args.output / "job-id.txt").write_text(job_id + "\n") + item = { + "Job": job_id, "Name": shadow.WORK_ITEM, "State": "Passed", + "ExitCode": 0, "MachineName": "test-worker", + } + item.update(item_changes or {}) + if report is None: + report = ( + '' + '' + '' + ) + if files is None: + files = [{"Name": "testResults.xml", "Link": "https://example.invalid/report"}] + job = {"Name": job_id, "QueueId": self.queues[0]["QueueId"]} + job.update(job_changes or {}) + with (patch.object(shadow, "completed_work_item", return_value=item), + patch.object(shadow, "get_json", side_effect=lambda url: job if url == f"{shadow.API}/jobs/{job_id}" else files), + patch.object(shadow, "download", side_effect=lambda url, path: path.write_text(report)), + redirect_stdout(io.StringIO())): + shadow.collect(args) + + def test_shadow_cold_store_and_restored_entries_compare_actual_outcomes(self): + import shutil + + first_job = "00000000-0000-0000-0000-000000000001" + second_job = "00000000-0000-0000-0000-000000000002" + first = self.prepare_shadow() + self.collect_shadow(first, first_job) + first_summary = shadow.read_json(first.output / "summary.json") + self.assertEqual("miss", first_summary["lookup"]) + self.assertEqual(2, first_summary["tests"]) + restored = self.root / "restored-cache" + shutil.copytree(self.cache / "entries", restored / "entries") + self.cache.rename(self.root / "previous-cache") + self.cache = restored + second = self.prepare_shadow("second") + report = ( + '' + '' + '' + ) + self.collect_shadow(second, second_job, report) + summary = shadow.read_json(second.output / "summary.json") + self.assertEqual("hit", summary["lookup"]) + self.assertEqual("match", summary["comparison"]) + self.assertEqual(first_job, summary["cachedJobId"]) + self.assertEqual(second_job, summary["jobId"]) + self.assertEqual(first_summary["key"], summary["key"]) + + def test_shadow_reports_mismatches_without_overwriting_cached_provenance(self): + first = self.prepare_shadow() + self.collect_shadow(first, "00000000-0000-0000-0000-000000000001") + second = self.prepare_shadow("second") + changed = ( + '' + '' + '' + ) + with self.assertRaisesRegex(ValueError, "outcomes differ"): + self.collect_shadow(second, "00000000-0000-0000-0000-000000000002", changed) + summary = shadow.read_json(second.output / "summary.json") + self.assertEqual("mismatch", summary["comparison"]) + self.assertEqual("00000000-0000-0000-0000-000000000001", summary["cachedJobId"]) + + def test_shadow_does_not_admit_unsuccessful_remote_completions(self): + args = self.prepare_shadow() + for changes in ( + {"State": "Failed"}, {"FailureReason": {"Issue": {"Name": "WorkItemTimeout"}}}, + {"Errors": [{"Message": "Infrastructure failure"}]}, {"ExitCode": 1}, {"ExitCode": None}, + ): + with self.subTest(changes=changes), self.assertRaisesRegex(ValueError, "did not complete"): + self.collect_shadow(args, "00000000-0000-0000-0000-000000000001", item_changes=changes) + with self.assertRaisesRegex(ValueError, "target queue"): + self.collect_shadow(args, "00000000-0000-0000-0000-000000000001", job_changes={"QueueId": "other"}) + self.assertEqual([], list((self.cache / "entries").rglob("entry.json"))) + + def test_shadow_requires_one_uploaded_report_and_native_report_validation(self): + args = self.prepare_shadow() + for files in ([], [{"Name": "testResults.xml"}] * 2): + with self.subTest(files=files), self.assertRaises(ValueError): + self.collect_shadow(args, "00000000-0000-0000-0000-000000000001", files=files) + with self.assertRaises(subprocess.CalledProcessError): + self.collect_shadow(args, "00000000-0000-0000-0000-000000000001", report="") + self.assertEqual([], list((self.cache / "entries").rglob("entry.json"))) + + def test_shadow_polling_rejects_timeouts_unknown_states_and_identity_mismatches(self): + item = {"Job": "job", "Name": "suite", "State": "Running"} + with (patch.object(shadow, "get_json", return_value=item), + patch.object(shadow.time, "monotonic", side_effect=[0, 61]), + self.assertRaises(TimeoutError)): + shadow.completed_work_item("job", "suite", 60) + for changes in ({"State": "Canceled"}, {"Job": "other"}, {"Name": "other"}): + with (self.subTest(changes=changes), + patch.object(shadow, "get_json", return_value=dict(item, **changes)), + self.assertRaises(ValueError)): + shadow.completed_work_item("job", "suite", 60) + with (patch.object(shadow, "get_json", side_effect=[item, dict(item, State="Finished")]), + patch.object(shadow.time, "sleep") as sleep): + self.assertEqual("Finished", shadow.completed_work_item("job", "suite", 60)["State"]) + sleep.assert_called_once() + + +if __name__ == "__main__": + EXECUTABLE = str(Path(sys.argv.pop(1)).resolve()) + unittest.main() From a49224c78ab0c2267d1564e2c379c7fec79e4ca9 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Tue, 20 Oct 2026 18:14:13 +0000 Subject: [PATCH 2/8] Fix Helix shadow queue alias validation Read completed job details to validate the submitted queue alias while preserving the resolved physical queue in provenance. Cover alias resolution, conflicting aliases, and missing-alias fallback. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ee547d49-5419-4268-b01f-6b2757f99953 --- src/tools/helix-cache/README.md | 8 +++-- src/tools/helix-cache/shadow.py | 7 ++-- src/tools/helix-cache/tests/test_cache.py | 40 ++++++++++++++++++++++- 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/src/tools/helix-cache/README.md b/src/tools/helix-cache/README.md index 3affdccb1a5092..24bba66137e7e5 100644 --- a/src/tools/helix-cache/README.md +++ b/src/tools/helix-cache/README.md @@ -142,12 +142,14 @@ pinned Helix SDK without injecting extra reporters. Its default target only validates inputs; submission requires **`-t:Submit`**, which waits for completion and cancels the submitted job on task cancellation. -`collect` checks the public Helix API's job, host queue, work-item identity, terminal state, -infrastructure errors, failure reason and exit code, then downloads the uploaded +`collect` checks the public Helix API's job, submitted queue alias (or queue ID +when no alias is reported), work-item identity, terminal state, infrastructure +errors, failure reason and exit code, then downloads the uploaded xUnit XML and console log. It constructs the receipt from that completion, passes the report through native admission, and compares any hit with the fresh test identities/outcomes/skip reasons. Ordering, worker directory prefixes and -execution timings are ignored. The original cached job provenance is retained. +execution timings are ignored. The resolved physical queue ID and submitted +alias are preserved in provenance alongside the original cached job. Every hit still executes on Helix; no cached result is published as a new run. Evidence is included in the existing build-log artifact under diff --git a/src/tools/helix-cache/shadow.py b/src/tools/helix-cache/shadow.py index 34a94dab8809d0..fbb447a81202c6 100644 --- a/src/tools/helix-cache/shadow.py +++ b/src/tools/helix-cache/shadow.py @@ -126,9 +126,10 @@ def collect(args): job_id = str(uuid.UUID((output / "job-id.txt").read_text().strip())) summary = {"key": execution["key"], "lookup": lookup["status"], "jobId": job_id, "mode": "shadow"} write_json(output / "summary.json", summary) - job = get_json(f"{API}/jobs/{job_id}") + job = get_json(f"{API}/jobs/{job_id}/details") expected_queue = execution["inputs"]["environmentFingerprint"]["QueueId"] - if job["Name"] != job_id or job["QueueId"].casefold() != expected_queue.casefold(): + submitted_queue = job.get("QueueAlias") or job["QueueId"] + if job["Name"] != job_id or submitted_queue.casefold() != expected_queue.casefold(): raise ValueError("Helix returned a different job or target queue.") item = completed_work_item(job_id, execution["inputs"]["name"], args.wait_seconds) write_json(output / "work-item.json", item) @@ -159,6 +160,8 @@ def collect(args): write_json(results / "provenance.json", { "kind": "helix-execution", "jobId": job_id, + "queueAlias": job.get("QueueAlias"), + "resolvedQueueId": job["QueueId"], "machineName": item["MachineName"], "state": item["State"], }) diff --git a/src/tools/helix-cache/tests/test_cache.py b/src/tools/helix-cache/tests/test_cache.py index 1c3fcca56a6637..eea4bbd33f4510 100644 --- a/src/tools/helix-cache/tests/test_cache.py +++ b/src/tools/helix-cache/tests/test_cache.py @@ -346,8 +346,13 @@ def collect_shadow(self, args, job_id, report=None, item_changes=None, files=Non files = [{"Name": "testResults.xml", "Link": "https://example.invalid/report"}] job = {"Name": job_id, "QueueId": self.queues[0]["QueueId"]} job.update(job_changes or {}) + responses = { + f"{shadow.API}/jobs/{job_id}": {"Name": job["Name"], "QueueId": job["QueueId"]}, + f"{shadow.API}/jobs/{job_id}/details": job, + f"{shadow.API}/jobs/{job_id}/workitems/{shadow.WORK_ITEM}/files?latestOnly=true": files, + } with (patch.object(shadow, "completed_work_item", return_value=item), - patch.object(shadow, "get_json", side_effect=lambda url: job if url == f"{shadow.API}/jobs/{job_id}" else files), + patch.object(shadow, "get_json", side_effect=responses.__getitem__), patch.object(shadow, "download", side_effect=lambda url, path: path.write_text(report)), redirect_stdout(io.StringIO())): shadow.collect(args) @@ -407,6 +412,39 @@ def test_shadow_does_not_admit_unsuccessful_remote_completions(self): self.collect_shadow(args, "00000000-0000-0000-0000-000000000001", job_changes={"QueueId": "other"}) self.assertEqual([], list((self.cache / "entries").rglob("entry.json"))) + def test_shadow_queue_alias_validation_and_provenance(self): + expected = self.queues[0]["QueueId"] + cases = [ + ({"QueueId": expected + ".rt", "QueueAlias": expected.upper()}, True), + ({"QueueId": expected}, True), + ({"QueueId": expected, "QueueAlias": None}, True), + ({"QueueId": expected, "QueueAlias": ""}, True), + ({"QueueId": expected, "QueueAlias": "other"}, False), + ({"QueueId": "other", "QueueAlias": "other"}, False), + ({"QueueId": "other"}, False), + ({"Name": "other", "QueueId": expected + ".rt", "QueueAlias": expected}, False), + ] + for index, (job, accepted) in enumerate(cases): + with self.subTest(job=job): + self.cache = self.root / f"cache-{index}" + args = self.prepare_shadow(f"shadow-{index}") + if accepted: + self.collect_shadow(args, "00000000-0000-0000-0000-000000000001", job_changes=job) + provenance = shadow.read_json(args.output / "results/provenance.json") + self.assertEqual(job["QueueId"], provenance["resolvedQueueId"]) + self.assertEqual(job.get("QueueAlias"), provenance["queueAlias"]) + entry, = shadow.read_json(args.output / "lookup.json")["entries"] + receipt = shadow.read_json( + self.cache / "entries/v1" / entry["execution"]["key"][:2] + / entry["execution"]["key"] / "results/result.json" + ) + self.assertEqual(expected, receipt["targetQueue"]) + else: + with self.assertRaisesRegex(ValueError, "job or target queue"): + self.collect_shadow(args, "00000000-0000-0000-0000-000000000001", job_changes=job) + self.assertFalse((args.output / "results").exists()) + self.assertEqual([], list((self.cache / "entries").rglob("entry.json"))) + def test_shadow_requires_one_uploaded_report_and_native_report_validation(self): args = self.prepare_shadow() for files in ([], [{"Name": "testResults.xml"}] * 2): From 1d407ea357947dc79a3ca4906c07b245906a9209 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Wed, 21 Oct 2026 03:32:30 +0000 Subject: [PATCH 3/8] Refactor Helix cache into a typed native CLI Use separate JSON contexts and an explicit command, evaluation, and action pipeline for Helix cache operations. Replace the Python/resources shadow implementation with native shadow orchestration and C# integration tests, organized into sibling source and tests projects. Correct the opt-in gate for the shadow workflow. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ee547d49-5419-4268-b01f-6b2757f99953 --- .../libraries/helix-cache-shadow.yml | 14 +- .../testing/helix-cache-shadow.proj | 0 src/tools/helix-cache/JsonFile.cs | 114 ----- src/tools/helix-cache/LinuxEnvironment.cs | 101 ---- src/tools/helix-cache/Models.cs | 120 ----- src/tools/helix-cache/Program.cs | 82 --- src/tools/helix-cache/README.md | 48 +- src/tools/helix-cache/Results.cs | 94 ---- src/tools/helix-cache/SR.cs | 16 - src/tools/helix-cache/Strings.resx | 25 - src/tools/helix-cache/shadow.py | 207 -------- src/tools/helix-cache/{ => src}/Cache.cs | 95 ++-- src/tools/helix-cache/src/CommandLine.cs | 89 ++++ src/tools/helix-cache/src/ExecutionKey.cs | 51 ++ .../helix-cache/{ => src}/HelixCache.csproj | 3 + src/tools/helix-cache/src/HelixClient.cs | 94 ++++ src/tools/helix-cache/src/HelixModels.cs | 57 +++ src/tools/helix-cache/src/LinuxEnvironment.cs | 102 ++++ src/tools/helix-cache/src/Models.cs | 273 ++++++++++ src/tools/helix-cache/src/Program.cs | 121 +++++ src/tools/helix-cache/src/Results.cs | 119 +++++ src/tools/helix-cache/src/ShadowValidation.cs | 169 +++++++ src/tools/helix-cache/{ => src}/Snapshot.cs | 21 +- src/tools/helix-cache/tests/CacheFixture.cs | 157 ++++++ src/tools/helix-cache/tests/CacheTests.cs | 426 ++++++++++++++++ .../helix-cache/tests/HelixCache.Tests.csproj | 10 + src/tools/helix-cache/tests/ShadowTests.cs | 303 +++++++++++ src/tools/helix-cache/tests/test_cache.py | 476 ------------------ 28 files changed, 2077 insertions(+), 1310 deletions(-) rename src/tools/helix-cache/Shadow.proj => eng/testing/helix-cache-shadow.proj (100%) delete mode 100644 src/tools/helix-cache/JsonFile.cs delete mode 100644 src/tools/helix-cache/LinuxEnvironment.cs delete mode 100644 src/tools/helix-cache/Models.cs delete mode 100644 src/tools/helix-cache/Program.cs delete mode 100644 src/tools/helix-cache/Results.cs delete mode 100644 src/tools/helix-cache/SR.cs delete mode 100644 src/tools/helix-cache/Strings.resx delete mode 100644 src/tools/helix-cache/shadow.py rename src/tools/helix-cache/{ => src}/Cache.cs (59%) create mode 100644 src/tools/helix-cache/src/CommandLine.cs create mode 100644 src/tools/helix-cache/src/ExecutionKey.cs rename src/tools/helix-cache/{ => src}/HelixCache.csproj (86%) create mode 100644 src/tools/helix-cache/src/HelixClient.cs create mode 100644 src/tools/helix-cache/src/HelixModels.cs create mode 100644 src/tools/helix-cache/src/LinuxEnvironment.cs create mode 100644 src/tools/helix-cache/src/Models.cs create mode 100644 src/tools/helix-cache/src/Program.cs create mode 100644 src/tools/helix-cache/src/Results.cs create mode 100644 src/tools/helix-cache/src/ShadowValidation.cs rename src/tools/helix-cache/{ => src}/Snapshot.cs (84%) create mode 100644 src/tools/helix-cache/tests/CacheFixture.cs create mode 100644 src/tools/helix-cache/tests/CacheTests.cs create mode 100644 src/tools/helix-cache/tests/HelixCache.Tests.csproj create mode 100644 src/tools/helix-cache/tests/ShadowTests.cs delete mode 100644 src/tools/helix-cache/tests/test_cache.py diff --git a/eng/pipelines/libraries/helix-cache-shadow.yml b/eng/pipelines/libraries/helix-cache-shadow.yml index 73115a9357a198..a6bbe92c3ad980 100644 --- a/eng/pipelines/libraries/helix-cache-shadow.yml +++ b/eng/pipelines/libraries/helix-cache-shadow.yml @@ -13,7 +13,7 @@ parameters: shouldContinueOnError: false steps: -- ${{ if and(parameters.enabled, eq(parameters.platform, 'linux_x64'), eq(variables['System.TeamProject'], 'public')) }}: +- ${{ if and(eq(parameters.enabled, 'true'), eq(parameters.platform, 'linux_x64'), eq(variables['System.TeamProject'], 'public')) }}: # Persist completed entries only. Input snapshots are recreated in each job. - task: Cache@2 displayName: Helix shadow result cache @@ -32,10 +32,10 @@ steps: cache="$(Pipeline.Workspace)/.helix-cache" mkdir -p "$cache/entries" - "$dotnet" publish "$project/HelixCache.csproj" -c Release -r linux-x64 \ + "$dotnet" publish "$project/src/HelixCache.csproj" -c Release -r linux-x64 \ -o "$(Build.SourcesDirectory)/artifacts/tools/helix-cache" executable="$(Build.SourcesDirectory)/artifacts/tools/helix-cache/helix-cache" - python3 "$project/tests/test_cache.py" "$executable" + "$dotnet" test "$project/tests/HelixCache.Tests.csproj" -c Release "$dotnet" msbuild "$(Build.SourcesDirectory)/src/libraries/sendtohelix.proj" \ -t:PrepareCorrelationPayloadDirectory -p:Configuration=$(_BuildConfig) -p:RuntimeFlavor=CoreCLR @@ -43,17 +43,15 @@ steps: -nologo -p:Configuration=$(_BuildConfig) -getProperty:NetCoreAppCurrentBuildSettings)" archives="$("$dotnet" msbuild "$(Build.SourcesDirectory)/src/libraries/System.Memory.Data/tests/System.Memory.Data.Tests.csproj" \ -nologo -p:Configuration=$(_BuildConfig) -getProperty:TestArchiveTestsDir)" - python3 "$project/shadow.py" prepare \ - --executable "$executable" --cache "$cache" --output "$output" \ + "$executable" prepare --cache "$cache" --output "$output" \ --payload "$archives/System.Memory.Data.Tests.zip" \ --runtime "$(Build.SourcesDirectory)/artifacts/helix/runtime/test-runtime-$settings.zip" \ --queue AzureLinux.3.Amd64.Open # A cache hit never bypasses this submission. - "$dotnet" msbuild "$project/Shadow.proj" -t:Submit \ + "$dotnet" msbuild "$(Build.SourcesDirectory)/eng/testing/helix-cache-shadow.proj" -t:Submit \ -p:HelixCacheShadowDirectory="$output" -p:ShadowCreator=dotnet-bot - python3 "$project/shadow.py" collect \ - --executable "$executable" --cache "$cache" --output "$output" + "$executable" collect --cache "$cache" --output "$output" displayName: Validate Helix cache in shadow mode (always execute) continueOnError: true timeoutInMinutes: 75 diff --git a/src/tools/helix-cache/Shadow.proj b/eng/testing/helix-cache-shadow.proj similarity index 100% rename from src/tools/helix-cache/Shadow.proj rename to eng/testing/helix-cache-shadow.proj diff --git a/src/tools/helix-cache/JsonFile.cs b/src/tools/helix-cache/JsonFile.cs deleted file mode 100644 index e07132a81c5e02..00000000000000 --- a/src/tools/helix-cache/JsonFile.cs +++ /dev/null @@ -1,114 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Security.Cryptography; -using System.Text.Json; -using System.Text.Json.Serialization.Metadata; - -namespace HelixCache; - -internal static class JsonFile -{ - internal static T Read(string path, JsonTypeInfo type) - { - using FileStream stream = File.OpenRead(path); - using JsonDocument document = JsonDocument.Parse(stream); - using (Utf8JsonWriter validator = new(Stream.Null)) - { - WriteCanonical(validator, document.RootElement); - } - return JsonSerializer.Deserialize(document.RootElement, type) ?? throw new InvalidDataException(SR.Get("InvalidField", path)); - } - - internal static void Write(string path, T value, JsonTypeInfo type) - { - path = Path.GetFullPath(path); - Directory.CreateDirectory(Path.GetDirectoryName(path)!); - string temporary = path + "." + Guid.NewGuid().ToString("N") + ".tmp"; - try - { - using (FileStream stream = new(temporary, FileMode.CreateNew, FileAccess.Write)) - { - JsonSerializer.Serialize(stream, value, type); - stream.Flush(flushToDisk: true); - } - File.Move(temporary, path, overwrite: true); - } - finally - { - File.Delete(temporary); - } - } - - internal static string Digest(JsonElement value) - { - using MemoryStream stream = new(); - using (Utf8JsonWriter writer = new(stream)) - { - WriteCanonical(writer, value); - } - return Convert.ToHexStringLower(SHA256.HashData(stream.GetBuffer().AsSpan(0, checked((int)stream.Length)))); - } - - internal static void WriteCanonical(Utf8JsonWriter writer, JsonElement value) - { - switch (value.ValueKind) - { - case JsonValueKind.Object: - writer.WriteStartObject(); - HashSet names = new(StringComparer.Ordinal); - foreach (JsonProperty property in value.EnumerateObject().OrderBy(p => p.Name, StringComparer.Ordinal)) - { - if (!names.Add(property.Name)) - { - throw new InvalidDataException(SR.Get("Duplicate", property.Name)); - } - writer.WritePropertyName(property.Name); - WriteCanonical(writer, property.Value); - } - writer.WriteEndObject(); - break; - case JsonValueKind.Array: - writer.WriteStartArray(); - foreach (JsonElement item in value.EnumerateArray()) - { - Require(item.ValueKind != JsonValueKind.Null, nameof(item)); - WriteCanonical(writer, item); - } - writer.WriteEndArray(); - break; - default: - value.WriteTo(writer); - break; - } - } - - internal static void ValidateVersion(int version) - { - if (version != Cache.ProtocolVersion) - { - throw new InvalidDataException(SR.Get("Version", version, Cache.ProtocolVersion)); - } - } - - internal static void Require(bool condition, string field) - { - if (!condition) - { - throw new InvalidDataException(SR.Get("InvalidField", field)); - } - } - - internal static void RelativePath(string path, bool allowEmpty = false) - { - Require(allowEmpty || !string.IsNullOrWhiteSpace(path), nameof(path)); - if (Path.IsPathRooted(path) || path.Split('/').Any(p => p is "." or "..") || path.Contains('\\')) - { - throw new InvalidDataException(SR.Get("RelativePath", path)); - } - } -} diff --git a/src/tools/helix-cache/LinuxEnvironment.cs b/src/tools/helix-cache/LinuxEnvironment.cs deleted file mode 100644 index fc231da1df16f6..00000000000000 --- a/src/tools/helix-cache/LinuxEnvironment.cs +++ /dev/null @@ -1,101 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Net.Http; -using System.Text.Json; -using System.Threading.Tasks; - -namespace HelixCache; - -internal sealed class LinuxEnvironment : IDisposable -{ - private readonly JsonDocument _queues; - private readonly Dictionary _fingerprints = new(StringComparer.Ordinal); - - private LinuxEnvironment(JsonDocument queues) => _queues = queues; - - internal static async Task LoadAsync(string? queueInfo) - { - if (queueInfo is not null) - { - using FileStream file = File.OpenRead(queueInfo); - return new LinuxEnvironment(await JsonDocument.ParseAsync(file)); - } - - using HttpClient client = new() { Timeout = TimeSpan.FromSeconds(60) }; - using Stream stream = await client.GetStreamAsync("https://helix.dot.net/api/2019-06-17/info/queues"); - return new LinuxEnvironment(await JsonDocument.ParseAsync(stream)); - } - - internal JsonElement Resolve(string target) - { - if (_fingerprints.TryGetValue(target, out JsonElement fingerprint)) - { - return fingerprint; - } - - // Matches Arcade JobSender's (alias)host@container syntax; retain a digest's second '@'. - // https://github.com/dotnet/arcade/blob/7103cc604358625941ee1d105e662f0d697085c6/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs#L374-L399 - int separator = target.IndexOf('@'); - string host = separator < 0 ? target : target[..separator]; - string? container = separator < 0 ? null : target[(separator + 1)..]; - if (host.StartsWith('(')) - { - int close = host.IndexOf(')'); - JsonFile.Require(separator >= 0 && close > 1 && close < host.Length - 1, nameof(target)); - host = host[(close + 1)..]; - } - - if (container is not null) - { - int digestStart = container.LastIndexOf("@sha256:", StringComparison.Ordinal); - if (digestStart <= 0 || !Cache.IsDigest(container[(digestStart + "@sha256:".Length)..])) - { - throw new InvalidDataException(SR.Get("Container", container)); - } - } - - JsonElement? queue = null; - foreach (JsonElement candidate in _queues.RootElement.EnumerateArray()) - { - if (string.Equals(candidate.GetProperty("QueueId").GetString(), host, StringComparison.OrdinalIgnoreCase)) - { - JsonFile.Require(queue is null, nameof(target)); - queue = candidate; - } - } - - if (queue is not JsonElement info || - !string.Equals(info.GetProperty("OperatingSystemGroup").GetString(), "linux", StringComparison.OrdinalIgnoreCase) || - !string.Equals(info.GetProperty("Architecture").GetString(), "AMD64", StringComparison.OrdinalIgnoreCase) || - !info.GetProperty("IsAvailable").GetBoolean()) - { - throw new InvalidDataException(SR.Get("Queue", host)); - } - - using MemoryStream buffer = new(); - using (Utf8JsonWriter writer = new(buffer)) - { - writer.WriteStartObject(); - foreach (string property in new[] { "QueueId", "Architecture", "OperatingSystemGroup", "GalleryImage", "PreInstalledImage", "PreparedImage", "Artifacts", "WorkspacePath" }) - { - if (info.TryGetProperty(property, out JsonElement value)) - { - writer.WritePropertyName(property); - JsonFile.WriteCanonical(writer, value); - } - } - writer.WriteString("Container", container); - writer.WriteEndObject(); - } - using JsonDocument document = JsonDocument.Parse(buffer.ToArray()); - fingerprint = document.RootElement.Clone(); - _fingerprints.Add(target, fingerprint); - return fingerprint; - } - - public void Dispose() => _queues.Dispose(); -} diff --git a/src/tools/helix-cache/Models.cs b/src/tools/helix-cache/Models.cs deleted file mode 100644 index baa49be7963030..00000000000000 --- a/src/tools/helix-cache/Models.cs +++ /dev/null @@ -1,120 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace HelixCache; - -internal sealed class Submission -{ - public required int SchemaVersion { get; init; } - public required List WorkItems { get; init; } -} - -internal sealed class WorkItem -{ - public required string Name { get; init; } - public required List Queues { get; init; } - public required Payload Payload { get; init; } - public List CorrelationPayloads { get; init; } = []; - public required string Command { get; init; } - public List PreCommands { get; init; } = []; - public List PostCommands { get; init; } = []; - public Dictionary Environment { get; init; } = []; - public required int TimeoutSeconds { get; init; } - public required int ExpectedExitCode { get; init; } -} - -internal sealed class Payload -{ - public required string Path { get; init; } - public string Destination { get; init; } = ""; -} - -internal sealed class PayloadIdentity -{ - public required string Digest { get; init; } - public required string Destination { get; init; } - public required string Kind { get; init; } -} - -internal sealed class ExecutionInputs -{ - public required string Name { get; init; } - public required string TargetQueue { get; init; } - public required JsonElement EnvironmentFingerprint { get; init; } - public required PayloadIdentity Payload { get; init; } - public required List CorrelationPayloads { get; init; } - public required string Command { get; init; } - public required List PreCommands { get; init; } - public required List PostCommands { get; init; } - public required Dictionary Environment { get; init; } - public required int TimeoutSeconds { get; init; } - public required int ExpectedExitCode { get; init; } -} - -internal sealed class ExecutionRecord -{ - public required int SchemaVersion { get; init; } - public required string Key { get; init; } - public required ExecutionInputs Inputs { get; init; } - public required string PayloadPath { get; init; } - public required List CorrelationPayloadPaths { get; init; } -} - -internal sealed class LookupResult -{ - public int SchemaVersion { get; init; } = Cache.ProtocolVersion; - public List Entries { get; init; } = []; -} - -internal sealed class LookupEntry -{ - public required string Status { get; init; } - public required ExecutionRecord Execution { get; init; } - public string? ResultsPath { get; init; } -} - -internal sealed class ResultReceipt -{ - public required int SchemaVersion { get; init; } - public required bool Completed { get; init; } - public required bool InfrastructureSucceeded { get; init; } - public required int ExitCode { get; init; } - public required string JobId { get; init; } - public required string WorkItemName { get; init; } - public required string TargetQueue { get; init; } - public required List ResultFiles { get; init; } -} - -internal sealed class TestSummary -{ - public int Total { get; set; } - public int Passed { get; set; } - public int Skipped { get; set; } -} - -internal sealed class StoredEntry -{ - public required int SchemaVersion { get; init; } - public required ExecutionRecord Execution { get; init; } - public required DateTimeOffset CreatedUtc { get; init; } - public required string ResultsDigest { get; init; } - public required TestSummary Summary { get; init; } -} - -[JsonSourceGenerationOptions( - PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, - WriteIndented = true, - UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow, - RespectNullableAnnotations = true)] -[JsonSerializable(typeof(Submission))] -[JsonSerializable(typeof(ExecutionRecord))] -[JsonSerializable(typeof(ExecutionInputs))] -[JsonSerializable(typeof(LookupResult))] -[JsonSerializable(typeof(ResultReceipt))] -[JsonSerializable(typeof(StoredEntry))] -internal sealed partial class CacheJsonContext : JsonSerializerContext; diff --git a/src/tools/helix-cache/Program.cs b/src/tools/helix-cache/Program.cs deleted file mode 100644 index 9d2b772274468d..00000000000000 --- a/src/tools/helix-cache/Program.cs +++ /dev/null @@ -1,82 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Net.Http; -using System.Text.Json; -using System.Threading.Tasks; -using System.Xml; - -namespace HelixCache; - -internal static class Program -{ - private static async Task Main(string[] args) - { - try - { - if (args is ["--help"] or ["-h"]) - { - Console.WriteLine(SR.Get("Usage")); - return 0; - } - if (!OperatingSystem.IsLinux()) - { - throw new PlatformNotSupportedException(SR.Get("UnsupportedPlatform")); - } - if (args.Length < 2 || args[0] is not ("lookup" or "store") || args.Length % 2 != 0) - { - throw new InvalidDataException(SR.Get("Usage")); - } - - Dictionary options = new(StringComparer.Ordinal); - for (int i = 2; i < args.Length; i += 2) - { - if (!options.TryAdd(args[i], args[i + 1]) || string.IsNullOrWhiteSpace(args[i + 1])) - { - throw new InvalidDataException(SR.Get("Usage")); - } - } - - Cache cache = new(Take(options, "--cache")); - string manifest = Path.GetFullPath(args[1]); - if (args[0] == "lookup") - { - string output = Take(options, "--output"); - options.Remove("--queue-info", out string? queueInfo); - JsonFile.Require(options.Count == 0, nameof(options)); - Submission submission = JsonFile.Read(manifest, CacheJsonContext.Default.Submission); - using LinuxEnvironment environment = await LinuxEnvironment.LoadAsync(queueInfo); - LookupResult result = cache.Lookup(submission, Path.GetDirectoryName(manifest)!, environment); - JsonFile.Write(output, result, CacheJsonContext.Default.LookupResult); - } - else - { - string results = Take(options, "--results"); - JsonFile.Require(options.Count == 0, nameof(options)); - ExecutionRecord execution = JsonFile.Read(manifest, CacheJsonContext.Default.ExecutionRecord); - cache.Store(execution, results); - Console.Error.WriteLine(SR.Get("Stored", execution.Key)); - } - return 0; - } - catch (Exception ex) when (ex is InvalidDataException or IOException or UnauthorizedAccessException or JsonException or - HttpRequestException or TaskCanceledException or XmlException or ArgumentException or - InvalidOperationException or KeyNotFoundException or OverflowException or PlatformNotSupportedException) - { - Console.Error.WriteLine(SR.Get("Error", ex.Message)); - return 1; - } - } - - private static string Take(Dictionary options, string name) - { - if (!options.Remove(name, out string? value)) - { - throw new InvalidDataException(SR.Get("Usage")); - } - return value; - } -} diff --git a/src/tools/helix-cache/README.md b/src/tools/helix-cache/README.md index 24bba66137e7e5..9ac1e4415d6c20 100644 --- a/src/tools/helix-cache/README.md +++ b/src/tools/helix-cache/README.md @@ -9,15 +9,38 @@ submission and collection without replacing any test executions or results. From the repository root, using its configured SDK and NuGet feeds: ```sh -dotnet publish src/tools/helix-cache/HelixCache.csproj -c Release -r linux-x64 -python3 src/tools/helix-cache/tests/test_cache.py \ - artifacts/bin/HelixCache/Release/net11.0/linux-x64/publish/helix-cache +dotnet publish src/tools/helix-cache/src/HelixCache.csproj -c Release -r linux-x64 \ + -o artifacts/tools/helix-cache +dotnet test src/tools/helix-cache/tests/HelixCache.Tests.csproj -c Release ``` NativeAOT publishing requires a supported native compiler/linker and development libraries. For managed development, build with `-p:PublishAot=false` and run the generated apphost with the SDK runtime available. +## Structure + +`src/` contains the executable project; `tests/` is a sibling xUnit project. +Tests exercise the published NativeAOT executable and inject HTTP responses for +Helix collection. Neither project needs source-glob exclusions. + +The command pipeline is: + +1. `CommandLine` parses and validates arguments into command-specific records before any I/O. +2. `Submission` validates work items and payload declarations; `LinuxEnvironment` resolves the declared queue environment. +3. `Cache.Lookup` freezes shared inputs once per lookup, constructs execution identities, and checks completed entries. +4. `HelixClient` handles typed HTTP responses, completion polling, and result downloads. +5. `Results.Read` validates receipts and reports and produces a `TestRun` containing counts and comparable outcomes. +6. `ShadowValidation` compares a fresh run with any cached run before `Cache.Store` publishes a passing entry atomically. + +JSON contracts use source-generated System.Text.Json serialization. +`HelixJsonContext` uses the service's PascalCase names for transport models. +`CacheJsonContext` uses camelCase for local models, including environment +fingerprints. Image/provisioning descriptions remain opaque JSON because their +open-ended fields must all participate in the key. Canonical key encoding is +separate from deserialization and domain validation. Prototype formats and keys +may change during development; existing entries are disposable. + ## Lookup ```sh @@ -105,7 +128,7 @@ result collector from Helix's completed-work-item status: cancellations or timeouts. Submission success is not completion. This tool does not independently authenticate the receipt with Helix. -V1 accepts xUnit XML (`assemblies` / `assembly` / `test`) only. Each assembly must +The prototype accepts xUnit XML (`assemblies` / `assembly` / `test`) only. Each assembly must contain tests whose results agree with its total/passed/failed/skipped counts; failures, errors, empty or incomplete reports are rejected. Skips are retained. All files in the results directory are copied and checksummed. The execution key @@ -133,11 +156,10 @@ The cache task's build ID is **not** part of an execution digest. Only completed entries are persisted; frozen input snapshots are recreated in each job. Existing Azure Pipelines branch/PR cache scoping applies. -`shadow.py` is a thin Python standard-library orchestration/transport layer; all -input capture, digest computation, cache validation and admission use the C# -NativeAOT executable. Its `prepare` command records the full command, upload -post-command, timeout and queue, runs native lookup, and exports frozen paths -to `submission.props`. `Shadow.proj` submits those exact archives through the +All orchestration runs in the C# NativeAOT executable. Its `prepare` command +records the full command, upload post-command, timeout and queue, runs native +lookup, and exports frozen paths to `submission.props`. +`eng/testing/helix-cache-shadow.proj` submits those exact archives through the pinned Helix SDK without injecting extra reporters. Its default target only validates inputs; submission requires **`-t:Submit`**, which waits for completion and cancels the submitted job on task cancellation. @@ -150,7 +172,8 @@ passes the report through native admission, and compares any hit with the fresh test identities/outcomes/skip reasons. Ordering, worker directory prefixes and execution timings are ignored. The resolved physical queue ID and submitted alias are preserved in provenance alongside the original cached job. -Every hit still executes on Helix; no cached result is published as a new run. +On a hit, mismatched outcomes fail validation before store. Every hit still +executes on Helix; no cached result is published as a new run. Evidence is included in the existing build-log artifact under `artifacts/log//helix-cache-shadow/`, including `lookup.json`, @@ -162,14 +185,13 @@ by this first shadow lane. For local preparation without submitting, after building the archives: ```sh -python3 src/tools/helix-cache/shadow.py prepare \ - --executable artifacts/bin/HelixCache/Release/net11.0/linux-x64/publish/helix-cache \ +artifacts/tools/helix-cache/helix-cache prepare \ --cache artifacts/tmp/helix-cache \ --output artifacts/log/Release/helix-cache-shadow \ --payload artifacts/helix/tests/linux.AnyCPU.Release/System.Memory.Data.Tests.zip \ --runtime artifacts/helix/runtime/test-runtime-net11.0-linux-Release-x64.zip \ --queue AzureLinux.3.Amd64.Open -dotnet msbuild src/tools/helix-cache/Shadow.proj -t:Validate \ +dotnet msbuild eng/testing/helix-cache-shadow.proj -t:Validate \ -p:HelixCacheShadowDirectory="$PWD/artifacts/log/Release/helix-cache-shadow" ``` diff --git a/src/tools/helix-cache/Results.cs b/src/tools/helix-cache/Results.cs deleted file mode 100644 index dfccbef2179b07..00000000000000 --- a/src/tools/helix-cache/Results.cs +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Xml; -using System.Xml.Linq; - -namespace HelixCache; - -internal static class Results -{ - internal static TestSummary Validate(string directory, ExecutionRecord execution) - { - ResultReceipt receipt = JsonFile.Read(Path.Combine(directory, "result.json"), CacheJsonContext.Default.ResultReceipt); - JsonFile.ValidateVersion(receipt.SchemaVersion); - if (!receipt.Completed || !receipt.InfrastructureSucceeded || - receipt.ExitCode != execution.Inputs.ExpectedExitCode || - receipt.WorkItemName != execution.Inputs.Name || - receipt.TargetQueue != execution.Inputs.TargetQueue || - !Guid.TryParse(receipt.JobId, out _) || receipt.ResultFiles.Count == 0) - { - throw new InvalidDataException(SR.Get("Result", directory)); - } - - TestSummary summary = new(); - HashSet reports = new(StringComparer.Ordinal); - foreach (string report in receipt.ResultFiles) - { - JsonFile.RelativePath(report); - JsonFile.Require(reports.Add(report), nameof(receipt.ResultFiles)); - string path = Path.Combine(directory, report); - using XmlReader reader = XmlReader.Create(path, new XmlReaderSettings { DtdProcessing = DtdProcessing.Prohibit, XmlResolver = null }); - XDocument document = XDocument.Load(reader); - if (document.Root is not XElement root || root.Name != "assemblies" || !root.Elements("assembly").Any()) - { - throw new InvalidDataException(SR.Get("Report", report)); - } - - foreach (XElement assembly in root.Elements("assembly")) - { - TestSummary actual = new(); - foreach (XElement test in assembly.Descendants("test")) - { - checked - { - actual.Total++; - switch ((string?)test.Attribute("result")) - { - case "Pass": - actual.Passed++; - break; - case "Skip": - actual.Skipped++; - break; - default: - throw new InvalidDataException(SR.Get("Report", report)); - } - } - } - - if (actual.Total == 0 || Count(assembly, "total") != actual.Total || - Count(assembly, "passed") != actual.Passed || Count(assembly, "skipped") != actual.Skipped || - Count(assembly, "failed") != 0 || - assembly.Descendants("error").Any() || assembly.Descendants("failure").Any() || - (assembly.Attribute("errors") is not null && Count(assembly, "errors") != 0)) - { - throw new InvalidDataException(SR.Get("Report", report)); - } - - checked - { - summary.Total += actual.Total; - summary.Passed += actual.Passed; - summary.Skipped += actual.Skipped; - } - } - } - - return summary; - } - - private static int Count(XElement assembly, string name) - { - if (!int.TryParse((string?)assembly.Attribute(name), NumberStyles.None, CultureInfo.InvariantCulture, out int count)) - { - throw new InvalidDataException(SR.Get("Report", name)); - } - return count; - } -} diff --git a/src/tools/helix-cache/SR.cs b/src/tools/helix-cache/SR.cs deleted file mode 100644 index ddee04d4ff2a1e..00000000000000 --- a/src/tools/helix-cache/SR.cs +++ /dev/null @@ -1,16 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Globalization; -using System.Resources; - -namespace HelixCache; - -internal static class SR -{ - private static readonly ResourceManager s_resources = new("HelixCache.Strings", typeof(SR).Assembly); - - internal static string Get(string name, params object?[] arguments) => - string.Format(CultureInfo.CurrentCulture, - s_resources.GetString(name) ?? throw new MissingManifestResourceException(name), arguments); -} diff --git a/src/tools/helix-cache/Strings.resx b/src/tools/helix-cache/Strings.resx deleted file mode 100644 index 740f83f520b6f4..00000000000000 --- a/src/tools/helix-cache/Strings.resx +++ /dev/null @@ -1,25 +0,0 @@ - - - text/microsoft-resx - 2.0 - System.Resources.ResXResourceReader, System.Windows.Forms - System.Resources.ResXResourceWriter, System.Windows.Forms - Usage: helix-cache lookup submission.json --cache DIRECTORY --output lookup.json [--queue-info queues.json] - helix-cache store execution.json --results DIRECTORY --cache DIRECTORY -Only Linux x64 queues and digest-pinned container references are supported. - helix-cache: {0} - Invalid or missing manifest field: {0}. - Unsupported schema version {0}; expected {1}. - This prototype requires a Linux host. - Queue '{0}' is missing, unavailable, or not a Linux AMD64 queue. - Container reference '{0}' must end in an explicit @sha256: digest. - Duplicate name or property: {0}. - Payload '{0}' must be a directory or a ZIP archive. - Symbolic links are not supported in directory payloads or results: {0}. Use an archive for payloads containing links. - Cache or staging directory '{0}' must not be inside input directory '{1}'. - Cache entry or snapshot failed validation: {0}. - Results are incomplete, unsuccessful, or do not match the execution record: {0}. - Expected a completed, nonempty, passing xUnit report: {0}. - Expected a relative path without traversal: {0}. - Stored {0}. - diff --git a/src/tools/helix-cache/shadow.py b/src/tools/helix-cache/shadow.py deleted file mode 100644 index fbb447a81202c6..00000000000000 --- a/src/tools/helix-cache/shadow.py +++ /dev/null @@ -1,207 +0,0 @@ -# Licensed to the .NET Foundation under one or more agreements. -# The .NET Foundation licenses this file to you under the MIT license. - -"""Prepare frozen Helix submissions and collect public results; caching stays in the native CLI.""" - -import argparse -from collections import Counter -import json -from pathlib import Path -import subprocess -import time -from urllib.parse import quote -from urllib.request import urlopen -import uuid -import xml.etree.ElementTree as ET - - -API = "https://helix.dot.net/api/2019-06-17" -WORK_ITEM = "System.Memory.Data.Tests" -COMMAND = 'bash ./RunTests.sh --runtime-path "$HELIX_CORRELATION_PAYLOAD"' -POST_COMMAND = 'cp testResults.xml "$HELIX_WORKITEM_UPLOAD_ROOT/testResults.xml"' - - -def write_json(path, value): - path.write_text(json.dumps(value, indent=2) + "\n") - - -def read_json(path): - return json.loads(path.read_text()) - - -def invoke(executable, *args): - subprocess.run([str(executable), *map(str, args)], check=True) - - -def msbuild_literal(value): - for char in "%$@();'*?": - value = value.replace(char, f"%{ord(char):02X}") - return value - - -def prepare(args): - for archive in (args.payload, args.runtime): - if not archive.is_file() or archive.suffix.lower() != ".zip": - raise ValueError(f"The shadow submitter requires an existing ZIP archive: {archive}") - output = args.output.resolve() - output.mkdir(parents=True, exist_ok=False) - submission = { - "schemaVersion": 1, - "workItems": [{ - "name": WORK_ITEM, - "queues": [args.queue], - "payload": {"path": str(args.payload.resolve())}, - "correlationPayloads": [{"path": str(args.runtime.resolve())}], - "command": COMMAND, - "postCommands": [POST_COMMAND], - "timeoutSeconds": 900, - "expectedExitCode": 0, - }], - } - write_json(output / "submission.json", submission) - options = ["--queue-info", args.queue_info] if args.queue_info else [] - invoke(args.executable, "lookup", output / "submission.json", - "--cache", args.cache, "--output", output / "lookup.json", *options) - entry, = read_json(output / "lookup.json")["entries"] - execution = entry["execution"] - write_json(output / "execution.json", execution) - project = ET.Element("Project") - properties = ET.SubElement(project, "PropertyGroup") - for name, value in { - "ShadowName": execution["inputs"]["name"], - "ShadowQueue": execution["inputs"]["targetQueue"], - "ShadowPayload": execution["payloadPath"], - "ShadowRuntime": execution["correlationPayloadPaths"][0], - "ShadowCommand": execution["inputs"]["command"], - "ShadowPostCommand": execution["inputs"]["postCommands"][0], - "ShadowTimeout": "00:15:00", - }.items(): - ET.SubElement(properties, name).text = msbuild_literal(value) - ET.indent(project, space=" ") - ET.ElementTree(project).write(output / "submission.props", encoding="utf-8", xml_declaration=True) - print(f"Helix cache shadow: {entry['status']} {execution['key']}; execution will NOT be skipped.") - - -def get_json(url): - with urlopen(url, timeout=60) as response: - return json.load(response) - - -def download(url, destination): - with urlopen(url, timeout=60) as response: - destination.write_bytes(response.read()) - - -def completed_work_item(job_id, name, wait_seconds): - deadline = time.monotonic() + wait_seconds - url = f"{API}/jobs/{job_id}/workitems/{quote(name, safe='')}" - while True: - item = get_json(url) - if item["Job"] != job_id or item["Name"] != name: - raise ValueError("Helix returned a different job or work item.") - if item["State"] in ("Finished", "Passed", "Failed"): - return item - if item["State"] not in ("Unscheduled", "Waiting", "Running"): - raise ValueError(f"Unexpected Helix work item state: {item['State']}") - remaining = deadline - time.monotonic() - if remaining <= 0: - raise TimeoutError(f"Helix job {job_id} did not complete within {wait_seconds} seconds.") - time.sleep(min(15, remaining)) - - -def outcomes(path): - root = ET.parse(path).getroot() - return Counter( - (Path(assembly.attrib["name"]).name, test.attrib["name"], test.attrib["result"], - test.findtext("reason/message", default="") if test.attrib["result"] == "Skip" else "") - for assembly in root.findall("assembly") - for test in assembly.findall("collection/test") - ) - - -def collect(args): - output = args.output.resolve() - execution = read_json(output / "execution.json") - lookup, = read_json(output / "lookup.json")["entries"] - job_id = str(uuid.UUID((output / "job-id.txt").read_text().strip())) - summary = {"key": execution["key"], "lookup": lookup["status"], "jobId": job_id, "mode": "shadow"} - write_json(output / "summary.json", summary) - job = get_json(f"{API}/jobs/{job_id}/details") - expected_queue = execution["inputs"]["environmentFingerprint"]["QueueId"] - submitted_queue = job.get("QueueAlias") or job["QueueId"] - if job["Name"] != job_id or submitted_queue.casefold() != expected_queue.casefold(): - raise ValueError("Helix returned a different job or target queue.") - item = completed_work_item(job_id, execution["inputs"]["name"], args.wait_seconds) - write_json(output / "work-item.json", item) - if (item["State"] not in ("Finished", "Passed") or item.get("FailureReason") is not None - or item.get("Errors") or type(item.get("ExitCode")) is not int - or item["ExitCode"] != execution["inputs"]["expectedExitCode"]): - raise ValueError(f"Helix work item did not complete successfully; see {output / 'work-item.json'}.") - files = get_json(f"{API}/jobs/{job_id}/workitems/{quote(item['Name'], safe='')}/files?latestOnly=true") - reports = [file for file in files if file["Name"] == "testResults.xml"] - if len(reports) != 1: - raise ValueError(f"Expected exactly one uploaded testResults.xml; found {len(reports)}.") - report, = reports - results = output / "results" - results.mkdir() - download(report["Link"], results / "testResults.xml") - if item.get("ConsoleOutputUri"): - download(item["ConsoleOutputUri"], results / "console.log") - write_json(results / "result.json", { - "schemaVersion": 1, - "completed": True, - "infrastructureSucceeded": True, - "exitCode": item["ExitCode"], - "jobId": job_id, - "workItemName": item["Name"], - "targetQueue": execution["inputs"]["targetQueue"], - "resultFiles": ["testResults.xml"], - }) - write_json(results / "provenance.json", { - "kind": "helix-execution", - "jobId": job_id, - "queueAlias": job.get("QueueAlias"), - "resolvedQueueId": job["QueueId"], - "machineName": item["MachineName"], - "state": item["State"], - }) - invoke(args.executable, "store", output / "execution.json", "--results", results, "--cache", args.cache) - fresh = outcomes(results / "testResults.xml") - summary["tests"] = sum(fresh.values()) - summary["comparison"] = "not-applicable" - if lookup["status"] == "hit": - cached_results = Path(lookup["resultsPath"]) - cached = outcomes(cached_results / "testResults.xml") - summary["cachedJobId"] = read_json(cached_results / "result.json")["jobId"] - summary["comparison"] = "match" if cached == fresh else "mismatch" - write_json(output / "summary.json", summary) - if cached != fresh: - raise ValueError(f"Cached and fresh test outcomes differ; see {output / 'summary.json'}.") - summary["stored"] = True - write_json(output / "summary.json", summary) - print(json.dumps(summary, indent=2)) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - for name in ("prepare", "collect"): - command = commands.add_parser(name) - command.add_argument("--executable", type=Path, required=True) - command.add_argument("--cache", type=Path, required=True) - command.add_argument("--output", type=Path, required=True) - if name == "prepare": - command.add_argument("--payload", type=Path, required=True) - command.add_argument("--runtime", type=Path, required=True) - command.add_argument("--queue", required=True) - command.add_argument("--queue-info", type=Path) - else: - command.add_argument("--wait-seconds", type=int, default=3600) - args = parser.parse_args() - if args.command == "collect" and args.wait_seconds <= 0: - parser.error("--wait-seconds must be positive") - (prepare if args.command == "prepare" else collect)(args) - - -if __name__ == "__main__": - main() diff --git a/src/tools/helix-cache/Cache.cs b/src/tools/helix-cache/src/Cache.cs similarity index 59% rename from src/tools/helix-cache/Cache.cs rename to src/tools/helix-cache/src/Cache.cs index 6f1553620cb06d..b3e6a013a60364 100644 --- a/src/tools/helix-cache/Cache.cs +++ b/src/tools/helix-cache/src/Cache.cs @@ -13,48 +13,41 @@ internal sealed class Cache { internal const int ProtocolVersion = 1; private readonly string _root; - private readonly Dictionary _snapshots = new(StringComparer.Ordinal); internal Cache(string directory) => _root = Path.GetFullPath(directory); internal LookupResult Lookup(Submission submission, string baseDirectory, LinuxEnvironment environment) { - JsonFile.ValidateVersion(submission.SchemaVersion); - JsonFile.Require(submission.WorkItems.Count > 0, nameof(submission.WorkItems)); + submission.Validate(); + Dictionary fingerprints = submission.WorkItems.SelectMany(item => item.Queues) + .Distinct(StringComparer.Ordinal).ToDictionary(target => target, environment.Resolve, StringComparer.Ordinal); + Dictionary snapshots = new(StringComparer.Ordinal); LookupResult result = new(); - HashSet names = new(StringComparer.Ordinal); foreach (WorkItem item in submission.WorkItems) { - JsonFile.Require(!string.IsNullOrWhiteSpace(item.Name) && names.Add(item.Name), nameof(item.Name)); - JsonFile.Require(!string.IsNullOrWhiteSpace(item.Command), nameof(item.Command)); - JsonFile.Require(item.TimeoutSeconds > 0, nameof(item.TimeoutSeconds)); - JsonFile.Require(item.Queues.Count > 0 && item.Queues.Distinct(StringComparer.Ordinal).Count() == item.Queues.Count, nameof(item.Queues)); - JsonFile.Require(item.PreCommands.All(c => c is not null) && item.PostCommands.All(c => c is not null) && - item.Environment.All(p => !string.IsNullOrEmpty(p.Key) && p.Value is not null), nameof(item.Environment)); - Snapshot payload = Capture(item.Payload, baseDirectory); - List correlation = item.CorrelationPayloads.Select(p => Capture(p, baseDirectory)).ToList(); + Snapshot payload = Capture(item.Payload, baseDirectory, snapshots); + List correlation = item.CorrelationPayloads.Select(p => Capture(p, baseDirectory, snapshots)).ToList(); foreach (string target in item.Queues) { - JsonFile.Require(!string.IsNullOrWhiteSpace(target), nameof(item.Queues)); ExecutionInputs inputs = new() { Name = item.Name, TargetQueue = target, - EnvironmentFingerprint = environment.Resolve(target), + EnvironmentFingerprint = fingerprints[target], Payload = Identity(item.Payload, payload), CorrelationPayloads = item.CorrelationPayloads.Select((p, i) => Identity(p, correlation[i])).ToList(), Command = item.Command, - PreCommands = item.PreCommands, - PostCommands = item.PostCommands, - Environment = item.Environment, + PreCommands = [.. item.PreCommands], + PostCommands = [.. item.PostCommands], + Environment = new(item.Environment, StringComparer.Ordinal), TimeoutSeconds = item.TimeoutSeconds, ExpectedExitCode = item.ExpectedExitCode, }; ExecutionRecord execution = new() { SchemaVersion = ProtocolVersion, - Key = Key(inputs), + Key = ExecutionKey.Create(inputs), Inputs = inputs, PayloadPath = payload.Path, CorrelationPayloadPaths = correlation.Select(p => p.Path).ToList(), @@ -67,7 +60,7 @@ internal LookupResult Lookup(Submission submission, string baseDirectory, LinuxE } result.Entries.Add(new LookupEntry { - Status = hit ? "hit" : "miss", + Status = hit ? LookupStatus.Hit : LookupStatus.Miss, Execution = execution, ResultsPath = hit ? Path.Combine(entry, "results") : null, }); @@ -78,7 +71,7 @@ internal LookupResult Lookup(Submission submission, string baseDirectory, LinuxE internal void Store(ExecutionRecord execution, string results) { - ValidateRecord(execution); + execution.Validate(); VerifySnapshot(execution.PayloadPath, execution.Inputs.Payload.Digest); for (int i = 0; i < execution.CorrelationPayloadPaths.Count; i++) { @@ -90,7 +83,7 @@ internal void Store(ExecutionRecord execution, string results) Snapshot.EnsureOutside(results, temporary.Path); string stagedResults = Path.Combine(temporary.Path, "results"); Snapshot.Copy(results, stagedResults); - TestSummary summary = Results.Validate(stagedResults, execution); + TestSummary summary = Results.Read(stagedResults, execution).Summary; StoredEntry entry = new() { SchemaVersion = ProtocolVersion, @@ -99,7 +92,11 @@ internal void Store(ExecutionRecord execution, string results) ResultsDigest = Snapshot.Hash(stagedResults), Summary = summary, }; - JsonFile.Write(Path.Combine(temporary.Path, "entry.json"), entry, CacheJsonContext.Default.StoredEntry); + using (FileStream file = File.Create(Path.Combine(temporary.Path, "entry.json"))) + { + JsonSerializer.Serialize(file, entry, CacheJsonContext.Default.StoredEntry); + file.Flush(flushToDisk: true); + } string final = EntryPath(execution.Key); Directory.CreateDirectory(Path.GetDirectoryName(final)!); try @@ -112,16 +109,20 @@ internal void Store(ExecutionRecord execution, string results) } } - private Snapshot Capture(Payload payload, string baseDirectory) + internal TestRun ReadResults(ExecutionRecord execution) + { + execution.Validate(); + return ValidateEntry(EntryPath(execution.Key), execution.Key); + } + + private Snapshot Capture(Payload payload, string baseDirectory, Dictionary snapshots) { - JsonFile.Require(!string.IsNullOrWhiteSpace(payload.Path), nameof(payload.Path)); - JsonFile.RelativePath(payload.Destination, allowEmpty: true); string path = Path.GetFullPath(payload.Path, baseDirectory); - if (!_snapshots.TryGetValue(path, out Snapshot? snapshot)) + if (!snapshots.TryGetValue(path, out Snapshot? snapshot)) { Snapshot.EnsureOutside(path, _root); snapshot = Snapshot.Capture(path, _root); - _snapshots.Add(path, snapshot); + snapshots.Add(path, snapshot); } return snapshot; } @@ -133,45 +134,43 @@ private Snapshot Capture(Payload payload, string baseDirectory) Kind = snapshot.Kind, }; - private void ValidateEntry(string directory, string key) + private static TestRun ValidateEntry(string directory, string key) { - StoredEntry entry = JsonFile.Read(Path.Combine(directory, "entry.json"), CacheJsonContext.Default.StoredEntry); - JsonFile.ValidateVersion(entry.SchemaVersion); - ValidateRecord(entry.Execution); + using FileStream file = File.OpenRead(Path.Combine(directory, "entry.json")); + StoredEntry entry = JsonSerializer.Deserialize(file, CacheJsonContext.Default.StoredEntry) + ?? throw new InvalidDataException($"Cache entry failed validation: {directory}."); + ValidateVersion(entry.SchemaVersion); + entry.Execution.Validate(); string results = Path.Combine(directory, "results"); if (entry.Execution.Key != key || Snapshot.Hash(results) != entry.ResultsDigest) { - throw new InvalidDataException(SR.Get("Corrupt", directory)); - } - TestSummary summary = Results.Validate(results, entry.Execution); - if (summary.Total != entry.Summary.Total || summary.Passed != entry.Summary.Passed || summary.Skipped != entry.Summary.Skipped) - { - throw new InvalidDataException(SR.Get("Corrupt", directory)); + throw new InvalidDataException($"Cache entry failed validation: {directory}."); } - } - - private static void ValidateRecord(ExecutionRecord execution) - { - JsonFile.ValidateVersion(execution.SchemaVersion); - if (!IsDigest(execution.Key) || Key(execution.Inputs) != execution.Key || - execution.CorrelationPayloadPaths.Count != execution.Inputs.CorrelationPayloads.Count) + TestRun run = Results.Read(results, entry.Execution); + if (run.Summary.Total != entry.Summary.Total || run.Summary.Passed != entry.Summary.Passed || run.Summary.Skipped != entry.Summary.Skipped) { - throw new InvalidDataException(SR.Get("Corrupt", execution.Key)); + throw new InvalidDataException($"Cache entry failed validation: {directory}."); } + return run; } private static void VerifySnapshot(string path, string digest) { if (Snapshot.Hash(path) != digest) { - throw new InvalidDataException(SR.Get("Corrupt", path)); + throw new InvalidDataException($"Snapshot failed validation: {path}."); } } private string EntryPath(string key) => Path.Combine(_root, "entries", "v1", key[..2], key); - private static string Key(ExecutionInputs inputs) => - JsonFile.Digest(JsonSerializer.SerializeToElement(inputs, CacheJsonContext.Default.ExecutionInputs)); + internal static void ValidateVersion(int version) + { + if (version != ProtocolVersion) + { + throw new InvalidDataException($"Unsupported schema version {version}; expected {ProtocolVersion}."); + } + } internal static bool IsDigest(string value) => value.Length == 64 && value.All(c => c is >= '0' and <= '9' or >= 'a' and <= 'f'); diff --git a/src/tools/helix-cache/src/CommandLine.cs b/src/tools/helix-cache/src/CommandLine.cs new file mode 100644 index 00000000000000..de567b68c744a8 --- /dev/null +++ b/src/tools/helix-cache/src/CommandLine.cs @@ -0,0 +1,89 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; + +namespace HelixCache; + +internal abstract record Command(string CacheDirectory); +internal sealed record LookupCommand(string Manifest, string CacheDirectory, string Output, string? QueueInfo) : Command(CacheDirectory); +internal sealed record StoreCommand(string Manifest, string CacheDirectory, string Results) : Command(CacheDirectory); +internal sealed record PrepareCommand(string CacheDirectory, string Output, string Payload, string Runtime, string Queue, string? QueueInfo) : Command(CacheDirectory); +internal sealed record CollectCommand(string CacheDirectory, string Output, TimeSpan Wait) : Command(CacheDirectory); + +internal static class CommandLine +{ + internal const string Usage = """ + Usage: helix-cache lookup submission.json --cache DIRECTORY --output lookup.json [--queue-info queues.json] + helix-cache store execution.json --results DIRECTORY --cache DIRECTORY + helix-cache prepare --cache DIRECTORY --output DIRECTORY --payload tests.zip --runtime runtime.zip --queue QUEUE [--queue-info queues.json] + helix-cache collect --cache DIRECTORY --output DIRECTORY [--wait-seconds SECONDS] + Only Linux x64 queues and digest-pinned container references are supported. + """; + + internal static Command Parse(string[] args) + { + if (args.Length == 0 || args[0] is not ("lookup" or "store" or "prepare" or "collect")) + { + throw new InvalidDataException(Usage); + } + bool hasManifest = args[0] is "lookup" or "store"; + int firstOption = hasManifest ? 2 : 1; + if (args.Length < firstOption || + (hasManifest && (string.IsNullOrWhiteSpace(args[1]) || args[1].StartsWith("--", StringComparison.Ordinal)))) + { + throw new InvalidDataException($"Missing manifest for '{args[0]}'.\n{Usage}"); + } + Dictionary options = new(StringComparer.Ordinal); + for (int i = firstOption; i < args.Length; i += 2) + { + string name = args[i]; + if (!name.StartsWith("--", StringComparison.Ordinal) || i + 1 == args.Length || + string.IsNullOrWhiteSpace(args[i + 1]) || args[i + 1].StartsWith("--", StringComparison.Ordinal)) + { + throw new InvalidDataException($"Expected an option and value at '{name}'.\n{Usage}"); + } + if (!options.TryAdd(name, args[i + 1])) + { + throw new InvalidDataException($"Duplicate option '{name}'."); + } + } + + string cache = Required("--cache"); + Command command = args[0] switch + { + "lookup" => new LookupCommand(args[1], cache, Required("--output"), Optional("--queue-info")), + "store" => new StoreCommand(args[1], cache, Required("--results")), + "prepare" => new PrepareCommand(cache, Required("--output"), Required("--payload"), + Required("--runtime"), Required("--queue"), Optional("--queue-info")), + "collect" => new CollectCommand(cache, Required("--output"), ParseWait(Optional("--wait-seconds"))), + _ => throw new UnreachableException(), + }; + if (options.Count > 0) + { + throw new InvalidDataException($"Unknown option '{string.Join("', '", options.Keys)}'."); + } + return command; + + string Required(string name) => Optional(name) ?? throw new InvalidDataException($"Missing required option '{name}'.\n{Usage}"); + string? Optional(string name) => options.Remove(name, out string? value) ? value : null; + } + + private static TimeSpan ParseWait(string? value) + { + if (value is null) + { + return TimeSpan.FromHours(1); + } + if (!int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out int seconds) || + seconds <= 0 || seconds > (uint.MaxValue - 1) / 1000) + { + throw new InvalidDataException("--wait-seconds must be a positive number of seconds supported by a cancellation timer."); + } + return TimeSpan.FromSeconds(seconds); + } +} diff --git a/src/tools/helix-cache/src/ExecutionKey.cs b/src/tools/helix-cache/src/ExecutionKey.cs new file mode 100644 index 00000000000000..4613d7dc040e44 --- /dev/null +++ b/src/tools/helix-cache/src/ExecutionKey.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text.Json; + +namespace HelixCache; + +internal static class ExecutionKey +{ + internal static string Create(ExecutionInputs inputs) + { + using MemoryStream stream = new(); + using (Utf8JsonWriter writer = new(stream)) + { + WriteCanonical(writer, JsonSerializer.SerializeToElement(inputs, CacheJsonContext.Default.ExecutionInputs)); + } + return Convert.ToHexStringLower(SHA256.HashData(stream.GetBuffer().AsSpan(0, checked((int)stream.Length)))); + } + + // Property order is not execution input, including within opaque Helix provisioning metadata. + private static void WriteCanonical(Utf8JsonWriter writer, JsonElement value) + { + switch (value.ValueKind) + { + case JsonValueKind.Object: + writer.WriteStartObject(); + foreach (JsonProperty property in value.EnumerateObject().OrderBy(p => p.Name, StringComparer.Ordinal)) + { + writer.WritePropertyName(property.Name); + WriteCanonical(writer, property.Value); + } + writer.WriteEndObject(); + break; + case JsonValueKind.Array: + writer.WriteStartArray(); + foreach (JsonElement item in value.EnumerateArray()) + { + WriteCanonical(writer, item); + } + writer.WriteEndArray(); + break; + default: + value.WriteTo(writer); + break; + } + } +} diff --git a/src/tools/helix-cache/HelixCache.csproj b/src/tools/helix-cache/src/HelixCache.csproj similarity index 86% rename from src/tools/helix-cache/HelixCache.csproj rename to src/tools/helix-cache/src/HelixCache.csproj index 8fbae8b8951256..ad2071f61a0bb1 100644 --- a/src/tools/helix-cache/HelixCache.csproj +++ b/src/tools/helix-cache/src/HelixCache.csproj @@ -9,4 +9,7 @@ false true + + + diff --git a/src/tools/helix-cache/src/HelixClient.cs b/src/tools/helix-cache/src/HelixClient.cs new file mode 100644 index 00000000000000..2e85d64827bfae --- /dev/null +++ b/src/tools/helix-cache/src/HelixClient.cs @@ -0,0 +1,94 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace HelixCache; + +internal sealed class HelixClient(HttpClient client) +{ + private const string Api = "https://helix.dot.net/api/2019-06-17"; + + internal async Task> GetQueuesAsync(CancellationToken cancellationToken) => + await client.GetFromJsonAsync($"{Api}/info/queues", HelixJsonContext.Default.ListHelixQueue, cancellationToken) + ?? throw new InvalidDataException("Helix returned no queue metadata."); + + internal async Task GetJobAsync(string jobId, CancellationToken cancellationToken) => + await client.GetFromJsonAsync($"{Api}/jobs/{jobId}/details", HelixJsonContext.Default.HelixJob, cancellationToken) + ?? throw new InvalidDataException("Helix returned no job details."); + + internal async Task WaitForCompletionAsync(string jobId, string name, TimeSpan wait, + CancellationToken cancellationToken = default) + { + using CancellationTokenSource timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(wait); + try + { + while (true) + { + HelixWorkItem item = await client.GetFromJsonAsync( + $"{Api}/jobs/{jobId}/workitems/{Uri.EscapeDataString(name)}", + HelixJsonContext.Default.HelixWorkItem, timeout.Token) + ?? throw new InvalidDataException("Helix returned no work item."); + if (item.Job != jobId || item.Name != name) + { + throw new InvalidDataException("Helix returned a different job or work item."); + } + if (item.State is "Finished" or "Passed" or "Failed") + { + return item; + } + if (item.State is not ("Unscheduled" or "Waiting" or "Running")) + { + throw new InvalidDataException($"Unexpected Helix work item state: {item.State}."); + } + await Task.Delay(TimeSpan.FromSeconds(15), timeout.Token); + } + } + catch (OperationCanceledException ex) when (timeout.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + throw new TimeoutException($"Helix job {jobId} did not complete within {wait.TotalSeconds} seconds.", ex); + } + } + + internal async Task DownloadResultsAsync(HelixWorkItem item, string directory, CancellationToken cancellationToken) + { + List files = await client.GetFromJsonAsync( + $"{Api}/jobs/{item.Job}/workitems/{Uri.EscapeDataString(item.Name)}/files?latestOnly=true", + HelixJsonContext.Default.ListHelixFile, cancellationToken) + ?? throw new InvalidDataException("Helix returned no work item file list."); + if (files.Any(file => file is null)) + { + throw new InvalidDataException("Helix work item files must not contain null."); + } + List reports = files.Where(file => file.Name == "testResults.xml").ToList(); + if (reports.Count != 1) + { + throw new InvalidDataException($"Expected exactly one uploaded testResults.xml; found {reports.Count}."); + } + if (Directory.Exists(directory) || File.Exists(directory)) + { + throw new IOException($"The result download directory already exists: {directory}."); + } + Directory.CreateDirectory(directory); + await DownloadAsync(reports[0].Link, Path.Combine(directory, "testResults.xml"), cancellationToken); + if (!string.IsNullOrEmpty(item.ConsoleOutputUri)) + { + await DownloadAsync(item.ConsoleOutputUri, Path.Combine(directory, "console.log"), cancellationToken); + } + } + + private async Task DownloadAsync(string uri, string path, CancellationToken cancellationToken) + { + using Stream source = await client.GetStreamAsync(uri, cancellationToken); + using FileStream destination = new(path, FileMode.CreateNew, FileAccess.Write); + await source.CopyToAsync(destination, cancellationToken); + } +} diff --git a/src/tools/helix-cache/src/HelixModels.cs b/src/tools/helix-cache/src/HelixModels.cs new file mode 100644 index 00000000000000..5c34836c6199fa --- /dev/null +++ b/src/tools/helix-cache/src/HelixModels.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace HelixCache; + +internal sealed record HelixQueue +{ + public required string QueueId { get; init; } + public string? Architecture { get; init; } + public string? OperatingSystemGroup { get; init; } + public bool IsAvailable { get; init; } + + [JsonExtensionData] + public Dictionary Metadata { get; init; } = []; +} + +internal sealed record HelixJob +{ + public required string Name { get; init; } + public required string QueueId { get; init; } + public string? QueueAlias { get; init; } +} + +internal sealed record HelixWorkItem +{ + public required string Job { get; init; } + public required string Name { get; init; } + public required string State { get; init; } + public int? ExitCode { get; init; } + public string? MachineName { get; init; } + public string? ConsoleOutputUri { get; init; } + public HelixFailure? FailureReason { get; init; } + public List? Errors { get; init; } +} + +internal sealed class HelixFailure +{ + [JsonExtensionData] + public Dictionary Details { get; init; } = []; +} + +internal sealed class HelixFile +{ + public required string Name { get; init; } + public required string Link { get; init; } +} + +[JsonSourceGenerationOptions(AllowDuplicateProperties = false, RespectNullableAnnotations = true)] +[JsonSerializable(typeof(List))] +[JsonSerializable(typeof(HelixJob))] +[JsonSerializable(typeof(HelixWorkItem))] +[JsonSerializable(typeof(List))] +internal sealed partial class HelixJsonContext : JsonSerializerContext; diff --git a/src/tools/helix-cache/src/LinuxEnvironment.cs b/src/tools/helix-cache/src/LinuxEnvironment.cs new file mode 100644 index 00000000000000..d37a1928689dfb --- /dev/null +++ b/src/tools/helix-cache/src/LinuxEnvironment.cs @@ -0,0 +1,102 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace HelixCache; + +internal sealed class LinuxEnvironment +{ + private readonly List _queues; + private readonly Dictionary _fingerprints = new(StringComparer.Ordinal); + + internal LinuxEnvironment(List queues) => _queues = queues; + + internal static async Task LoadAsync(string? queueInfo, HelixClient client, CancellationToken cancellationToken) + { + if (queueInfo is not null) + { + using FileStream file = File.OpenRead(queueInfo); + return new LinuxEnvironment(await JsonSerializer.DeserializeAsync(file, HelixJsonContext.Default.ListHelixQueue, cancellationToken) + ?? throw new InvalidDataException($"Invalid or missing queue metadata: {queueInfo}.")); + } + + return new LinuxEnvironment(await client.GetQueuesAsync(cancellationToken)); + } + + internal EnvironmentFingerprint Resolve(string target) + { + if (_fingerprints.TryGetValue(target, out EnvironmentFingerprint? fingerprint)) + { + return fingerprint; + } + + // Matches Arcade JobSender's (alias)host@container syntax; retain a digest's second '@'. + // https://github.com/dotnet/arcade/blob/7103cc604358625941ee1d105e662f0d697085c6/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs#L374-L399 + int separator = target.IndexOf('@'); + string host = separator < 0 ? target : target[..separator]; + string? container = separator < 0 ? null : target[(separator + 1)..]; + if (host.StartsWith('(')) + { + int close = host.IndexOf(')'); + if (separator < 0 || close <= 1 || close >= host.Length - 1) + { + throw new InvalidDataException($"Invalid target queue: {target}."); + } + host = host[(close + 1)..]; + } + + if (container is not null) + { + int digestStart = container.LastIndexOf("@sha256:", StringComparison.Ordinal); + if (digestStart <= 0 || !Cache.IsDigest(container[(digestStart + "@sha256:".Length)..])) + { + throw new InvalidDataException($"Container reference '{container}' must end in an explicit @sha256: digest."); + } + } + + HelixQueue? queue = null; + foreach (HelixQueue candidate in _queues) + { + if (candidate is null) + { + throw new InvalidDataException("Helix queue metadata must not contain null."); + } + if (string.Equals(candidate.QueueId, host, StringComparison.OrdinalIgnoreCase)) + { + if (queue is not null) + { + throw new InvalidDataException($"Duplicate target queue: {target}."); + } + queue = candidate; + } + } + + if (queue is not { Architecture: string architecture, OperatingSystemGroup: string operatingSystem } || + !string.Equals(queue.OperatingSystemGroup, "linux", StringComparison.OrdinalIgnoreCase) || + !string.Equals(queue.Architecture, "AMD64", StringComparison.OrdinalIgnoreCase) || + !queue.IsAvailable) + { + throw new InvalidDataException($"Queue '{host}' is missing, unavailable, or not a Linux AMD64 queue."); + } + + fingerprint = new EnvironmentFingerprint + { + QueueId = queue.QueueId, + Architecture = architecture, + OperatingSystemGroup = operatingSystem, + Container = container, + Provisioning = queue.Metadata.Where(p => p.Key is + "GalleryImage" or "PreInstalledImage" or "PreparedImage" or "Artifacts" or "WorkspacePath") + .ToDictionary(p => p.Key, p => p.Value, StringComparer.Ordinal), + }; + _fingerprints.Add(target, fingerprint); + return fingerprint; + } +} diff --git a/src/tools/helix-cache/src/Models.cs b/src/tools/helix-cache/src/Models.cs new file mode 100644 index 00000000000000..ca497dab54bbea --- /dev/null +++ b/src/tools/helix-cache/src/Models.cs @@ -0,0 +1,273 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace HelixCache; + +internal enum PayloadKind +{ + [JsonStringEnumMemberName("directory")] + Directory, + [JsonStringEnumMemberName("archive")] + Archive, +} + +internal enum LookupStatus +{ + [JsonStringEnumMemberName("miss")] + Miss, + [JsonStringEnumMemberName("hit")] + Hit, +} + +internal enum OutcomeComparison +{ + [JsonStringEnumMemberName("not-applicable")] + NotApplicable, + [JsonStringEnumMemberName("match")] + Match, + [JsonStringEnumMemberName("mismatch")] + Mismatch, +} + +internal sealed record Submission +{ + public required int SchemaVersion { get; init; } + public required List WorkItems { get; init; } + + internal void Validate() + { + Cache.ValidateVersion(SchemaVersion); + HashSet names = new(StringComparer.Ordinal); + if (WorkItems.Count == 0) + { + throw new InvalidDataException("A submission must contain at least one work item."); + } + foreach (WorkItem item in WorkItems) + { + if (item is null || string.IsNullOrWhiteSpace(item.Name) || !names.Add(item.Name)) + { + throw new InvalidDataException("Work items must be nonnull and have unique, nonempty names."); + } + item.Validate(); + } + } +} + +internal sealed record WorkItem +{ + public required string Name { get; init; } + public required List Queues { get; init; } + public required Payload Payload { get; init; } + public List CorrelationPayloads { get; init; } = []; + public required string Command { get; init; } + public List PreCommands { get; init; } = []; + public List PostCommands { get; init; } = []; + public Dictionary Environment { get; init; } = []; + public required int TimeoutSeconds { get; init; } + public required int ExpectedExitCode { get; init; } + + internal void Validate() + { + ValidateExecution(Command, TimeoutSeconds, PreCommands, PostCommands, Environment); + if (Queues.Count == 0 || Queues.Any(string.IsNullOrWhiteSpace) || + Queues.Distinct(StringComparer.Ordinal).Count() != Queues.Count) + { + throw new InvalidDataException($"Work item '{Name}' requires unique, nonempty target queues."); + } + Payload.Validate(); + foreach (Payload payload in CorrelationPayloads) + { + if (payload is null) + { + throw new InvalidDataException($"Work item '{Name}' has a null correlation payload."); + } + payload.Validate(); + } + } + + internal static void ValidateExecution(string command, int timeoutSeconds, List preCommands, + List postCommands, Dictionary environment) + { + if (string.IsNullOrWhiteSpace(command) || timeoutSeconds <= 0 || + preCommands.Any(c => c is null) || postCommands.Any(c => c is null) || + environment.Any(p => string.IsNullOrEmpty(p.Key) || p.Value is null)) + { + throw new InvalidDataException("Execution requires a nonempty command, positive timeout, nonnull commands and environment values."); + } + } +} + +internal sealed record Payload +{ + public required string Path { get; init; } + public string Destination { get; init; } = ""; + + internal void Validate() + { + if (string.IsNullOrWhiteSpace(Path)) + { + throw new InvalidDataException("A payload requires a nonempty path."); + } + Snapshot.ValidateRelativePath(Destination, allowEmpty: true); + } +} + +internal sealed record PayloadIdentity +{ + public required string Digest { get; init; } + public required string Destination { get; init; } + public required PayloadKind Kind { get; init; } + + internal void Validate() + { + if (!Cache.IsDigest(Digest) || Kind is not (PayloadKind.Directory or PayloadKind.Archive)) + { + throw new InvalidDataException("A payload identity requires a SHA-256 digest and a directory or archive kind."); + } + Snapshot.ValidateRelativePath(Destination, allowEmpty: true); + } +} + +internal sealed class EnvironmentFingerprint +{ + public required string QueueId { get; init; } + public required string Architecture { get; init; } + public required string OperatingSystemGroup { get; init; } + public string? Container { get; init; } + + [JsonExtensionData] + public Dictionary Provisioning { get; init; } = []; +} + +internal sealed record ExecutionInputs +{ + public required string Name { get; init; } + public required string TargetQueue { get; init; } + public required EnvironmentFingerprint EnvironmentFingerprint { get; init; } + public required PayloadIdentity Payload { get; init; } + public required List CorrelationPayloads { get; init; } + public required string Command { get; init; } + public required List PreCommands { get; init; } + public required List PostCommands { get; init; } + public required Dictionary Environment { get; init; } + public required int TimeoutSeconds { get; init; } + public required int ExpectedExitCode { get; init; } +} + +internal sealed record ExecutionRecord +{ + public required int SchemaVersion { get; init; } + public required string Key { get; init; } + public required ExecutionInputs Inputs { get; init; } + public required string PayloadPath { get; init; } + public required List CorrelationPayloadPaths { get; init; } + + internal void Validate() + { + Cache.ValidateVersion(SchemaVersion); + WorkItem.ValidateExecution(Inputs.Command, Inputs.TimeoutSeconds, Inputs.PreCommands, Inputs.PostCommands, Inputs.Environment); + Inputs.Payload.Validate(); + foreach (PayloadIdentity payload in Inputs.CorrelationPayloads) + { + if (payload is null) + { + throw new InvalidDataException("Execution correlation payload identities must not contain null."); + } + payload.Validate(); + } + if (!Cache.IsDigest(Key) || ExecutionKey.Create(Inputs) != Key || + string.IsNullOrWhiteSpace(Inputs.Name) || string.IsNullOrWhiteSpace(Inputs.TargetQueue) || + string.IsNullOrWhiteSpace(PayloadPath) || CorrelationPayloadPaths.Any(string.IsNullOrWhiteSpace) || + CorrelationPayloadPaths.Count != Inputs.CorrelationPayloads.Count) + { + throw new InvalidDataException($"Execution record failed validation: {Key}."); + } + } +} + +internal sealed class LookupResult +{ + public int SchemaVersion { get; init; } = Cache.ProtocolVersion; + public List Entries { get; init; } = []; +} + +internal sealed class LookupEntry +{ + public required LookupStatus Status { get; init; } + public required ExecutionRecord Execution { get; init; } + public string? ResultsPath { get; init; } +} + +internal sealed record ResultReceipt +{ + public required int SchemaVersion { get; init; } + public required bool Completed { get; init; } + public required bool InfrastructureSucceeded { get; init; } + public required int ExitCode { get; init; } + public required string JobId { get; init; } + public required string WorkItemName { get; init; } + public required string TargetQueue { get; init; } + public required List ResultFiles { get; init; } +} + +internal sealed class TestSummary +{ + public int Total { get; set; } + public int Passed { get; set; } + public int Skipped { get; set; } +} + +internal sealed class StoredEntry +{ + public required int SchemaVersion { get; init; } + public required ExecutionRecord Execution { get; init; } + public required DateTimeOffset CreatedUtc { get; init; } + public required string ResultsDigest { get; init; } + public required TestSummary Summary { get; init; } +} + +internal sealed record ShadowSummary +{ + public required string Key { get; init; } + public required LookupStatus Lookup { get; init; } + public required string JobId { get; init; } + public string Mode { get; init; } = "shadow"; + public int? Tests { get; init; } + public OutcomeComparison? Comparison { get; init; } + public string? CachedJobId { get; init; } + public bool Stored { get; init; } +} + +internal sealed class ExecutionProvenance +{ + public string Kind { get; init; } = "helix-execution"; + public required string JobId { get; init; } + public string? QueueAlias { get; init; } + public required string ResolvedQueueId { get; init; } + public string? MachineName { get; init; } + public required string State { get; init; } +} + +[JsonSourceGenerationOptions( + AllowDuplicateProperties = false, + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + WriteIndented = true, + UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow, + UseStringEnumConverter = true, + RespectNullableAnnotations = true)] +[JsonSerializable(typeof(Submission))] +[JsonSerializable(typeof(ExecutionRecord))] +[JsonSerializable(typeof(ExecutionInputs))] +[JsonSerializable(typeof(LookupResult))] +[JsonSerializable(typeof(ResultReceipt))] +[JsonSerializable(typeof(StoredEntry))] +[JsonSerializable(typeof(ShadowSummary))] +[JsonSerializable(typeof(ExecutionProvenance))] +internal sealed partial class CacheJsonContext : JsonSerializerContext; diff --git a/src/tools/helix-cache/src/Program.cs b/src/tools/helix-cache/src/Program.cs new file mode 100644 index 00000000000000..5505408e46447e --- /dev/null +++ b/src/tools/helix-cache/src/Program.cs @@ -0,0 +1,121 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using System.Xml; + +namespace HelixCache; + +internal static class Program +{ + private static async Task Main(string[] args) + { + using CancellationTokenSource cancellation = new(); + ConsoleCancelEventHandler cancel = (_, e) => + { + e.Cancel = true; + cancellation.Cancel(); + }; + Console.CancelKeyPress += cancel; + try + { + if (args is ["--help" or "-h"] or ["lookup" or "store" or "prepare" or "collect", "--help" or "-h"]) + { + Console.WriteLine(CommandLine.Usage); + return 0; + } + Command command = CommandLine.Parse(args); + if (!OperatingSystem.IsLinux()) + { + throw new PlatformNotSupportedException("This prototype requires a Linux host."); + } + await ExecuteAsync(command, cancellation.Token); + return 0; + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + Console.Error.WriteLine("helix-cache: Cancelled."); + return 130; + } + catch (Exception ex) when (ex is InvalidDataException or IOException or UnauthorizedAccessException or JsonException or + HttpRequestException or TaskCanceledException or XmlException or ArgumentException or + OverflowException or PlatformNotSupportedException or TimeoutException) + { + Console.Error.WriteLine($"helix-cache: {ex.Message}"); + return 1; + } + finally + { + Console.CancelKeyPress -= cancel; + } + } + + private static async Task ExecuteAsync(Command command, CancellationToken cancellationToken) + { + Cache cache = new(command.CacheDirectory); + using HttpClient http = new() { Timeout = TimeSpan.FromSeconds(60) }; + HelixClient helix = new(http); + switch (command) + { + case LookupCommand lookup: + { + string manifest = Path.GetFullPath(lookup.Manifest); + using FileStream input = File.OpenRead(manifest); + Submission submission = await JsonSerializer.DeserializeAsync(input, CacheJsonContext.Default.Submission, cancellationToken) + ?? throw new InvalidDataException($"Missing submission: {manifest}."); + submission.Validate(); + LinuxEnvironment environment = await LinuxEnvironment.LoadAsync(lookup.QueueInfo, helix, cancellationToken); + LookupResult result = cache.Lookup(submission, Path.GetDirectoryName(manifest)!, environment); + WriteLookup(lookup.Output, result); + break; + } + case StoreCommand store: + { + using FileStream input = File.OpenRead(store.Manifest); + ExecutionRecord execution = await JsonSerializer.DeserializeAsync(input, CacheJsonContext.Default.ExecutionRecord, cancellationToken) + ?? throw new InvalidDataException($"Missing execution record: {store.Manifest}."); + cache.Store(execution, store.Results); + Console.Error.WriteLine($"Stored {execution.Key}."); + break; + } + case PrepareCommand prepare: + { + LinuxEnvironment environment = await LinuxEnvironment.LoadAsync(prepare.QueueInfo, helix, cancellationToken); + LookupEntry entry = ShadowValidation.Prepare(cache, prepare.Output, prepare.Payload, prepare.Runtime, prepare.Queue, environment); + Console.WriteLine($"Helix cache shadow: {entry.Status.ToString().ToLowerInvariant()} {entry.Execution.Key}; execution will NOT be skipped."); + break; + } + case CollectCommand collect: + { + ShadowSummary summary = await ShadowValidation.CollectAsync(cache, collect.Output, collect.Wait, helix, cancellationToken); + Console.WriteLine(JsonSerializer.Serialize(summary, CacheJsonContext.Default.ShadowSummary)); + break; + } + } + } + + private static void WriteLookup(string path, LookupResult result) + { + string output = Path.GetFullPath(path); + Directory.CreateDirectory(Path.GetDirectoryName(output)!); + string temporary = output + "." + Guid.NewGuid().ToString("N") + ".tmp"; + try + { + using (FileStream file = File.Create(temporary)) + { + JsonSerializer.Serialize(file, result, CacheJsonContext.Default.LookupResult); + file.Flush(flushToDisk: true); + } + File.Move(temporary, output, overwrite: true); + } + finally + { + File.Delete(temporary); + } + } +} diff --git a/src/tools/helix-cache/src/Results.cs b/src/tools/helix-cache/src/Results.cs new file mode 100644 index 00000000000000..ae4437d1c98b4e --- /dev/null +++ b/src/tools/helix-cache/src/Results.cs @@ -0,0 +1,119 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Xml; +using System.Xml.Linq; + +namespace HelixCache; + +internal readonly record struct TestOutcome(string Assembly, string Name, string Result, string SkipReason); + +internal sealed record TestRun(ResultReceipt Receipt, TestSummary Summary, Dictionary Outcomes) +{ + internal bool HasSameOutcomes(TestRun other) => + Outcomes.Count == other.Outcomes.Count && + Outcomes.All(p => other.Outcomes.TryGetValue(p.Key, out int count) && count == p.Value); +} + +internal static class Results +{ + internal static TestRun Read(string directory, ExecutionRecord execution) + { + using FileStream file = File.OpenRead(Path.Combine(directory, "result.json")); + ResultReceipt receipt = JsonSerializer.Deserialize(file, CacheJsonContext.Default.ResultReceipt) + ?? throw new InvalidDataException($"Missing result receipt: {directory}."); + Cache.ValidateVersion(receipt.SchemaVersion); + if (!receipt.Completed || !receipt.InfrastructureSucceeded || + receipt.ExitCode != execution.Inputs.ExpectedExitCode || + receipt.WorkItemName != execution.Inputs.Name || receipt.TargetQueue != execution.Inputs.TargetQueue || + !Guid.TryParse(receipt.JobId, out _) || receipt.ResultFiles.Count == 0 || + receipt.ResultFiles.Any(string.IsNullOrWhiteSpace)) + { + throw new InvalidDataException($"Results are incomplete, unsuccessful, or do not match the execution record: {directory}."); + } + + TestSummary summary = new(); + Dictionary outcomes = []; + HashSet reports = new(StringComparer.Ordinal); + foreach (string report in receipt.ResultFiles) + { + Snapshot.ValidateRelativePath(report); + if (!reports.Add(report)) + { + throw new InvalidDataException($"Duplicate report in {nameof(receipt.ResultFiles)}: {report}."); + } + ReadReport(Path.Combine(directory, report), summary, outcomes); + } + return new TestRun(receipt, summary, outcomes); + } + + private static void ReadReport(string path, TestSummary summary, Dictionary outcomes) + { + using XmlReader reader = XmlReader.Create(path, new XmlReaderSettings { DtdProcessing = DtdProcessing.Prohibit, XmlResolver = null }); + XDocument document = XDocument.Load(reader); + if (document.Root is not XElement root || root.Name != "assemblies" || !root.Elements("assembly").Any() || + document.Descendants("error").Any() || document.Descendants("failure").Any()) + { + throw InvalidReport(path); + } + foreach (XElement assembly in root.Elements("assembly")) + { + string? assemblyName = Path.GetFileName((string?)assembly.Attribute("name")); + if (string.IsNullOrWhiteSpace(assemblyName)) + { + throw InvalidReport(path); + } + TestSummary actual = new(); + foreach (XElement test in assembly.Elements("collection").Elements("test")) + { + string? name = (string?)test.Attribute("name"); + string? result = (string?)test.Attribute("result"); + if (string.IsNullOrWhiteSpace(name) || result is not ("Pass" or "Skip")) + { + throw InvalidReport(path); + } + TestOutcome outcome = new(assemblyName, name, result, + result == "Skip" ? (string?)test.Element("reason")?.Element("message") ?? "" : ""); + outcomes.TryGetValue(outcome, out int count); + checked + { + outcomes[outcome] = count + 1; + actual.Total++; + actual.Passed += result == "Pass" ? 1 : 0; + actual.Skipped += result == "Skip" ? 1 : 0; + } + } + if (actual.Total == 0 || assembly.Descendants("test").Count() != actual.Total || + Count(assembly, "total") != actual.Total || Count(assembly, "passed") != actual.Passed || + Count(assembly, "skipped") != actual.Skipped || Count(assembly, "failed") != 0 || + (assembly.Attribute("errors") is not null && Count(assembly, "errors") != 0)) + { + throw InvalidReport(path); + } + checked + { + summary.Total += actual.Total; + summary.Passed += actual.Passed; + summary.Skipped += actual.Skipped; + } + } + } + + private static InvalidDataException InvalidReport(string path) => + new($"Expected a completed, nonempty, passing xUnit report with named tests and assemblies: {path}."); + + private static int Count(XElement assembly, string name) + { + if (!int.TryParse((string?)assembly.Attribute(name), NumberStyles.None, CultureInfo.InvariantCulture, out int count)) + { + throw new InvalidDataException($"Invalid xUnit report count: {name}."); + } + return count; + } +} diff --git a/src/tools/helix-cache/src/ShadowValidation.cs b/src/tools/helix-cache/src/ShadowValidation.cs new file mode 100644 index 00000000000000..05f8851bb3bd3b --- /dev/null +++ b/src/tools/helix-cache/src/ShadowValidation.cs @@ -0,0 +1,169 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; + +namespace HelixCache; + +internal static class ShadowValidation +{ + internal const string WorkItem = "System.Memory.Data.Tests"; + internal const string Command = "bash ./RunTests.sh --runtime-path \"$HELIX_CORRELATION_PAYLOAD\""; + internal const string PostCommand = "cp testResults.xml \"$HELIX_WORKITEM_UPLOAD_ROOT/testResults.xml\""; + + internal static LookupEntry Prepare(Cache cache, string output, string payload, string runtime, string queue, LinuxEnvironment environment) + { + payload = Path.GetFullPath(payload); + runtime = Path.GetFullPath(runtime); + foreach (string archive in new[] { payload, runtime }) + { + if (!File.Exists(archive) || !archive.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidDataException($"The shadow submitter requires an existing ZIP archive: {archive}."); + } + } + output = Path.GetFullPath(output); + if (Directory.Exists(output) || File.Exists(output)) + { + throw new IOException($"The shadow output directory already exists: {output}."); + } + Submission submission = new() + { + SchemaVersion = Cache.ProtocolVersion, + WorkItems = [new WorkItem + { + Name = WorkItem, + Queues = [queue], + Payload = new Payload { Path = payload }, + CorrelationPayloads = [new Payload { Path = runtime }], + Command = Command, + PostCommands = [PostCommand], + TimeoutSeconds = 900, + ExpectedExitCode = 0, + }], + }; + LookupResult lookup = cache.Lookup(submission, output, environment); + LookupEntry entry = lookup.Entries.Single(); + + Directory.CreateDirectory(output); + File.WriteAllText(Path.Combine(output, "submission.json"), JsonSerializer.Serialize(submission, CacheJsonContext.Default.Submission)); + File.WriteAllText(Path.Combine(output, "lookup.json"), JsonSerializer.Serialize(lookup, CacheJsonContext.Default.LookupResult)); + File.WriteAllText(Path.Combine(output, "execution.json"), JsonSerializer.Serialize(entry.Execution, CacheJsonContext.Default.ExecutionRecord)); + WriteSubmissionProperties(output, entry.Execution); + return entry; + } + + internal static async Task CollectAsync(Cache cache, string output, TimeSpan wait, HelixClient helix, + CancellationToken cancellationToken = default) + { + output = Path.GetFullPath(output); + LookupResult lookup = JsonSerializer.Deserialize(File.ReadAllText(Path.Combine(output, "lookup.json")), + CacheJsonContext.Default.LookupResult) ?? throw new InvalidDataException("Missing shadow lookup result."); + Cache.ValidateVersion(lookup.SchemaVersion); + if (lookup.Entries.Count != 1 || lookup.Entries[0] is not LookupEntry entry || + entry.Status is not (LookupStatus.Hit or LookupStatus.Miss)) + { + throw new InvalidDataException("Expected exactly one shadow lookup entry with hit or miss status."); + } + ExecutionRecord execution = entry.Execution; + execution.Validate(); + TestRun? cached = entry.Status == LookupStatus.Hit ? cache.ReadResults(execution) : null; + if (!Guid.TryParse(File.ReadAllText(Path.Combine(output, "job-id.txt")).Trim(), out Guid job)) + { + throw new InvalidDataException("Expected a Helix job GUID in job-id.txt."); + } + string jobId = job.ToString(); + ShadowSummary summary = new() { Key = execution.Key, Lookup = entry.Status, JobId = jobId }; + string summaryPath = Path.Combine(output, "summary.json"); + File.WriteAllText(summaryPath, JsonSerializer.Serialize(summary, CacheJsonContext.Default.ShadowSummary)); + + HelixJob jobDetails = await helix.GetJobAsync(jobId, cancellationToken); + string submittedQueue = string.IsNullOrEmpty(jobDetails.QueueAlias) ? jobDetails.QueueId : jobDetails.QueueAlias; + if (jobDetails.Name != jobId || + !string.Equals(submittedQueue, execution.Inputs.EnvironmentFingerprint.QueueId, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidDataException("Helix returned a different job or target queue."); + } + HelixWorkItem item = await helix.WaitForCompletionAsync(jobId, execution.Inputs.Name, wait, cancellationToken); + File.WriteAllText(Path.Combine(output, "work-item.json"), JsonSerializer.Serialize(item, HelixJsonContext.Default.HelixWorkItem)); + if (item.State is not ("Finished" or "Passed") || item.FailureReason is not null || + item.Errors is { Count: > 0 } || item.ExitCode is null || item.ExitCode != execution.Inputs.ExpectedExitCode) + { + throw new InvalidDataException($"Helix work item did not complete successfully; see {Path.Combine(output, "work-item.json")}."); + } + + string results = Path.Combine(output, "results"); + await helix.DownloadResultsAsync(item, results, cancellationToken); + ResultReceipt receipt = new() + { + SchemaVersion = Cache.ProtocolVersion, + Completed = true, + InfrastructureSucceeded = true, + ExitCode = item.ExitCode.Value, + JobId = jobId, + WorkItemName = item.Name, + TargetQueue = execution.Inputs.TargetQueue, + ResultFiles = ["testResults.xml"], + }; + ExecutionProvenance provenance = new() + { + JobId = jobId, + QueueAlias = jobDetails.QueueAlias, + ResolvedQueueId = jobDetails.QueueId, + MachineName = item.MachineName, + State = item.State, + }; + File.WriteAllText(Path.Combine(results, "result.json"), JsonSerializer.Serialize(receipt, CacheJsonContext.Default.ResultReceipt)); + File.WriteAllText(Path.Combine(results, "provenance.json"), JsonSerializer.Serialize(provenance, CacheJsonContext.Default.ExecutionProvenance)); + + TestRun fresh = Results.Read(results, execution); + summary = summary with + { + Tests = fresh.Summary.Total, + CachedJobId = cached?.Receipt.JobId, + Comparison = cached is null ? OutcomeComparison.NotApplicable : + cached.HasSameOutcomes(fresh) ? OutcomeComparison.Match : OutcomeComparison.Mismatch, + }; + if (summary.Comparison == OutcomeComparison.Mismatch) + { + File.WriteAllText(summaryPath, JsonSerializer.Serialize(summary, CacheJsonContext.Default.ShadowSummary)); + throw new InvalidDataException($"Cached and fresh test outcomes differ; see {summaryPath}."); + } + cache.Store(execution, results); + summary = summary with { Stored = true }; + File.WriteAllText(summaryPath, JsonSerializer.Serialize(summary, CacheJsonContext.Default.ShadowSummary)); + return summary; + } + + private static void WriteSubmissionProperties(string output, ExecutionRecord execution) + { + XElement properties = new("PropertyGroup"); + foreach ((string name, string value) in new Dictionary + { + ["ShadowName"] = execution.Inputs.Name, + ["ShadowQueue"] = execution.Inputs.TargetQueue, + ["ShadowPayload"] = execution.PayloadPath, + ["ShadowRuntime"] = execution.CorrelationPayloadPaths[0], + ["ShadowCommand"] = execution.Inputs.Command, + ["ShadowPostCommand"] = execution.Inputs.PostCommands[0], + ["ShadowTimeout"] = TimeSpan.FromSeconds(execution.Inputs.TimeoutSeconds).ToString("c", CultureInfo.InvariantCulture), + }) + { + string escaped = value; + foreach (char character in "%$@();'*?") + { + escaped = escaped.Replace(character.ToString(), $"%{(int)character:X2}", StringComparison.Ordinal); + } + properties.Add(new XElement(name, escaped)); + } + new XDocument(new XElement("Project", properties)).Save(Path.Combine(output, "submission.props")); + } +} diff --git a/src/tools/helix-cache/Snapshot.cs b/src/tools/helix-cache/src/Snapshot.cs similarity index 84% rename from src/tools/helix-cache/Snapshot.cs rename to src/tools/helix-cache/src/Snapshot.cs index 310a06cf2b8697..23d046eeea657f 100644 --- a/src/tools/helix-cache/Snapshot.cs +++ b/src/tools/helix-cache/src/Snapshot.cs @@ -15,7 +15,7 @@ internal sealed class Snapshot internal required string Path { get; init; } internal required string Digest { get; init; } - internal required string Kind { get; init; } + internal required PayloadKind Kind { get; init; } internal static Snapshot Capture(string source, string cache) { @@ -25,7 +25,7 @@ internal static Snapshot Capture(string source, string cache) bool directory = (attributes & FileAttributes.Directory) != 0; if (!directory && !source.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) { - throw new InvalidDataException(SR.Get("Payload", source)); + throw new InvalidDataException($"Payload '{source}' must be a directory or a ZIP archive."); } using TemporaryDirectory temporary = new(System.IO.Path.Combine(cache, "tmp")); @@ -46,11 +46,11 @@ internal static Snapshot Capture(string source, string cache) string existing = System.IO.Path.Combine(final, name); if (Hash(existing) != digest) { - throw new InvalidDataException(SR.Get("Corrupt", existing)); + throw new InvalidDataException($"Snapshot failed validation: {existing}."); } } - return new Snapshot { Path = System.IO.Path.Combine(final, name), Digest = digest, Kind = directory ? "directory" : "archive" }; + return new Snapshot { Path = System.IO.Path.Combine(final, name), Digest = digest, Kind = directory ? PayloadKind.Directory : PayloadKind.Archive }; } internal static void EnsureOutside(string source, string destination) @@ -58,7 +58,16 @@ internal static void EnsureOutside(string source, string destination) if (Directory.Exists(source) && (destination == source || destination.StartsWith(System.IO.Path.TrimEndingDirectorySeparator(source) + "/", StringComparison.Ordinal))) { - throw new InvalidDataException(SR.Get("NestedCache", destination, source)); + throw new InvalidDataException($"Cache or staging directory '{destination}' must not be inside input directory '{source}'."); + } + } + + internal static void ValidateRelativePath(string path, bool allowEmpty = false) + { + if ((!allowEmpty && string.IsNullOrWhiteSpace(path)) || System.IO.Path.IsPathRooted(path) || + path.Split('/').Any(p => p is "." or "..") || path.Contains('\\')) + { + throw new InvalidDataException($"Expected a relative path without traversal: {path}."); } } @@ -129,7 +138,7 @@ private static void RejectLink(string path, FileAttributes attributes) { if ((attributes & FileAttributes.ReparsePoint) != 0) { - throw new InvalidDataException(SR.Get("Symlink", path)); + throw new InvalidDataException($"Symbolic links are not supported in directory payloads or results: {path}. Use an archive for payloads containing links."); } } } diff --git a/src/tools/helix-cache/tests/CacheFixture.cs b/src/tools/helix-cache/tests/CacheFixture.cs new file mode 100644 index 00000000000000..d3f8c085cafe57 --- /dev/null +++ b/src/tools/helix-cache/tests/CacheFixture.cs @@ -0,0 +1,157 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Text.Json; +using System.Text.Json.Serialization.Metadata; +using System.Threading.Tasks; +using Xunit; + +namespace HelixCache.Tests; + +internal sealed class CacheFixture : IDisposable +{ + private const UnixFileMode ExecutableMode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherExecute; + internal const string FirstJob = "00000000-0000-0000-0000-000000000001"; + internal const string SecondJob = "00000000-0000-0000-0000-000000000002"; + internal const string Report = """ + + + + """; + + internal string Root { get; } = Directory.CreateTempSubdirectory("helix-cache-test-").FullName; + internal string CacheDirectory { get; set; } + internal string PayloadDirectory { get; } + internal string Archive { get; } + internal List Queues { get; } + internal Submission Submission { get; set; } + internal WorkItem Item + { + get => Submission.WorkItems[0]; + set => Submission.WorkItems[0] = value; + } + + internal CacheFixture() + { + CacheDirectory = Path.Combine(Root, "cache"); + PayloadDirectory = Directory.CreateDirectory(Path.Combine(Root, "payload")).FullName; + File.SetUnixFileMode(PayloadDirectory, ExecutableMode); + File.WriteAllText(Path.Combine(PayloadDirectory, "run.sh"), "#!/bin/sh\nexit 0\n"); + File.SetUnixFileMode(Path.Combine(PayloadDirectory, "run.sh"), ExecutableMode); + Archive = Path.Combine(Root, "shared.zip"); + File.WriteAllText(Archive, "opaque archive bytes"); + Queues = [new HelixQueue + { + QueueId = "linux.test.open", Architecture = "AMD64", OperatingSystemGroup = "linux", IsAvailable = true, + Metadata = new Dictionary + { + ["GalleryImage"] = JsonSerializer.SerializeToElement(new { Version = "1" }), + ["Artifacts"] = JsonSerializer.SerializeToElement(new[] { new { Id = "prepare", Parameters = new { version = "1" } } }), + }, + }]; + Submission = new Submission + { + SchemaVersion = 1, + WorkItems = [new WorkItem + { + Name = "suite", Queues = [Queues[0].QueueId], Payload = new Payload { Path = "payload" }, + CorrelationPayloads = [new Payload { Path = "shared.zip", Destination = "runtime" }], + Command = "./run.sh", Environment = new() { ["SECOND"] = "2", ["FIRST"] = "1" }, + TimeoutSeconds = 60, ExpectedExitCode = 0, + }], + }; + } + + internal string Write(string name, T value, JsonTypeInfo type) + { + string path = Path.Combine(Root, name); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, JsonSerializer.Serialize(value, type)); + return path; + } + + internal static T Read(string path, JsonTypeInfo type) => + JsonSerializer.Deserialize(File.ReadAllText(path), type)!; + + internal LookupEntry Lookup(bool success = true) + { + Invoke(success, "lookup", Write("submission.json", Submission, CacheJsonContext.Default.Submission), + "--cache", CacheDirectory, "--output", Path.Combine(Root, "lookup.json"), + "--queue-info", Write("queues.json", Queues, HelixJsonContext.Default.ListHelixQueue)); + return success ? Read(Path.Combine(Root, "lookup.json"), CacheJsonContext.Default.LookupResult).Entries[0] : null!; + } + + internal string CreateResults(ExecutionRecord execution, string jobId = FirstJob) + { + string results = Directory.CreateDirectory(Path.Combine(Root, "results")).FullName; + Write("results/result.json", new ResultReceipt + { + SchemaVersion = 1, + Completed = true, + InfrastructureSucceeded = true, + ExitCode = 0, + JobId = jobId, + WorkItemName = execution.Inputs.Name, + TargetQueue = execution.Inputs.TargetQueue, + ResultFiles = ["tests.xml"], + }, CacheJsonContext.Default.ResultReceipt); + File.WriteAllText(Path.Combine(results, "tests.xml"), Report); + return results; + } + + internal void Store(ExecutionRecord execution, bool success = true) => + Invoke(success, "store", Write("execution.json", execution, CacheJsonContext.Default.ExecutionRecord), + "--results", Path.Combine(Root, "results"), "--cache", CacheDirectory); + + internal string Prepare(string name = "shadow") + { + string output = Path.Combine(Root, name); + ShadowValidation.Prepare(new Cache(CacheDirectory), output, Archive, Archive, Queues[0].QueueId, new LinuxEnvironment(Queues)); + return output; + } + + internal static LookupEntry ShadowLookup(string output) => + Assert.Single(Read(Path.Combine(output, "lookup.json"), CacheJsonContext.Default.LookupResult).Entries); + + internal string Invoke(bool success, params string[] arguments) + { + string? executable = Environment.GetEnvironmentVariable("HELIX_CACHE_EXECUTABLE"); + if (executable is null) + { + DirectoryInfo? directory = new(Environment.CurrentDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "global.json"))) + { + directory = directory.Parent; + } + Assert.NotNull(directory); + executable = Path.Combine(directory.FullName, "artifacts/tools/helix-cache/helix-cache"); + } + ProcessStartInfo start = new(executable) { RedirectStandardOutput = true, RedirectStandardError = true }; + foreach (string argument in arguments) + { + start.ArgumentList.Add(argument); + } + using Process process = Process.Start(start)!; + Task stdout = process.StandardOutput.ReadToEndAsync(); + Task stderr = process.StandardError.ReadToEndAsync(); + if (!process.WaitForExit(30_000)) + { + process.Kill(entireProcessTree: true); + Assert.Fail("The native cache process timed out."); + } + string error = stderr.GetAwaiter().GetResult(); + Assert.True(process.ExitCode == (success ? 0 : 1), error); + if (!success) + { + Assert.Contains("helix-cache:", error); + } + return stdout.GetAwaiter().GetResult(); + } + + public void Dispose() => Directory.Delete(Root, recursive: true); +} diff --git a/src/tools/helix-cache/tests/CacheTests.cs b/src/tools/helix-cache/tests/CacheTests.cs new file mode 100644 index 00000000000000..c75d891bcbdb63 --- /dev/null +++ b/src/tools/helix-cache/tests/CacheTests.cs @@ -0,0 +1,426 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Xunit; + +namespace HelixCache.Tests; + +public sealed class CacheTests +{ + [Fact] + public void RoundTripPreservesFrozenInputsAndResults() + { + using CacheFixture f = new(); + LookupEntry miss = f.Lookup(); + Assert.Equal(LookupStatus.Miss, miss.Status); + Assert.NotEqual(f.PayloadDirectory, miss.Execution.PayloadPath); + JsonObject fingerprint = JsonNode.Parse(File.ReadAllText(Path.Combine(f.Root, "lookup.json")))! + ["entries"]![0]!["execution"]!["inputs"]!["environmentFingerprint"]!.AsObject(); + Assert.Equal(f.Queues[0].QueueId, (string?)fingerprint["queueId"]); + Assert.Equal("AMD64", (string?)fingerprint["architecture"]); + Assert.Equal("linux", (string?)fingerprint["operatingSystemGroup"]); + Assert.Contains("container", fingerprint.Select(p => p.Key)); + f.CreateResults(miss.Execution); + f.Store(miss.Execution); + LookupEntry hit = f.Lookup(); + Assert.Equal(LookupStatus.Hit, hit.Status); + Assert.Equal(miss.Execution.Key, hit.Execution.Key); + Assert.Equal(File.ReadAllBytes(Path.Combine(f.Root, "results/tests.xml")), + File.ReadAllBytes(Path.Combine(hit.ResultsPath!, "tests.xml"))); + File.WriteAllText(Path.Combine(f.PayloadDirectory, "run.sh"), "changed"); + Assert.Equal("#!/bin/sh\nexit 0\n", File.ReadAllText(Path.Combine(miss.Execution.PayloadPath, "run.sh"))); + Assert.Equal(LookupStatus.Miss, f.Lookup().Status); + } + + [Fact] + public void RepeatedLookupsOnOneCacheCaptureChangedInputs() + { + using CacheFixture f = new(); + Cache cache = new(f.CacheDirectory); + LinuxEnvironment environment = new(f.Queues); + string before = Assert.Single(cache.Lookup(f.Submission, f.Root, environment).Entries).Execution.Key; + File.WriteAllText(Path.Combine(f.PayloadDirectory, "run.sh"), "changed"); + string after = Assert.Single(cache.Lookup(f.Submission, f.Root, environment).Entries).Execution.Key; + Assert.NotEqual(before, after); + } + + [Fact] + public void WholeSubmissionIsValidatedBeforeCapturingInputs() + { + using CacheFixture f = new(); + f.Submission.WorkItems.Add(f.Item with { Name = "invalid", TimeoutSeconds = 0 }); + Assert.Throws(() => new Cache(f.CacheDirectory).Lookup(f.Submission, f.Root, new LinuxEnvironment(f.Queues))); + Assert.False(Directory.Exists(f.CacheDirectory)); + } + + [Fact] + public void OpaqueProvisioningPreservesNullArrayValues() + { + using CacheFixture f = new(); + f.Queues[0].Metadata["PreparedImage"] = JsonSerializer.Deserialize("""{"FutureField":[null,1]}"""); + LookupEntry miss = f.Lookup(); + f.CreateResults(miss.Execution); + f.Store(miss.Execution); + Assert.Equal(LookupStatus.Hit, f.Lookup().Status); + f.Queues[0].Metadata["PreparedImage"] = JsonSerializer.Deserialize("""{"FutureField":[null,2]}"""); + Assert.NotEqual(miss.Execution.Key, f.Lookup().Execution.Key); + } + + [Theory] + [InlineData("command")] + [InlineData("preCommands")] + [InlineData("postCommands")] + [InlineData("timeout")] + [InlineData("exitCode")] + [InlineData("environment")] + public void ExecutionChangesInvalidateKey(string field) + { + using CacheFixture f = new(); + string baseline = f.Lookup().Execution.Key; + f.Item = field switch + { + "command" => f.Item with { Command = "./run.sh other" }, + "preCommands" => f.Item with { PreCommands = ["export STRESS=1"] }, + "postCommands" => f.Item with { PostCommands = ["echo done"] }, + "timeout" => f.Item with { TimeoutSeconds = 61 }, + "exitCode" => f.Item with { ExpectedExitCode = 100 }, + "environment" => f.Item with { Environment = new() { ["FIRST"] = "changed" } }, + _ => throw new InvalidOperationException(), + }; + Assert.NotEqual(baseline, f.Lookup().Execution.Key); + } + + [Theory] + [InlineData("GalleryImage", """{"Version":"2"}""")] + [InlineData("Artifacts", """[{"Id":"prepare","Parameters":{"version":"2"}}]""")] + [InlineData("PreparedImage", """{"Version":"new","FutureField":{"b":2,"a":1}}""")] + public void ImageAndProvisioningChangesInvalidateKey(string field, string value) + { + using CacheFixture f = new(); + string baseline = f.Lookup().Execution.Key; + f.Queues[0].Metadata[field] = JsonSerializer.Deserialize(value); + Assert.NotEqual(baseline, f.Lookup().Execution.Key); + } + + [Fact] + public void ObjectOrderingLiveQueueStateAndTimestampsDoNotChangeKey() + { + using CacheFixture f = new(); + f.Queues[0].Metadata["PreparedImage"] = JsonSerializer.Deserialize("""{"b":{"z":2,"a":1},"a":null}"""); + string baseline = f.Lookup().Execution.Key; + f.Queues[0].Metadata["PreparedImage"] = JsonSerializer.Deserialize("""{"a":null,"b":{"a":1,"z":2}}"""); + f.Queues[0].Metadata["QueueDepth"] = JsonSerializer.SerializeToElement(300); + f.Queues[0].Metadata["IsInProtectedMode"] = JsonSerializer.SerializeToElement(true); + f.Item = f.Item with { Environment = new() { ["FIRST"] = "1", ["SECOND"] = "2" } }; + File.SetLastWriteTimeUtc(Path.Combine(f.PayloadDirectory, "run.sh"), DateTime.UnixEpoch); + Assert.Equal(baseline, f.Lookup().Execution.Key); + } + + [Fact] + public void RelocationDoesNotChangeKeyButModesAndEmptyDirectoriesDo() + { + using CacheFixture f = new(); + string baseline = f.Lookup().Execution.Key; + string relocated = Path.Combine(f.Root, "relocated"); + Directory.Move(f.PayloadDirectory, relocated); + f.Item = f.Item with { Payload = f.Item.Payload with { Path = relocated } }; + Assert.Equal(baseline, f.Lookup().Execution.Key); + string script = Path.Combine(relocated, "run.sh"); + UnixFileMode mode = File.GetUnixFileMode(script); + File.SetUnixFileMode(script, mode & ~UnixFileMode.UserExecute); + Assert.NotEqual(baseline, f.Lookup().Execution.Key); + File.SetUnixFileMode(script, mode); + Directory.CreateDirectory(Path.Combine(relocated, "empty")); + Assert.NotEqual(baseline, f.Lookup().Execution.Key); + } + + [Fact] + public void QueuesAndPinnedContainersArePartitioned() + { + using CacheFixture f = new(); + string target = "(Debian.Amd64)linux.test.open@registry/image:tag@sha256:" + new string('a', 64); + f.Item.Queues.Add(target); + LookupEntry first = f.Lookup(); + LookupResult lookup = CacheFixture.Read(Path.Combine(f.Root, "lookup.json"), CacheJsonContext.Default.LookupResult); + Assert.Equal(2, lookup.Entries.Count); + Assert.NotEqual(first.Execution.Key, lookup.Entries[1].Execution.Key); + Assert.Equal(target[(target.IndexOf('@') + 1)..], lookup.Entries[1].Execution.Inputs.EnvironmentFingerprint.Container); + f.CreateResults(first.Execution); + f.Store(first.Execution); + f.Lookup(); + lookup = CacheFixture.Read(Path.Combine(f.Root, "lookup.json"), CacheJsonContext.Default.LookupResult); + Assert.Equal([LookupStatus.Hit, LookupStatus.Miss], lookup.Entries.Select(e => e.Status)); + } + + [Theory] + [InlineData("linux.test.open@registry/image:latest")] + [InlineData("(alias)linux.test.open")] + public void UnpinnedOrMalformedContainerTargetsAreRejected(string target) + { + using CacheFixture f = new(); + f.Item = f.Item with { Queues = [target] }; + f.Lookup(success: false); + } + + [Fact] + public void SharedPayloadBytesDestinationAndOverlayOrderAffectKey() + { + using CacheFixture f = new(); + string baseline = f.Lookup().Execution.Key; + File.WriteAllText(f.Archive, "new archive"); + Assert.NotEqual(baseline, f.Lookup().Execution.Key); + File.WriteAllText(f.Archive, "opaque archive bytes"); + f.Item.CorrelationPayloads[0] = f.Item.CorrelationPayloads[0] with { Destination = "other" }; + Assert.NotEqual(baseline, f.Lookup().Execution.Key); + f.Item.CorrelationPayloads.Add(new Payload { Path = f.Archive, Destination = "second" }); + string before = f.Lookup().Execution.Key; + f.Item.CorrelationPayloads.Reverse(); + Assert.NotEqual(before, f.Lookup().Execution.Key); + } + + [Theory] + [InlineData("completed")] + [InlineData("infrastructure")] + [InlineData("exitCode")] + [InlineData("workItem")] + [InlineData("queue")] + [InlineData("job")] + [InlineData("empty")] + [InlineData("traversal")] + [InlineData("duplicate")] + public void UnsuccessfulReceiptsAreNotStored(string field) + { + using CacheFixture f = new(); + ExecutionRecord execution = f.Lookup().Execution; + f.CreateResults(execution); + ResultReceipt receipt = CacheFixture.Read(Path.Combine(f.Root, "results/result.json"), CacheJsonContext.Default.ResultReceipt); + receipt = field switch + { + "completed" => receipt with { Completed = false }, + "infrastructure" => receipt with { InfrastructureSucceeded = false }, + "exitCode" => receipt with { ExitCode = 1 }, + "workItem" => receipt with { WorkItemName = "other" }, + "queue" => receipt with { TargetQueue = "other" }, + "job" => receipt with { JobId = "not-a-guid" }, + "empty" => receipt with { ResultFiles = [] }, + "traversal" => receipt with { ResultFiles = ["../outside.xml"] }, + "duplicate" => receipt with { ResultFiles = ["tests.xml", "tests.xml"] }, + _ => throw new InvalidOperationException(), + }; + f.Write("results/result.json", receipt, CacheJsonContext.Default.ResultReceipt); + f.Store(execution, success: false); + Assert.Equal(LookupStatus.Miss, f.Lookup().Status); + } + + [Theory] + [InlineData("")] + [InlineData("""""")] + [InlineData("""""")] + [InlineData("""""")] + [InlineData("""""")] + [InlineData("""]>&x;""")] + [InlineData("""""")] + [InlineData("""""")] + [InlineData("""""")] + public void ReportsMustBeCompleteNonemptyAndPassing(string report) + { + using CacheFixture f = new(); + ExecutionRecord execution = f.Lookup().Execution; + f.CreateResults(execution); + File.WriteAllText(Path.Combine(f.Root, "results/tests.xml"), report); + f.Store(execution, success: false); + Assert.Equal(LookupStatus.Miss, f.Lookup().Status); + } + + [Fact] + public void CorruptResultsAreAnErrorRatherThanAHitOrMiss() + { + using CacheFixture f = new(); + ExecutionRecord execution = f.Lookup().Execution; + f.CreateResults(execution); + f.Store(execution); + string results = f.Lookup().ResultsPath!; + File.WriteAllText(Path.Combine(results, "tests.xml"), "corrupted"); + f.Lookup(success: false); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ForgedKeysAndModifiedSnapshotsCannotBeStored(bool forgeKey) + { + using CacheFixture f = new(); + ExecutionRecord execution = f.Lookup().Execution; + f.CreateResults(execution); + if (forgeKey) + { + execution = execution with { Key = new string('0', 64) }; + } + else + { + File.WriteAllText(Path.Combine(execution.PayloadPath, "run.sh"), "changed"); + } + f.Store(execution, success: false); + } + + [Fact] + public void ResultsCannotContainTheirStagingDirectory() + { + using CacheFixture f = new(); + ExecutionRecord execution = f.Lookup().Execution; + f.Invoke(false, "store", f.Write("execution.json", execution, CacheJsonContext.Default.ExecutionRecord), + "--results", Path.Combine(f.CacheDirectory, "tmp"), "--cache", f.CacheDirectory); + Assert.Empty(Directory.EnumerateFileSystemEntries(Path.Combine(f.CacheDirectory, "tmp"))); + Assert.Equal(LookupStatus.Miss, f.Lookup().Status); + } + + [Fact] + public async Task DuplicateStoresAreAtomicAndIdempotent() + { + using CacheFixture f = new(); + ExecutionRecord execution = f.Lookup().Execution; + f.CreateResults(execution); + string record = f.Write("execution.json", execution, CacheJsonContext.Default.ExecutionRecord); + Task[] operations = Enumerable.Range(0, 2).Select(_ => Task.Run(() => f.Invoke(true, + "store", record, "--results", Path.Combine(f.Root, "results"), "--cache", f.CacheDirectory))).ToArray(); + await Task.WhenAll(operations); + Assert.Equal(LookupStatus.Hit, f.Lookup().Status); + Assert.Empty(Directory.EnumerateFileSystemEntries(Path.Combine(f.CacheDirectory, "tmp"))); + } + + [Theory] + [InlineData("architecture")] + [InlineData("os")] + [InlineData("availability")] + public void UnsupportedQueuesAreRejected(string field) + { + using CacheFixture f = new(); + f.Queues[0] = field switch + { + "architecture" => f.Queues[0] with { Architecture = "ARM64" }, + "os" => f.Queues[0] with { OperatingSystemGroup = "windows" }, + "availability" => f.Queues[0] with { IsAvailable = false }, + _ => throw new InvalidOperationException(), + }; + f.Lookup(success: false); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void LinksAndNestedCachesAreRejected(bool link) + { + using CacheFixture f = new(); + if (link) + { + File.CreateSymbolicLink(Path.Combine(f.PayloadDirectory, "link"), f.Archive); + } + else + { + f.CacheDirectory = Path.Combine(f.PayloadDirectory, "cache"); + } + f.Lookup(success: false); + } + + [Theory] + [InlineData("version")] + [InlineData("empty")] + [InlineData("nullItem")] + [InlineData("unknown")] + [InlineData("missing")] + [InlineData("nullCommand")] + [InlineData("timeout")] + [InlineData("nullCorrelation")] + public void InvalidManifestsAreRejected(string mutation) + { + using CacheFixture f = new(); + JsonNode manifest = JsonNode.Parse(JsonSerializer.Serialize(f.Submission, CacheJsonContext.Default.Submission))!; + JsonNode item = manifest["workItems"]![0]!; + switch (mutation) + { + case "version": manifest["schemaVersion"] = 2; break; + case "empty": manifest["workItems"] = new JsonArray(); break; + case "nullItem": manifest["workItems"]![0] = null; break; + case "unknown": manifest["unrecognized"] = true; break; + case "missing": item.AsObject().Remove("expectedExitCode"); break; + case "nullCommand": item["command"] = null; break; + case "timeout": item["timeoutSeconds"] = 0; break; + case "nullCorrelation": item["correlationPayloads"]![0] = null; break; + default: throw new InvalidOperationException(); + } + string path = Path.Combine(f.Root, "invalid.json"); + File.WriteAllText(path, manifest.ToJsonString()); + f.Invoke(false, "lookup", path, "--cache", f.CacheDirectory, "--output", Path.Combine(f.Root, "lookup.json"), + "--queue-info", f.Write("queues.json", f.Queues, HelixJsonContext.Default.ListHelixQueue)); + } + + [Theory] + [InlineData("""{"schemaVersion":1,"schemaVersion":1,"workItems":[]}""")] + [InlineData("""{"schemaVersion":1,"workItems":[],"workItems":[]}""")] + public void DuplicateJsonPropertiesAreRejected(string manifest) + { + using CacheFixture f = new(); + string path = Path.Combine(f.Root, "duplicate.json"); + File.WriteAllText(path, manifest); + f.Invoke(false, "lookup", path, "--cache", f.CacheDirectory, "--output", Path.Combine(f.Root, "lookup.json"), + "--queue-info", f.Write("queues.json", f.Queues, HelixJsonContext.Default.ListHelixQueue)); + } + + [Fact] + public void CliRejectsMissingAndDuplicateOptions() + { + using CacheFixture f = new(); + f.Invoke(false, "store", "missing", "--cache", f.CacheDirectory); + f.Invoke(false, "lookup", "missing", "--cache", f.CacheDirectory, "--cache", f.CacheDirectory); + f.Invoke(false, "collect", "--cache", f.CacheDirectory, "--output", f.Root, "--wait-seconds", "0"); + } + + [Theory] + [InlineData("--help")] + [InlineData("lookup --help")] + [InlineData("store -h")] + [InlineData("prepare --help")] + [InlineData("collect --help")] + public void HelpDoesNotRequireInputsOrCreateCache(string arguments) + { + using CacheFixture f = new(); + Assert.Contains("Usage: helix-cache", f.Invoke(true, arguments.Split(' '))); + Assert.False(Directory.Exists(f.CacheDirectory)); + } + + [Theory] + [InlineData("lookup", "Missing manifest")] + [InlineData("lookup --cache cache --output output", "Missing manifest")] + [InlineData("lookup missing.json --cache cache --output output --unknown value", "Unknown option '--unknown'")] + [InlineData("lookup missing.json --cache cache --cache cache", "Duplicate option '--cache'")] + [InlineData("store missing.json --cache cache", "Missing required option '--results'")] + [InlineData("collect --cache cache --output output --wait-seconds -1", "--wait-seconds")] + [InlineData("collect --cache --output output", "Expected an option and value")] + public void CommandLineErrorsIdentifyTheInvalidArgument(string arguments, string diagnostic) + { + InvalidDataException error = Assert.Throws(() => CommandLine.Parse(arguments.Split(' '))); + Assert.Contains(diagnostic, error.Message); + } + + [Fact] + public void NativePrepareWritesFrozenSubmissionProperties() + { + using CacheFixture f = new(); + string output = Path.Combine(f.Root, "native-shadow"); + f.Invoke(true, "prepare", "--cache", f.CacheDirectory, "--output", output, "--payload", f.Archive, + "--runtime", f.Archive, "--queue", f.Queues[0].QueueId, + "--queue-info", f.Write("queues.json", f.Queues, HelixJsonContext.Default.ListHelixQueue)); + LookupEntry entry = CacheFixture.ShadowLookup(output); + Assert.Equal(LookupStatus.Miss, entry.Status); + Assert.NotEqual(f.Archive, entry.Execution.PayloadPath); + Assert.Equal(ShadowValidation.Command, entry.Execution.Inputs.Command); + Assert.Contains("%24", File.ReadAllText(Path.Combine(output, "submission.props"))); + } +} diff --git a/src/tools/helix-cache/tests/HelixCache.Tests.csproj b/src/tools/helix-cache/tests/HelixCache.Tests.csproj new file mode 100644 index 00000000000000..a521f2c3ab8d57 --- /dev/null +++ b/src/tools/helix-cache/tests/HelixCache.Tests.csproj @@ -0,0 +1,10 @@ + + + $(NetCoreAppToolCurrent) + enable + false + + + + + diff --git a/src/tools/helix-cache/tests/ShadowTests.cs b/src/tools/helix-cache/tests/ShadowTests.cs new file mode 100644 index 00000000000000..2cfe4734d90198 --- /dev/null +++ b/src/tools/helix-cache/tests/ShadowTests.cs @@ -0,0 +1,303 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace HelixCache.Tests; + +public sealed class ShadowTests +{ + [Fact] + public async Task ColdStoreAndRestoredEntriesCompareActualOutcomes() + { + using CacheFixture f = new(); + string first = f.Prepare(); + await Collect(f, first, CacheFixture.FirstJob); + ShadowSummary cold = CacheFixture.Read(Path.Combine(first, "summary.json"), CacheJsonContext.Default.ShadowSummary); + Assert.Equal(LookupStatus.Miss, cold.Lookup); + Assert.Equal(2, cold.Tests); + string restored = Path.Combine(f.Root, "restored-cache"); + Snapshot.Copy(Path.Combine(f.CacheDirectory, "entries"), Path.Combine(restored, "entries")); + Directory.Move(f.CacheDirectory, Path.Combine(f.Root, "previous-cache")); + f.CacheDirectory = restored; + string second = f.Prepare("second"); + Assert.Equal(LookupStatus.Hit, CacheFixture.ShadowLookup(second).Status); + string reordered = """ + + + + """; + await Collect(f, second, CacheFixture.SecondJob, report: reordered); + ShadowSummary summary = CacheFixture.Read(Path.Combine(second, "summary.json"), CacheJsonContext.Default.ShadowSummary); + Assert.Equal(OutcomeComparison.Match, summary.Comparison); + Assert.Equal(CacheFixture.FirstJob, summary.CachedJobId); + Assert.Equal(CacheFixture.SecondJob, summary.JobId); + Assert.Equal(cold.Key, summary.Key); + } + + [Theory] + [InlineData("name")] + [InlineData("skipReason")] + [InlineData("duplicate")] + public async Task MismatchesDoNotOverwriteCachedProvenance(string change) + { + using CacheFixture f = new(); + string first = f.Prepare(); + await Collect(f, first, CacheFixture.FirstJob); + string second = f.Prepare("second"); + string changed = change switch + { + "name" => CacheFixture.Report.Replace("name=\"one\"", "name=\"different-test\"", StringComparison.Ordinal), + "skipReason" => CacheFixture.Report.Replace("", + "different", StringComparison.Ordinal), + "duplicate" => CacheFixture.Report.Replace("total=\"2\" passed=\"1\"", "total=\"3\" passed=\"2\"", StringComparison.Ordinal) + .Replace("", "", StringComparison.Ordinal), + _ => throw new InvalidOperationException(), + }; + InvalidDataException error = await Assert.ThrowsAsync( + () => Collect(f, second, CacheFixture.SecondJob, report: changed)); + Assert.Contains("outcomes differ", error.Message); + ShadowSummary summary = CacheFixture.Read(Path.Combine(second, "summary.json"), CacheJsonContext.Default.ShadowSummary); + Assert.Equal(OutcomeComparison.Mismatch, summary.Comparison); + Assert.False(summary.Stored); + Assert.Equal(CacheFixture.FirstJob, summary.CachedJobId); + string cached = CacheFixture.ShadowLookup(second).ResultsPath!; + Assert.Equal(CacheFixture.FirstJob, + CacheFixture.Read(Path.Combine(cached, "result.json"), CacheJsonContext.Default.ResultReceipt).JobId); + } + + [Theory] + [InlineData("failed")] + [InlineData("failureReason")] + [InlineData("errors")] + [InlineData("exitCode")] + [InlineData("missingExitCode")] + public async Task UnsuccessfulRemoteCompletionsAreNotAdmitted(string failure) + { + using CacheFixture f = new(); + string output = f.Prepare(); + HelixWorkItem item = PassedItem(CacheFixture.FirstJob); + item = failure switch + { + "failed" => item with { State = "Failed" }, + "failureReason" => item with { FailureReason = new HelixFailure() }, + "errors" => item with { Errors = [new HelixFailure()] }, + "exitCode" => item with { ExitCode = 1 }, + "missingExitCode" => item with { ExitCode = null }, + _ => throw new InvalidOperationException(), + }; + InvalidDataException error = await Assert.ThrowsAsync( + () => Collect(f, output, CacheFixture.FirstJob, item: item)); + Assert.Contains("did not complete", error.Message); + Assert.False(Directory.Exists(Path.Combine(output, "results"))); + Assert.Empty(Directory.EnumerateFiles(f.CacheDirectory, "entry.json", SearchOption.AllDirectories)); + } + + [Theory] + [InlineData("linux.test.open.rt", "LINUX.TEST.OPEN", false, true)] + [InlineData("linux.test.open", null, false, true)] + [InlineData("linux.test.open", "", false, true)] + [InlineData("linux.test.open", "other", false, false)] + [InlineData("other", "other", false, false)] + [InlineData("other", null, false, false)] + [InlineData("linux.test.open.rt", "linux.test.open", true, false)] + public async Task QueueAliasesAndJobIdentityAreValidatedAndRetained( + string queueId, string? alias, bool wrongJob, bool accepted) + { + using CacheFixture f = new(); + string output = f.Prepare(); + HelixJob job = new() + { + Name = wrongJob ? CacheFixture.SecondJob : CacheFixture.FirstJob, + QueueId = queueId, + QueueAlias = alias, + }; + if (!accepted) + { + InvalidDataException error = await Assert.ThrowsAsync( + () => Collect(f, output, CacheFixture.FirstJob, job: job)); + Assert.Contains("job or target queue", error.Message); + Assert.False(Directory.Exists(Path.Combine(output, "results"))); + Assert.Empty(Directory.EnumerateFiles(f.CacheDirectory, "entry.json", SearchOption.AllDirectories)); + return; + } + await Collect(f, output, CacheFixture.FirstJob, job: job); + string entry = Directory.GetFiles(f.CacheDirectory, "entry.json", SearchOption.AllDirectories)[0]; + string results = Path.Combine(Path.GetDirectoryName(entry)!, "results"); + ExecutionProvenance provenance = CacheFixture.Read(Path.Combine(results, "provenance.json"), CacheJsonContext.Default.ExecutionProvenance); + Assert.Equal(queueId, provenance.ResolvedQueueId); + Assert.Equal(alias, provenance.QueueAlias); + Assert.Equal("linux.test.open", + CacheFixture.Read(Path.Combine(results, "result.json"), CacheJsonContext.Default.ResultReceipt).TargetQueue); + } + + [Fact] + public void MissingQueueAliasIsSupportedByTypedDeserialization() + { + HelixJob job = JsonSerializer.Deserialize("""{"Name":"job","QueueId":"queue"}""", HelixJsonContext.Default.HelixJob)!; + Assert.Null(job.QueueAlias); + } + + [Theory] + [InlineData(0)] + [InlineData(2)] + public async Task ExactlyOneUploadedReportIsRequired(int count) + { + using CacheFixture f = new(); + string output = f.Prepare(); + List files = []; + for (int i = 0; i < count; i++) + { + files.Add(new HelixFile { Name = "testResults.xml", Link = "https://example.invalid/report" }); + } + InvalidDataException error = await Assert.ThrowsAsync( + () => Collect(f, output, CacheFixture.FirstJob, files: files)); + Assert.Contains("exactly one", error.Message); + Assert.False(Directory.Exists(Path.Combine(output, "results"))); + } + + [Theory] + [InlineData("")] + [InlineData("""""")] + public async Task UploadedXmlMustPassNativeAdmission(string report) + { + using CacheFixture f = new(); + string output = f.Prepare(); + await Assert.ThrowsAsync(() => Collect(f, output, CacheFixture.FirstJob, report: report)); + Assert.Empty(Directory.EnumerateFiles(f.CacheDirectory, "entry.json", SearchOption.AllDirectories)); + } + + [Fact] + public async Task CorruptionBetweenPrepareAndCollectIsNotOverwritten() + { + using CacheFixture f = new(); + string first = f.Prepare(); + await Collect(f, first, CacheFixture.FirstJob); + string second = f.Prepare("second"); + string cachedReport = Path.Combine(CacheFixture.ShadowLookup(second).ResultsPath!, "testResults.xml"); + File.WriteAllText(cachedReport, "corrupt"); + await Assert.ThrowsAsync(() => Collect(f, second, CacheFixture.SecondJob)); + Assert.False(Directory.Exists(Path.Combine(second, "results"))); + Assert.Equal("corrupt", File.ReadAllText(cachedReport)); + } + + [Theory] + [InlineData("Canceled", false, false)] + [InlineData("Passed", true, false)] + [InlineData("Passed", false, true)] + public async Task PollingRejectsUnknownStatesAndWrongIdentity(string state, bool wrongJob, bool wrongName) + { + HelixWorkItem item = PassedItem(CacheFixture.FirstJob) with + { + State = state, + Job = wrongJob ? CacheFixture.SecondJob : CacheFixture.FirstJob, + Name = wrongName ? "other" : ShadowValidation.WorkItem, + }; + using HttpClient client = Client(new HelixJob { Name = CacheFixture.FirstJob, QueueId = "queue" }, item); + await Assert.ThrowsAsync(() => + new HelixClient(client).WaitForCompletionAsync(CacheFixture.FirstJob, ShadowValidation.WorkItem, TimeSpan.FromMinutes(1))); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task PollingHonorsTimeoutAndCallerCancellation(bool cancelCaller) + { + using HttpClient client = new(new Stub(async (_, cancellation) => + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellation); + throw new InvalidOperationException(); + })); + using CancellationTokenSource cancellation = new(); + if (cancelCaller) + { + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => new HelixClient(client).WaitForCompletionAsync( + CacheFixture.FirstJob, ShadowValidation.WorkItem, TimeSpan.FromMinutes(1), cancellation.Token)); + } + else + { + await Assert.ThrowsAsync(() => new HelixClient(client).WaitForCompletionAsync( + CacheFixture.FirstJob, ShadowValidation.WorkItem, TimeSpan.FromMilliseconds(20))); + } + } + + [Fact] + public async Task PollingWaitsForTerminalCompletion() + { + int requests = 0; + using HttpClient client = new(new Stub((_, _) => + { + HelixWorkItem item = PassedItem(CacheFixture.FirstJob) with { State = ++requests == 1 ? "Running" : "Finished" }; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = JsonContent.Create(item, HelixJsonContext.Default.HelixWorkItem), + }); + })); + HelixWorkItem completed = await new HelixClient(client).WaitForCompletionAsync( + CacheFixture.FirstJob, ShadowValidation.WorkItem, TimeSpan.FromMinutes(1)); + Assert.Equal("Finished", completed.State); + Assert.Equal(2, requests); + } + + [Fact] + public async Task TransportTimeoutIsNotReportedAsTheJobDeadline() + { + using HttpClient client = new(new Stub((_, _) => throw new TaskCanceledException("HTTP request timed out"))); + TaskCanceledException error = await Assert.ThrowsAsync(() => + new HelixClient(client).WaitForCompletionAsync(CacheFixture.FirstJob, ShadowValidation.WorkItem, TimeSpan.FromMinutes(1))); + Assert.Equal("HTTP request timed out", error.Message); + } + + private static async Task Collect(CacheFixture f, string output, string jobId, + HelixJob? job = null, HelixWorkItem? item = null, List? files = null, string? report = null) + { + File.WriteAllText(Path.Combine(output, "job-id.txt"), jobId + "\n"); + using HttpClient client = Client(job ?? new HelixJob { Name = jobId, QueueId = f.Queues[0].QueueId }, + item ?? PassedItem(jobId), files, report); + await ShadowValidation.CollectAsync(new Cache(f.CacheDirectory), output, TimeSpan.FromMinutes(1), new HelixClient(client)); + } + + private static HelixWorkItem PassedItem(string jobId) => new() + { + Job = jobId, + Name = ShadowValidation.WorkItem, + State = "Passed", + ExitCode = 0, + MachineName = "test-worker", + }; + + private static HttpClient Client(HelixJob job, HelixWorkItem item, List? files = null, string? report = null) => + new(new Stub((request, _) => + { + HttpContent content = request.RequestUri!.AbsolutePath switch + { + string path when path.EndsWith("/details", StringComparison.Ordinal) => + JsonContent.Create(job, HelixJsonContext.Default.HelixJob), + string path when path.EndsWith("/files", StringComparison.Ordinal) => + JsonContent.Create(files ?? [new HelixFile { Name = "testResults.xml", Link = "https://example.invalid/report" }], + HelixJsonContext.Default.ListHelixFile), + "/report" => new StringContent(report ?? CacheFixture.Report, Encoding.UTF8, "application/xml"), + string path when path.Contains("/workitems/", StringComparison.Ordinal) => + JsonContent.Create(item, HelixJsonContext.Default.HelixWorkItem), + _ => throw new InvalidOperationException($"Unexpected HTTP request: {request.RequestUri}"), + }; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = content }); + })); + + private sealed class Stub(Func> send) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => + send(request, cancellationToken); + } +} diff --git a/src/tools/helix-cache/tests/test_cache.py b/src/tools/helix-cache/tests/test_cache.py deleted file mode 100644 index eea4bbd33f4510..00000000000000 --- a/src/tools/helix-cache/tests/test_cache.py +++ /dev/null @@ -1,476 +0,0 @@ -# Licensed to the .NET Foundation under one or more agreements. -# The .NET Foundation licenses this file to you under the MIT license. - -"""Run against the published native binary: python3 test_cache.py /path/to/helix-cache.""" - -import argparse -from contextlib import redirect_stdout -import copy -import io -import json -import os -from pathlib import Path -import subprocess -import sys -import tempfile -import unittest -from concurrent.futures import ThreadPoolExecutor -from unittest.mock import patch - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -import shadow - - -EXECUTABLE = None - - -class CacheTests(unittest.TestCase): - def setUp(self): - self.temporary = tempfile.TemporaryDirectory(prefix="helix-cache-test-") - self.addCleanup(self.temporary.cleanup) - self.root = Path(self.temporary.name) - self.cache = self.root / "cache" - self.payload = self.root / "payload" - self.payload.mkdir() - (self.payload / "run.sh").write_text("#!/bin/sh\nexit 0\n") - (self.payload / "run.sh").chmod(0o755) - self.shared = self.root / "shared.zip" - self.shared.write_bytes(b"opaque archive bytes") - self.queues = [{ - "QueueId": "linux.test.open", - "Architecture": "AMD64", - "OperatingSystemGroup": "linux", - "IsAvailable": True, - "GalleryImage": {"Version": "1"}, - "Artifacts": [{"Id": "prepare", "Parameters": {"version": "1"}}], - "QueueDepth": 0, - }] - self.submission = { - "schemaVersion": 1, - "workItems": [{ - "name": "suite", - "queues": ["linux.test.open"], - "payload": {"path": "payload"}, - "correlationPayloads": [{"path": "shared.zip", "destination": "runtime"}], - "command": "./run.sh", - "environment": {"SECOND": "2", "FIRST": "1"}, - "timeoutSeconds": 60, - "expectedExitCode": 0, - }], - } - - def write_json(self, name, value): - path = self.root / name - path.write_text(json.dumps(value)) - return path - - def invoke(self, *args, success=True): - result = subprocess.run( - [EXECUTABLE, *map(str, args)], capture_output=True, text=True, timeout=30 - ) - self.assertEqual(0 if success else 1, result.returncode, result.stderr) - self.assertEqual("", result.stdout) - if not success: - self.assertIn("helix-cache:", result.stderr) - return result - - def lookup(self, success=True): - self.invoke( - "lookup", self.write_json("submission.json", self.submission), - "--cache", self.cache, "--output", self.root / "lookup.json", - "--queue-info", self.write_json("queues.json", self.queues), - success=success, - ) - if success: - return json.loads((self.root / "lookup.json").read_text())["entries"] - return None - - def create_results(self, execution): - results = self.root / "results" - results.mkdir(exist_ok=True) - receipt = { - "schemaVersion": 1, - "completed": True, - "infrastructureSucceeded": True, - "exitCode": 0, - "jobId": "00000000-0000-0000-0000-000000000001", - "workItemName": execution["inputs"]["name"], - "targetQueue": execution["inputs"]["targetQueue"], - "resultFiles": ["tests.xml"], - } - self.write_json("results/result.json", receipt) - (results / "tests.xml").write_text( - '' - '' - '' - ) - return results - - def store(self, execution, success=True): - return self.invoke( - "store", self.write_json("execution.json", execution), - "--results", self.root / "results", "--cache", self.cache, - success=success, - ) - - def test_round_trip_uses_frozen_inputs_and_preserves_results(self): - first = self.lookup()[0] - self.assertEqual("miss", first["status"]) - execution = first["execution"] - snapshot = Path(execution["payloadPath"]) - self.assertNotEqual(self.payload, snapshot) - self.assertEqual(0o755, (snapshot / "run.sh").stat().st_mode & 0o777) - self.create_results(execution) - self.store(execution) - hit = self.lookup()[0] - self.assertEqual("hit", hit["status"]) - self.assertEqual(execution["key"], hit["execution"]["key"]) - self.assertEqual( - (self.root / "results/tests.xml").read_bytes(), - (Path(hit["resultsPath"]) / "tests.xml").read_bytes(), - ) - (self.payload / "run.sh").write_text("changed") - self.assertEqual("#!/bin/sh\nexit 0\n", (snapshot / "run.sh").read_text()) - self.assertEqual("miss", self.lookup()[0]["status"]) - - def test_execution_and_environment_changes_invalidate_keys(self): - baseline = self.lookup()[0]["execution"]["key"] - original = copy.deepcopy(self.submission) - for field, value in [ - ("command", "./run.sh other"), - ("preCommands", ["export STRESS=1"]), - ("postCommands", ["echo done"]), - ("timeoutSeconds", 61), - ("expectedExitCode", 100), - ("environment", {"FIRST": "changed"}), - ]: - with self.subTest(field=field): - self.submission = copy.deepcopy(original) - self.submission["workItems"][0][field] = value - self.assertNotEqual(baseline, self.lookup()[0]["execution"]["key"]) - self.submission = original - self.queues[0]["GalleryImage"]["Version"] = "2" - self.assertNotEqual(baseline, self.lookup()[0]["execution"]["key"]) - self.queues[0]["GalleryImage"]["Version"] = "1" - self.queues[0]["Artifacts"][0]["Parameters"]["version"] = "2" - self.assertNotEqual(baseline, self.lookup()[0]["execution"]["key"]) - - def test_irrelevant_order_and_live_queue_state_do_not_change_key(self): - baseline = self.lookup()[0]["execution"]["key"] - self.queues[0]["QueueDepth"] = 300 - self.queues[0]["IsInProtectedMode"] = True - self.submission["workItems"][0]["environment"] = {"FIRST": "1", "SECOND": "2"} - os.utime(self.payload / "run.sh", (1000000000, 1000000000)) - self.assertEqual(baseline, self.lookup()[0]["execution"]["key"]) - - def test_paths_do_not_affect_keys_but_modes_and_empty_directories_do(self): - baseline = self.lookup()[0]["execution"]["key"] - relocated = self.root / "relocated" - self.payload.rename(relocated) - self.submission["workItems"][0]["payload"]["path"] = "relocated" - self.assertEqual(baseline, self.lookup()[0]["execution"]["key"]) - (relocated / "run.sh").chmod(0o644) - self.assertNotEqual(baseline, self.lookup()[0]["execution"]["key"]) - (relocated / "run.sh").chmod(0o755) - (relocated / "empty").mkdir() - self.assertNotEqual(baseline, self.lookup()[0]["execution"]["key"]) - - def test_queue_partitioning_and_digest_pinned_containers(self): - target = "(Debian.Amd64)linux.test.open@registry/image:tag@sha256:" + "a" * 64 - self.submission["workItems"][0]["queues"].append(target) - entries = self.lookup() - self.assertEqual(2, len(entries)) - self.assertNotEqual(entries[0]["execution"]["key"], entries[1]["execution"]["key"]) - self.assertEqual( - "registry/image:tag@sha256:" + "a" * 64, - entries[1]["execution"]["inputs"]["environmentFingerprint"]["Container"], - ) - self.create_results(entries[0]["execution"]) - self.store(entries[0]["execution"]) - self.assertEqual(["hit", "miss"], [e["status"] for e in self.lookup()]) - self.submission["workItems"][0]["queues"] = ["linux.test.open@registry/image:latest"] - self.lookup(success=False) - self.submission["workItems"][0]["queues"] = ["(alias)linux.test.open"] - self.lookup(success=False) - - def test_shared_payload_bytes_destination_and_overlay_order_matter(self): - baseline = self.lookup()[0]["execution"]["key"] - self.shared.write_bytes(b"new archive") - self.assertNotEqual(baseline, self.lookup()[0]["execution"]["key"]) - self.shared.write_bytes(b"opaque archive bytes") - self.submission["workItems"][0]["correlationPayloads"][0]["destination"] = "other" - self.assertNotEqual(baseline, self.lookup()[0]["execution"]["key"]) - correlations = self.submission["workItems"][0]["correlationPayloads"] - correlations.append({"path": "shared.zip", "destination": "second"}) - before = self.lookup()[0]["execution"]["key"] - correlations.reverse() - self.assertNotEqual(before, self.lookup()[0]["execution"]["key"]) - - def test_unsuccessful_receipts_are_not_stored(self): - execution = self.lookup()[0]["execution"] - self.create_results(execution) - original = json.loads((self.root / "results/result.json").read_text()) - for field, value in [ - ("completed", False), ("infrastructureSucceeded", False), - ("exitCode", 1), ("workItemName", "other"), - ("targetQueue", "other"), ("jobId", "not-a-guid"), - ("resultFiles", []), ("resultFiles", ["../outside.xml"]), - ("resultFiles", ["tests.xml", "tests.xml"]), - ]: - with self.subTest(field=field, value=value): - receipt = dict(original, **{field: value}) - self.write_json("results/result.json", receipt) - self.store(execution, success=False) - self.assertEqual("miss", self.lookup()[0]["status"]) - - def test_reports_must_be_complete_nonempty_and_passing(self): - execution = self.lookup()[0]["execution"] - self.create_results(execution) - for report in [ - "", - '', - '', - '' - '', - '' - '', - ']>' - '&x;', - ]: - with self.subTest(report=report): - (self.root / "results/tests.xml").write_text(report) - self.store(execution, success=False) - self.assertEqual("miss", self.lookup()[0]["status"]) - - def test_corruption_is_an_error_not_a_hit_or_silent_miss(self): - execution = self.lookup()[0]["execution"] - self.create_results(execution) - self.store(execution) - hit = self.lookup()[0] - (Path(hit["resultsPath"]) / "tests.xml").write_text("corrupted") - self.lookup(success=False) - - def test_modified_snapshot_or_forged_key_cannot_be_stored(self): - execution = self.lookup()[0]["execution"] - self.create_results(execution) - forged = dict(execution, key="0" * 64) - self.store(forged, success=False) - (Path(execution["payloadPath"]) / "run.sh").write_text("modified after lookup") - self.store(execution, success=False) - - def test_store_rejects_results_containing_its_staging_directory(self): - execution = self.lookup()[0]["execution"] - result = self.invoke( - "store", self.write_json("execution.json", execution), - "--results", self.cache / "tmp", "--cache", self.cache, - success=False, - ) - self.assertIn("must not be inside input directory", result.stderr) - self.assertEqual([], list((self.cache / "tmp").iterdir())) - self.assertEqual("miss", self.lookup()[0]["status"]) - - def test_duplicate_stores_are_atomic_and_idempotent(self): - execution = self.lookup()[0]["execution"] - self.create_results(execution) - record = self.write_json("execution.json", execution) - with ThreadPoolExecutor(max_workers=2) as pool: - operations = [ - pool.submit( - self.invoke, "store", record, "--results", self.root / "results", - "--cache", self.cache, - ) for _ in range(2) - ] - for operation in operations: - operation.result() - self.assertEqual("hit", self.lookup()[0]["status"]) - self.assertEqual([], list((self.cache / "tmp").iterdir())) - - def test_rejects_unsupported_queues_links_and_nested_cache(self): - for field, value in [ - ("Architecture", "ARM64"), ("OperatingSystemGroup", "windows"), - ("IsAvailable", False), - ]: - with self.subTest(field=field): - original = self.queues[0][field] - self.queues[0][field] = value - self.lookup(success=False) - self.queues[0][field] = original - (self.payload / "link").symlink_to(self.shared) - self.lookup(success=False) - (self.payload / "link").unlink() - self.cache = self.payload / "cache" - self.lookup(success=False) - - def test_strict_manifest_and_cli_validation(self): - original = copy.deepcopy(self.submission) - for transform in [ - lambda s: s.update(schemaVersion=2), - lambda s: s.update(workItems=[]), - lambda s: s.update(workItems=[None]), - lambda s: s.update(unrecognized=True), - lambda s: s["workItems"][0].pop("expectedExitCode"), - lambda s: s["workItems"][0].update(command=None), - lambda s: s["workItems"][0].update(timeoutSeconds=0), - ]: - with self.subTest(transform=transform): - self.submission = copy.deepcopy(original) - transform(self.submission) - self.lookup(success=False) - self.invoke("store", "missing", "--cache", self.cache, success=False) - self.invoke("lookup", "missing", "--cache", self.cache, "--cache", self.cache, success=False) - - def prepare_shadow(self, name="shadow"): - args = argparse.Namespace( - executable=Path(EXECUTABLE), cache=self.cache, output=self.root / name, - payload=self.shared, runtime=self.shared, queue=self.queues[0]["QueueId"], - queue_info=self.write_json("queues.json", self.queues), wait_seconds=60, - ) - with redirect_stdout(io.StringIO()): - shadow.prepare(args) - return args - - def collect_shadow(self, args, job_id, report=None, item_changes=None, files=None, job_changes=None): - (args.output / "job-id.txt").write_text(job_id + "\n") - item = { - "Job": job_id, "Name": shadow.WORK_ITEM, "State": "Passed", - "ExitCode": 0, "MachineName": "test-worker", - } - item.update(item_changes or {}) - if report is None: - report = ( - '' - '' - '' - ) - if files is None: - files = [{"Name": "testResults.xml", "Link": "https://example.invalid/report"}] - job = {"Name": job_id, "QueueId": self.queues[0]["QueueId"]} - job.update(job_changes or {}) - responses = { - f"{shadow.API}/jobs/{job_id}": {"Name": job["Name"], "QueueId": job["QueueId"]}, - f"{shadow.API}/jobs/{job_id}/details": job, - f"{shadow.API}/jobs/{job_id}/workitems/{shadow.WORK_ITEM}/files?latestOnly=true": files, - } - with (patch.object(shadow, "completed_work_item", return_value=item), - patch.object(shadow, "get_json", side_effect=responses.__getitem__), - patch.object(shadow, "download", side_effect=lambda url, path: path.write_text(report)), - redirect_stdout(io.StringIO())): - shadow.collect(args) - - def test_shadow_cold_store_and_restored_entries_compare_actual_outcomes(self): - import shutil - - first_job = "00000000-0000-0000-0000-000000000001" - second_job = "00000000-0000-0000-0000-000000000002" - first = self.prepare_shadow() - self.collect_shadow(first, first_job) - first_summary = shadow.read_json(first.output / "summary.json") - self.assertEqual("miss", first_summary["lookup"]) - self.assertEqual(2, first_summary["tests"]) - restored = self.root / "restored-cache" - shutil.copytree(self.cache / "entries", restored / "entries") - self.cache.rename(self.root / "previous-cache") - self.cache = restored - second = self.prepare_shadow("second") - report = ( - '' - '' - '' - ) - self.collect_shadow(second, second_job, report) - summary = shadow.read_json(second.output / "summary.json") - self.assertEqual("hit", summary["lookup"]) - self.assertEqual("match", summary["comparison"]) - self.assertEqual(first_job, summary["cachedJobId"]) - self.assertEqual(second_job, summary["jobId"]) - self.assertEqual(first_summary["key"], summary["key"]) - - def test_shadow_reports_mismatches_without_overwriting_cached_provenance(self): - first = self.prepare_shadow() - self.collect_shadow(first, "00000000-0000-0000-0000-000000000001") - second = self.prepare_shadow("second") - changed = ( - '' - '' - '' - ) - with self.assertRaisesRegex(ValueError, "outcomes differ"): - self.collect_shadow(second, "00000000-0000-0000-0000-000000000002", changed) - summary = shadow.read_json(second.output / "summary.json") - self.assertEqual("mismatch", summary["comparison"]) - self.assertEqual("00000000-0000-0000-0000-000000000001", summary["cachedJobId"]) - - def test_shadow_does_not_admit_unsuccessful_remote_completions(self): - args = self.prepare_shadow() - for changes in ( - {"State": "Failed"}, {"FailureReason": {"Issue": {"Name": "WorkItemTimeout"}}}, - {"Errors": [{"Message": "Infrastructure failure"}]}, {"ExitCode": 1}, {"ExitCode": None}, - ): - with self.subTest(changes=changes), self.assertRaisesRegex(ValueError, "did not complete"): - self.collect_shadow(args, "00000000-0000-0000-0000-000000000001", item_changes=changes) - with self.assertRaisesRegex(ValueError, "target queue"): - self.collect_shadow(args, "00000000-0000-0000-0000-000000000001", job_changes={"QueueId": "other"}) - self.assertEqual([], list((self.cache / "entries").rglob("entry.json"))) - - def test_shadow_queue_alias_validation_and_provenance(self): - expected = self.queues[0]["QueueId"] - cases = [ - ({"QueueId": expected + ".rt", "QueueAlias": expected.upper()}, True), - ({"QueueId": expected}, True), - ({"QueueId": expected, "QueueAlias": None}, True), - ({"QueueId": expected, "QueueAlias": ""}, True), - ({"QueueId": expected, "QueueAlias": "other"}, False), - ({"QueueId": "other", "QueueAlias": "other"}, False), - ({"QueueId": "other"}, False), - ({"Name": "other", "QueueId": expected + ".rt", "QueueAlias": expected}, False), - ] - for index, (job, accepted) in enumerate(cases): - with self.subTest(job=job): - self.cache = self.root / f"cache-{index}" - args = self.prepare_shadow(f"shadow-{index}") - if accepted: - self.collect_shadow(args, "00000000-0000-0000-0000-000000000001", job_changes=job) - provenance = shadow.read_json(args.output / "results/provenance.json") - self.assertEqual(job["QueueId"], provenance["resolvedQueueId"]) - self.assertEqual(job.get("QueueAlias"), provenance["queueAlias"]) - entry, = shadow.read_json(args.output / "lookup.json")["entries"] - receipt = shadow.read_json( - self.cache / "entries/v1" / entry["execution"]["key"][:2] - / entry["execution"]["key"] / "results/result.json" - ) - self.assertEqual(expected, receipt["targetQueue"]) - else: - with self.assertRaisesRegex(ValueError, "job or target queue"): - self.collect_shadow(args, "00000000-0000-0000-0000-000000000001", job_changes=job) - self.assertFalse((args.output / "results").exists()) - self.assertEqual([], list((self.cache / "entries").rglob("entry.json"))) - - def test_shadow_requires_one_uploaded_report_and_native_report_validation(self): - args = self.prepare_shadow() - for files in ([], [{"Name": "testResults.xml"}] * 2): - with self.subTest(files=files), self.assertRaises(ValueError): - self.collect_shadow(args, "00000000-0000-0000-0000-000000000001", files=files) - with self.assertRaises(subprocess.CalledProcessError): - self.collect_shadow(args, "00000000-0000-0000-0000-000000000001", report="") - self.assertEqual([], list((self.cache / "entries").rglob("entry.json"))) - - def test_shadow_polling_rejects_timeouts_unknown_states_and_identity_mismatches(self): - item = {"Job": "job", "Name": "suite", "State": "Running"} - with (patch.object(shadow, "get_json", return_value=item), - patch.object(shadow.time, "monotonic", side_effect=[0, 61]), - self.assertRaises(TimeoutError)): - shadow.completed_work_item("job", "suite", 60) - for changes in ({"State": "Canceled"}, {"Job": "other"}, {"Name": "other"}): - with (self.subTest(changes=changes), - patch.object(shadow, "get_json", return_value=dict(item, **changes)), - self.assertRaises(ValueError)): - shadow.completed_work_item("job", "suite", 60) - with (patch.object(shadow, "get_json", side_effect=[item, dict(item, State="Finished")]), - patch.object(shadow.time, "sleep") as sleep): - self.assertEqual("Finished", shadow.completed_work_item("job", "suite", 60)["State"]) - sleep.assert_called_once() - - -if __name__ == "__main__": - EXECUTABLE = str(Path(sys.argv.pop(1)).resolve()) - unittest.main() From 0a45ebbb34faa5bf4fee2ae384f2f52c2f902c66 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Wed, 21 Oct 2026 05:29:12 +0000 Subject: [PATCH 4/8] Temporarily enable Helix cache shadow validation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ee547d49-5419-4268-b01f-6b2757f99953 --- eng/pipelines/runtime.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 032aae21615fb2..69258abf6f7934 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -2,7 +2,7 @@ parameters: - name: helixCacheShadow displayName: Validate Linux Helix result caching (shadow mode) type: boolean - default: false + default: true # Setting batch to true, triggers one build at a time. # if there is a push while a build in progress, it will wait, From 8c218de90cde1c89c9fa05ba1fa6b04e8de173a4 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Wed, 21 Oct 2026 20:14:22 +0000 Subject: [PATCH 5/8] Restore opt-in Helix cache shadow validation Keep shadow validation disabled by default after the enabled CI run. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ee547d49-5419-4268-b01f-6b2757f99953 --- eng/pipelines/runtime.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 69258abf6f7934..032aae21615fb2 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -2,7 +2,7 @@ parameters: - name: helixCacheShadow displayName: Validate Linux Helix result caching (shadow mode) type: boolean - default: true + default: false # Setting batch to true, triggers one build at a time. # if there is a push while a build in progress, it will wait, From 6287dfee7a2c31e0ddac464a424821b09ea3e5d4 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Wed, 21 Oct 2026 20:19:11 +0000 Subject: [PATCH 6/8] Keep Helix cache validation enabled during draft testing Leave the shadow lane enabled while validating cache reuse in the draft PR. The synchronous experimental harness is intentional; disable it before merge. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ee547d49-5419-4268-b01f-6b2757f99953 --- eng/pipelines/runtime.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 032aae21615fb2..69258abf6f7934 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -2,7 +2,7 @@ parameters: - name: helixCacheShadow displayName: Validate Linux Helix result caching (shadow mode) type: boolean - default: false + default: true # Setting batch to true, triggers one build at a time. # if there is a push while a build in progress, it will wait, From 3ba38d5ae77b30330c461da1c67ae5d612dbab29 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Sat, 24 Oct 2026 02:45:39 +0000 Subject: [PATCH 7/8] Normalize Helix cache snapshots and fingerprint the SDK Canonicalize submitted ZIP snapshots without discarding file contents or Unix modes. Include the command-generator package digest in execution keys and verify it before submission. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ee547d49-5419-4268-b01f-6b2757f99953 --- .../libraries/helix-cache-shadow.yml | 4 +- eng/testing/helix-cache-shadow.proj | 10 +- src/tools/helix-cache/README.md | 42 ++++-- src/tools/helix-cache/src/Cache.cs | 1 + src/tools/helix-cache/src/CommandLine.cs | 6 +- src/tools/helix-cache/src/Models.cs | 13 +- src/tools/helix-cache/src/Program.cs | 3 +- src/tools/helix-cache/src/ShadowValidation.cs | 5 +- src/tools/helix-cache/src/Snapshot.cs | 61 ++++++++- src/tools/helix-cache/tests/CacheFixture.cs | 24 +++- src/tools/helix-cache/tests/CacheTests.cs | 128 +++++++++++++++++- 11 files changed, 265 insertions(+), 32 deletions(-) diff --git a/eng/pipelines/libraries/helix-cache-shadow.yml b/eng/pipelines/libraries/helix-cache-shadow.yml index a6bbe92c3ad980..0909d5a9f66ec3 100644 --- a/eng/pipelines/libraries/helix-cache-shadow.yml +++ b/eng/pipelines/libraries/helix-cache-shadow.yml @@ -43,10 +43,12 @@ steps: -nologo -p:Configuration=$(_BuildConfig) -getProperty:NetCoreAppCurrentBuildSettings)" archives="$("$dotnet" msbuild "$(Build.SourcesDirectory)/src/libraries/System.Memory.Data/tests/System.Memory.Data.Tests.csproj" \ -nologo -p:Configuration=$(_BuildConfig) -getProperty:TestArchiveTestsDir)" + sdk="$("$dotnet" msbuild "$(Build.SourcesDirectory)/eng/testing/helix-cache-shadow.proj" \ + -nologo -getProperty:HelixCacheSdkPackage)" "$executable" prepare --cache "$cache" --output "$output" \ --payload "$archives/System.Memory.Data.Tests.zip" \ --runtime "$(Build.SourcesDirectory)/artifacts/helix/runtime/test-runtime-$settings.zip" \ - --queue AzureLinux.3.Amd64.Open + --queue AzureLinux.3.Amd64.Open --helix-sdk "$sdk" # A cache hit never bypasses this submission. "$dotnet" msbuild "$(Build.SourcesDirectory)/eng/testing/helix-cache-shadow.proj" -t:Submit \ diff --git a/eng/testing/helix-cache-shadow.proj b/eng/testing/helix-cache-shadow.proj index 313e7857c79b48..73b4a2bc98cea5 100644 --- a/eng/testing/helix-cache-shadow.proj +++ b/eng/testing/helix-cache-shadow.proj @@ -1,8 +1,9 @@ - + $(ShadowQueue) + $([System.IO.Path]::GetFullPath('$(MicrosoftDotNetHelixSdkTasksAssembly)/../../../microsoft.dotnet.helix.sdk.$(MicrosoftDotNetHelixSdkPackageVersion).nupkg')) @@ -20,10 +21,15 @@ Text="Expected a prepared System.Memory.Data.Tests shadow submission." /> + + + + - + new LookupCommand(args[1], cache, Required("--output"), Optional("--queue-info")), "store" => new StoreCommand(args[1], cache, Required("--results")), "prepare" => new PrepareCommand(cache, Required("--output"), Required("--payload"), - Required("--runtime"), Required("--queue"), Optional("--queue-info")), + Required("--runtime"), Required("--queue"), Required("--helix-sdk"), Optional("--queue-info")), "collect" => new CollectCommand(cache, Required("--output"), ParseWait(Optional("--wait-seconds"))), _ => throw new UnreachableException(), }; diff --git a/src/tools/helix-cache/src/Models.cs b/src/tools/helix-cache/src/Models.cs index ca497dab54bbea..f46d2a2055f85d 100644 --- a/src/tools/helix-cache/src/Models.cs +++ b/src/tools/helix-cache/src/Models.cs @@ -69,13 +69,14 @@ internal sealed record WorkItem public required string Command { get; init; } public List PreCommands { get; init; } = []; public List PostCommands { get; init; } = []; + public string? CommandGeneratorDigest { get; init; } public Dictionary Environment { get; init; } = []; public required int TimeoutSeconds { get; init; } public required int ExpectedExitCode { get; init; } internal void Validate() { - ValidateExecution(Command, TimeoutSeconds, PreCommands, PostCommands, Environment); + ValidateExecution(Command, TimeoutSeconds, PreCommands, PostCommands, Environment, CommandGeneratorDigest); if (Queues.Count == 0 || Queues.Any(string.IsNullOrWhiteSpace) || Queues.Distinct(StringComparer.Ordinal).Count() != Queues.Count) { @@ -93,7 +94,7 @@ internal void Validate() } internal static void ValidateExecution(string command, int timeoutSeconds, List preCommands, - List postCommands, Dictionary environment) + List postCommands, Dictionary environment, string? commandGeneratorDigest) { if (string.IsNullOrWhiteSpace(command) || timeoutSeconds <= 0 || preCommands.Any(c => c is null) || postCommands.Any(c => c is null) || @@ -101,6 +102,10 @@ internal static void ValidateExecution(string command, int timeoutSeconds, List< { throw new InvalidDataException("Execution requires a nonempty command, positive timeout, nonnull commands and environment values."); } + if (commandGeneratorDigest is not null && !Cache.IsDigest(commandGeneratorDigest)) + { + throw new InvalidDataException("A command generator digest must be a lowercase SHA-256 digest."); + } } } @@ -156,6 +161,7 @@ internal sealed record ExecutionInputs public required string Command { get; init; } public required List PreCommands { get; init; } public required List PostCommands { get; init; } + public string? CommandGeneratorDigest { get; init; } public required Dictionary Environment { get; init; } public required int TimeoutSeconds { get; init; } public required int ExpectedExitCode { get; init; } @@ -172,7 +178,8 @@ internal sealed record ExecutionRecord internal void Validate() { Cache.ValidateVersion(SchemaVersion); - WorkItem.ValidateExecution(Inputs.Command, Inputs.TimeoutSeconds, Inputs.PreCommands, Inputs.PostCommands, Inputs.Environment); + WorkItem.ValidateExecution(Inputs.Command, Inputs.TimeoutSeconds, Inputs.PreCommands, Inputs.PostCommands, Inputs.Environment, + Inputs.CommandGeneratorDigest); Inputs.Payload.Validate(); foreach (PayloadIdentity payload in Inputs.CorrelationPayloads) { diff --git a/src/tools/helix-cache/src/Program.cs b/src/tools/helix-cache/src/Program.cs index 5505408e46447e..067c190bd0ff6e 100644 --- a/src/tools/helix-cache/src/Program.cs +++ b/src/tools/helix-cache/src/Program.cs @@ -86,7 +86,8 @@ private static async Task ExecuteAsync(Command command, CancellationToken cancel case PrepareCommand prepare: { LinuxEnvironment environment = await LinuxEnvironment.LoadAsync(prepare.QueueInfo, helix, cancellationToken); - LookupEntry entry = ShadowValidation.Prepare(cache, prepare.Output, prepare.Payload, prepare.Runtime, prepare.Queue, environment); + LookupEntry entry = ShadowValidation.Prepare(cache, prepare.Output, prepare.Payload, prepare.Runtime, prepare.Queue, + prepare.HelixSdk, environment); Console.WriteLine($"Helix cache shadow: {entry.Status.ToString().ToLowerInvariant()} {entry.Execution.Key}; execution will NOT be skipped."); break; } diff --git a/src/tools/helix-cache/src/ShadowValidation.cs b/src/tools/helix-cache/src/ShadowValidation.cs index 05f8851bb3bd3b..cc0908df29e410 100644 --- a/src/tools/helix-cache/src/ShadowValidation.cs +++ b/src/tools/helix-cache/src/ShadowValidation.cs @@ -19,7 +19,8 @@ internal static class ShadowValidation internal const string Command = "bash ./RunTests.sh --runtime-path \"$HELIX_CORRELATION_PAYLOAD\""; internal const string PostCommand = "cp testResults.xml \"$HELIX_WORKITEM_UPLOAD_ROOT/testResults.xml\""; - internal static LookupEntry Prepare(Cache cache, string output, string payload, string runtime, string queue, LinuxEnvironment environment) + internal static LookupEntry Prepare(Cache cache, string output, string payload, string runtime, string queue, string helixSdk, + LinuxEnvironment environment) { payload = Path.GetFullPath(payload); runtime = Path.GetFullPath(runtime); @@ -46,6 +47,7 @@ internal static LookupEntry Prepare(Cache cache, string output, string payload, CorrelationPayloads = [new Payload { Path = runtime }], Command = Command, PostCommands = [PostCommand], + CommandGeneratorDigest = Snapshot.HashFile(helixSdk), TimeoutSeconds = 900, ExpectedExitCode = 0, }], @@ -154,6 +156,7 @@ private static void WriteSubmissionProperties(string output, ExecutionRecord exe ["ShadowRuntime"] = execution.CorrelationPayloadPaths[0], ["ShadowCommand"] = execution.Inputs.Command, ["ShadowPostCommand"] = execution.Inputs.PostCommands[0], + ["ShadowCommandGeneratorDigest"] = execution.Inputs.CommandGeneratorDigest!, ["ShadowTimeout"] = TimeSpan.FromSeconds(execution.Inputs.TimeoutSeconds).ToString("c", CultureInfo.InvariantCulture), }) { diff --git a/src/tools/helix-cache/src/Snapshot.cs b/src/tools/helix-cache/src/Snapshot.cs index 23d046eeea657f..cdd58cce08aa5d 100644 --- a/src/tools/helix-cache/src/Snapshot.cs +++ b/src/tools/helix-cache/src/Snapshot.cs @@ -2,7 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Security.Cryptography; using System.Text.Json; @@ -32,7 +34,16 @@ internal static Snapshot Capture(string source, string cache) EnsureOutside(source, temporary.Path); string name = directory ? "payload" : System.IO.Path.GetFileName(source); string staged = System.IO.Path.Combine(temporary.Path, name); - Copy(source, staged); + if (directory) + { + Copy(source, staged); + } + else + { + NormalizeArchive(source, staged); + File.SetLastWriteTimeUtc(staged, s_timestamp); + File.SetUnixFileMode(staged, File.GetUnixFileMode(source)); + } string digest = Hash(staged); string parent = System.IO.Path.Combine(cache, "inputs", "v1"); Directory.CreateDirectory(parent); @@ -109,6 +120,51 @@ internal static string Hash(string path) return Convert.ToHexStringLower(SHA256.HashData(manifest.GetBuffer().AsSpan(0, checked((int)manifest.Length)))); } + internal static string HashFile(string path) + { + using FileStream file = File.OpenRead(path); + return Convert.ToHexStringLower(SHA256.HashData(file)); + } + + private static void NormalizeArchive(string source, string destination) + { + using ZipArchive input = ZipFile.OpenRead(source); + Dictionary paths = new(StringComparer.Ordinal); + foreach (ZipArchiveEntry entry in input.Entries) + { + string path = entry.FullName.TrimEnd('/'); + ValidateRelativePath(path); + if (path.Split('/').Any(string.IsNullOrEmpty) || entry.FullName.EndsWith("//", StringComparison.Ordinal) || + path.Contains('\0') || !paths.TryAdd(path, entry) || + (entry.FullName.EndsWith('/') && entry.Length != 0)) + { + throw new InvalidDataException($"Invalid or duplicate ZIP entry '{entry.FullName}' in '{source}'."); + } + } + foreach (string path in paths.Keys) + { + for (int separator = path.IndexOf('/'); separator >= 0; separator = path.IndexOf('/', separator + 1)) + { + if (paths.TryGetValue(path[..separator], out ZipArchiveEntry? parent) && !parent.FullName.EndsWith('/')) + { + throw new InvalidDataException($"ZIP entry '{path}' has a non-directory parent in '{source}'."); + } + } + } + + using FileStream file = new(destination, FileMode.CreateNew, FileAccess.Write); + using ZipArchive output = new(file, ZipArchiveMode.Create); + foreach (ZipArchiveEntry entry in input.Entries.OrderBy(e => e.FullName, StringComparer.Ordinal)) + { + ZipArchiveEntry normalized = output.CreateEntry(entry.FullName, CompressionLevel.Fastest); + normalized.LastWriteTime = new DateTimeOffset(s_timestamp); + normalized.ExternalAttributes = entry.ExternalAttributes; + using Stream contents = entry.Open(); + using Stream target = normalized.Open(); + contents.CopyTo(target); + } + } + private static void WriteTree(Utf8JsonWriter writer, string path, string relative) { FileAttributes attributes = File.GetAttributes(path); @@ -120,8 +176,7 @@ private static void WriteTree(Utf8JsonWriter writer, string path, string relativ writer.WriteNumberValue((int)File.GetUnixFileMode(path)); if (!directory) { - using FileStream file = File.OpenRead(path); - writer.WriteStringValue(Convert.ToHexStringLower(SHA256.HashData(file))); + writer.WriteStringValue(HashFile(path)); } writer.WriteEndArray(); if (directory) diff --git a/src/tools/helix-cache/tests/CacheFixture.cs b/src/tools/helix-cache/tests/CacheFixture.cs index d3f8c085cafe57..5a32c837b1173b 100644 --- a/src/tools/helix-cache/tests/CacheFixture.cs +++ b/src/tools/helix-cache/tests/CacheFixture.cs @@ -5,6 +5,8 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.IO.Compression; +using System.Linq; using System.Text.Json; using System.Text.Json.Serialization.Metadata; using System.Threading.Tasks; @@ -14,6 +16,10 @@ namespace HelixCache.Tests; internal sealed class CacheFixture : IDisposable { + internal const int ArchiveFileMode = 0x81A4; + internal const int ArchiveExecutableMode = 0x81ED; + internal const int ArchiveDirectoryMode = 0x41ED; + internal const int ArchiveLinkMode = 0xA1FF; private const UnixFileMode ExecutableMode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherExecute; internal const string FirstJob = "00000000-0000-0000-0000-000000000001"; @@ -44,7 +50,7 @@ internal CacheFixture() File.WriteAllText(Path.Combine(PayloadDirectory, "run.sh"), "#!/bin/sh\nexit 0\n"); File.SetUnixFileMode(Path.Combine(PayloadDirectory, "run.sh"), ExecutableMode); Archive = Path.Combine(Root, "shared.zip"); - File.WriteAllText(Archive, "opaque archive bytes"); + WriteArchive(Archive, [("shared.txt", "shared payload", ArchiveFileMode)]); Queues = [new HelixQueue { QueueId = "linux.test.open", Architecture = "AMD64", OperatingSystemGroup = "linux", IsAvailable = true, @@ -67,6 +73,20 @@ internal CacheFixture() }; } + internal static void WriteArchive(string path, (string Name, string Content, int Mode)[] entries, bool repack = false) + { + using FileStream file = File.Create(path); + using ZipArchive archive = new(file, ZipArchiveMode.Create); + foreach ((string name, string content, int mode) in repack ? entries.Reverse() : entries) + { + ZipArchiveEntry entry = archive.CreateEntry(name, repack ? CompressionLevel.NoCompression : CompressionLevel.Optimal); + entry.LastWriteTime = new DateTimeOffset(repack ? 2025 : 2024, 1, 1, 0, 0, 0, TimeSpan.Zero); + entry.ExternalAttributes = mode << 16; + using StreamWriter writer = new(entry.Open()); + writer.Write(content); + } + } + internal string Write(string name, T value, JsonTypeInfo type) { string path = Path.Combine(Root, name); @@ -111,7 +131,7 @@ internal void Store(ExecutionRecord execution, bool success = true) => internal string Prepare(string name = "shadow") { string output = Path.Combine(Root, name); - ShadowValidation.Prepare(new Cache(CacheDirectory), output, Archive, Archive, Queues[0].QueueId, new LinuxEnvironment(Queues)); + ShadowValidation.Prepare(new Cache(CacheDirectory), output, Archive, Archive, Queues[0].QueueId, Archive, new LinuxEnvironment(Queues)); return output; } diff --git a/src/tools/helix-cache/tests/CacheTests.cs b/src/tools/helix-cache/tests/CacheTests.cs index c75d891bcbdb63..63c38292d6d4d6 100644 --- a/src/tools/helix-cache/tests/CacheTests.cs +++ b/src/tools/helix-cache/tests/CacheTests.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Text.Json; using System.Text.Json.Nodes; @@ -80,6 +81,7 @@ public void OpaqueProvisioningPreservesNullArrayValues() [InlineData("timeout")] [InlineData("exitCode")] [InlineData("environment")] + [InlineData("commandGenerator")] public void ExecutionChangesInvalidateKey(string field) { using CacheFixture f = new(); @@ -92,6 +94,7 @@ public void ExecutionChangesInvalidateKey(string field) "timeout" => f.Item with { TimeoutSeconds = 61 }, "exitCode" => f.Item with { ExpectedExitCode = 100 }, "environment" => f.Item with { Environment = new() { ["FIRST"] = "changed" } }, + "commandGenerator" => f.Item with { CommandGeneratorDigest = new string('a', 64) }, _ => throw new InvalidOperationException(), }; Assert.NotEqual(baseline, f.Lookup().Execution.Key); @@ -174,9 +177,9 @@ public void SharedPayloadBytesDestinationAndOverlayOrderAffectKey() { using CacheFixture f = new(); string baseline = f.Lookup().Execution.Key; - File.WriteAllText(f.Archive, "new archive"); + CacheFixture.WriteArchive(f.Archive, [("shared.txt", "changed payload", CacheFixture.ArchiveFileMode)]); Assert.NotEqual(baseline, f.Lookup().Execution.Key); - File.WriteAllText(f.Archive, "opaque archive bytes"); + CacheFixture.WriteArchive(f.Archive, [("shared.txt", "shared payload", CacheFixture.ArchiveFileMode)]); f.Item.CorrelationPayloads[0] = f.Item.CorrelationPayloads[0] with { Destination = "other" }; Assert.NotEqual(baseline, f.Lookup().Execution.Key); f.Item.CorrelationPayloads.Add(new Payload { Path = f.Archive, Destination = "second" }); @@ -185,6 +188,123 @@ public void SharedPayloadBytesDestinationAndOverlayOrderAffectKey() Assert.NotEqual(before, f.Lookup().Execution.Key); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RepackedArchivesReuseFrozenBytesAndResults(bool workItemPayload) + { + using CacheFixture f = new(); + if (workItemPayload) + { + f.Item = f.Item with { Payload = new Payload { Path = f.Archive } }; + } + (string, string, int)[] entries = + [ + ("run.sh", "#!/bin/sh\nexit 0\n", CacheFixture.ArchiveExecutableMode), + ("data.txt", "payload", CacheFixture.ArchiveFileMode), + ("empty/", "", CacheFixture.ArchiveDirectoryMode), + ("link", "data.txt", CacheFixture.ArchiveLinkMode), + ]; + CacheFixture.WriteArchive(f.Archive, entries); + byte[] original = File.ReadAllBytes(f.Archive); + LookupEntry first = f.Lookup(); + string frozen = first.Execution.CorrelationPayloadPaths[0]; + Assert.Equal(original, File.ReadAllBytes(f.Archive)); + f.CreateResults(first.Execution); + f.Store(first.Execution); + + CacheFixture.WriteArchive(f.Archive, entries, repack: true); + Assert.False(original.SequenceEqual(File.ReadAllBytes(f.Archive))); + LookupEntry second = f.Lookup(); + Assert.Equal(first.Execution.Key, second.Execution.Key); + Assert.Equal(LookupStatus.Hit, second.Status); + Assert.Equal(frozen, second.Execution.CorrelationPayloadPaths[0]); + using ZipArchive archive = ZipFile.OpenRead(frozen); + Assert.Equal(["data.txt", "empty/", "link", "run.sh"], archive.Entries.Select(e => e.FullName)); + Assert.All(archive.Entries, e => Assert.Equal(new DateTime(2000, 1, 1), e.LastWriteTime.DateTime)); + foreach ((string name, string content, int mode) in entries) + { + ZipArchiveEntry entry = archive.GetEntry(name)!; + Assert.Equal(mode << 16, entry.ExternalAttributes); + using StreamReader reader = new(entry.Open()); + Assert.Equal(content, reader.ReadToEnd()); + } + } + + [Theory] + [InlineData("content")] + [InlineData("mode")] + [InlineData("emptyDirectory")] + [InlineData("linkTarget")] + public void ArchiveExecutionChangesInvalidateKey(string change) + { + using CacheFixture f = new(); + (string, string, int)[] entries = + [ + ("run.sh", "original", CacheFixture.ArchiveExecutableMode), + ("link", "run.sh", CacheFixture.ArchiveLinkMode), + ]; + CacheFixture.WriteArchive(f.Archive, entries); + string before = f.Lookup().Execution.Key; + entries = change switch + { + "content" => [("run.sh", "changed", CacheFixture.ArchiveExecutableMode), entries[1]], + "mode" => [("run.sh", "original", CacheFixture.ArchiveFileMode), entries[1]], + "emptyDirectory" => [.. entries, ("empty/", "", CacheFixture.ArchiveDirectoryMode)], + "linkTarget" => [entries[0], ("link", "other", CacheFixture.ArchiveLinkMode)], + _ => throw new InvalidOperationException(), + }; + CacheFixture.WriteArchive(f.Archive, entries); + Assert.NotEqual(before, f.Lookup().Execution.Key); + } + + [Theory] + [InlineData("../outside", "other")] + [InlineData("/absolute", "other")] + [InlineData("a//b", "other")] + [InlineData("a\\b", "other")] + [InlineData("same", "same")] + [InlineData("same", "same/")] + [InlineData("parent", "parent/child")] + public void AmbiguousArchivePathsAreRejected(string first, string second) + { + using CacheFixture f = new(); + CacheFixture.WriteArchive(f.Archive, [(first, "", CacheFixture.ArchiveFileMode), (second, "", CacheFixture.ArchiveFileMode)]); + f.Lookup(success: false); + } + + [Fact] + public void InvalidArchivesAreRejected() + { + using CacheFixture f = new(); + File.WriteAllText(f.Archive, "not a ZIP archive"); + f.Lookup(success: false); + } + + [Fact] + public void NativePrepareFingerprintsTheCommandGeneratorPackage() + { + using CacheFixture f = new(); + string package = Path.Combine(f.Root, "sdk.nupkg"); + File.WriteAllText(package, "command generator"); + string first = Prepare("first"); + string relocated = Path.Combine(f.Root, "relocated.nupkg"); + File.Copy(package, relocated); + package = relocated; + Assert.Equal(first, Prepare("relocated")); + File.AppendAllText(package, " changed"); + Assert.NotEqual(first, Prepare("changed")); + + string Prepare(string name) + { + string output = Path.Combine(f.Root, name); + f.Invoke(true, "prepare", "--cache", f.CacheDirectory, "--output", output, "--payload", f.Archive, + "--runtime", f.Archive, "--queue", f.Queues[0].QueueId, "--helix-sdk", package, + "--queue-info", f.Write("queues.json", f.Queues, HelixJsonContext.Default.ListHelixQueue)); + return CacheFixture.ShadowLookup(output).Execution.Key; + } + } + [Theory] [InlineData("completed")] [InlineData("infrastructure")] @@ -338,6 +458,7 @@ public void LinksAndNestedCachesAreRejected(bool link) [InlineData("nullCommand")] [InlineData("timeout")] [InlineData("nullCorrelation")] + [InlineData("commandGenerator")] public void InvalidManifestsAreRejected(string mutation) { using CacheFixture f = new(); @@ -353,6 +474,7 @@ public void InvalidManifestsAreRejected(string mutation) case "nullCommand": item["command"] = null; break; case "timeout": item["timeoutSeconds"] = 0; break; case "nullCorrelation": item["correlationPayloads"]![0] = null; break; + case "commandGenerator": item["commandGeneratorDigest"] = "invalid"; break; default: throw new InvalidOperationException(); } string path = Path.Combine(f.Root, "invalid.json"); @@ -415,7 +537,7 @@ public void NativePrepareWritesFrozenSubmissionProperties() using CacheFixture f = new(); string output = Path.Combine(f.Root, "native-shadow"); f.Invoke(true, "prepare", "--cache", f.CacheDirectory, "--output", output, "--payload", f.Archive, - "--runtime", f.Archive, "--queue", f.Queues[0].QueueId, + "--runtime", f.Archive, "--queue", f.Queues[0].QueueId, "--helix-sdk", f.Archive, "--queue-info", f.Write("queues.json", f.Queues, HelixJsonContext.Default.ListHelixQueue)); LookupEntry entry = CacheFixture.ShadowLookup(output); Assert.Equal(LookupStatus.Miss, entry.Status); From 0e162be9662e2c3e4cc68d82b14b936720d5d15a Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Tue, 27 Oct 2026 18:42:06 +0000 Subject: [PATCH 8/8] Expand Helix cache shadow validation to four suites Submit fast library suites in one Helix job with a shared runtime snapshot. Record per-suite lookups and outcomes, and validate every report and comparison before admitting results. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ee547d49-5419-4268-b01f-6b2757f99953 --- .../libraries/helix-cache-shadow.yml | 3 + eng/testing/helix-cache-shadow.proj | 18 +- src/tools/helix-cache/README.md | 21 +- src/tools/helix-cache/src/CommandLine.cs | 14 +- src/tools/helix-cache/src/Models.cs | 10 +- src/tools/helix-cache/src/Program.cs | 7 +- src/tools/helix-cache/src/ShadowValidation.cs | 231 ++++++++++++------ src/tools/helix-cache/tests/CacheFixture.cs | 8 +- src/tools/helix-cache/tests/CacheTests.cs | 63 +++++ src/tools/helix-cache/tests/ShadowTests.cs | 196 +++++++++++++-- 10 files changed, 451 insertions(+), 120 deletions(-) diff --git a/eng/pipelines/libraries/helix-cache-shadow.yml b/eng/pipelines/libraries/helix-cache-shadow.yml index 0909d5a9f66ec3..2d0e81d456a6f1 100644 --- a/eng/pipelines/libraries/helix-cache-shadow.yml +++ b/eng/pipelines/libraries/helix-cache-shadow.yml @@ -47,6 +47,9 @@ steps: -nologo -getProperty:HelixCacheSdkPackage)" "$executable" prepare --cache "$cache" --output "$output" \ --payload "$archives/System.Memory.Data.Tests.zip" \ + --payload "$archives/System.Runtime.CompilerServices.Unsafe.Tests.zip" \ + --payload "$archives/System.IO.Hashing.Tests.zip" \ + --payload "$archives/System.Text.Encodings.Web.Tests.zip" \ --runtime "$(Build.SourcesDirectory)/artifacts/helix/runtime/test-runtime-$settings.zip" \ --queue AzureLinux.3.Amd64.Open --helix-sdk "$sdk" diff --git a/eng/testing/helix-cache-shadow.proj b/eng/testing/helix-cache-shadow.proj index 73b4a2bc98cea5..65431ac8d67c17 100644 --- a/eng/testing/helix-cache-shadow.proj +++ b/eng/testing/helix-cache-shadow.proj @@ -8,19 +8,15 @@ - - $(ShadowPayload) - $(ShadowCommand) - $(ShadowPostCommand) - $(ShadowTimeout) - - - + + + @@ -44,7 +40,7 @@ Lines="$(ShadowJobId)" Overwrite="true" /> - 1 + @(ShadowWorkItem->Count()) $(ShadowCancellationToken) diff --git a/src/tools/helix-cache/README.md b/src/tools/helix-cache/README.md index 6f223feb5be3d3..a520558818ea5e 100644 --- a/src/tools/helix-cache/README.md +++ b/src/tools/helix-cache/README.md @@ -152,19 +152,27 @@ Queue `eng/pipelines/runtime.yml` with its `helixCacheShadow` boolean parameter enabled. This draft temporarily defaults to **true** for validation; disable it before merging. The additional steps run only in the public project's Linux x64 `CoreCLR_AllSubsets` job, after its normal submissions. -They use the existing build's `System.Memory.Data.Tests.zip` and runtime archive -and submit one additional work item to `AzureLinux.3.Amd64.Open`. +They use the existing build's test archives for `System.Memory.Data.Tests`, +`System.Runtime.CompilerServices.Unsafe.Tests`, `System.IO.Hashing.Tests`, and +`System.Text.Encodings.Web.Tests`. All four work items are submitted in one +additional job to `AzureLinux.3.Amd64.Open`, sharing one frozen runtime archive. `Cache@2` restores `$(Pipeline.Workspace)/.helix-cache/entries`, using a rolling per-build snapshot key and a stable restore prefix, as the compiler caches do. The cache task's build ID is **not** part of an execution digest. Only completed entries are persisted; frozen input snapshots are recreated in each job. Existing Azure Pipelines branch/PR cache scoping applies. +An Azure restore hit is not a native result hit. Check the post-job cache-save +log before concluding that a new snapshot persisted: the agent can print +`Failed to cache item.` followed by `Saved item.` while leaving the task green. All orchestration runs in the C# NativeAOT executable. Its `prepare` command records the full command, upload post-command, timeout, queue and SHA-256 digest of the pinned Helix SDK package, runs native lookup, and exports frozen paths to -`submission.props`. `--helix-sdk` must identify that installed SDK's `.nupkg`. +`submission.props`. Repeat `--payload` for each suite; work-item names come from +archive basenames and must be unique (case-insensitively), using ASCII letters, +digits, dots, underscores or hyphens, excluding empty names, `.` and `..`. +`--helix-sdk` must identify that installed SDK's `.nupkg`. `eng/testing/helix-cache-shadow.proj` submits those exact archives through the pinned Helix SDK without injecting extra reporters. The SDK generates its own command wrapper, including chmod and exit-code handling; its package digest @@ -184,12 +192,14 @@ passes the report through native admission, and compares any hit with the fresh test identities/outcomes/skip reasons. Ordering, worker directory prefixes and execution timings are ignored. The resolved physical queue ID and submitted alias are preserved in provenance alongside the original cached job. -On a hit, mismatched outcomes fail validation before store. Every hit still +Every suite's report and comparison must pass before any suite is stored. +Each entry is published atomically, not the entire batch. Every hit still executes on Helix; no cached result is published as a new run. Evidence is included in the existing build-log artifact under `artifacts/log//helix-cache-shadow/`, including `lookup.json`, -`execution.json`, `work-item.json`, `summary.json`, and fresh results. Shadow +`summary.json` with per-suite keys, hit/miss status, test counts and comparisons, +and `work-items//{execution.json,work-item.json,results/}`. Shadow errors are visible but use `continueOnError`, so they do not replace or weaken the normal test decision. Internal queues and authentication are not supported by this first shadow lane. @@ -202,6 +212,7 @@ artifacts/tools/helix-cache/helix-cache prepare \ --cache artifacts/tmp/helix-cache \ --output artifacts/log/Release/helix-cache-shadow \ --payload artifacts/helix/tests/linux.AnyCPU.Release/System.Memory.Data.Tests.zip \ + --payload artifacts/helix/tests/linux.AnyCPU.Release/System.IO.Hashing.Tests.zip \ --runtime artifacts/helix/runtime/test-runtime-net11.0-linux-Release-x64.zip \ --queue AzureLinux.3.Amd64.Open --helix-sdk "$sdk" dotnet msbuild eng/testing/helix-cache-shadow.proj -t:Validate \ diff --git a/src/tools/helix-cache/src/CommandLine.cs b/src/tools/helix-cache/src/CommandLine.cs index 2e5109ce19aa2a..5252a5e5354d36 100644 --- a/src/tools/helix-cache/src/CommandLine.cs +++ b/src/tools/helix-cache/src/CommandLine.cs @@ -12,7 +12,7 @@ namespace HelixCache; internal abstract record Command(string CacheDirectory); internal sealed record LookupCommand(string Manifest, string CacheDirectory, string Output, string? QueueInfo) : Command(CacheDirectory); internal sealed record StoreCommand(string Manifest, string CacheDirectory, string Results) : Command(CacheDirectory); -internal sealed record PrepareCommand(string CacheDirectory, string Output, string Payload, string Runtime, string Queue, string HelixSdk, string? QueueInfo) : Command(CacheDirectory); +internal sealed record PrepareCommand(string CacheDirectory, string Output, List Payloads, string Runtime, string Queue, string HelixSdk, string? QueueInfo) : Command(CacheDirectory); internal sealed record CollectCommand(string CacheDirectory, string Output, TimeSpan Wait) : Command(CacheDirectory); internal static class CommandLine @@ -20,7 +20,7 @@ internal static class CommandLine internal const string Usage = """ Usage: helix-cache lookup submission.json --cache DIRECTORY --output lookup.json [--queue-info queues.json] helix-cache store execution.json --results DIRECTORY --cache DIRECTORY - helix-cache prepare --cache DIRECTORY --output DIRECTORY --payload tests.zip --runtime runtime.zip --queue QUEUE --helix-sdk SDK.nupkg [--queue-info queues.json] + helix-cache prepare --cache DIRECTORY --output DIRECTORY --payload tests.zip [--payload more-tests.zip ...] --runtime runtime.zip --queue QUEUE --helix-sdk SDK.nupkg [--queue-info queues.json] helix-cache collect --cache DIRECTORY --output DIRECTORY [--wait-seconds SECONDS] Only Linux x64 queues and digest-pinned container references are supported. """; @@ -39,6 +39,7 @@ internal static Command Parse(string[] args) throw new InvalidDataException($"Missing manifest for '{args[0]}'.\n{Usage}"); } Dictionary options = new(StringComparer.Ordinal); + List payloads = []; for (int i = firstOption; i < args.Length; i += 2) { string name = args[i]; @@ -47,7 +48,11 @@ internal static Command Parse(string[] args) { throw new InvalidDataException($"Expected an option and value at '{name}'.\n{Usage}"); } - if (!options.TryAdd(name, args[i + 1])) + if (args[0] == "prepare" && name == "--payload") + { + payloads.Add(args[i + 1]); + } + else if (!options.TryAdd(name, args[i + 1])) { throw new InvalidDataException($"Duplicate option '{name}'."); } @@ -58,7 +63,8 @@ internal static Command Parse(string[] args) { "lookup" => new LookupCommand(args[1], cache, Required("--output"), Optional("--queue-info")), "store" => new StoreCommand(args[1], cache, Required("--results")), - "prepare" => new PrepareCommand(cache, Required("--output"), Required("--payload"), + "prepare" => new PrepareCommand(cache, Required("--output"), + payloads.Count > 0 ? payloads : throw new InvalidDataException($"Missing required option '--payload'.\n{Usage}"), Required("--runtime"), Required("--queue"), Required("--helix-sdk"), Optional("--queue-info")), "collect" => new CollectCommand(cache, Required("--output"), ParseWait(Optional("--wait-seconds"))), _ => throw new UnreachableException(), diff --git a/src/tools/helix-cache/src/Models.cs b/src/tools/helix-cache/src/Models.cs index f46d2a2055f85d..8313ea27214b13 100644 --- a/src/tools/helix-cache/src/Models.cs +++ b/src/tools/helix-cache/src/Models.cs @@ -242,10 +242,16 @@ internal sealed class StoredEntry internal sealed record ShadowSummary { - public required string Key { get; init; } - public required LookupStatus Lookup { get; init; } public required string JobId { get; init; } public string Mode { get; init; } = "shadow"; + public required List WorkItems { get; init; } +} + +internal sealed record ShadowWorkItemSummary +{ + public required string Name { get; init; } + public required string Key { get; init; } + public required LookupStatus Lookup { get; init; } public int? Tests { get; init; } public OutcomeComparison? Comparison { get; init; } public string? CachedJobId { get; init; } diff --git a/src/tools/helix-cache/src/Program.cs b/src/tools/helix-cache/src/Program.cs index 067c190bd0ff6e..8837e638c58b43 100644 --- a/src/tools/helix-cache/src/Program.cs +++ b/src/tools/helix-cache/src/Program.cs @@ -86,9 +86,12 @@ private static async Task ExecuteAsync(Command command, CancellationToken cancel case PrepareCommand prepare: { LinuxEnvironment environment = await LinuxEnvironment.LoadAsync(prepare.QueueInfo, helix, cancellationToken); - LookupEntry entry = ShadowValidation.Prepare(cache, prepare.Output, prepare.Payload, prepare.Runtime, prepare.Queue, + LookupResult result = ShadowValidation.Prepare(cache, prepare.Output, prepare.Payloads, prepare.Runtime, prepare.Queue, prepare.HelixSdk, environment); - Console.WriteLine($"Helix cache shadow: {entry.Status.ToString().ToLowerInvariant()} {entry.Execution.Key}; execution will NOT be skipped."); + foreach (LookupEntry entry in result.Entries) + { + Console.WriteLine($"Helix cache shadow: {entry.Execution.Inputs.Name}: {entry.Status.ToString().ToLowerInvariant()} {entry.Execution.Key}; execution will NOT be skipped."); + } break; } case CollectCommand collect: diff --git a/src/tools/helix-cache/src/ShadowValidation.cs b/src/tools/helix-cache/src/ShadowValidation.cs index cc0908df29e410..b7a8e2e1c0d75f 100644 --- a/src/tools/helix-cache/src/ShadowValidation.cs +++ b/src/tools/helix-cache/src/ShadowValidation.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; @@ -15,16 +16,15 @@ namespace HelixCache; internal static class ShadowValidation { - internal const string WorkItem = "System.Memory.Data.Tests"; internal const string Command = "bash ./RunTests.sh --runtime-path \"$HELIX_CORRELATION_PAYLOAD\""; internal const string PostCommand = "cp testResults.xml \"$HELIX_WORKITEM_UPLOAD_ROOT/testResults.xml\""; - internal static LookupEntry Prepare(Cache cache, string output, string payload, string runtime, string queue, string helixSdk, + internal static LookupResult Prepare(Cache cache, string output, List payloads, string runtime, string queue, string helixSdk, LinuxEnvironment environment) { - payload = Path.GetFullPath(payload); + payloads = payloads.Select(Path.GetFullPath).ToList(); runtime = Path.GetFullPath(runtime); - foreach (string archive in new[] { payload, runtime }) + foreach (string archive in payloads.Append(runtime)) { if (!File.Exists(archive) || !archive.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) { @@ -36,31 +36,50 @@ internal static LookupEntry Prepare(Cache cache, string output, string payload, { throw new IOException($"The shadow output directory already exists: {output}."); } + HashSet names = new(StringComparer.OrdinalIgnoreCase); + foreach (string payload in payloads) + { + string name = Path.GetFileNameWithoutExtension(payload); + ValidateName(name); + if (!names.Add(name)) + { + throw new InvalidDataException($"Duplicate shadow work item name: {name}."); + } + } + if (payloads.Count == 0) + { + throw new InvalidDataException("A shadow submission requires at least one test payload."); + } + string sdkDigest = Snapshot.HashFile(helixSdk); Submission submission = new() { SchemaVersion = Cache.ProtocolVersion, - WorkItems = [new WorkItem + WorkItems = payloads.Select(payload => new WorkItem { - Name = WorkItem, + Name = Path.GetFileNameWithoutExtension(payload), Queues = [queue], Payload = new Payload { Path = payload }, CorrelationPayloads = [new Payload { Path = runtime }], Command = Command, PostCommands = [PostCommand], - CommandGeneratorDigest = Snapshot.HashFile(helixSdk), + CommandGeneratorDigest = sdkDigest, TimeoutSeconds = 900, ExpectedExitCode = 0, - }], + }).ToList(), }; LookupResult lookup = cache.Lookup(submission, output, environment); - LookupEntry entry = lookup.Entries.Single(); Directory.CreateDirectory(output); File.WriteAllText(Path.Combine(output, "submission.json"), JsonSerializer.Serialize(submission, CacheJsonContext.Default.Submission)); File.WriteAllText(Path.Combine(output, "lookup.json"), JsonSerializer.Serialize(lookup, CacheJsonContext.Default.LookupResult)); - File.WriteAllText(Path.Combine(output, "execution.json"), JsonSerializer.Serialize(entry.Execution, CacheJsonContext.Default.ExecutionRecord)); - WriteSubmissionProperties(output, entry.Execution); - return entry; + foreach (LookupEntry entry in lookup.Entries) + { + string directory = Directory.CreateDirectory(Path.Combine(output, "work-items", entry.Execution.Inputs.Name)).FullName; + File.WriteAllText(Path.Combine(directory, "execution.json"), + JsonSerializer.Serialize(entry.Execution, CacheJsonContext.Default.ExecutionRecord)); + } + WriteSubmissionProperties(output, lookup); + return lookup; } internal static async Task CollectAsync(Cache cache, string output, TimeSpan wait, HelixClient helix, @@ -70,103 +89,157 @@ internal static async Task CollectAsync(Cache cache, string outpu LookupResult lookup = JsonSerializer.Deserialize(File.ReadAllText(Path.Combine(output, "lookup.json")), CacheJsonContext.Default.LookupResult) ?? throw new InvalidDataException("Missing shadow lookup result."); Cache.ValidateVersion(lookup.SchemaVersion); - if (lookup.Entries.Count != 1 || lookup.Entries[0] is not LookupEntry entry || - entry.Status is not (LookupStatus.Hit or LookupStatus.Miss)) + if (lookup.Entries.Count == 0) + { + throw new InvalidDataException("Expected at least one shadow lookup entry."); + } + HashSet names = new(StringComparer.OrdinalIgnoreCase); + foreach (LookupEntry entry in lookup.Entries) { - throw new InvalidDataException("Expected exactly one shadow lookup entry with hit or miss status."); + if (entry is null || entry.Status is not (LookupStatus.Hit or LookupStatus.Miss)) + { + throw new InvalidDataException("Expected shadow lookup entries with hit or miss status."); + } + entry.Execution.Validate(); + ValidateName(entry.Execution.Inputs.Name); + if (!names.Add(entry.Execution.Inputs.Name)) + { + throw new InvalidDataException($"Duplicate shadow work item name: {entry.Execution.Inputs.Name}."); + } } - ExecutionRecord execution = entry.Execution; - execution.Validate(); - TestRun? cached = entry.Status == LookupStatus.Hit ? cache.ReadResults(execution) : null; + TestRun?[] cached = lookup.Entries.Select(entry => + entry.Status == LookupStatus.Hit ? cache.ReadResults(entry.Execution) : null).ToArray(); if (!Guid.TryParse(File.ReadAllText(Path.Combine(output, "job-id.txt")).Trim(), out Guid job)) { throw new InvalidDataException("Expected a Helix job GUID in job-id.txt."); } string jobId = job.ToString(); - ShadowSummary summary = new() { Key = execution.Key, Lookup = entry.Status, JobId = jobId }; + ShadowSummary summary = new() + { + JobId = jobId, + WorkItems = lookup.Entries.Select(entry => new ShadowWorkItemSummary + { + Name = entry.Execution.Inputs.Name, + Key = entry.Execution.Key, + Lookup = entry.Status, + }).ToList(), + }; string summaryPath = Path.Combine(output, "summary.json"); - File.WriteAllText(summaryPath, JsonSerializer.Serialize(summary, CacheJsonContext.Default.ShadowSummary)); + WriteSummary(); HelixJob jobDetails = await helix.GetJobAsync(jobId, cancellationToken); string submittedQueue = string.IsNullOrEmpty(jobDetails.QueueAlias) ? jobDetails.QueueId : jobDetails.QueueAlias; if (jobDetails.Name != jobId || - !string.Equals(submittedQueue, execution.Inputs.EnvironmentFingerprint.QueueId, StringComparison.OrdinalIgnoreCase)) + lookup.Entries.Any(entry => !string.Equals(submittedQueue, + entry.Execution.Inputs.EnvironmentFingerprint.QueueId, StringComparison.OrdinalIgnoreCase))) { throw new InvalidDataException("Helix returned a different job or target queue."); } - HelixWorkItem item = await helix.WaitForCompletionAsync(jobId, execution.Inputs.Name, wait, cancellationToken); - File.WriteAllText(Path.Combine(output, "work-item.json"), JsonSerializer.Serialize(item, HelixJsonContext.Default.HelixWorkItem)); - if (item.State is not ("Finished" or "Passed") || item.FailureReason is not null || - item.Errors is { Count: > 0 } || item.ExitCode is null || item.ExitCode != execution.Inputs.ExpectedExitCode) + Stopwatch elapsed = Stopwatch.StartNew(); + for (int i = 0; i < lookup.Entries.Count; i++) { - throw new InvalidDataException($"Helix work item did not complete successfully; see {Path.Combine(output, "work-item.json")}."); - } + ExecutionRecord execution = lookup.Entries[i].Execution; + TimeSpan remaining = wait - elapsed.Elapsed; + if (remaining <= TimeSpan.Zero) + { + throw new TimeoutException("Timed out waiting for the shadow job."); + } + HelixWorkItem item = await helix.WaitForCompletionAsync(jobId, execution.Inputs.Name, remaining, cancellationToken); + string directory = Directory.CreateDirectory(Path.Combine(output, "work-items", execution.Inputs.Name)).FullName; + string workItemPath = Path.Combine(directory, "work-item.json"); + File.WriteAllText(workItemPath, JsonSerializer.Serialize(item, HelixJsonContext.Default.HelixWorkItem)); + if (item.State is not ("Finished" or "Passed") || item.FailureReason is not null || + item.Errors is { Count: > 0 } || item.ExitCode is null || item.ExitCode != execution.Inputs.ExpectedExitCode) + { + throw new InvalidDataException($"Helix work item did not complete successfully; see {workItemPath}."); + } - string results = Path.Combine(output, "results"); - await helix.DownloadResultsAsync(item, results, cancellationToken); - ResultReceipt receipt = new() - { - SchemaVersion = Cache.ProtocolVersion, - Completed = true, - InfrastructureSucceeded = true, - ExitCode = item.ExitCode.Value, - JobId = jobId, - WorkItemName = item.Name, - TargetQueue = execution.Inputs.TargetQueue, - ResultFiles = ["testResults.xml"], - }; - ExecutionProvenance provenance = new() - { - JobId = jobId, - QueueAlias = jobDetails.QueueAlias, - ResolvedQueueId = jobDetails.QueueId, - MachineName = item.MachineName, - State = item.State, - }; - File.WriteAllText(Path.Combine(results, "result.json"), JsonSerializer.Serialize(receipt, CacheJsonContext.Default.ResultReceipt)); - File.WriteAllText(Path.Combine(results, "provenance.json"), JsonSerializer.Serialize(provenance, CacheJsonContext.Default.ExecutionProvenance)); + string results = Path.Combine(directory, "results"); + await helix.DownloadResultsAsync(item, results, cancellationToken); + ResultReceipt receipt = new() + { + SchemaVersion = Cache.ProtocolVersion, + Completed = true, + InfrastructureSucceeded = true, + ExitCode = item.ExitCode.Value, + JobId = jobId, + WorkItemName = item.Name, + TargetQueue = execution.Inputs.TargetQueue, + ResultFiles = ["testResults.xml"], + }; + ExecutionProvenance provenance = new() + { + JobId = jobId, + QueueAlias = jobDetails.QueueAlias, + ResolvedQueueId = jobDetails.QueueId, + MachineName = item.MachineName, + State = item.State, + }; + File.WriteAllText(Path.Combine(results, "result.json"), JsonSerializer.Serialize(receipt, CacheJsonContext.Default.ResultReceipt)); + File.WriteAllText(Path.Combine(results, "provenance.json"), JsonSerializer.Serialize(provenance, CacheJsonContext.Default.ExecutionProvenance)); - TestRun fresh = Results.Read(results, execution); - summary = summary with - { - Tests = fresh.Summary.Total, - CachedJobId = cached?.Receipt.JobId, - Comparison = cached is null ? OutcomeComparison.NotApplicable : - cached.HasSameOutcomes(fresh) ? OutcomeComparison.Match : OutcomeComparison.Mismatch, - }; - if (summary.Comparison == OutcomeComparison.Mismatch) + TestRun fresh = Results.Read(results, execution); + TestRun? previous = cached[i]; + summary.WorkItems[i] = summary.WorkItems[i] with + { + Tests = fresh.Summary.Total, + CachedJobId = previous?.Receipt.JobId, + Comparison = previous is null ? OutcomeComparison.NotApplicable : + previous.HasSameOutcomes(fresh) ? OutcomeComparison.Match : OutcomeComparison.Mismatch, + }; + WriteSummary(); + if (summary.WorkItems[i].Comparison == OutcomeComparison.Mismatch) + { + throw new InvalidDataException($"Cached and fresh test outcomes differ for {execution.Inputs.Name}; see {summaryPath}."); + } + } + for (int i = 0; i < lookup.Entries.Count; i++) { - File.WriteAllText(summaryPath, JsonSerializer.Serialize(summary, CacheJsonContext.Default.ShadowSummary)); - throw new InvalidDataException($"Cached and fresh test outcomes differ; see {summaryPath}."); + ExecutionRecord execution = lookup.Entries[i].Execution; + cache.Store(execution, Path.Combine(output, "work-items", execution.Inputs.Name, "results")); + summary.WorkItems[i] = summary.WorkItems[i] with { Stored = true }; + WriteSummary(); } - cache.Store(execution, results); - summary = summary with { Stored = true }; - File.WriteAllText(summaryPath, JsonSerializer.Serialize(summary, CacheJsonContext.Default.ShadowSummary)); return summary; + + void WriteSummary() => File.WriteAllText(summaryPath, JsonSerializer.Serialize(summary, CacheJsonContext.Default.ShadowSummary)); } - private static void WriteSubmissionProperties(string output, ExecutionRecord execution) + private static void ValidateName(string name) { - XElement properties = new("PropertyGroup"); - foreach ((string name, string value) in new Dictionary - { - ["ShadowName"] = execution.Inputs.Name, - ["ShadowQueue"] = execution.Inputs.TargetQueue, - ["ShadowPayload"] = execution.PayloadPath, - ["ShadowRuntime"] = execution.CorrelationPayloadPaths[0], - ["ShadowCommand"] = execution.Inputs.Command, - ["ShadowPostCommand"] = execution.Inputs.PostCommands[0], - ["ShadowCommandGeneratorDigest"] = execution.Inputs.CommandGeneratorDigest!, - ["ShadowTimeout"] = TimeSpan.FromSeconds(execution.Inputs.TimeoutSeconds).ToString("c", CultureInfo.InvariantCulture), - }) + if (name.Length == 0 || name is "." or ".." || name.Any(c => !char.IsAsciiLetterOrDigit(c) && c is not ('.' or '_' or '-'))) + { + throw new InvalidDataException($"Shadow work item names must use ASCII letters, digits, '.', '_' or '-' and cannot be empty, '.' or '..': {name}."); + } + } + + private static void WriteSubmissionProperties(string output, LookupResult lookup) + { + ExecutionRecord first = lookup.Entries[0].Execution; + XElement properties = new("PropertyGroup", + Property("ShadowQueue", first.Inputs.TargetQueue), + Property("ShadowRuntime", first.CorrelationPayloadPaths[0]), + Property("ShadowCommandGeneratorDigest", first.Inputs.CommandGeneratorDigest!)); + XElement items = new("ItemGroup"); + foreach (LookupEntry entry in lookup.Entries) + { + ExecutionRecord execution = entry.Execution; + items.Add(new XElement("ShadowWorkItem", new XAttribute("Include", execution.Inputs.Name), + Property("PayloadArchive", execution.PayloadPath), + Property("Command", execution.Inputs.Command), + Property("PostCommands", execution.Inputs.PostCommands[0]), + Property("Timeout", TimeSpan.FromSeconds(execution.Inputs.TimeoutSeconds).ToString("c", CultureInfo.InvariantCulture)))); + } + new XDocument(new XElement("Project", properties, items)).Save(Path.Combine(output, "submission.props")); + + static XElement Property(string name, string value) { string escaped = value; foreach (char character in "%$@();'*?") { escaped = escaped.Replace(character.ToString(), $"%{(int)character:X2}", StringComparison.Ordinal); } - properties.Add(new XElement(name, escaped)); + return new XElement(name, escaped); } - new XDocument(new XElement("Project", properties)).Save(Path.Combine(output, "submission.props")); } } diff --git a/src/tools/helix-cache/tests/CacheFixture.cs b/src/tools/helix-cache/tests/CacheFixture.cs index 5a32c837b1173b..e6aaf9bf5a21ba 100644 --- a/src/tools/helix-cache/tests/CacheFixture.cs +++ b/src/tools/helix-cache/tests/CacheFixture.cs @@ -24,6 +24,7 @@ internal sealed class CacheFixture : IDisposable UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherExecute; internal const string FirstJob = "00000000-0000-0000-0000-000000000001"; internal const string SecondJob = "00000000-0000-0000-0000-000000000002"; + internal const string WorkItem = "shared"; internal const string Report = """ @@ -128,16 +129,19 @@ internal void Store(ExecutionRecord execution, bool success = true) => Invoke(success, "store", Write("execution.json", execution, CacheJsonContext.Default.ExecutionRecord), "--results", Path.Combine(Root, "results"), "--cache", CacheDirectory); - internal string Prepare(string name = "shadow") + internal string Prepare(string name = "shadow", params string[] payloads) { string output = Path.Combine(Root, name); - ShadowValidation.Prepare(new Cache(CacheDirectory), output, Archive, Archive, Queues[0].QueueId, Archive, new LinuxEnvironment(Queues)); + ShadowValidation.Prepare(new Cache(CacheDirectory), output, payloads.Length == 0 ? [Archive] : [.. payloads], + Archive, Queues[0].QueueId, Archive, new LinuxEnvironment(Queues)); return output; } internal static LookupEntry ShadowLookup(string output) => Assert.Single(Read(Path.Combine(output, "lookup.json"), CacheJsonContext.Default.LookupResult).Entries); + internal static string ShadowResults(string output, string name = WorkItem) => Path.Combine(output, "work-items", name, "results"); + internal string Invoke(bool success, params string[] arguments) { string? executable = Environment.GetEnvironmentVariable("HELIX_CACHE_EXECUTABLE"); diff --git a/src/tools/helix-cache/tests/CacheTests.cs b/src/tools/helix-cache/tests/CacheTests.cs index 63c38292d6d4d6..3e6f39f3d81def 100644 --- a/src/tools/helix-cache/tests/CacheTests.cs +++ b/src/tools/helix-cache/tests/CacheTests.cs @@ -9,6 +9,7 @@ using System.Text.Json; using System.Text.Json.Nodes; using System.Threading.Tasks; +using System.Xml.Linq; using Xunit; namespace HelixCache.Tests; @@ -525,12 +526,74 @@ public void HelpDoesNotRequireInputsOrCreateCache(string arguments) [InlineData("store missing.json --cache cache", "Missing required option '--results'")] [InlineData("collect --cache cache --output output --wait-seconds -1", "--wait-seconds")] [InlineData("collect --cache --output output", "Expected an option and value")] + [InlineData("prepare --cache cache --output output", "Missing required option '--payload'")] public void CommandLineErrorsIdentifyTheInvalidArgument(string arguments, string diagnostic) { InvalidDataException error = Assert.Throws(() => CommandLine.Parse(arguments.Split(' '))); Assert.Contains(diagnostic, error.Message); } + [Fact] + public void NativePrepareSupportsSeveralPayloadsAndSharesTheRuntimeSnapshot() + { + using CacheFixture f = new(); + string second = Path.Combine(f.Root, "Second.Tests.zip"); + File.Copy(f.Archive, second); + string output = Path.Combine(f.Root, "batch"); + f.Invoke(true, "prepare", "--cache", f.CacheDirectory, "--output", output, + "--payload", f.Archive, "--payload", second, + "--runtime", f.Archive, "--queue", f.Queues[0].QueueId, "--helix-sdk", f.Archive, + "--queue-info", f.Write("queues.json", f.Queues, HelixJsonContext.Default.ListHelixQueue)); + + LookupResult lookup = CacheFixture.Read(Path.Combine(output, "lookup.json"), CacheJsonContext.Default.LookupResult); + Assert.Equal(new[] { "shared", "Second.Tests" }, lookup.Entries.Select(e => e.Execution.Inputs.Name)); + Assert.All(lookup.Entries, entry => Assert.Equal(LookupStatus.Miss, entry.Status)); + Assert.Equal(lookup.Entries[0].Execution.CorrelationPayloadPaths, lookup.Entries[1].Execution.CorrelationPayloadPaths); + Assert.NotEqual(lookup.Entries[0].Execution.Key, lookup.Entries[1].Execution.Key); + Assert.All(lookup.Entries, entry => Assert.True(File.Exists( + Path.Combine(output, "work-items", entry.Execution.Inputs.Name, "execution.json")))); + XDocument props = XDocument.Load(Path.Combine(output, "submission.props")); + Assert.Equal(new[] { "shared", "Second.Tests" }, + props.Descendants("ShadowWorkItem").Select(item => item.Attribute("Include")!.Value)); + Assert.Equal(lookup.Entries.Select(e => e.Execution.PayloadPath), + props.Descendants("PayloadArchive").Select(item => item.Value)); + } + + [Theory] + [InlineData("shared.zip")] + [InlineData("SHARED.zip")] + public void NativePrepareRejectsDuplicateSuiteNamesBeforeCreatingSnapshots(string name) + { + using CacheFixture f = new(); + string payload = Path.Combine(Directory.CreateDirectory(Path.Combine(f.Root, "other")).FullName, name); + File.Copy(f.Archive, payload); + string output = Path.Combine(f.Root, "batch"); + f.Invoke(false, "prepare", "--cache", f.CacheDirectory, "--output", output, + "--payload", f.Archive, "--payload", payload, + "--runtime", f.Archive, "--queue", f.Queues[0].QueueId, "--helix-sdk", f.Archive, + "--queue-info", f.Write("queues.json", f.Queues, HelixJsonContext.Default.ListHelixQueue)); + Assert.False(Directory.Exists(output)); + Assert.False(Directory.Exists(f.CacheDirectory)); + } + + [Theory] + [InlineData(".zip")] + [InlineData("...zip")] + [InlineData("invalid name.zip")] + [InlineData("invalid;name.zip")] + public void NativePrepareRejectsUnsafeSuiteNamesBeforeCreatingSnapshots(string name) + { + using CacheFixture f = new(); + string payload = Path.Combine(f.Root, name); + File.Copy(f.Archive, payload); + string output = Path.Combine(f.Root, "batch"); + f.Invoke(false, "prepare", "--cache", f.CacheDirectory, "--output", output, "--payload", payload, + "--runtime", f.Archive, "--queue", f.Queues[0].QueueId, "--helix-sdk", f.Archive, + "--queue-info", f.Write("queues.json", f.Queues, HelixJsonContext.Default.ListHelixQueue)); + Assert.False(Directory.Exists(output)); + Assert.False(Directory.Exists(f.CacheDirectory)); + } + [Fact] public void NativePrepareWritesFrozenSubmissionProperties() { diff --git a/src/tools/helix-cache/tests/ShadowTests.cs b/src/tools/helix-cache/tests/ShadowTests.cs index 2cfe4734d90198..00c9bd90d2ff6b 100644 --- a/src/tools/helix-cache/tests/ShadowTests.cs +++ b/src/tools/helix-cache/tests/ShadowTests.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Json; @@ -17,13 +18,129 @@ namespace HelixCache.Tests; public sealed class ShadowTests { + [Fact] + public async Task SeveralSuitesShareOneJobAndCompareMixedHitsAndMisses() + { + using CacheFixture f = new(); + string secondPayload = Path.Combine(f.Root, "Second.Tests.zip"); + File.Copy(f.Archive, secondPayload); + string first = f.Prepare("first", f.Archive, secondPayload); + Dictionary reports = SuiteReports(CacheFixture.WorkItem, "Second.Tests"); + ShadowSummary cold = await CollectBatch(f, first, CacheFixture.FirstJob, reports); + Assert.Equal(2, cold.WorkItems.Count); + Assert.All(cold.WorkItems, item => + { + Assert.Equal(LookupStatus.Miss, item.Lookup); + Assert.Equal(OutcomeComparison.NotApplicable, item.Comparison); + Assert.Equal(2, item.Tests); + Assert.True(item.Stored); + }); + + string restored = Path.Combine(f.Root, "restored-cache"); + Snapshot.Copy(Path.Combine(f.CacheDirectory, "entries"), Path.Combine(restored, "entries")); + Directory.Move(f.CacheDirectory, Path.Combine(f.Root, "previous-cache")); + f.CacheDirectory = restored; + CacheFixture.WriteArchive(secondPayload, [("changed.txt", "changed input", CacheFixture.ArchiveFileMode)]); + string second = f.Prepare("second", f.Archive, secondPayload); + ShadowSummary mixed = await CollectBatch(f, second, CacheFixture.SecondJob, reports); + Assert.Equal(CacheFixture.SecondJob, mixed.JobId); + Assert.Equal(LookupStatus.Hit, mixed.WorkItems[0].Lookup); + Assert.Equal(OutcomeComparison.Match, mixed.WorkItems[0].Comparison); + Assert.Equal(CacheFixture.FirstJob, mixed.WorkItems[0].CachedJobId); + Assert.Equal(cold.WorkItems[0].Key, mixed.WorkItems[0].Key); + Assert.Equal(LookupStatus.Miss, mixed.WorkItems[1].Lookup); + Assert.Equal(OutcomeComparison.NotApplicable, mixed.WorkItems[1].Comparison); + Assert.Null(mixed.WorkItems[1].CachedJobId); + Assert.NotEqual(cold.WorkItems[1].Key, mixed.WorkItems[1].Key); + Assert.All(mixed.WorkItems, item => + { + Assert.True(item.Stored); + string results = CacheFixture.ShadowResults(second, item.Name); + Assert.Equal(reports[item.Name], File.ReadAllText(Path.Combine(results, "testResults.xml"))); + ResultReceipt receipt = CacheFixture.Read(Path.Combine(results, "result.json"), CacheJsonContext.Default.ResultReceipt); + Assert.Equal(item.Name, receipt.WorkItemName); + Assert.Equal(CacheFixture.SecondJob, receipt.JobId); + ExecutionProvenance provenance = CacheFixture.Read(Path.Combine(results, "provenance.json"), CacheJsonContext.Default.ExecutionProvenance); + Assert.Equal(item.Name + "-worker", provenance.MachineName); + }); + LookupResult lookup = CacheFixture.Read(Path.Combine(second, "lookup.json"), CacheJsonContext.Default.LookupResult); + Assert.Equal(CacheFixture.FirstJob, new Cache(f.CacheDirectory).ReadResults(lookup.Entries[0].Execution).Receipt.JobId); + Assert.Equal(CacheFixture.SecondJob, new Cache(f.CacheDirectory).ReadResults(lookup.Entries[1].Execution).Receipt.JobId); + } + + [Theory] + [InlineData("remote")] + [InlineData("report")] + [InlineData("mismatch")] + public async Task LaterSuiteFailuresPreventAdmissionOfTheWholeBatch(string failure) + { + using CacheFixture f = new(); + string secondPayload = Path.Combine(f.Root, "Second.Tests.zip"); + File.Copy(f.Archive, secondPayload); + string seeded = f.Prepare("seed", secondPayload); + await CollectBatch(f, seeded, CacheFixture.FirstJob, SuiteReports("Second.Tests")); + string output = f.Prepare("batch", f.Archive, secondPayload); + Dictionary reports = SuiteReports(CacheFixture.WorkItem, "Second.Tests"); + if (failure is "report" or "mismatch") + { + reports["Second.Tests"] = failure == "report" ? "" : + reports["Second.Tests"].Replace("Second.Tests.one", "different", StringComparison.Ordinal); + } + await Assert.ThrowsAsync(() => + CollectBatch(f, output, CacheFixture.SecondJob, reports, failure == "remote" ? "Second.Tests" : null)); + ShadowSummary summary = CacheFixture.Read(Path.Combine(output, "summary.json"), CacheJsonContext.Default.ShadowSummary); + Assert.All(summary.WorkItems, item => Assert.False(item.Stored)); + Assert.Equal(OutcomeComparison.NotApplicable, summary.WorkItems[0].Comparison); + if (failure == "mismatch") + { + Assert.Equal(OutcomeComparison.Mismatch, summary.WorkItems[1].Comparison); + } + Assert.Single(Directory.EnumerateFiles(f.CacheDirectory, "entry.json", SearchOption.AllDirectories)); + LookupResult lookup = CacheFixture.Read(Path.Combine(output, "lookup.json"), CacheJsonContext.Default.LookupResult); + Assert.Equal(CacheFixture.FirstJob, new Cache(f.CacheDirectory).ReadResults(lookup.Entries[1].Execution).Receipt.JobId); + Assert.Equal(LookupStatus.Miss, + Assert.Single(ShadowValidation.Prepare(new Cache(f.CacheDirectory), Path.Combine(f.Root, "retry"), + [f.Archive], f.Archive, f.Queues[0].QueueId, f.Archive, new LinuxEnvironment(f.Queues)).Entries).Status); + } + + [Theory] + [InlineData("duplicate")] + [InlineData("unsafe")] + [InlineData("status")] + public async Task InvalidLaterLookupEntriesAreRejectedBeforeContactingHelix(string invalid) + { + using CacheFixture f = new(); + string secondPayload = Path.Combine(f.Root, "Second.Tests.zip"); + File.Copy(f.Archive, secondPayload); + string output = f.Prepare("batch", f.Archive, secondPayload); + LookupResult lookup = CacheFixture.Read(Path.Combine(output, "lookup.json"), CacheJsonContext.Default.LookupResult); + ExecutionRecord execution = lookup.Entries[1].Execution; + ExecutionInputs inputs = execution.Inputs with + { + Name = invalid == "duplicate" ? CacheFixture.WorkItem : invalid == "unsafe" ? "../escape" : execution.Inputs.Name, + }; + lookup.Entries[1] = new LookupEntry + { + Status = invalid == "status" ? (LookupStatus)123 : LookupStatus.Miss, + Execution = execution with { Inputs = inputs, Key = ExecutionKey.Create(inputs) }, + }; + File.WriteAllText(Path.Combine(output, "lookup.json"), JsonSerializer.Serialize(lookup, CacheJsonContext.Default.LookupResult)); + File.WriteAllText(Path.Combine(output, "job-id.txt"), CacheFixture.FirstJob); + using HttpClient client = new(new Stub((_, _) => throw new InvalidOperationException("Unexpected HTTP request"))); + await Assert.ThrowsAsync(() => + ShadowValidation.CollectAsync(new Cache(f.CacheDirectory), output, TimeSpan.FromMinutes(1), new HelixClient(client))); + Assert.All(lookup.Entries, item => Assert.False(Directory.Exists(CacheFixture.ShadowResults(output, item.Execution.Inputs.Name)))); + Assert.Empty(Directory.EnumerateFiles(f.CacheDirectory, "entry.json", SearchOption.AllDirectories)); + } + [Fact] public async Task ColdStoreAndRestoredEntriesCompareActualOutcomes() { using CacheFixture f = new(); string first = f.Prepare(); await Collect(f, first, CacheFixture.FirstJob); - ShadowSummary cold = CacheFixture.Read(Path.Combine(first, "summary.json"), CacheJsonContext.Default.ShadowSummary); + ShadowWorkItemSummary cold = Assert.Single( + CacheFixture.Read(Path.Combine(first, "summary.json"), CacheJsonContext.Default.ShadowSummary).WorkItems); Assert.Equal(LookupStatus.Miss, cold.Lookup); Assert.Equal(2, cold.Tests); string restored = Path.Combine(f.Root, "restored-cache"); @@ -38,10 +155,11 @@ public async Task ColdStoreAndRestoredEntriesCompareActualOutcomes() """; await Collect(f, second, CacheFixture.SecondJob, report: reordered); - ShadowSummary summary = CacheFixture.Read(Path.Combine(second, "summary.json"), CacheJsonContext.Default.ShadowSummary); + ShadowSummary batch = CacheFixture.Read(Path.Combine(second, "summary.json"), CacheJsonContext.Default.ShadowSummary); + ShadowWorkItemSummary summary = Assert.Single(batch.WorkItems); Assert.Equal(OutcomeComparison.Match, summary.Comparison); Assert.Equal(CacheFixture.FirstJob, summary.CachedJobId); - Assert.Equal(CacheFixture.SecondJob, summary.JobId); + Assert.Equal(CacheFixture.SecondJob, batch.JobId); Assert.Equal(cold.Key, summary.Key); } @@ -67,7 +185,8 @@ public async Task MismatchesDoNotOverwriteCachedProvenance(string change) InvalidDataException error = await Assert.ThrowsAsync( () => Collect(f, second, CacheFixture.SecondJob, report: changed)); Assert.Contains("outcomes differ", error.Message); - ShadowSummary summary = CacheFixture.Read(Path.Combine(second, "summary.json"), CacheJsonContext.Default.ShadowSummary); + ShadowWorkItemSummary summary = Assert.Single( + CacheFixture.Read(Path.Combine(second, "summary.json"), CacheJsonContext.Default.ShadowSummary).WorkItems); Assert.Equal(OutcomeComparison.Mismatch, summary.Comparison); Assert.False(summary.Stored); Assert.Equal(CacheFixture.FirstJob, summary.CachedJobId); @@ -99,7 +218,7 @@ public async Task UnsuccessfulRemoteCompletionsAreNotAdmitted(string failure) InvalidDataException error = await Assert.ThrowsAsync( () => Collect(f, output, CacheFixture.FirstJob, item: item)); Assert.Contains("did not complete", error.Message); - Assert.False(Directory.Exists(Path.Combine(output, "results"))); + Assert.False(Directory.Exists(CacheFixture.ShadowResults(output))); Assert.Empty(Directory.EnumerateFiles(f.CacheDirectory, "entry.json", SearchOption.AllDirectories)); } @@ -127,7 +246,7 @@ public async Task QueueAliasesAndJobIdentityAreValidatedAndRetained( InvalidDataException error = await Assert.ThrowsAsync( () => Collect(f, output, CacheFixture.FirstJob, job: job)); Assert.Contains("job or target queue", error.Message); - Assert.False(Directory.Exists(Path.Combine(output, "results"))); + Assert.False(Directory.Exists(CacheFixture.ShadowResults(output))); Assert.Empty(Directory.EnumerateFiles(f.CacheDirectory, "entry.json", SearchOption.AllDirectories)); return; } @@ -163,7 +282,7 @@ public async Task ExactlyOneUploadedReportIsRequired(int count) InvalidDataException error = await Assert.ThrowsAsync( () => Collect(f, output, CacheFixture.FirstJob, files: files)); Assert.Contains("exactly one", error.Message); - Assert.False(Directory.Exists(Path.Combine(output, "results"))); + Assert.False(Directory.Exists(CacheFixture.ShadowResults(output))); } [Theory] @@ -187,7 +306,7 @@ public async Task CorruptionBetweenPrepareAndCollectIsNotOverwritten() string cachedReport = Path.Combine(CacheFixture.ShadowLookup(second).ResultsPath!, "testResults.xml"); File.WriteAllText(cachedReport, "corrupt"); await Assert.ThrowsAsync(() => Collect(f, second, CacheFixture.SecondJob)); - Assert.False(Directory.Exists(Path.Combine(second, "results"))); + Assert.False(Directory.Exists(CacheFixture.ShadowResults(second))); Assert.Equal("corrupt", File.ReadAllText(cachedReport)); } @@ -201,11 +320,11 @@ public async Task PollingRejectsUnknownStatesAndWrongIdentity(string state, bool { State = state, Job = wrongJob ? CacheFixture.SecondJob : CacheFixture.FirstJob, - Name = wrongName ? "other" : ShadowValidation.WorkItem, + Name = wrongName ? "other" : CacheFixture.WorkItem, }; using HttpClient client = Client(new HelixJob { Name = CacheFixture.FirstJob, QueueId = "queue" }, item); await Assert.ThrowsAsync(() => - new HelixClient(client).WaitForCompletionAsync(CacheFixture.FirstJob, ShadowValidation.WorkItem, TimeSpan.FromMinutes(1))); + new HelixClient(client).WaitForCompletionAsync(CacheFixture.FirstJob, CacheFixture.WorkItem, TimeSpan.FromMinutes(1))); } [Theory] @@ -223,12 +342,12 @@ public async Task PollingHonorsTimeoutAndCallerCancellation(bool cancelCaller) { cancellation.Cancel(); await Assert.ThrowsAnyAsync(() => new HelixClient(client).WaitForCompletionAsync( - CacheFixture.FirstJob, ShadowValidation.WorkItem, TimeSpan.FromMinutes(1), cancellation.Token)); + CacheFixture.FirstJob, CacheFixture.WorkItem, TimeSpan.FromMinutes(1), cancellation.Token)); } else { await Assert.ThrowsAsync(() => new HelixClient(client).WaitForCompletionAsync( - CacheFixture.FirstJob, ShadowValidation.WorkItem, TimeSpan.FromMilliseconds(20))); + CacheFixture.FirstJob, CacheFixture.WorkItem, TimeSpan.FromMilliseconds(20))); } } @@ -245,7 +364,7 @@ public async Task PollingWaitsForTerminalCompletion() }); })); HelixWorkItem completed = await new HelixClient(client).WaitForCompletionAsync( - CacheFixture.FirstJob, ShadowValidation.WorkItem, TimeSpan.FromMinutes(1)); + CacheFixture.FirstJob, CacheFixture.WorkItem, TimeSpan.FromMinutes(1)); Assert.Equal("Finished", completed.State); Assert.Equal(2, requests); } @@ -255,10 +374,57 @@ public async Task TransportTimeoutIsNotReportedAsTheJobDeadline() { using HttpClient client = new(new Stub((_, _) => throw new TaskCanceledException("HTTP request timed out"))); TaskCanceledException error = await Assert.ThrowsAsync(() => - new HelixClient(client).WaitForCompletionAsync(CacheFixture.FirstJob, ShadowValidation.WorkItem, TimeSpan.FromMinutes(1))); + new HelixClient(client).WaitForCompletionAsync(CacheFixture.FirstJob, CacheFixture.WorkItem, TimeSpan.FromMinutes(1))); Assert.Equal("HTTP request timed out", error.Message); } + private static Dictionary SuiteReports(params string[] names) => + names.ToDictionary(name => name, name => CacheFixture.Report.Replace("name=\"one\"", $"name=\"{name}.one\"", StringComparison.Ordinal)); + + private static async Task CollectBatch(CacheFixture f, string output, string jobId, + Dictionary reports, string? failingSuite = null) + { + File.WriteAllText(Path.Combine(output, "job-id.txt"), jobId); + int jobRequests = 0; + using HttpClient client = new(new Stub((request, _) => + { + string path = request.RequestUri!.AbsolutePath; + string[] segments = path.Split('/', StringSplitOptions.RemoveEmptyEntries); + HttpContent content; + if (path.EndsWith("/details", StringComparison.Ordinal)) + { + jobRequests++; + content = JsonContent.Create(new HelixJob { Name = jobId, QueueId = f.Queues[0].QueueId }, HelixJsonContext.Default.HelixJob); + } + else if (path.StartsWith("/reports/", StringComparison.Ordinal)) + { + content = new StringContent(reports[segments[^1]], Encoding.UTF8, "application/xml"); + } + else if (path.EndsWith("/files", StringComparison.Ordinal)) + { + content = JsonContent.Create(new List + { + new() { Name = "testResults.xml", Link = $"https://example.invalid/reports/{segments[^2]}" }, + }, HelixJsonContext.Default.ListHelixFile); + } + else + { + string name = segments[^1]; + Assert.Contains(name, reports.Keys); + content = JsonContent.Create(PassedItem(jobId) with + { + Name = name, + MachineName = name + "-worker", + State = name == failingSuite ? "Failed" : "Passed", + }, HelixJsonContext.Default.HelixWorkItem); + } + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = content }); + })); + ShadowSummary summary = await ShadowValidation.CollectAsync(new Cache(f.CacheDirectory), output, TimeSpan.FromMinutes(1), new HelixClient(client)); + Assert.Equal(1, jobRequests); + return summary; + } + private static async Task Collect(CacheFixture f, string output, string jobId, HelixJob? job = null, HelixWorkItem? item = null, List? files = null, string? report = null) { @@ -271,7 +437,7 @@ private static async Task Collect(CacheFixture f, string output, string jobId, private static HelixWorkItem PassedItem(string jobId) => new() { Job = jobId, - Name = ShadowValidation.WorkItem, + Name = CacheFixture.WorkItem, State = "Passed", ExitCode = 0, MachineName = "test-worker",