diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 3800429ba5a906a..569f96a2c63581f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,13 +3,13 @@ - + https://github.com/dotnet/arcade - 4b95fb1a9307265eb75f62d4937be50e6786e94e + 25da4e8f08fab05a39c27443df99b96d738f3c82 - + https://github.com/dotnet/arcade - 4b95fb1a9307265eb75f62d4937be50e6786e94e + 25da4e8f08fab05a39c27443df99b96d738f3c82 diff --git a/eng/Versions.props b/eng/Versions.props index c4109ee63278335..4a8c55a1db6c86d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -7,7 +7,7 @@ - 8.0.0-beta.26278.3 + 8.0.0-beta.26431.7 0x030902F0 diff --git a/eng/common/Get-GitHubAppToken.ps1 b/eng/common/Get-GitHubAppToken.ps1 new file mode 100644 index 000000000000000..ea776bd6bc28a82 --- /dev/null +++ b/eng/common/Get-GitHubAppToken.ps1 @@ -0,0 +1,165 @@ +# Mints a short-lived GitHub App installation access token by signing a JWT +# with a private key stored in Azure Key Vault (RSA, RS256). The signed JWT is +# exchanged with the GitHub API for a token scoped to a single installation. +# +# Requirements: +# - A GitHub App whose private key has been uploaded into Key Vault as an RSA +# key (the PEM converted to a Key Vault *key*, NOT stored as a secret). +# - The caller (the federated Azure service connection used to run this script) +# must have the `Key Vault Crypto User` role (or at minimum the `Sign` +# action) on that key. +# - The App must be installed on the target organization/account +# (`InstallationOwner`) with the permissions/repositories it needs. +# +# Installation tokens (ghs_*) are exempt from the enterprise classic-PAT +# lifetime policy, which is why this replaces the long-lived PAT. + +[CmdletBinding()] +param( + # Name of the Key Vault that holds the GitHub App's RSA signing key. + [Parameter(Mandatory = $true)] + [string] $KeyVaultName, + + # Name of the RSA key inside the Key Vault (the App's private key). + [Parameter(Mandatory = $true)] + [string] $KeyName, + + # The GitHub App's Client ID (the value to put in the `iss` JWT claim). + [Parameter(Mandatory = $true)] + [string] $AppClientId, + + # Login of the organization or user account whose installation we should + # mint the token for (e.g. `dotnet`, `microsoft`). + [Parameter(Mandatory = $true)] + [string] $InstallationOwner, + + # Optional Azure DevOps pipeline variable name to set with the installation + # token (marked as a secret). When not specified, the token is written to + # stdout instead. + [Parameter(Mandatory = $false)] + [string] $OutputVariableName +) + +$ErrorActionPreference = 'Stop' +$PSNativeCommandUseErrorActionPreference = $true + +. $PSScriptRoot\pipeline-logging-functions.ps1 + +function ConvertTo-Base64Url([byte[]] $bytes) { + return [Convert]::ToBase64String($bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_') +} + +# Build JWT header and payload. Use [ordered] hashtables so JSON +# serialization is deterministic. +$jwtHeader = [ordered]@{ + alg = 'RS256' + typ = 'JWT' +} +$now = [System.DateTimeOffset]::UtcNow +$jwtPayload = [ordered]@{ + iat = $now.AddMinutes(-1).ToUnixTimeSeconds() + exp = $now.AddMinutes(5).ToUnixTimeSeconds() + iss = $AppClientId +} + +$headerEncoded = ConvertTo-Base64Url ([System.Text.Encoding]::UTF8.GetBytes(($jwtHeader | ConvertTo-Json -Compress))) +$payloadEncoded = ConvertTo-Base64Url ([System.Text.Encoding]::UTF8.GetBytes(($jwtPayload | ConvertTo-Json -Compress))) +$signingInput = "$headerEncoded.$payloadEncoded" + +# Key Vault `sign` expects the *digest* (base64), not the raw bytes. +$sha256 = [System.Security.Cryptography.SHA256]::Create() +$digestBytes = $sha256.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($signingInput)) +$digestBase64 = [Convert]::ToBase64String($digestBytes) + +Write-Host "Signing JWT with key '$KeyName' in vault '$KeyVaultName'..." +$previousNativeCommandErrorPreference = $PSNativeCommandUseErrorActionPreference +try { + # Azure CLI can emit non-fatal Python warnings to stderr even when signing succeeds. + # Use the exit code to determine success for this invocation. + $PSNativeCommandUseErrorActionPreference = $false + $signatureBase64 = az keyvault key sign ` + --vault-name $KeyVaultName ` + --name $KeyName ` + --algorithm RS256 ` + --digest $digestBase64 ` + --query signature ` + --output tsv ` + --only-show-errors + $signExitCode = $LASTEXITCODE +} +catch { + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to sign the JWT via Key Vault (key '$KeyName', vault '$KeyVaultName'): $_. Verify the service connection identity has the 'Key Vault Crypto User' role (Sign action) on the key." + exit 1 +} +finally { + $PSNativeCommandUseErrorActionPreference = $previousNativeCommandErrorPreference +} +if ($signExitCode -ne 0 -or [string]::IsNullOrWhiteSpace($signatureBase64)) { + Write-PipelineTelemetryError -Category 'Build' -Message "'az keyvault key sign' exited with code $signExitCode for key '$KeyName' in vault '$KeyVaultName'. Verify the service connection identity has the 'Key Vault Crypto User' role (Sign action) on the key." + exit 1 +} +$signatureUrl = $signatureBase64.Trim().TrimEnd('=').Replace('+', '-').Replace('/', '_') +$jwt = "$signingInput.$signatureUrl" + +$headers = @{ + Authorization = "Bearer $jwt" + 'X-GitHub-Api-Version' = '2022-11-28' + Accept = 'application/vnd.github+json' + 'User-Agent' = 'dotnet-arcade-onelocbuild' +} + +Write-Host "Looking up installation for '$InstallationOwner'..." +try { + $installations = [System.Collections.Generic.List[object]]::new() + $page = 1 + do { + $pageResponse = Invoke-RestMethod ` + -Uri "https://api.github.com/app/installations?per_page=100&page=$page" ` + -Headers $headers ` + -Method Get + $pageInstallationCount = 0 + foreach ($installation in $pageResponse) { + $installations.Add($installation) + $pageInstallationCount++ + } + $page++ + } while ($pageInstallationCount -eq 100) +} +catch { + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to list GitHub App installations: $_. The signed JWT may be invalid or the App's Client ID ('$AppClientId') may be incorrect." + exit 1 +} +$matchingInstallations = @($installations | Where-Object { $_.account.login -ieq $InstallationOwner }) +if ($matchingInstallations.Count -eq 0) { + $found = ($installations | ForEach-Object { $_.account.login }) -join ', ' + Write-PipelineTelemetryError -Category 'Build' -Message "No installation found for '$InstallationOwner'. App is installed on: $found" + exit 1 +} +if ($matchingInstallations.Count -ne 1) { + $matchingIds = ($matchingInstallations | ForEach-Object { $_.id }) -join ', ' + Write-PipelineTelemetryError -Category 'Build' -Message "Found multiple installations for '$InstallationOwner': $matchingIds" + exit 1 +} +$installation = $matchingInstallations[0] +Write-Host "Using installation $($installation.id) for '$($installation.account.login)'." + +try { + $tokenResponse = Invoke-RestMethod ` + -Uri "https://api.github.com/app/installations/$($installation.id)/access_tokens" ` + -Headers $headers ` + -Method Post ` + -ContentType 'application/json' +} +catch { + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to mint an installation access token for '$InstallationOwner' (installation $($installation.id)): $_" + exit 1 +} + +Write-Host "Got installation token for '$InstallationOwner' (expires $($tokenResponse.expires_at))." +if ($OutputVariableName) { + Write-Host "Setting pipeline variable '$OutputVariableName'." + Write-Host "##vso[task.setvariable variable=$OutputVariableName;issecret=true]$($tokenResponse.token)" +} +else { + Write-Host $tokenResponse.token -ForegroundColor Green +} diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 59b2d55e1a330e9..8f1ac28e1191b07 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -1,17 +1,10 @@ -# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds. -# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080 +# This script adds internal feeds required to build commits that depend on internal package sources. For instance, +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables +# disabled internal Maestro (darc-int*) feeds. # -# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry -# under for each Maestro managed private feed. Two additional credential -# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport. +# Optionally, this script also adds a credential entry for each of the internal feeds if supplied. # -# This script needs to be called in every job that will restore packages and which the base repo has -# private AzDO feeds in the NuGet.config. -# -# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)` -# from the AzureDevOps-Artifact-Feeds-Pats variable group. -# -# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing +# See example call for this script below. # # - task: PowerShell@2 # displayName: Setup Private Feeds Credentials @@ -21,11 +14,17 @@ # arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config -Password $Env:Token # env: # Token: $(dn-bot-dnceng-artifact-feeds-rw) +# +# Note that the NuGetAuthenticate task should be called after SetupNugetSources. +# This ensures that: +# - Appropriate creds are set for the added internal feeds (if not supplied to the script) +# - The credential provider is installed. +# [CmdletBinding()] param ( [Parameter(Mandatory = $true)][string]$ConfigFile, - [Parameter(Mandatory = $true)][string]$Password + $Password ) $ErrorActionPreference = "Stop" @@ -48,11 +47,17 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern else { Write-Host "Package source $SourceName already present." } + AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd } # Add a credential node for the specified source function AddCredential($creds, $source, $username, $pwd) { + # If no cred supplied, don't do anything. + if (!$pwd) { + return; + } + # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -110,11 +115,6 @@ if (!(Test-Path $ConfigFile -PathType Leaf)) { ExitWithExitCode 1 } -if (!$Password) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Please supply a valid PAT' - ExitWithExitCode 1 -} - # Load NuGet.config $doc = New-Object System.Xml.XmlDocument $filename = (Get-Item $ConfigFile).FullName @@ -127,11 +127,14 @@ if ($sources -eq $null) { $doc.DocumentElement.AppendChild($sources) | Out-Null } -# Looks for a node. Create it if none is found. -$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") -if ($creds -eq $null) { - $creds = $doc.CreateElement("packageSourceCredentials") - $doc.DocumentElement.AppendChild($creds) | Out-Null +$creds = $null +if ($Password) { + # Looks for a node. Create it if none is found. + $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") + if ($creds -eq $null) { + $creds = $doc.CreateElement("packageSourceCredentials") + $doc.DocumentElement.AppendChild($creds) | Out-Null + } } # Check for disabledPackageSources; we'll enable any darc-int ones we find there @@ -164,4 +167,4 @@ foreach ($dotnetVersion in $dotnetVersions) { } } -$doc.Save($filename) \ No newline at end of file +$doc.Save($filename) diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index c0e7bbef21c4677..5c1c644280e6de5 100755 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -1,28 +1,26 @@ #!/usr/bin/env bash -# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds. -# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080 +# This script adds internal feeds required to build commits that depend on internal package sources. For instance, +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables +# disabled internal Maestro (darc-int*) feeds. +# +# Optionally, this script also adds a credential entry for each of the internal feeds if supplied. # -# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry -# under for each Maestro's managed private feed. Two additional credential -# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport. -# -# This script needs to be called in every job that will restore packages and which the base repo has -# private AzDO feeds in the NuGet.config. -# -# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)` -# from the AzureDevOps-Artifact-Feeds-Pats variable group. -# -# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing. +# See example call for this script below. # # - task: Bash@3 -# displayName: Setup Private Feeds Credentials +# displayName: Setup Internal Feeds # inputs: # filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh -# arguments: $(System.DefaultWorkingDirectory)/NuGet.config $Token +# arguments: $(System.DefaultWorkingDirectory)/NuGet.config # condition: ne(variables['Agent.OS'], 'Windows_NT') -# env: -# Token: $(dn-bot-dnceng-artifact-feeds-rw) +# - task: NuGetAuthenticate@1 +# +# Note that the NuGetAuthenticate task should be called after SetupNugetSources. +# This ensures that: +# - Appropriate creds are set for the added internal feeds (if not supplied to the script) +# - The credential provider is installed. +# ConfigFile=$1 CredToken=$2 @@ -48,11 +46,6 @@ if [ ! -f "$ConfigFile" ]; then ExitWithExitCode 1 fi -if [ -z "$CredToken" ]; then - Write-PipelineTelemetryError -category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Please supply a valid PAT" - ExitWithExitCode 1 -fi - if [[ `uname -s` == "Darwin" ]]; then NL=$'\\\n' TB='' @@ -140,18 +133,20 @@ PackageSources+="$IFS" PackageSources+=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"') IFS=$PrevIFS -for FeedName in ${PackageSources[@]} ; do - # Check if there is no existing credential for this FeedName - grep -i "<$FeedName>" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding credentials for $FeedName." +if [ "$CredToken" ]; then + for FeedName in ${PackageSources[@]} ; do + # Check if there is no existing credential for this FeedName + grep -i "<$FeedName>" $ConfigFile + if [ "$?" != "0" ]; then + echo "Adding credentials for $FeedName." - PackageSourceCredentialsNodeFooter="" - NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}" + PackageSourceCredentialsNodeFooter="" + NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}" - sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile - fi -done + sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile + fi + done +fi # Re-enable any entries in disabledPackageSources where the feed name contains darc-int grep -i "" $ConfigFile diff --git a/eng/common/core-templates/steps/get-github-app-token.yml b/eng/common/core-templates/steps/get-github-app-token.yml new file mode 100644 index 000000000000000..6d42a48d3c3671f --- /dev/null +++ b/eng/common/core-templates/steps/get-github-app-token.yml @@ -0,0 +1,79 @@ +# Mints a short-lived GitHub App installation access token by signing a JWT +# with a private key stored in Azure Key Vault (RSA, RS256). The JWT is +# exchanged with the GitHub API for a token scoped to a single installation. +# +# Requirements (per GitHub App you want to authenticate as): +# - A GitHub App with its private key uploaded into Key Vault as an RSA key +# (PEM converted to a key, NOT stored as a secret). +# - The Azure service connection passed via `azureSubscription` must be +# granted the `Key Vault Crypto User` role (or at minimum `Sign` action) +# on that key. +# - The App must be installed on the target organization/account +# (`installationOwner`) with the permissions/repositories you need. +# +# Output: a secret pipeline variable named ${{ parameters.outputVariableName }} +# containing the installation access token. Token lifetime is ~1 hour and is +# automatically scrubbed from logs. Installation tokens are exempt from the +# enterprise classic-PAT lifetime policy. + +parameters: +# Azure DevOps service connection (federated) that can call +# `az keyvault key sign` on the App's signing key. +- name: azureSubscription + type: string + +# Name of the Key Vault that holds the GitHub App's RSA signing key. +- name: keyVaultName + type: string + +# Name of the RSA key inside the Key Vault (the App's private key). +- name: keyName + type: string + +# The GitHub App's Client ID (the value to put in the `iss` JWT claim). +# Prefer this over the numeric App ID; GitHub accepts either, but Client ID +# is the documented form going forward. +- name: appClientId + type: string + +# Login of the organization or user account whose installation we should +# mint the token for (e.g. `dotnet`, `microsoft`). +- name: installationOwner + type: string + +# Name of the pipeline variable that will receive the installation token. +- name: outputVariableName + type: string + +- name: is1ESPipeline + type: boolean + +- name: stepName + type: string + default: getGitHubAppInstallationToken + +- name: condition + type: string + default: '' + +- name: displayName + type: string + default: Get GitHub App installation token + +steps: +- task: AzureCLI@2 + displayName: ${{ parameters.displayName }} + name: ${{ parameters.stepName }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + inputs: + azureSubscription: ${{ parameters.azureSubscription }} + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + & "$(System.DefaultWorkingDirectory)/eng/common/Get-GitHubAppToken.ps1" ` + -KeyVaultName '${{ parameters.keyVaultName }}' ` + -KeyName '${{ parameters.keyName }}' ` + -AppClientId '${{ parameters.appClientId }}' ` + -InstallationOwner '${{ parameters.installationOwner }}' ` + -OutputVariableName '${{ parameters.outputVariableName }}' diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1 deleted file mode 100644 index f031ed5b25e9fad..000000000000000 --- a/eng/common/sdl/extract-artifact-packages.ps1 +++ /dev/null @@ -1,82 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where artifact packages are stored - [Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -$disableConfigureToolsetImport = $true - -function ExtractArtifacts { - if (!(Test-Path $InputPath)) { - Write-Host "Input Path does not exist: $InputPath" - ExitWithExitCode 0 - } - $Jobs = @() - Get-ChildItem "$InputPath\*.nupkg" | - ForEach-Object { - $Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName - } - - foreach ($Job in $Jobs) { - Wait-Job -Id $Job.Id | Receive-Job - } -} - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - $ExtractPackage = { - param( - [string] $PackagePath # Full path to a NuGet package - ) - - if (!(Test-Path $PackagePath)) { - Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath" - ExitWithExitCode 1 - } - - $RelevantExtensions = @('.dll', '.exe', '.pdb') - Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...' - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath); - - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $TargetPath = Join-Path -Path $ExtractPath -ChildPath (Split-Path -Path $_.FullName) - [System.IO.Directory]::CreateDirectory($TargetPath); - - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.FullName - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile) - } - } - catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 - } - finally { - $zip.Dispose() - } - } - Measure-Command { ExtractArtifacts } -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml index 68e7a65605c5562..5840fe2fff8538c 100644 --- a/eng/common/templates-official/job/onelocbuild.yml +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -8,6 +8,21 @@ parameters: CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + # Service connection for WIF-based Entra authentication to ceapex feeds (replaces CeapexPat). + # The internal and DevDiv projects each provide a project-scoped connection with this name. + # Other projects, and any pipeline that sets this to '', fall back to CeapexPat. + CeapexServiceConnection: 'dnceng-onelocbuild-ceapex' + + # GitHub App authentication for the OneLoc check-in PR. + # dnceng/internal and DevDiv/DevDiv are enabled by default with their project-scoped service + # connections. Other projects must explicitly opt in after provisioning equivalent infrastructure. + UseGitHubAppAuthentication: true + UseGitHubAppAuthenticationInOtherProjects: false + GitHubAppServiceConnection: 'dnceng-oneloc-githubapp' + GitHubAppClientId: 'Iv23lijBU8x3gc9lDOc9' + GitHubAppKeyVaultName: 'EngKeyVault' + GitHubAppKeyName: 'oneloc-localization-app-key' + SourcesDirectory: $(System.DefaultWorkingDirectory) CreatePr: true AutoCompletePr: false @@ -68,6 +83,30 @@ jobs: displayName: Generate LocProject.json condition: ${{ parameters.condition }} + # Acquire an Entra token for ceapex feed access in the supported internal and DevDiv projects. + - ${{ if and(ne(parameters.CeapexServiceConnection, ''), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'))) }}: + - template: /eng/common/templates-official/steps/get-federated-access-token.yml + parameters: + federatedServiceConnection: ${{ parameters.CeapexServiceConnection }} + outputVariableName: 'CeapexEntraToken' + condition: ${{ parameters.condition }} + + # Mint a short-lived GitHub App installation token for the loc check-in PR. Use the connection + # provisioned in each supported project; other projects must explicitly opt in and override it. + - ${{ if and(eq(parameters.RepoType, 'gitHub'), eq(parameters.UseGitHubAppAuthentication, true), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.UseGitHubAppAuthenticationInOtherProjects, true))) }}: + - template: /eng/common/templates-official/steps/get-github-app-token.yml + parameters: + ${{ if and(eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.GitHubAppServiceConnection, 'dnceng-oneloc-githubapp')) }}: + azureSubscription: 'devdiv-oneloc-githubapp' + ${{ else }}: + azureSubscription: ${{ parameters.GitHubAppServiceConnection }} + keyVaultName: ${{ parameters.GitHubAppKeyVaultName }} + keyName: ${{ parameters.GitHubAppKeyName }} + appClientId: ${{ parameters.GitHubAppClientId }} + installationOwner: ${{ parameters.GitHubOrg }} + outputVariableName: 'GitHubAppInstallationToken' + condition: ${{ parameters.condition }} + - task: OneLocBuild@2 displayName: OneLocBuild env: @@ -84,10 +123,16 @@ jobs: ${{ if eq(parameters.RepoType, 'gitHub') }}: isShouldReusePrSelected: ${{ parameters.ReusePr }} packageSourceAuth: patAuth - patVariable: ${{ parameters.CeapexPat }} + ${{ if and(ne(parameters.CeapexServiceConnection, ''), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'))) }}: + patVariable: $(CeapexEntraToken) + ${{ if or(eq(parameters.CeapexServiceConnection, ''), and(ne(variables['System.TeamProject'], 'internal'), ne(variables['System.TeamProject'], 'DevDiv'))) }}: + patVariable: ${{ parameters.CeapexPat }} ${{ if eq(parameters.RepoType, 'gitHub') }}: repoType: ${{ parameters.RepoType }} - gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if and(eq(parameters.UseGitHubAppAuthentication, true), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.UseGitHubAppAuthenticationInOtherProjects, true))) }}: + gitHubPatVariable: "$(GitHubAppInstallationToken)" + ${{ else }}: + gitHubPatVariable: "${{ parameters.GithubPat }}" ${{ if ne(parameters.MirrorRepo, '') }}: isMirrorRepoSelected: true gitHubOrganization: ${{ parameters.GitHubOrg }} diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index 67a0de433a3dac3..4842d4c723ead7b 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -48,8 +48,6 @@ jobs: variables: - template: /eng/common/templates-official/variables/pool-providers.yml - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: Publish-Build-Assets - - group: AzureDevOps-Artifact-Feeds-Pats - name: runCodesignValidationInjection value: false - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml index f6476912a861f98..b62247fe7031ed5 100644 --- a/eng/common/templates-official/jobs/codeql-build.yml +++ b/eng/common/templates-official/jobs/codeql-build.yml @@ -17,7 +17,6 @@ jobs: enableTelemetry: true variables: - - group: Publish-Build-Assets # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in # sync with the packages.config file. - name: DefaultGuardianVersion diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml index 173914f2364a709..c4b8507bb454bd0 100644 --- a/eng/common/templates-official/post-build/common-variables.yml +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -1,6 +1,4 @@ variables: - - group: Publish-Build-Assets - # Whether the build is internal or not - name: IsInternalBuild value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} diff --git a/eng/common/templates-official/steps/get-github-app-token.yml b/eng/common/templates-official/steps/get-github-app-token.yml new file mode 100644 index 000000000000000..c89f3641a4dbe00 --- /dev/null +++ b/eng/common/templates-official/steps/get-github-app-token.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/get-github-app-token.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml index c307825c91222fb..1695b6dcf37f241 100644 --- a/eng/common/templates-official/steps/source-build.yml +++ b/eng/common/templates-official/steps/source-build.yml @@ -18,19 +18,22 @@ parameters: steps: # Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) - script: | - set -x df -h - # If building on the internal project, the artifact feeds variable may be available (usually only if needed) - # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # If building on the dnceng internal project, call the feed setup script to add internal feeds corresponding to public ones. # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those # changes. internalRestoreArgs= - if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + if [ '${{ eq(variables['System.TeamProject'], 'internal') }}' = 'True' ]; then # Temporarily work around https://github.com/dotnet/arcade/issues/7709 chmod +x $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh - $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh $(System.DefaultWorkingDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + # Authenticate to internal feeds using the build identity (System.AccessToken) + if [ -z "$(System.AccessToken)" ]; then + echo "##vso[task.logissue type=error]System.AccessToken is empty; cannot authenticate to internal feeds for source-build." + exit 1 + fi + "$(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh" "$(System.DefaultWorkingDirectory)/NuGet.config" "$(System.AccessToken)" internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. @@ -50,6 +53,9 @@ steps: internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' fi + # Enable command tracing for the build-argument assembly below (no secrets are evaluated here). + set -x + buildConfig=Release # Check if AzDO substitutes in a build config from a variable, and use it if so. if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then @@ -86,6 +92,9 @@ steps: assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml fi + # internalRuntimeDownloadArgs embeds the dotnetbuilds-internal-container-read-token-base64 secret, + # so disable tracing before the build invocation so the token isn't echoed via xtrace. + set +x ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ --configuration $buildConfig \ --restore --build --pack $publishArgs -bl \ diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 28ccaaa482064e4..7152ff120adf834 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -36,7 +36,6 @@ jobs: displayName: Run SDL tool condition: and(succeededOrFailed(), eq( ${{ parameters.enable }}, 'true')) variables: - - group: DotNet-VSTS-Bot - name: AzDOProjectName value: ${{ parameters.AzDOProjectName }} - name: AzDOPipelineId @@ -110,18 +109,6 @@ jobs: displayName: Trim the version from the NuGet packages continueOnError: ${{ parameters.sdlContinueOnError }} - - powershell: eng/common/sdl/extract-artifact-packages.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts - displayName: Extract Blob Artifacts - continueOnError: ${{ parameters.sdlContinueOnError }} - - - powershell: eng/common/sdl/extract-artifact-packages.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts - displayName: Extract Package Artifacts - continueOnError: ${{ parameters.sdlContinueOnError }} - - ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}: - powershell: eng/common/sdl/extract-artifact-archives.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml index 0d9a54751c81af5..1a8a07ca0168ca7 100644 --- a/eng/common/templates/job/onelocbuild.yml +++ b/eng/common/templates/job/onelocbuild.yml @@ -8,6 +8,21 @@ parameters: CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + # Service connection for WIF-based Entra authentication to ceapex feeds (replaces CeapexPat). + # The internal and DevDiv projects each provide a project-scoped connection with this name. + # Other projects, and any pipeline that sets this to '', fall back to CeapexPat. + CeapexServiceConnection: 'dnceng-onelocbuild-ceapex' + + # GitHub App authentication for the OneLoc check-in PR. + # dnceng/internal and DevDiv/DevDiv are enabled by default with their project-scoped service + # connections. Other projects must explicitly opt in after provisioning equivalent infrastructure. + UseGitHubAppAuthentication: true + UseGitHubAppAuthenticationInOtherProjects: false + GitHubAppServiceConnection: 'dnceng-oneloc-githubapp' + GitHubAppClientId: 'Iv23lijBU8x3gc9lDOc9' + GitHubAppKeyVaultName: 'EngKeyVault' + GitHubAppKeyName: 'oneloc-localization-app-key' + SourcesDirectory: $(System.DefaultWorkingDirectory) CreatePr: true AutoCompletePr: false @@ -65,6 +80,30 @@ jobs: displayName: Generate LocProject.json condition: ${{ parameters.condition }} + # Acquire an Entra token for ceapex feed access in the supported internal and DevDiv projects. + - ${{ if and(ne(parameters.CeapexServiceConnection, ''), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'))) }}: + - template: /eng/common/templates/steps/get-federated-access-token.yml + parameters: + federatedServiceConnection: ${{ parameters.CeapexServiceConnection }} + outputVariableName: 'CeapexEntraToken' + condition: ${{ parameters.condition }} + + # Mint a short-lived GitHub App installation token for the loc check-in PR. Use the connection + # provisioned in each supported project; other projects must explicitly opt in and override it. + - ${{ if and(eq(parameters.RepoType, 'gitHub'), eq(parameters.UseGitHubAppAuthentication, true), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.UseGitHubAppAuthenticationInOtherProjects, true))) }}: + - template: /eng/common/templates/steps/get-github-app-token.yml + parameters: + ${{ if and(eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.GitHubAppServiceConnection, 'dnceng-oneloc-githubapp')) }}: + azureSubscription: 'devdiv-oneloc-githubapp' + ${{ else }}: + azureSubscription: ${{ parameters.GitHubAppServiceConnection }} + keyVaultName: ${{ parameters.GitHubAppKeyVaultName }} + keyName: ${{ parameters.GitHubAppKeyName }} + appClientId: ${{ parameters.GitHubAppClientId }} + installationOwner: ${{ parameters.GitHubOrg }} + outputVariableName: 'GitHubAppInstallationToken' + condition: ${{ parameters.condition }} + - task: OneLocBuild@2 displayName: OneLocBuild env: @@ -81,10 +120,16 @@ jobs: ${{ if eq(parameters.RepoType, 'gitHub') }}: isShouldReusePrSelected: ${{ parameters.ReusePr }} packageSourceAuth: patAuth - patVariable: ${{ parameters.CeapexPat }} + ${{ if and(ne(parameters.CeapexServiceConnection, ''), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'))) }}: + patVariable: $(CeapexEntraToken) + ${{ if or(eq(parameters.CeapexServiceConnection, ''), and(ne(variables['System.TeamProject'], 'internal'), ne(variables['System.TeamProject'], 'DevDiv'))) }}: + patVariable: ${{ parameters.CeapexPat }} ${{ if eq(parameters.RepoType, 'gitHub') }}: repoType: ${{ parameters.RepoType }} - gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if and(eq(parameters.UseGitHubAppAuthentication, true), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.UseGitHubAppAuthenticationInOtherProjects, true))) }}: + gitHubPatVariable: "$(GitHubAppInstallationToken)" + ${{ else }}: + gitHubPatVariable: "${{ parameters.GithubPat }}" ${{ if ne(parameters.MirrorRepo, '') }}: isMirrorRepoSelected: true gitHubOrganization: ${{ parameters.GitHubOrg }} diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 9234ccab1a45d63..42c154b1304c870 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -48,8 +48,6 @@ jobs: variables: - template: /eng/common/templates/variables/pool-providers.yml - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: Publish-Build-Assets - - group: AzureDevOps-Artifact-Feeds-Pats - name: runCodesignValidationInjection value: false - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: diff --git a/eng/common/templates/jobs/codeql-build.yml b/eng/common/templates/jobs/codeql-build.yml index e8b43e3b4cba16e..a89c1f026825aa8 100644 --- a/eng/common/templates/jobs/codeql-build.yml +++ b/eng/common/templates/jobs/codeql-build.yml @@ -17,7 +17,6 @@ jobs: enableTelemetry: true variables: - - group: Publish-Build-Assets # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in # sync with the packages.config file. - name: DefaultGuardianVersion diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index 173914f2364a709..c4b8507bb454bd0 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -1,6 +1,4 @@ variables: - - group: Publish-Build-Assets - # Whether the build is internal or not - name: IsInternalBuild value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} diff --git a/eng/common/templates/steps/get-github-app-token.yml b/eng/common/templates/steps/get-github-app-token.yml new file mode 100644 index 000000000000000..79e182c64167afb --- /dev/null +++ b/eng/common/templates/steps/get-github-app-token.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/get-github-app-token.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index d08a0e92caa49e4..c1e517f38d02dda 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -18,19 +18,22 @@ parameters: steps: # Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) - script: | - set -x df -h - # If building on the internal project, the artifact feeds variable may be available (usually only if needed) - # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # If building on the dnceng internal project, call the feed setup script to add internal feeds corresponding to public ones. # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those # changes. internalRestoreArgs= - if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + if [ '${{ eq(variables['System.TeamProject'], 'internal') }}' = 'True' ]; then # Temporarily work around https://github.com/dotnet/arcade/issues/7709 chmod +x $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh - $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh $(System.DefaultWorkingDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + # Authenticate to internal feeds using the build identity (System.AccessToken) + if [ -z "$(System.AccessToken)" ]; then + echo "##vso[task.logissue type=error]System.AccessToken is empty; cannot authenticate to internal feeds for source-build." + exit 1 + fi + "$(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh" "$(System.DefaultWorkingDirectory)/NuGet.config" "$(System.AccessToken)" internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. @@ -50,6 +53,9 @@ steps: internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' fi + # Enable command tracing for the build-argument assembly below (no secrets are evaluated here). + set -x + buildConfig=Release # Check if AzDO substitutes in a build config from a variable, and use it if so. if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then @@ -86,6 +92,9 @@ steps: assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml fi + # internalRuntimeDownloadArgs embeds the dotnetbuilds-internal-container-read-token-base64 secret, + # so disable tracing before the build invocation so the token isn't echoed via xtrace. + set +x ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ --configuration $buildConfig \ --restore --build --pack $publishArgs -bl \ diff --git a/global.json b/global.json index 8a140b5b5cb2d24..4327c4c7856b737 100644 --- a/global.json +++ b/global.json @@ -6,7 +6,7 @@ "dotnet": "8.0.126" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.26278.3", + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.26431.7", "Microsoft.Build.NoTargets": "3.5.0", "Microsoft.Build.Traversal": "2.0.34" }