Skip to content

Commit

Permalink
Merge commit 'd20ce761422e7178512bb89bfa3fdce797924616'
Browse files Browse the repository at this point in the history
  • Loading branch information
Mirroring committed Jul 11, 2024
2 parents dc9cf7d + d20ce76 commit 73adbdc
Show file tree
Hide file tree
Showing 16 changed files with 288 additions and 30 deletions.
8 changes: 4 additions & 4 deletions eng/Version.Details.xml
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,13 @@
</Dependency>
</ProductDependencies>
<ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.24266.4">
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.24360.7">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>2eab07c3d7b78219d10099b19fafeef2ecae1779</Sha>
<Sha>fbc993a9e8fb4926ce04c95ba2e48852c9d9df65</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.24266.4">
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.24360.7">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>2eab07c3d7b78219d10099b19fafeef2ecae1779</Sha>
<Sha>fbc993a9e8fb4926ce04c95ba2e48852c9d9df65</Sha>
</Dependency>
</ToolsetDependencies>
</Dependencies>
8 changes: 8 additions & 0 deletions eng/common/templates-official/job/source-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ parameters:
# container and pool.
platform: {}

# If set to true and running on a non-public project,
# Internal blob storage locations will be enabled.
# This is not enabled by default because many repositories do not need internal sources
# and do not need to have the required service connections approved in the pipeline.
enableInternalSources: false

jobs:
- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
displayName: Source-Build (${{ parameters.platform.name }})
Expand Down Expand Up @@ -59,6 +65,8 @@ jobs:
clean: all

steps:
- ${{ if eq(parameters.enableInternalSources, true) }}:
- template: /eng/common/templates-official/steps/enable-internal-runtimes.yml
- template: /eng/common/templates-official/steps/source-build.yml
parameters:
platform: ${{ parameters.platform }}
35 changes: 25 additions & 10 deletions eng/common/templates-official/job/source-index-stage1.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
parameters:
runAsPublic: false
sourceIndexPackageVersion: 1.0.1-20240320.1
sourceIndexUploadPackageVersion: 2.0.0-20240502.12
sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2
sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: []
Expand All @@ -17,14 +18,14 @@ jobs:
dependsOn: ${{ parameters.dependsOn }}
condition: ${{ parameters.condition }}
variables:
- name: SourceIndexPackageVersion
value: ${{ parameters.sourceIndexPackageVersion }}
- name: SourceIndexUploadPackageVersion
value: ${{ parameters.sourceIndexUploadPackageVersion }}
- name: SourceIndexProcessBinlogPackageVersion
value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }}
- name: SourceIndexPackageSource
value: ${{ parameters.sourceIndexPackageSource }}
- name: BinlogPath
value: ${{ parameters.binlogPath }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- group: source-dot-net stage1 variables

pool: ${{ parameters.pool }}
steps:
Expand All @@ -40,8 +41,8 @@ jobs:
workingDirectory: $(Agent.TempDirectory)

- script: |
$(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
$(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
$(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
$(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
displayName: Download Tools
# Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
workingDirectory: $(Agent.TempDirectory)
Expand All @@ -53,7 +54,21 @@ jobs:
displayName: Process Binlog into indexable sln

- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name)
- task: AzureCLI@2
displayName: Get stage 1 auth token
inputs:
azureSubscription: 'SourceDotNet Stage1 Publish'
addSpnToEnvironment: true
scriptType: 'ps'
scriptLocation: 'inlineScript'
inlineScript: |
echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId"
echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken"
echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId"
- script: |
az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN)
displayName: "Login to Azure"
- script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
displayName: Upload stage1 artifacts to source index
env:
BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)
8 changes: 8 additions & 0 deletions eng/common/templates-official/jobs/source-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@ parameters:
# one job runs on 'defaultManagedPlatform'.
platforms: []

# If set to true and running on a non-public project,
# Internal nuget and blob storage locations will be enabled.
# This is not enabled by default because many repositories do not need internal sources
# and do not need to have the required service connections approved in the pipeline.
enableInternalSources: false

jobs:

- ${{ if ne(parameters.allCompletedJobId, '') }}:
Expand All @@ -38,9 +44,11 @@ jobs:
parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ platform }}
enableInternalSources: ${{ parameters.enableInternalSources }}

- ${{ if eq(length(parameters.platforms), 0) }}:
- template: /eng/common/templates-official/job/source-build.yml
parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ parameters.defaultManagedPlatform }}
enableInternalSources: ${{ parameters.enableInternalSources }}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ variables:
- group: AzureDevOps-Artifact-Feeds-Pats
- group: DotNet-Blob-Feed
- group: DotNet-DotNetCli-Storage
- group: DotNet-MSRC-Storage
- group: Publish-Build-Assets

# Whether the build is internal or not
Expand Down
28 changes: 28 additions & 0 deletions eng/common/templates-official/steps/enable-internal-runtimes.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64'
# variable with the base64-encoded SAS token, by default

parameters:
- name: federatedServiceConnection
type: string
default: 'dotnetbuilds-internal-read'
- name: outputVariableName
type: string
default: 'dotnetbuilds-internal-container-read-token-base64'
- name: expiryInHours
type: number
default: 1
- name: base64Encode
type: boolean
default: true

steps:
- ${{ if ne(variables['System.TeamProject'], 'public') }}:
- template: /eng/common/templates-official/steps/get-delegation-sas.yml
parameters:
federatedServiceConnection: ${{ parameters.federatedServiceConnection }}
outputVariableName: ${{ parameters.outputVariableName }}
expiryInHours: ${{ parameters.expiryInHours }}
base64Encode: ${{ parameters.base64Encode }}
storageAccount: dotnetbuilds
container: internal
permissions: rl
43 changes: 43 additions & 0 deletions eng/common/templates-official/steps/get-delegation-sas.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
parameters:
- name: federatedServiceConnection
type: string
- name: outputVariableName
type: string
- name: expiryInHours
type: number
default: 1
- name: base64Encode
type: boolean
default: false
- name: storageAccount
type: string
- name: container
type: string
- name: permissions
type: string
default: 'rl'

steps:
- task: AzureCLI@2
displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}'
inputs:
azureSubscription: ${{ parameters.federatedServiceConnection }}
scriptType: 'pscore'
scriptLocation: 'inlineScript'
inlineScript: |
# Calculate the expiration of the SAS token and convert to UTC
$expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
$sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
if ($LASTEXITCODE -ne 0) {
Write-Error "Failed to generate SAS token."
exit 1
}
if ('${{ parameters.base64Encode }}' -eq 'true') {
$sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas))
}
Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas"
28 changes: 28 additions & 0 deletions eng/common/templates-official/steps/get-federated-access-token.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
parameters:
- name: federatedServiceConnection
type: string
- name: outputVariableName
type: string
# Resource to get a token for. Common values include:
# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps
# - 'https://storage.azure.com/' for storage
# Defaults to Azure DevOps
- name: resource
type: string
default: '499b84ac-1321-427f-aa17-267ca6975798'

steps:
- task: AzureCLI@2
displayName: 'Getting federated access token for feeds'
inputs:
azureSubscription: ${{ parameters.federatedServiceConnection }}
scriptType: 'pscore'
scriptLocation: 'inlineScript'
inlineScript: |
$accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv
if ($LASTEXITCODE -ne 0) {
Write-Error "Failed to get access token for resource '${{ parameters.resource }}'"
exit 1
}
Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken"
8 changes: 8 additions & 0 deletions eng/common/templates/job/source-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ parameters:
# container and pool.
platform: {}

# If set to true and running on a non-public project,
# Internal blob storage locations will be enabled.
# This is not enabled by default because many repositories do not need internal sources
# and do not need to have the required service connections approved in the pipeline.
enableInternalSources: false

jobs:
- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
displayName: Source-Build (${{ parameters.platform.name }})
Expand Down Expand Up @@ -58,6 +64,8 @@ jobs:
clean: all

steps:
- ${{ if eq(parameters.enableInternalSources, true) }}:
- template: /eng/common/templates/steps/enable-internal-runtimes.yml
- template: /eng/common/templates/steps/source-build.yml
parameters:
platform: ${{ parameters.platform }}
35 changes: 25 additions & 10 deletions eng/common/templates/job/source-index-stage1.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
parameters:
runAsPublic: false
sourceIndexPackageVersion: 1.0.1-20240320.1
sourceIndexUploadPackageVersion: 2.0.0-20240502.12
sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2
sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: []
Expand All @@ -15,14 +16,14 @@ jobs:
dependsOn: ${{ parameters.dependsOn }}
condition: ${{ parameters.condition }}
variables:
- name: SourceIndexPackageVersion
value: ${{ parameters.sourceIndexPackageVersion }}
- name: SourceIndexUploadPackageVersion
value: ${{ parameters.sourceIndexUploadPackageVersion }}
- name: SourceIndexProcessBinlogPackageVersion
value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }}
- name: SourceIndexPackageSource
value: ${{ parameters.sourceIndexPackageSource }}
- name: BinlogPath
value: ${{ parameters.binlogPath }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- group: source-dot-net stage1 variables

pool: ${{ parameters.pool }}
steps:
Expand All @@ -38,8 +39,8 @@ jobs:
workingDirectory: $(Agent.TempDirectory)

- script: |
$(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
$(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
$(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
$(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
displayName: Download Tools
# Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
workingDirectory: $(Agent.TempDirectory)
Expand All @@ -51,7 +52,21 @@ jobs:
displayName: Process Binlog into indexable sln

- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name)
- task: AzureCLI@2
displayName: Get stage 1 auth token
inputs:
azureSubscription: 'SourceDotNet Stage1 Publish'
addSpnToEnvironment: true
scriptType: 'ps'
scriptLocation: 'inlineScript'
inlineScript: |
echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId"
echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken"
echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId"
- script: |
az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN)
displayName: "Login to Azure"
- script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
displayName: Upload stage1 artifacts to source index
env:
BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)
8 changes: 8 additions & 0 deletions eng/common/templates/jobs/source-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@ parameters:
# one job runs on 'defaultManagedPlatform'.
platforms: []

# If set to true and running on a non-public project,
# Internal nuget and blob storage locations will be enabled.
# This is not enabled by default because many repositories do not need internal sources
# and do not need to have the required service connections approved in the pipeline.
enableInternalSources: false

jobs:

- ${{ if ne(parameters.allCompletedJobId, '') }}:
Expand All @@ -38,9 +44,11 @@ jobs:
parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ platform }}
enableInternalSources: ${{ parameters.enableInternalSources }}

- ${{ if eq(length(parameters.platforms), 0) }}:
- template: /eng/common/templates/job/source-build.yml
parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ parameters.defaultManagedPlatform }}
enableInternalSources: ${{ parameters.enableInternalSources }}
1 change: 0 additions & 1 deletion eng/common/templates/post-build/common-variables.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ variables:
- group: AzureDevOps-Artifact-Feeds-Pats
- group: DotNet-Blob-Feed
- group: DotNet-DotNetCli-Storage
- group: DotNet-MSRC-Storage
- group: Publish-Build-Assets

# Whether the build is internal or not
Expand Down
28 changes: 28 additions & 0 deletions eng/common/templates/steps/enable-internal-runtimes.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64'
# variable with the base64-encoded SAS token, by default

parameters:
- name: federatedServiceConnection
type: string
default: 'dotnetbuilds-internal-read'
- name: outputVariableName
type: string
default: 'dotnetbuilds-internal-container-read-token-base64'
- name: expiryInHours
type: number
default: 1
- name: base64Encode
type: boolean
default: true

steps:
- ${{ if ne(variables['System.TeamProject'], 'public') }}:
- template: /eng/common/templates/steps/get-delegation-sas.yml
parameters:
federatedServiceConnection: ${{ parameters.federatedServiceConnection }}
outputVariableName: ${{ parameters.outputVariableName }}
expiryInHours: ${{ parameters.expiryInHours }}
base64Encode: ${{ parameters.base64Encode }}
storageAccount: dotnetbuilds
container: internal
permissions: rl
Loading

0 comments on commit 73adbdc

Please sign in to comment.