Skip to content

Commit cf16d98

Browse files
aishwaryabhaishwaryabh
authored
Convert consolidated pipeline to a YAML based pipeline and enable nightly builds (#4292)
* add tag to nightly build * consolidate-artifacts.yml file * making artifact assembler run parallel and update consolidate artifacts yaml * fixing typo * update read me * addressing comments; lets see if this compiles lol * fixing typos * making sure each job is alphanumeric * display name * adding assemble artifacts path * trying to assemble artifact input * add artifact name * fix input * trying to print out conetnt * fix typo * fixing path * addressing more comments * remove no build param * fix nuget package script * fix conditional syntax * pls work * adding variable * adding variable * see if this works * fixing syntax hopefully * pls work fr * idk if this will work * see if this works * removing extra logging
1 parent 2cc8779 commit cf16d98

File tree

9 files changed

+399
-21
lines changed

9 files changed

+399
-21
lines changed

‎eng/ci/consolidate-artifacts.yml

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
schedules:
2+
- cron: "30 2 * * *"
3+
displayName: Nightly Consolidation
4+
always: true
5+
branches:
6+
include:
7+
- main
8+
9+
name: $(Build.SourceBranchName)_$(Build.Reason)
10+
11+
trigger: none
12+
13+
# This pipeline is triggered by the completion of scheduled builds
14+
# in the core-tools-inproc and core-tools-default pipelines
15+
resources:
16+
repositories:
17+
- repository: 1es
18+
type: git
19+
name: 1ESPipelineTemplates/1ESPipelineTemplates
20+
ref: refs/tags/release
21+
- repository: eng
22+
type: git
23+
name: engineering
24+
ref: refs/tags/release
25+
pipelines:
26+
- pipeline: core-tools-host
27+
source: azure\azure-functions-core-tools\core-tools-host.official
28+
29+
- pipeline: core-tools-inproc
30+
source: azure\azure-functions-core-tools\core-tools.official
31+
branch: in-proc # in-proc branch of core-tools.official
32+
tags:
33+
- nightly-build
34+
trigger:
35+
branches:
36+
include:
37+
- in-proc
38+
39+
- pipeline: core-tools-default
40+
source: azure\azure-functions-core-tools\core-tools.official
41+
branch: main # main branch of core-tools.official
42+
tags:
43+
- nightly-build
44+
trigger:
45+
branches:
46+
include:
47+
- main
48+
49+
variables:
50+
- template: /ci/variables/cfs.yml@eng
51+
- name: archs
52+
value: 'win-x64,win-x86,min.win-x64,osx-x64,linux-x64,osx-arm64,win-arm64,min.win-arm64,min.win-x86'
53+
54+
extends:
55+
template: v1/1ES.Official.PipelineTemplate.yml@1es
56+
parameters:
57+
featureFlags:
58+
incrementalSDLBinaryAnalysis: true
59+
incrementalSDLSourceAnalysis: true
60+
pool:
61+
name: 1es-pool-azfunc
62+
image: 1es-windows-2022
63+
os: windows
64+
sdl:
65+
codeql:
66+
compiled:
67+
enabled: true
68+
runSourceLanguagesInSourceAnalysis: true
69+
70+
stages:
71+
- stage: ConsolidateArticacts
72+
displayName: "Assemble Artifacts"
73+
jobs:
74+
- template: /eng/ci/templates/official/jobs/process-nuget-package.yml@self
75+
76+
- ${{ each arch in split(variables.archs,',') }}:
77+
- template: /eng/ci/templates/official/jobs/assemble-artifacts.yml@self
78+
parameters:
79+
arch: ${{ arch }}
80+
displayName: ${{ replace(replace(arch, '-', ''), '.', '') }}

‎eng/ci/templates/official/jobs/assemble-artifacts.yml

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
parameters:
2+
- name: arch
3+
type: string
4+
- name: displayName
5+
type: string
6+
7+
jobs:
8+
- job: AssembleArtifact${{ parameters.displayName }}
9+
displayName: "Assemble ${{ parameters.arch }}"
10+
templateContext:
11+
inputs:
12+
- input: pipelineArtifact
13+
pipeline: core-tools-host
14+
artifactName: drop-coretools-host-linux-signed
15+
targetPath: $(Pipeline.Workspace)\core-tools-host\drop-coretools-host-linux-signed
16+
- input: pipelineArtifact
17+
pipeline: core-tools-host
18+
artifactName: drop-coretools-host-windows
19+
targetPath: $(Pipeline.Workspace)\core-tools-host\drop-coretools-host-windows
20+
- input: pipelineArtifact
21+
pipeline: core-tools-inproc
22+
artifactName: drop-inproc6
23+
targetPath: $(Pipeline.Workspace)\core-tools-inproc\drop-inproc6
24+
- input: pipelineArtifact
25+
pipeline: core-tools-inproc
26+
artifactName: drop-inproc8
27+
targetPath: $(Pipeline.Workspace)\core-tools-inproc\drop-inproc8
28+
- input: pipelineArtifact
29+
pipeline: core-tools-default
30+
artifactName: drop
31+
targetPath: $(Pipeline.Workspace)\core-tools-default\drop
32+
33+
outputParentDirectory: $(Build.ArtifactStagingDirectory)
34+
outputs:
35+
- output: pipelineArtifact
36+
displayName: Publish assembled artifact ${{ parameters.arch }}
37+
targetPath: $(Build.ArtifactStagingDirectory)/core-tools
38+
artifact: drop-${{ parameters.arch }}
39+
- output: pipelineArtifact
40+
displayName: Publish metadata.json
41+
targetPath: $(Build.ArtifactStagingDirectory)/metadata.json
42+
artifact: drop-metadata-json
43+
condition: and(succeeded(), eq('${{ parameters.arch }}', 'min.win-x64'))
44+
45+
steps:
46+
- checkout: self
47+
48+
- task: DotNetCoreCLI@2
49+
displayName: "Run ArtifactAssembler"
50+
inputs:
51+
command: run
52+
projects: "$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler/Azure.Functions.ArtifactAssembler.csproj"
53+
arguments: '-c release -- "Azure.Functions.Cli.${{ parameters.arch }}"'
54+
workingDirectory: '$(Pipeline.Workspace)'
55+
56+
- ${{ if eq(parameters.arch, 'min.win-x64') }}:
57+
- task: PowerShell@2
58+
displayName: 'Generate metadata.json file'
59+
inputs:
60+
targetType: filePath
61+
filePath: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler/PipelineHelpers/generateMetadataFile.ps1'
62+
arguments: '-StagingDirectory "$(Pipeline.Workspace)\staging"'
63+
workingDirectory: '$(Pipeline.Workspace)'
64+
65+
- ${{ if startsWith(parameters.arch, 'win') }}:
66+
- task: PowerShell@2
67+
displayName: 'Generate MSI files'
68+
inputs:
69+
targetType: filePath
70+
filePath: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler/PipelineHelpers/generateMsiFiles.ps1'
71+
arguments: '-ArtifactsPath "$(Pipeline.Workspace)\staging\coretools-cli"'
72+
workingDirectory: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler'
73+
74+
- ${{ if startsWith(parameters.arch, 'win-x') }}:
75+
- template: ci/sign-files.yml@eng
76+
parameters:
77+
displayName: Sign MSI files
78+
folderPath: $(Pipeline.Workspace)\staging\coretools-cli
79+
pattern: '*.msi'
80+
signType: 'inline'
81+
inlineOperation: |
82+
[
83+
{
84+
"KeyCode": "CP-230012",
85+
"OperationCode": "SigntoolSign",
86+
"Parameters": {
87+
"OpusName": "Microsoft",
88+
"OpusInfo": "http://www.microsoft.com",
89+
"FileDigest": "/fd \"SHA256\"",
90+
"PageHash": "/NPH",
91+
"TimeStamp": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256"
92+
},
93+
"ToolName": "sign",
94+
"ToolVersion": "1.0"
95+
},
96+
{
97+
"KeyCode": "CP-230012",
98+
"OperationCode": "SigntoolVerify",
99+
"Parameters": {},
100+
"ToolName": "sign",
101+
"ToolVersion": "1.0"
102+
}
103+
]
104+
105+
- ${{ if eq(parameters.arch, 'min.win-x64') }}:
106+
- task: PowerShell@2
107+
displayName: 'Test Artifacts - Visual Studio'
108+
inputs:
109+
targetType: filePath
110+
filePath: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler/PipelineHelpers/testVsArtifacts.ps1'
111+
arguments: '-StagingDirectory "$(Pipeline.Workspace)\staging\coretools-visualstudio"'
112+
workingDirectory: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler'
113+
114+
- ${{ if startsWith(parameters.arch, 'win') }}:
115+
- task: PowerShell@2
116+
displayName: 'Test Artifacts'
117+
inputs:
118+
targetType: filePath
119+
filePath: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler/PipelineHelpers/testArtifacts.ps1'
120+
arguments: '-StagingDirectory "$(Pipeline.Workspace)\staging\coretools-cli"'
121+
workingDirectory: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler'
122+
123+
- task: DotNetCoreCLI@2
124+
displayName: 'Zip Artifacts'
125+
inputs:
126+
command: run
127+
projects: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler/Azure.Functions.ArtifactAssembler.csproj'
128+
arguments: '--no-build -c release -- zip'
129+
workingDirectory: '$(Pipeline.Workspace)'
130+
131+
- task: PowerShell@2
132+
displayName: 'Generate SHA files'
133+
inputs:
134+
targetType: filePath
135+
filePath: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler/PipelineHelpers/generateSha.ps1'
136+
arguments: '-CurrentDirectory "$(Pipeline.Workspace)"'
137+
138+
- task: CopyFiles@2
139+
displayName: 'Copy Files to Artifact Staging Directory'
140+
inputs:
141+
SourceFolder: '$(Pipeline.Workspace)/staging'
142+
Contents: 'coretools-*/**'
143+
TargetFolder: '$(Build.ArtifactStagingDirectory)/core-tools'
144+
145+
- ${{ if eq(parameters.arch, 'min.win-x64') }}:
146+
- task: CopyFiles@2
147+
displayName: 'Copy metadata.json to Artifact Staging Directory'
148+
inputs:
149+
SourceFolder: '$(Pipeline.Workspace)/staging'
150+
Contents: 'metadata.json'
151+
TargetFolder: '$(Build.ArtifactStagingDirectory)'

‎eng/ci/templates/official/jobs/build-test.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,10 @@ jobs:
8181
- pwsh: |
8282
.\check-vulnerabilities.ps1
8383
displayName: "Check for security vulnerabilities"
84+
- ${{ if eq(variables['Build.Reason'], 'Schedule') }}:
85+
- pwsh: |
86+
# Check if this is a scheduled build
87+
Write-Host "##vso[build.addbuildtag]nightly-build"
8488
- pwsh: |
8589
.\build.ps1
8690
env:

‎eng/ci/templates/official/jobs/process-nuget-package.yml

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
jobs:
2+
- job: ProcessNuGetPackage
3+
displayName: "Process NuGet Package"
4+
templateContext:
5+
inputs:
6+
- input: pipelineArtifact
7+
pipeline: core-tools-default
8+
artifactName: drop
9+
targetPath: $(Pipeline.Workspace)\core-tools-default\drop
10+
outputs:
11+
- output: pipelineArtifact
12+
displayName: Publish nuget package
13+
targetPath: $(Pipeline.Workspace)/nugetPackage
14+
artifact: drop-nuget-package
15+
steps:
16+
- checkout: self
17+
18+
- task: PowerShell@2
19+
displayName: 'Process and move nuget package to staging directory'
20+
inputs:
21+
targetType: filePath
22+
filePath: '$(Build.SourcesDirectory)/src/Azure.Functions.ArtifactAssembler/PipelineHelpers/moveNugetPackage.ps1'
23+
arguments: '-CurrentDirectory "$(Pipeline.Workspace)"'

0 commit comments

Comments
 (0)