diff --git a/.azure-pipelines/ci-build.yml b/.azure-pipelines/ci-build.yml index c916a3bc6..75875f913 100644 --- a/.azure-pipelines/ci-build.yml +++ b/.azure-pipelines/ci-build.yml @@ -7,6 +7,7 @@ trigger: include: - main - support/v1 + - task/move-to-deploy-stage tags: include: - 'v*' @@ -15,14 +16,15 @@ pr: include: - main - support/v1 + - task/move-to-deploy-stage variables: buildPlatform: 'Any CPU' buildConfiguration: 'Release' ProductBinPath: '$(Build.SourcesDirectory)\src\Microsoft.OpenApi\bin\$(BuildConfiguration)' - REGISTRY: 'msgraphprodregistry.azurecr.io' + REGISTRY: 'msgraphpperegistry.azurecr.io' IMAGE_NAME: 'public/openapi/hidi' - PREVIEW_BRANCH: 'refs/heads/main' + PREVIEW_BRANCH: 'refs/heads/task/move-to-deploy-stage' resources: repositories: @@ -41,6 +43,10 @@ extends: - ES365AIMigrationTooling stages: - stage: build + pool: + name: Azure-Pipelines-1ESPT-ExDShared + image: ubuntu-latest + os: linux jobs: - job: build templateContext: @@ -49,16 +55,28 @@ extends: displayName: 'Publish Artifact: Nugets' artifactName: Nugets targetPath: '$(Build.ArtifactStagingDirectory)/Nugets' + - output: pipelineArtifact + displayName: 'Publish Artifact: DockerImage' + artifactName: DockerImage + targetPath: '$(Build.ArtifactStagingDirectory)/DockerImage' + - output: pipelineArtifact + displayName: 'Publish Artifact: ImageConfig' + artifactName: ImageConfig + targetPath: '$(Build.ArtifactStagingDirectory)/ImageConfig' steps: - task: UseDotNet@2 displayName: 'Use .NET 6' inputs: - version: 6.x + version: 6.0.x + packageType: sdk + installationPath: $(Agent.ToolsDirectory)/dotnet - task: UseDotNet@2 displayName: 'Use .NET 8' inputs: - version: 8.x + version: 8.0.x + packageType: sdk + installationPath: $(Agent.ToolsDirectory)/dotnet # Install the nuget tool. - task: NuGetToolInstaller@1 @@ -71,7 +89,7 @@ extends: - task: DotNetCoreCLI@2 displayName: 'build' inputs: - projects: '$(Build.SourcesDirectory)\Microsoft.OpenApi.sln' + projects: '$(Build.SourcesDirectory)/Microsoft.OpenApi.sln' arguments: '--configuration $(BuildConfiguration) --no-incremental' # Run the Unit test @@ -79,7 +97,7 @@ extends: displayName: 'test' inputs: command: test - projects: '$(Build.SourcesDirectory)\Microsoft.OpenApi.sln' + projects: '$(Build.SourcesDirectory)/Microsoft.OpenApi.sln' arguments: '--configuration $(BuildConfiguration) --no-build' - task: EsrpCodeSigning@5 @@ -91,7 +109,7 @@ extends: AuthAKVName: 'akv-prod-eastus' AuthCertName: 'ReferenceLibraryPrivateCert' AuthSignCertName: 'ReferencePackagePublisherCertificate' - FolderPath: '$(Build.SourcesDirectory)\src' + FolderPath: '$(Build.SourcesDirectory)/src' signConfigType: 'inlineSignParams' inlineOperation: | [ @@ -198,239 +216,268 @@ extends: targetFolder: $(Build.ArtifactStagingDirectory)/Nugets sourceFolder: $(Build.ArtifactStagingDirectory) content: '*.nupkg' + + # Instead of copying repository files, build Docker images here + - script: | + docker run --privileged --rm tonistiigi/binfmt --install all + displayName: "Enable multi-platform builds" + + - script: | + docker buildx create --use --name mybuilder + displayName: "Set up Docker BuildX" + + - script: | + docker buildx inspect --bootstrap + displayName: "Ensure BuildX is working" + + - bash: | + # Extract version from project properties + versionInfo=$(pwsh -Command "(Get-Content '$(Build.SourcesDirectory)/Directory.Build.props' | Select-String -Pattern '(.*)').Matches.Groups[1].Value") + echo "##vso[task.setvariable variable=VERSION]$versionInfo" + + # Get date for tagging + buildDate=$(date +'%Y%m%d') + echo "##vso[task.setvariable variable=BUILDDATE]$buildDate" + + # Extract run number + runNumber=$(echo "$(Build.BuildNumber)" | grep -o '[0-9]\+$' || date +"%S%N" | cut -c1-3) + echo "##vso[task.setvariable variable=RUNNUMBER]$runNumber" + + # Create directories for Docker image artifacts and config + mkdir -p "$(Build.ArtifactStagingDirectory)/DockerImage" + mkdir -p "$(Build.ArtifactStagingDirectory)/ImageConfig" + + # Save configuration for deploy stage + echo "$versionInfo" > "$(Build.ArtifactStagingDirectory)/ImageConfig/version.txt" + echo "$buildDate" > "$(Build.ArtifactStagingDirectory)/ImageConfig/builddate.txt" + echo "$runNumber" > "$(Build.ArtifactStagingDirectory)/ImageConfig/runnumber.txt" + echo "$(Build.SourceBranch)" > "$(Build.ArtifactStagingDirectory)/ImageConfig/sourcebranch.txt" + displayName: 'Extract version and build info' + + - bash: | + # Create image tag based on branch/tag + if [[ "$(Build.SourceBranch)" == "$(PREVIEW_BRANCH)" ]]; then + # For main branch + IMAGE_TAGS="--tag $(REGISTRY)/$(IMAGE_NAME):nightly --tag $(REGISTRY)/$(IMAGE_NAME):$(VERSION).$(BUILDDATE)$(RUNNUMBER)" + echo "nightly" > "$(Build.ArtifactStagingDirectory)/ImageConfig/imagetype.txt" + elif [[ "$(Build.SourceBranch)" == refs/tags/v* ]]; then + # For release tags + IMAGE_TAGS="--tag $(REGISTRY)/$(IMAGE_NAME):latest --tag $(REGISTRY)/$(IMAGE_NAME):$(VERSION)" + echo "release" > "$(Build.ArtifactStagingDirectory)/ImageConfig/imagetype.txt" + else + # For other branches/PRs + IMAGE_TAGS="--tag $(REGISTRY)/$(IMAGE_NAME):$(VERSION).$(BUILDDATE)$(RUNNUMBER)" + echo "branch" > "$(Build.ArtifactStagingDirectory)/ImageConfig/imagetype.txt" + fi + + echo "$IMAGE_TAGS" > "$(Build.ArtifactStagingDirectory)/ImageConfig/imagetags.txt" + echo "##vso[task.setvariable variable=IMAGE_TAGS]$IMAGE_TAGS" + displayName: 'Set Docker image tags' + + - task: AzureCLI@2 + displayName: 'Login to Azure Container Registry' + inputs: + azureSubscription: 'ACR Push Test' + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + az acr login --name msgraphpperegistry + + - bash: | + # Build image with BuildX but don't push yet - save to local tar file + docker buildx build \ + --platform linux/amd64,linux/arm64/v8 \ + --output "type=docker,dest=$(Build.ArtifactStagingDirectory)/DockerImage/docker-image.tar" \ + $(IMAGE_TAGS) \ + $(Build.SourcesDirectory) + displayName: 'Build Docker image and save to artifact' + + # Copy Docker image to staging directory using CopyFiles + - task: CopyFiles@2 + displayName: 'Prepare Docker image for upload' + inputs: + sourceFolder: $(Build.ArtifactStagingDirectory)/DockerImage + contents: '**/*' + targetFolder: $(Build.ArtifactStagingDirectory)/DockerImage + + # Copy Docker image config to staging directory + - task: CopyFiles@2 + displayName: 'Prepare Docker image config for upload' + inputs: + sourceFolder: $(Build.ArtifactStagingDirectory)/ImageConfig + contents: '**/*' + targetFolder: $(Build.ArtifactStagingDirectory)/ImageConfig - stage: deploy - condition: and(contains(variables['build.sourceBranch'], 'refs/tags/v'), succeeded()) + condition: and(or(contains(variables['Build.SourceBranch'], 'refs/tags/v'), eq(variables['Build.SourceBranch'], variables['PREVIEW_BRANCH'])), succeeded()) dependsOn: build + pool: + name: Azure-Pipelines-1ESPT-ExDShared + os: linux + image: ubuntu-latest jobs: - - deployment: deploy_hidi - templateContext: - type: releaseJob - isProduction: true - inputs: - - input: pipelineArtifact - artifactName: Nugets - targetPath: '$(Pipeline.Workspace)' - dependsOn: [] - environment: nuget-org - strategy: - runOnce: - deploy: - pool: - vmImage: ubuntu-latest - steps: - - task: 1ES.PublishNuget@1 - displayName: 'NuGet push' - inputs: - packagesToPush: '$(Pipeline.Workspace)/Microsoft.OpenApi.Hidi.*.nupkg' - packageParentPath: '$(Pipeline.Workspace)' - nuGetFeedType: external - publishFeedCredentials: 'OpenAPI Nuget Connection' + # - deployment: deploy_hidi + # condition: and(contains(variables['build.SourceBranch'], 'refs/tags/v'), succeeded()) + # templateContext: + # type: releaseJob + # isProduction: true + # inputs: + # - input: pipelineArtifact + # artifactName: Nugets + # targetPath: '$(Pipeline.Workspace)' + # dependsOn: [] + # environment: nuget-org + # strategy: + # runOnce: + # deploy: + # pool: + # vmImage: ubuntu-latest + # steps: + # - task: 1ES.PublishNuget@1 + # displayName: 'NuGet push' + # inputs: + # packagesToPush: '$(Pipeline.Workspace)/Microsoft.OpenApi.Hidi.*.nupkg' + # packageParentPath: '$(Pipeline.Workspace)' + # nuGetFeedType: external + # publishFeedCredentials: 'OpenAPI Nuget Connection' - - deployment: deploy_lib - templateContext: - type: releaseJob - isProduction: true - inputs: - - input: pipelineArtifact - artifactName: Nugets - targetPath: '$(Pipeline.Workspace)' - dependsOn: [] - environment: nuget-org - strategy: - runOnce: - deploy: - pool: - vmImage: ubuntu-latest - steps: - - powershell: | - $fileNames = "$(Pipeline.Workspace)/Microsoft.OpenApi.Hidi.*.nupkg", "$(Pipeline.Workspace)/Microsoft.OpenApi.YamlReader.*.nupkg", "$(Pipeline.Workspace)/Microsoft.OpenApi.Workbench.*.nupkg" - foreach($fileName in $fileNames) { - if(Test-Path $fileName) { - rm $fileName -Verbose - } - } - displayName: remove other nupkgs to avoid duplication - - task: 1ES.PublishNuget@1 - displayName: 'NuGet push' - inputs: - packagesToPush: '$(Pipeline.Workspace)/Microsoft.OpenApi.*.nupkg' - packageParentPath: '$(Pipeline.Workspace)' - nuGetFeedType: external - publishFeedCredentials: 'OpenAPI Nuget Connection' + # - deployment: deploy_lib + # condition: and(contains(variables['build.SourceBranch'], 'refs/tags/v'), succeeded()) + # templateContext: + # type: releaseJob + # isProduction: true + # inputs: + # - input: pipelineArtifact + # artifactName: Nugets + # targetPath: '$(Pipeline.Workspace)' + # dependsOn: [] + # environment: nuget-org + # strategy: + # runOnce: + # deploy: + # pool: + # vmImage: ubuntu-latest + # steps: + # - powershell: | + # $fileNames = "$(Pipeline.Workspace)/Microsoft.OpenApi.Hidi.*.nupkg", "$(Pipeline.Workspace)/Microsoft.OpenApi.YamlReader.*.nupkg", "$(Pipeline.Workspace)/Microsoft.OpenApi.Workbench.*.nupkg" + # foreach($fileName in $fileNames) { + # if(Test-Path $fileName) { + # rm $fileName -Verbose + # } + # } + # displayName: remove other nupkgs to avoid duplication + # - task: 1ES.PublishNuget@1 + # displayName: 'NuGet push' + # inputs: + # packagesToPush: '$(Pipeline.Workspace)/Microsoft.OpenApi.*.nupkg' + # packageParentPath: '$(Pipeline.Workspace)' + # nuGetFeedType: external + # publishFeedCredentials: 'OpenAPI Nuget Connection' - - deployment: deploy_yaml_reader - templateContext: - type: releaseJob - isProduction: true - inputs: - - input: pipelineArtifact - artifactName: Nugets - targetPath: '$(Pipeline.Workspace)' - dependsOn: deploy_lib - environment: nuget-org - strategy: - runOnce: - deploy: - pool: - vmImage: ubuntu-latest - steps: - - task: 1ES.PublishNuget@1 - displayName: 'NuGet push' - inputs: - packagesToPush: '$(Pipeline.Workspace)/Microsoft.OpenApi.YamlReader.*.nupkg' - packageParentPath: '$(Pipeline.Workspace)' - nuGetFeedType: external - publishFeedCredentials: 'OpenAPI Nuget Connection' + # - deployment: deploy_yaml_reader + # condition: and(contains(variables['build.SourceBranch'], 'refs/tags/v'), succeeded()) + # templateContext: + # type: releaseJob + # isProduction: true + # inputs: + # - input: pipelineArtifact + # artifactName: Nugets + # targetPath: '$(Pipeline.Workspace)' + # dependsOn: deploy_lib + # environment: nuget-org + # strategy: + # runOnce: + # deploy: + # pool: + # vmImage: ubuntu-latest + # steps: + # - task: 1ES.PublishNuget@1 + # displayName: 'NuGet push' + # inputs: + # packagesToPush: '$(Pipeline.Workspace)/Microsoft.OpenApi.YamlReader.*.nupkg' + # packageParentPath: '$(Pipeline.Workspace)' + # nuGetFeedType: external + # publishFeedCredentials: 'OpenAPI Nuget Connection' + + # - deployment: create_github_release + # templateContext: + # type: releaseJob + # isProduction: true + # inputs: + # - input: pipelineArtifact + # artifactName: Nugets + # targetPath: '$(Pipeline.Workspace)' + # dependsOn: [] + # environment: kiota-github-releases + # strategy: + # runOnce: + # deploy: + # pool: + # vmImage: ubuntu-latest + # steps: + # - pwsh: | + # $artifactName = Get-ChildItem -Path $(Pipeline.Workspace) -Filter Microsoft.OpenApi.*.nupkg -recurse | select -First 1 + # $artifactVersion= $artifactName.Name -replace "Microsoft.OpenApi.", "" -replace ".nupkg", "" + # #Set Variable $artifactName and $artifactVersion + # Write-Host "##vso[task.setvariable variable=artifactVersion; isSecret=false;]$artifactVersion" + # echo "$artifactVersion" + # displayName: 'Fetch Artifact Name' + # - task: GitHubRelease@1 + # displayName: 'GitHub release (edit)' + # condition: succeededOrFailed() + # inputs: + # gitHubConnection: 'Github-MaggieKimani1' + # action: edit + # tagSource: userSpecifiedTag + # tag: 'v$(artifactVersion)' + # releaseNotesSource: inline + # assets: '$(Pipeline.Workspace)\**\*.exe' + # addChangeLog: false - - deployment: create_github_release + - deployment: deploy_docker_image + environment: docker-images-deploy templateContext: type: releaseJob isProduction: true inputs: - input: pipelineArtifact - artifactName: Nugets - targetPath: '$(Pipeline.Workspace)' - dependsOn: [] - environment: kiota-github-releases + artifactName: DockerImage + targetPath: '$(Pipeline.Workspace)/DockerImage' + - input: pipelineArtifact + artifactName: ImageConfig + targetPath: '$(Pipeline.Workspace)/ImageConfig' strategy: runOnce: deploy: pool: - vmImage: ubuntu-latest + vmImage: 'ubuntu-latest' steps: - - pwsh: | - $artifactName = Get-ChildItem -Path $(Pipeline.Workspace) -Filter Microsoft.OpenApi.*.nupkg -recurse | select -First 1 - $artifactVersion= $artifactName.Name -replace "Microsoft.OpenApi.", "" -replace ".nupkg", "" - #Set Variable $artifactName and $artifactVersion - Write-Host "##vso[task.setvariable variable=artifactVersion; isSecret=false;]$artifactVersion" - echo "$artifactVersion" - displayName: 'Fetch Artifact Name' - - task: GitHubRelease@1 - displayName: 'GitHub release (edit)' - condition: succeededOrFailed() - inputs: - gitHubConnection: 'Github-MaggieKimani1' - action: edit - tagSource: userSpecifiedTag - tag: 'v$(artifactVersion)' - releaseNotesSource: inline - assets: '$(Pipeline.Workspace)\**\*.exe' - addChangeLog: false - - - stage: Build_and_deploy_docker_images - displayName: 'Build and deploy docker images' - condition: or(eq(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['build.sourceBranch'], variables['PREVIEW_BRANCH'])) - dependsOn: build - pool: - name: Azure-Pipelines-1ESPT-ExDShared - image: ubuntu-latest - os: linux - jobs: - - job: buildAndPush - steps: - - task: AzureCLI@2 - displayName: 'Login to Azure Container Registry' - inputs: - azureSubscription: 'ACR Images Push Service Connection' - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - az acr login --name msgraphprodregistry - - - powershell: | - $content = [XML](Get-Content ./Directory.Build.props) - Write-Host "XML loaded, finding version..." - - # Handle PropertyGroup as either a single element or array - $version = $null - if ($content.Project.PropertyGroup -is [array]) { - Write-Host "PropertyGroup is an array, checking each entry..." - foreach ($pg in $content.Project.PropertyGroup) { - if ($pg.Version) { - $version = $pg.Version.ToString().Trim() - Write-Host "Found version in PropertyGroup array: $version" - break - } - } - } else { - # Single PropertyGroup - $version = $content.Project.PropertyGroup.Version - if ($version) { - $version = $version.ToString().Trim() - Write-Host "Found version in PropertyGroup: $version" - } - } - - if (-not $version) { - Write-Host "##vso[task.logissue type=error]Version not found in Directory.Build.props" - exit 1 - } - - Write-Host "Version found: $version" - Write-Host "##vso[task.setvariable variable=version;isoutput=true]$version" - Write-Host "##vso[task.setvariable variable=VERSION]$version" - displayName: 'Get version from csproj' - name: getversion - - - bash: | - # Debug output to verify version variable - echo "Version from previous step: $VERSION" - displayName: 'Verify version variable' - - - bash: | - echo "Build Number: $(Build.BuildNumber)" - # Extract the last 3 characters for the run number - runnumber=$(echo "$(Build.BuildNumber)" | grep -o '[0-9]\+$') - echo "Extracted Run Number: $runnumber" - - # If extraction fails, set a default - if [ -z "$runnumber" ]; then - echo "Extraction failed, using default value" - runnumber=$(date +"%S%N" | cut -c1-3) - echo "Generated fallback run number: $runnumber" - fi - - # Set the variable for later steps - echo "##vso[task.setvariable variable=RUNNUMBER]$runnumber" - echo "##vso[task.setvariable variable=RUNNUMBER;isOutput=true]$runnumber" - displayName: 'Get truncated run number' - name: getrunnumber - condition: eq(variables['Build.SourceBranch'], variables['PREVIEW_BRANCH']) - - - bash: | - date=$(date +'%Y%m%d') - echo "Date value: $date" - echo "##vso[task.setvariable variable=BUILDDATE;isOutput=true]$date" - echo "##vso[task.setvariable variable=BUILDDATE]$date" - displayName: 'Get current date' - name: setdate - condition: eq(variables['Build.SourceBranch'], variables['PREVIEW_BRANCH']) - - - bash: | - echo "Building Docker image..." - echo "Using build date: ${BUILDDATE}" - # Using quotes around tags to prevent flag interpretation - docker build \ - -t "$(REGISTRY)/$(IMAGE_NAME):nightly" \ - -t "$(REGISTRY)/$(IMAGE_NAME):${VERSION}.${BUILDDATE}${RUNNUMBER}" \ - "$(Build.SourcesDirectory)" - - echo "Pushing Docker image with nightly tag..." - docker push "$(REGISTRY)/$(IMAGE_NAME):nightly" - docker push "$(REGISTRY)/$(IMAGE_NAME):${VERSION}.${BUILDDATE}${RUNNUMBER}" - displayName: 'Build and Push Nightly Image' - condition: eq(variables['Build.SourceBranch'], variables['PREVIEW_BRANCH']) - - - bash: | - echo "Building Docker image for release..." - docker build \ - -t "$(REGISTRY)/$(IMAGE_NAME):latest" \ - -t "$(REGISTRY)/$(IMAGE_NAME):${VERSION}.${BUILDDATE}${RUNNUMBER}" \ - "$(Build.SourcesDirectory)" + - task: AzureCLI@2 + displayName: 'Login to Azure Container Registry' + inputs: + azureSubscription: 'ACR Push Test' + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + az acr login --name msgraphpperegistry - echo "Pushing Docker image with latest and version tags..." - docker push "$(REGISTRY)/$(IMAGE_NAME):latest" - docker push "$(REGISTRY)/$(IMAGE_NAME):${VERSION}.${BUILDDATE}${RUNNUMBER}" - displayName: 'Build and Push Release Image' - condition: startsWith(variables['Build.SourceBranch'], 'refs/tags/v') \ No newline at end of file + - bash: | + # Load the Docker image from tar file + docker load -i $(Pipeline.Workspace)/DockerImage/docker-image.tar + + # Read config values from artifact + VERSION=$(cat $(Pipeline.Workspace)/ImageConfig/version.txt) + BUILDDATE=$(cat $(Pipeline.Workspace)/ImageConfig/builddate.txt) + RUNNUMBER=$(cat $(Pipeline.Workspace)/ImageConfig/runnumber.txt) + + # Push the image tags + if [[ "$(Build.SourceBranch)" == "$(PREVIEW_BRANCH)" ]]; then + docker push $(REGISTRY)/$(IMAGE_NAME):nightly + docker push $(REGISTRY)/$(IMAGE_NAME):$VERSION.$BUILDDATE$RUNNUMBER + elif [[ "$(Build.SourceBranch)" == refs/tags/v* ]]; then + docker push $(REGISTRY)/$(IMAGE_NAME):latest + docker push $(REGISTRY)/$(IMAGE_NAME):$VERSION + else + docker push $(REGISTRY)/$(IMAGE_NAME):$VERSION.$BUILDDATE$RUNNUMBER + fi + displayName: 'Push Docker Image to Registry' \ No newline at end of file