Skip to content

Commit

Permalink
Merge pull request #35 from aks-lts/haitao/windows-jobsv2
Browse files Browse the repository at this point in the history
add CAPZ windows prow jobs
  • Loading branch information
haitch authored Jan 27, 2025
2 parents bce51e4 + f1c49c5 commit 9fec5e2
Show file tree
Hide file tree
Showing 6 changed files with 319 additions and 8 deletions.
26 changes: 25 additions & 1 deletion .github/workflows/deploy-lts-prow.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,14 @@ jobs:
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

- name: create resourceGroup
- name: create resourceGroups
run: |
if [ $(az group exists --name ${{ secrets.AZURE_RG }}) = false ]; then
az group create --name ${{ secrets.AZURE_RG }} --location ${{ secrets.AZURE_LOCATION }}
fi
if [ $(az group exists --name ${{ secrets.CAPZ_RG }}) = false ]; then
az group create --name ${{ secrets.CAPZ_RG }} --location ${{ secrets.AZURE_LOCATION }}
fi
- name: Deploy Prow cluster Bicep
id: bicep
Expand All @@ -53,12 +56,23 @@ jobs:
parameters: aks_cluster_region=${{ secrets.AZURE_LOCATION }} aks_cluster_admin_groups="${{ secrets.PROW_ADMIN_GROUPS }}" aks_cluster_admin_users="${{ secrets.PROW_ADMIN_USERS }}"
failOnStdErr: false

- name: Deploy CAPZ Bicep
id: capzbicep
uses: azure/arm-deploy@v2
with:
subscriptionId: ${{ vars.AZURE_SUBSCRIPTION_ID }}
resourceGroupName: ${{ secrets.CAPZ_RG }}
template: ./config/capz/capz.bicep
parameters: location=${{ secrets.AZURE_LOCATION }}
failOnStdErr: false

- name: Fetch config
run: |
echo "PROW_HOST=${{ steps.bicep.outputs.prowHostName }}" >> "$GITHUB_ENV"
echo "AZURE_STORAGE_ACCOUNT_USER=${{ steps.bicep.outputs.storageAccountName }}" >> "$GITHUB_ENV"
echo "PUBLIC_IP_NAME=${{ steps.bicep.outputs.publicIpName }}" >> "$GITHUB_ENV"
echo "CLUSTER_RG=${{ steps.bicep.outputs.resourceGroupName }}" >> "$GITHUB_ENV"
echo "CAPZ_SA=${{ steps.capzbicep.outputs.capzsastorage_name }}" >> "$GITHUB_ENV"
- name: Install Kubectl
uses: azure/setup-kubectl@v4
Expand Down Expand Up @@ -102,6 +116,12 @@ jobs:
PUBLIC_IP_ADDRESS=$(az network public-ip show -g ${{ secrets.AZURE_RG }} -n ${{ steps.bicep.outputs.publicIpName }} | jq -r '.ipAddress')
echo "::add-mask::$PUBLIC_IP_ADDRESS"
echo "PUBLIC_IP_ADDRESS=$PUBLIC_IP_ADDRESS" >> "$GITHUB_ENV"
CAPZ_CI_REGISTRY=$(az acr show -g ${{ secrets.CAPZ_RG }} -n ${{ steps.capzbicep.outputs.capzci_registry_name }} | jq .loginServer)
echo "::add-mask::$CAPZ_CI_REGISTRY"
echo "CAPZ_CI_REGISTRY=$CAPZ_CI_REGISTRY" >> "$GITHUB_ENV"
echo "AZURE_SUBSCRIPTION_ID=${{ secrets.AZURE_SUBSCRIPTION_ID }}" >> "$GITHUB_ENV"
echo "::add-mask::${{ steps.capzbicep.outputs.capz_gmsa_kv_name }}"
echo "CAPZ_GMSA_KV=${{ steps.capzbicep.outputs.capz_gmsa_kv_name }}" >> "$GITHUB_ENV"
- name: 'Create job configs'
run: |
Expand All @@ -110,6 +130,10 @@ jobs:
envsubst < config/prow/release-branch-jobs/1.28.yaml >> cm.yaml
kubectl create configmap config -n prow --from-file=config.yaml=cm.yaml -o yaml --dry-run=client | kubectl apply -f -
rm cm.yaml
env:
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
CAPZ_RG: ${{ secrets.CAPZ_RG }}
CAPZ_GMSA_KV: ${{ steps.capzbicep.outputs.capz_gmsa_kv_name }}

- name: 'Apply Prowjob CRD'
run: for f in config/prow/k8s/prowjob/*.yaml; do kubectl apply --server-side=true -f $f; done
Expand Down
155 changes: 155 additions & 0 deletions config/capz/capz.bicep
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
param resource_prefix string = 'capz'
@secure()
param location string = resourceGroup().location

param random_suffix string = substring(uniqueString(resourceGroup().id, location), 0, 8)

// https://github.com/kubernetes/k8s.io/tree/main/infra/azure/terraform/capz
// https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-sigs/sig-windows/release-master-windows.yaml
// https://github.com/kubernetes-sigs/windows-testing/tree/master/capz

resource cloudproviderId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
name: 'cloud-provider-user-identity'
location: location
}

resource domainVMId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
name: 'domain-vm-identity'
location: location
}

resource gmsaId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
name: 'gmsa-user-identity'
location: location
}

resource gmsa_kv 'Microsoft.KeyVault/vaults@2023-07-01' = {
name: '${resource_prefix}gmsakv${random_suffix}'
location: location
properties: {
sku: {
family: 'A'
name: 'standard'
}
tenantId: subscription().tenantId
accessPolicies: [
{
tenantId: subscription().tenantId
objectId: domainVMId.properties.principalId
permissions: {
secrets: ['set']
}
}
{
tenantId: subscription().tenantId
objectId: gmsaId.properties.principalId
permissions: {
secrets: ['get']
}
}
]
}
}

resource capzci_registry 'Microsoft.ContainerRegistry/registries@2023-01-01-preview' = {
name: '${resource_prefix}ci${random_suffix}'
location: location
sku: {
name: 'Premium'
}
properties:{
anonymousPullEnabled: true
policies: {
retentionPolicy: {
days: 7
status: 'enabled'
}
}
}
}

resource registrytask 'Microsoft.ContainerRegistry/registries/tasks@2019-06-01-preview' = {
name: 'midnight_capz_purge'
parent: capzci_registry
location: location
properties: {
platform: {
os: 'Linux'
architecture: 'amd64'
}
trigger:{
timerTriggers: [
{
name: 't1'
schedule: '0 0 * * *'
status: 'enabled'
}
]
baseImageTrigger: {
name: 'defaultBaseimageTriggerName'
baseImageTriggerType: 'Runtime'
updateTriggerPayloadType: 'Default'
}
}
agentConfiguration: {
cpu: 2
}

step: {
type: 'EncodedTask'
encodedTaskContent: base64('''
version: v1.1.0
steps:
- cmd: acr purge --filter azdisk:* --filter azure-cloud-controller-manager:* --filter azure-cloud-node-manager-arm64:* --filter azure-cloud-node-manager:* --filter cluster-api-azure:* --ago 1d --untagged
disableWorkingDirectoryOverride: true
timeout: 3600
''')
}
}
}

resource e2eprivatecommunity 'Microsoft.ContainerRegistry/registries@2023-01-01-preview' = {
name: '${resource_prefix}e2e${random_suffix}'
location: location
sku: {
name: 'Premium'
}
properties:{
anonymousPullEnabled: true
policies: {
retentionPolicy: {
days: 7
status: 'enabled'
}
}
}
}

resource capzsa 'Microsoft.Storage/storageAccounts@2022-05-01' = {
name: '${resource_prefix}sa${random_suffix}'
location: location
sku: {
name: 'Standard_ZRS'
}
kind: 'StorageV2'
properties: {
accessTier: 'Hot'
minimumTlsVersion: 'TLS1_2'
supportsHttpsTrafficOnly: true
}
}

resource cloudproviderblobcontributor 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
name: guid('storage-rbac', cloudproviderId.id, capzsa.id, 'Storage Blob Data Contributor')
scope: capzsa
properties: {
principalId: cloudproviderId.properties.principalId
roleDefinitionId: tenantResourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe') // Storage Blob Data Contributor
principalType: 'ServicePrincipal'
description: 'Allow controlPlane VM to download private build from storage account'
}
}

output capzci_registry_name string = capzci_registry.name
output capz_gmsa_kv_name string = gmsa_kv.name
output capzsastorage_name string = capzsa.name
4 changes: 2 additions & 2 deletions config/prow/cluster/prow-cluster.bicep
Original file line number Diff line number Diff line change
Expand Up @@ -99,11 +99,11 @@ resource aks 'Microsoft.ContainerService/managedClusters@2024-06-01' = {
}
}
oidcIssuerProfile: {
enabled: true
enabled: true // windows capz prowjob need this
}
securityProfile: {
workloadIdentity: {
enabled: true
enabled: true // windows capz prowjob need this
}
}
}
Expand Down
12 changes: 12 additions & 0 deletions config/prow/k8s/test-pods/capz.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: capz-clusteradmin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: azure
namespace: test-pods
59 changes: 58 additions & 1 deletion config/prow/release-branch-jobs/1.27.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,7 @@
memory: 9Gi
securityContext:
privileged: true
- always_run: true
- always_run: true # pull-kubernetes-verify
branches:
- release-1.27-lts
context: pull-kubernetes-verify
Expand Down Expand Up @@ -345,5 +345,62 @@
requests:
cpu: "7"
memory: 16Gi
securityContext:
privileged: true
- always_run: false # pull-kubernetes-e2e-capz-windows
branches:
- release-1.27-lts
name: pull-kubernetes-e2e-capz-windows
decorate: true
decoration_config:
timeout: 4h
labels:
preset-dind-enabled: "true"
preset-kind-volume-mounts: "true"
preset-capz-windows-common-pull: "true"
preset-capz-windows-2019: "true"
preset-capz-containerd-1-7-latest: "true"
preset-azure-community: "true"
extra_refs:
- base_ref: release-1.16
org: kubernetes-sigs
path_alias: sigs.k8s.io/cluster-api-provider-azure
repo: cluster-api-provider-azure
workdir: false
- base_ref: release-1.27
org: kubernetes-sigs
path_alias: sigs.k8s.io/cloud-provider-azure
repo: cloud-provider-azure
- org: haitch
repo: windows-testing
base_ref: master
path_alias: k8s.io/windows-testing
workdir: true
optional: true
path_alias: k8s.io/kubernetes
run_if_changed: azure.*\.go$|.*windows\.go$|test/e2e/windows/.*
spec:
serviceAccountName: azure
containers:
- command:
- "runner.sh"
- "env"
- "./capz/run-capz-e2e.sh"
image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20240705-131cd74733-1.27
name: ""
env:
- name: CAPI_VERSION
value: "v1.7.3" # newer version (v1.7.4 and beyond) use kubeadm.k8s.io/v1beta4, which is not support by 1.27 kubeadm.
- name: AZURE_LOCATION
value: "australiaeast"
- name: "KUBERNETES_VERSION"
value: "1.27"
resources:
requests:
cpu: "2"
memory: 9Gi
limits:
cpu: "2"
memory: 9Gi
securityContext:
privileged: true
Loading

0 comments on commit 9fec5e2

Please sign in to comment.