Update SAR source #1790
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update SAR source | |
| on: | |
| schedule: | |
| - cron: '0 12 * * *' | |
| workflow_dispatch: | |
| jobs: | |
| update: | |
| name: Update | |
| runs-on: macos-13 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v1 | |
| - name: Auth gcloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| project_id: ${{ secrets.GCP_PROJECT_ID }} | |
| create_credentials_file: true | |
| credentials_json: ${{ secrets.GCP_KEY_JSON }} | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: Azure Login | |
| uses: azure/login@v1 | |
| with: | |
| creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| - name: Manual | |
| run: | | |
| curl -v "https://docs.aws.amazon.com/service-authorization/latest/reference/reference_policies_actions-resources-contextkeys.html" | |
| pip3 install bs4 requests pyyaml inflect | |
| python3 -u util/aws_parliament_update_iam_data.py > util/aws_js/iam_definition.json | |
| sleep 1 | |
| mkdir -p gcp/roles/ | |
| cd gcp | |
| rm -rf google-api-go-client | |
| git clone --depth 1 https://github.com/googleapis/google-api-go-client.git | |
| rm -rf google-api-go-client/.git | |
| rm -rf google-cloud-node | |
| git clone --depth 1 https://github.com/googleapis/google-cloud-node.git | |
| rm -rf google-cloud-node/.git | |
| cd .. | |
| gcloud iam roles list --show-deleted --format json > gcp/predefined_roles.json | |
| basic_roles=$(for role in roles/reader roles/writer roles/admin; do | |
| gcloud iam roles describe "$role" --format json | jq 'del(.includedPermissions)' | |
| done | jq -s -c 'flatten') | |
| jq --argjson basic_roles "$basic_roles" '. + $basic_roles' gcp/predefined_roles.json > gcp/predefined_roles.json.tmp && mv gcp/predefined_roles.json.tmp gcp/predefined_roles.json | |
| gcloud iam roles list --show-deleted --format json | jq -r '.[]|[.name,(.name | split("/")[-1])] | @tsv' | | |
| while IFS=$'\t' read -r name shortname; do | |
| echo "Writing $shortname.json" | |
| gcloud iam roles describe "$name" --format json > gcp/roles/$shortname.json || echo "Failure to write $shortname.json" | |
| done | |
| git clone https://github.com/iann0036/iam-privilege-catalog.git | |
| python3 util/gcp_get_risks.py || true | |
| rm -rf iam-privilege-catalog | |
| python3 util/gcp_get_methods.py || true | |
| python3 util/gcp_get_permissions.py || true | |
| python3 util/gcp_compare_datasources.py || true | |
| python3 util/gcp_crawl.py || true | |
| rm -rf gcp/google-api-go-client | |
| rm -rf gcp/google-cloud-node | |
| rm gcreds || true | |
| git clone https://github.com/Azure/azure-rest-api-specs.git | |
| python3 util/azure_process_spec.py | |
| az provider operation list > azure/provider-operations.json | |
| python3 util/azure_clean_provider_ops.py | |
| az role definition list --query '[?roleType == `BuiltInRole`].{assignableScopes: assignableScopes, description: description, name:name, permissions: permissions, roleName: roleName, roleType: roleType, type: type}' > azure/built-in-roles-raw.json | |
| python3 util/azure_process_roles.py > azure/built-in-roles.json | |
| python3 util/azure_generate_map.py | |
| python3 util/aws_patch_iam_definition.py | |
| sleep 1 | |
| rm -rf docs/ update_iam_data.py | |
| # sudo apt -y install git nodejs | |
| git clone https://github.com/z0ph/MAMIP.git | |
| git clone https://github.com/primeharbor/sensitive_iam_actions.git | |
| python3 util/aws_add_managed_policies.py | |
| python3 util/aws_generate_tags.py | |
| cd util | |
| rm -rf aws-sdk-ruby | |
| git clone --depth 1 https://github.com/aws/aws-sdk-ruby.git | |
| cd .. | |
| cd util/aws_js | |
| npm i | |
| npm update aws-sdk | |
| node aws_add_docs.js | |
| cd ../.. | |
| rm -f aws/docs.json | |
| mv util/aws_js/docs.json aws/docs.json | |
| python3 util/aws_update_counts.py | |
| rm -f gha-* || true | |
| - name: Commit files | |
| env: | |
| BOT_GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} | |
| run: | | |
| set -eux | |
| git config --local user.email "[email protected]" | |
| git config --local user.name "Ian Mckay [bot]" | |
| git add . | |
| git commit -m "Update SAR data" -a || exit 0 | |
| remote_repo="https://iann0036-bot:${BOT_GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" | |
| git push "${remote_repo}" HEAD:main |