Generate stats #1220
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generate stats | |
| on: | |
| push: | |
| branches: | |
| - dev | |
| schedule: | |
| - cron: '0 1 * * *' | |
| workflow_dispatch: # for triggering builds manually | |
| jobs: | |
| generate_stats: | |
| runs-on: ubuntu-22.04 # supports python 3.7 | |
| steps: | |
| - name: Maximize build space | |
| uses: easimon/maximize-build-space@master | |
| with: | |
| root-reserve-mb: 512 | |
| swap-size-mb: 1024 | |
| remove-dotnet: 'true' | |
| remove-android: 'true' | |
| remove-haskell: 'true' | |
| - name: Checkout this repo | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python 3.7 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.7 | |
| cache: 'pip' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Install some extras | |
| run: | | |
| cd helpers | |
| git clone --branch version-2.03 https://github.com/IATI/IATI-Rulesets.git | |
| ln -s IATI-Rulesets/rulesets . | |
| ./get_codelist_mapping.sh | |
| ./get_codelists.sh | |
| ./get_schemas.sh | |
| wget -q https://raw.githubusercontent.com/codeforIATI/IATI-Dashboard/main/registry_id_relationships.csv | |
| wget -q https://codeforiati.org/imf-exchangerates/imf_exchangerates_A_ENDA_USD.csv -O currency_conversion/exchange_rates.csv | |
| cd .. | |
| - name: Download IATI data | |
| run: | | |
| wget -q -O iati_dump.zip $(wget -q -O - https://iati-data-dump.codeforiati.org/download) | |
| unzip -q iati_dump.zip | |
| mv iati-data-main/* . | |
| rm -rf iati_dump.zip iati-data-main | |
| - name: Checkout output (production) | |
| if: github.ref == 'refs/heads/main' | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| repository: codeforIATI/IATI-Stats-public | |
| path: out | |
| - name: Checkout output (dev) | |
| if: github.ref == 'refs/heads/dev' | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| repository: codeforIATI/IATI-Stats-dev | |
| path: out | |
| - name: Delete out/current | |
| run: rm -rf out/current | |
| - name: Generate ckan.json | |
| run: | | |
| python helpers/ckan.py | |
| cp ckan.json out | |
| mv ckan.json helpers | |
| mv metadata.json out | |
| mv licenses.json out | |
| - name: Run loop | |
| run: python calculate_stats.py --output out/current --multi 2 loop | |
| - name: Delete data and metadata | |
| run: rm -rf data metadata | |
| - name: Run aggregate | |
| run: python calculate_stats.py --output out/current aggregate | |
| - name: Run invert | |
| run: python calculate_stats.py --output out/current invert | |
| - name: Run gitaggregate | |
| run: python statsrunner/gitaggregate.py dated | |
| - name: Run gitaggregate-publisher | |
| run: python statsrunner/gitaggregate-publisher.py dated | |
| # - name: Add static files to output | |
| # run: cp -r static/* out | |
| - name: Deploy (production) 🚀 | |
| if: github.ref == 'refs/heads/main' | |
| uses: JamesIves/[email protected] | |
| with: | |
| GIT_CONFIG_NAME: Code for IATI bot | |
| GIT_CONFIG_EMAIL: [email protected] | |
| GITHUB_TOKEN: ${{ secrets.TOKEN }} | |
| REPOSITORY_NAME: codeforIATI/IATI-Stats-public | |
| BRANCH: gh-pages | |
| FOLDER: out | |
| CLEAN: true | |
| - name: Trigger a build on the analytics repo | |
| if: github.ref == 'refs/heads/main' | |
| uses: peter-evans/repository-dispatch@v1 | |
| with: | |
| repository: codeforIATI/analytics | |
| token: ${{ secrets.TOKEN }} | |
| event-type: build | |
| - name: Deploy (dev) 🚀 | |
| if: github.ref == 'refs/heads/dev' | |
| uses: JamesIves/[email protected] | |
| with: | |
| GIT_CONFIG_NAME: Code for IATI bot | |
| GIT_CONFIG_EMAIL: [email protected] | |
| GITHUB_TOKEN: ${{ secrets.TOKEN }} | |
| REPOSITORY_NAME: codeforIATI/IATI-Stats-dev | |
| BRANCH: gh-pages | |
| FOLDER: out | |
| CLEAN: true |