End to end tests (just of traceability so far)

#21

.github/workflows/end_to_end.yml

@@ -0,0 +1,58 @@
+name: End to end tests
+on: [push, pull_request]
+jobs:
+  generate_stats:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout this repo
+      uses: actions/checkout@v2
+      with:
+        persist-credentials: false
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+    - uses: actions/cache@v2
+      name: Cache dependencies
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Install some extras
+      run: |
+        cd helpers
+        git clone --branch version-2.03 https://github.com/IATI/IATI-Rulesets.git
+        ln -s IATI-Rulesets/rulesets .
+        ./get_codelist_mapping.sh
+        ./get_codelists.sh
+        ./get_schemas.sh
+        wget -q "https://raw.githubusercontent.com/codeforIATI/IATI-Dashboard/main/registry_id_relationships.csv"
+        wget -q https://codeforiati.org/imf-exchangerates/imf_exchangerates_A_ENDA_USD.csv -O currency_conversion/exchange_rates.csv
+        cd ..
+    - name: Symlink fixtures for IATI data
+      run: ln -s tests_end_to_end/fixtures/{data,metadata,metadata.json} .
+    - name: Create output dir
+      run: mkdir out
+    - name: Generate ckan.json
+      run: |
+        echo '{}' > helpers/ckan.json
+#       python helpers/ckan.py
+#       cp ckan.json out
+#       mv ckan.json helpers
+#       mv metadata.json out
+#       mv licenses.json out
+    - name: Run loop
+      run: python calculate_stats.py --output out/current --multi 2 loop
+    - name: Run aggregate
+      run: python calculate_stats.py --output out/current aggregate
+    - name: Run invert
+      run: python calculate_stats.py --output out/current invert
+    - name: Create traceable_percentages csv
+      run: python traceable_percentages.py > traceable_percentages.csv
+    - name: Create traceable_percentages csv
+      run: diff traceable_percentages.csv.expected traceable_percentages.csv

run_end_to_end-tracaebility-test.sh

@@ -0,0 +1 @@
+rm -r out/current; python calculate_stats.py --output out/current loop && python calculate_stats.py --output out/current aggregate && python calculate_stats.py --output out/current invert && python traceable_percentages.py > traceable_percentages.csv && diff traceable_percentages.csv.expected traceable_percentages.csv && echo 'Success'

tests_end_to_end/fixtures/data/funder/funder-activities.xml

@@ -0,0 +1,5 @@
+<iati-activities version="2.03">
+    <iati-activity>
+        <iati-identifier>XE-EXAMPLE-BB-01</iati-identifier>
+    </iati-activity>
+</iati-activities>

tests_end_to_end/fixtures/data/funder2/funder2-activities.xml

@@ -0,0 +1,26 @@
+<iati-activities version="2.03">
+    <iati-activity default-currency="USD">
+        <iati-identifier>XE-EXAMPLE-CC-01</iati-identifier>
+        <transaction>
+            <transaction-type code="2"/>
+            <transaction-date iso-date="2022-01-01" />
+            <value>1000</value>
+        </transaction>
+    </iati-activity>
+    <iati-activity>
+        <iati-identifier>XE-EXAMPLE-CC-02</iati-identifier>
+        <transaction>
+            <transaction-type code="2"/>
+            <transaction-date iso-date="2012-01-01" />
+            <value currency="EUR">1000</value>
+        </transaction>
+    </iati-activity>
+    <iati-activity>
+        <iati-identifier>XE-EXAMPLE-CC-03</iati-identifier>
+        <transaction>
+            <transaction-type code="2"/>
+            <transaction-date iso-date="2012-01-01" />
+            <value currency="GBP">1000</value>
+        </transaction>
+    </iati-activity>
+</iati-activities>

tests_end_to_end/fixtures/data/recipient/recipient-activities.xml

@@ -0,0 +1,23 @@
+<iati-activities version="2.03" default-currency="USD">
+    <iati-activity>
+        <iati-identifier>XE-EXAMPLE-AA-01</iati-identifier>
+        <transaction>
+            <transaction-type code="1"/>
+            <transaction-date iso-date="2022-01-01" />
+            <value>1000</value>
+            <provider-org provider-activity-id="XE-EXAMPLE-BB-01" type="10" ref="XE-EXAMPLE-BB" />
+        </transaction>
+        <transaction>
+            <transaction-type code="1"/>
+            <transaction-date iso-date="2022-01-01" />
+            <value>1000</value>
+            <provider-org provider-activity-id="XE-EXAMPLE-CC-01" type="10" ref="XE-EXAMPLE-CC" />
+        </transaction>
+        <transaction>
+            <transaction-type code="1"/>
+            <transaction-date iso-date="2022-01-01" />
+            <value>1000</value>
+            <provider-org provider-activity-id="XE-EXAMPLE-CC-02" type="10" ref="XE-EXAMPLE-CC" />
+        </transaction>
+    </iati-activity>
+</iati-activities>

tests_end_to_end/fixtures/metadata/funder/funder-activities.json

@@ -0,0 +1,3 @@
+{
+    "extras": [{"key":"filetype","value":"activity"}]
+}

tests_end_to_end/fixtures/metadata/funder2/funder2-activities.json

@@ -0,0 +1,3 @@
+{
+    "extras": [{"key":"filetype","value":"activity"}]
+}

tests_end_to_end/fixtures/metadata/recipient/recipient-activities.json

@@ -0,0 +1,3 @@
+{
+    "extras": [{"key":"filetype","value":"activity"}]
+}

traceable_percentages.csv.expected

@@ -0,0 +1,2 @@
+funder,1,1,100.0,0,0,
+funder2,2,3,67,2284.4666822899517,3869.2116470760116,59

traceable_percentages.py

@@ -0,0 +1,29 @@
+import csv
+import json
+import sys
+
+
+traceable_activities_by_publisher = json.load(open("out/current/aggregated/traceable_activities_by_publisher_id.json"))
+# This may be different from the total number of activity identifiers
+total_activities_by_publisher = json.load(open("out/current/aggregated/traceable_activities_by_publisher_id_denominator.json"))
+traceable_spend_by_publisher = json.load(open("out/current/aggregated/traceable_sum_commitments_and_disbursements_by_publisher_id.json"))
+total_spend_by_publisher = json.load(open("out/current/aggregated/traceable_sum_commitments_and_disbursements_by_publisher_id_denominator.json"))
+
+csvwriter = csv.writer(sys.stdout)
+
+for publisher, total_activities in total_activities_by_publisher.items():
+    traceable_activities = traceable_activities_by_publisher.get(publisher, 0)
+    percentage_activities = traceable_activities / total_activities * 100
+    traceable_spend = traceable_spend_by_publisher.get(publisher, 0)
+    total_spend = total_spend_by_publisher.get(publisher, 0)
+    if total_activities == 0 or traceable_activities == 0:
+        continue
+    csvwriter.writerow([
+        publisher,
+        traceable_activities,
+        total_activities,
+        f"{percentage_activities}" if percentage_activities == 100 else f"{percentage_activities:.2g}",
+        traceable_spend,
+        total_spend,
+        "" if total_spend == 0 else f"{(traceable_spend / total_spend * 100):.2g}",
+    ])