diff --git a/.github/workflows/end_to_end.yml b/.github/workflows/end_to_end.yml
new file mode 100644
index 00000000000..3a6f3fcceb6
--- /dev/null
+++ b/.github/workflows/end_to_end.yml
@@ -0,0 +1,58 @@
+name: End to end tests
+on: [push, pull_request]
+jobs:
+ generate_stats:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout this repo
+ uses: actions/checkout@v2
+ with:
+ persist-credentials: false
+ - name: Set up Python 3.7
+ uses: actions/setup-python@v2
+ with:
+ python-version: 3.7
+ - uses: actions/cache@v2
+ name: Cache dependencies
+ with:
+ path: ~/.cache/pip
+ key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+ restore-keys: |
+ ${{ runner.os }}-pip-
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install -r requirements.txt
+ - name: Install some extras
+ run: |
+ cd helpers
+ git clone --branch version-2.03 https://github.com/IATI/IATI-Rulesets.git
+ ln -s IATI-Rulesets/rulesets .
+ ./get_codelist_mapping.sh
+ ./get_codelists.sh
+ ./get_schemas.sh
+ wget -q "https://raw.githubusercontent.com/codeforIATI/IATI-Dashboard/main/registry_id_relationships.csv"
+ wget -q https://codeforiati.org/imf-exchangerates/imf_exchangerates_A_ENDA_USD.csv -O currency_conversion/exchange_rates.csv
+ cd ..
+ - name: Symlink fixtures for IATI data
+ run: ln -s tests_end_to_end/fixtures/{data,metadata,metadata.json} .
+ - name: Create output dir
+ run: mkdir out
+ - name: Generate ckan.json
+ run: |
+ echo '{}' > helpers/ckan.json
+# python helpers/ckan.py
+# cp ckan.json out
+# mv ckan.json helpers
+# mv metadata.json out
+# mv licenses.json out
+ - name: Run loop
+ run: python calculate_stats.py --output out/current --multi 2 loop
+ - name: Run aggregate
+ run: python calculate_stats.py --output out/current aggregate
+ - name: Run invert
+ run: python calculate_stats.py --output out/current invert
+ - name: Create traceable_percentages csv
+ run: python traceable_percentages.py > traceable_percentages.csv
+ - name: Create traceable_percentages csv
+ run: diff traceable_percentages.csv.expected traceable_percentages.csv
diff --git a/run_end_to_end-tracaebility-test.sh b/run_end_to_end-tracaebility-test.sh
new file mode 100755
index 00000000000..8d477385a51
--- /dev/null
+++ b/run_end_to_end-tracaebility-test.sh
@@ -0,0 +1 @@
+rm -r out/current; python calculate_stats.py --output out/current loop && python calculate_stats.py --output out/current aggregate && python calculate_stats.py --output out/current invert && python traceable_percentages.py > traceable_percentages.csv && diff traceable_percentages.csv.expected traceable_percentages.csv && echo 'Success'
diff --git a/tests_end_to_end/fixtures/data/funder/funder-activities.xml b/tests_end_to_end/fixtures/data/funder/funder-activities.xml
new file mode 100644
index 00000000000..10dff43f9a9
--- /dev/null
+++ b/tests_end_to_end/fixtures/data/funder/funder-activities.xml
@@ -0,0 +1,5 @@
+
+
+ XE-EXAMPLE-BB-01
+
+
diff --git a/tests_end_to_end/fixtures/data/funder2/funder2-activities.xml b/tests_end_to_end/fixtures/data/funder2/funder2-activities.xml
new file mode 100644
index 00000000000..bcca9cffc3f
--- /dev/null
+++ b/tests_end_to_end/fixtures/data/funder2/funder2-activities.xml
@@ -0,0 +1,26 @@
+
+
+ XE-EXAMPLE-CC-01
+
+
+
+ 1000
+
+
+
+ XE-EXAMPLE-CC-02
+
+
+
+ 1000
+
+
+
+ XE-EXAMPLE-CC-03
+
+
+
+ 1000
+
+
+
diff --git a/tests_end_to_end/fixtures/data/recipient/recipient-activities.xml b/tests_end_to_end/fixtures/data/recipient/recipient-activities.xml
new file mode 100644
index 00000000000..c1503c82ce5
--- /dev/null
+++ b/tests_end_to_end/fixtures/data/recipient/recipient-activities.xml
@@ -0,0 +1,23 @@
+
+
+ XE-EXAMPLE-AA-01
+
+
+
+ 1000
+
+
+
+
+
+ 1000
+
+
+
+
+
+ 1000
+
+
+
+
diff --git a/tests_end_to_end/fixtures/metadata/funder/funder-activities.json b/tests_end_to_end/fixtures/metadata/funder/funder-activities.json
new file mode 100644
index 00000000000..c9447fab322
--- /dev/null
+++ b/tests_end_to_end/fixtures/metadata/funder/funder-activities.json
@@ -0,0 +1,3 @@
+{
+ "extras": [{"key":"filetype","value":"activity"}]
+}
diff --git a/tests_end_to_end/fixtures/metadata/funder2/funder2-activities.json b/tests_end_to_end/fixtures/metadata/funder2/funder2-activities.json
new file mode 100644
index 00000000000..c9447fab322
--- /dev/null
+++ b/tests_end_to_end/fixtures/metadata/funder2/funder2-activities.json
@@ -0,0 +1,3 @@
+{
+ "extras": [{"key":"filetype","value":"activity"}]
+}
diff --git a/tests_end_to_end/fixtures/metadata/recipient/recipient-activities.json b/tests_end_to_end/fixtures/metadata/recipient/recipient-activities.json
new file mode 100644
index 00000000000..c9447fab322
--- /dev/null
+++ b/tests_end_to_end/fixtures/metadata/recipient/recipient-activities.json
@@ -0,0 +1,3 @@
+{
+ "extras": [{"key":"filetype","value":"activity"}]
+}
diff --git a/traceable_percentages.csv.expected b/traceable_percentages.csv.expected
new file mode 100644
index 00000000000..18ce047fcb7
--- /dev/null
+++ b/traceable_percentages.csv.expected
@@ -0,0 +1,2 @@
+funder,1,1,100.0,0,0,
+funder2,2,3,67,2284.4666822899517,3869.2116470760116,59
diff --git a/traceable_percentages.py b/traceable_percentages.py
new file mode 100644
index 00000000000..95fe26902ee
--- /dev/null
+++ b/traceable_percentages.py
@@ -0,0 +1,29 @@
+import csv
+import json
+import sys
+
+
+traceable_activities_by_publisher = json.load(open("out/current/aggregated/traceable_activities_by_publisher_id.json"))
+# This may be different from the total number of activity identifiers
+total_activities_by_publisher = json.load(open("out/current/aggregated/traceable_activities_by_publisher_id_denominator.json"))
+traceable_spend_by_publisher = json.load(open("out/current/aggregated/traceable_sum_commitments_and_disbursements_by_publisher_id.json"))
+total_spend_by_publisher = json.load(open("out/current/aggregated/traceable_sum_commitments_and_disbursements_by_publisher_id_denominator.json"))
+
+csvwriter = csv.writer(sys.stdout)
+
+for publisher, total_activities in total_activities_by_publisher.items():
+ traceable_activities = traceable_activities_by_publisher.get(publisher, 0)
+ percentage_activities = traceable_activities / total_activities * 100
+ traceable_spend = traceable_spend_by_publisher.get(publisher, 0)
+ total_spend = total_spend_by_publisher.get(publisher, 0)
+ if total_activities == 0 or traceable_activities == 0:
+ continue
+ csvwriter.writerow([
+ publisher,
+ traceable_activities,
+ total_activities,
+ f"{percentage_activities}" if percentage_activities == 100 else f"{percentage_activities:.2g}",
+ traceable_spend,
+ total_spend,
+ "" if total_spend == 0 else f"{(traceable_spend / total_spend * 100):.2g}",
+ ])