feat: acceptance tests (#848)

Author: lionel-nj

.github/workflows/acceptance_test.yml

@@ -54,7 +54,6 @@ jobs:
       - name: Prepare version name
         id: prep
         run: |
-          echo ${{ github.event.head_commit.message }}
           VERSION_TAG=edge
           if [[ $GITHUB_REF == refs/tags/* ]]; then
             VERSION_TAG=${GITHUB_REF#refs/tags/}
@@ -88,6 +87,11 @@ jobs:
         with:
           name: gtfs-validator-snapshot
           path: main/build/libs/gtfs-validator-${{ steps.prep.outputs.versionTag }}_cli.jar
+      - name: Persist comparator snapshot jar
+        uses: actions/upload-artifact@v2
+        with:
+          name: comparator-snapshot
+          path: output-comparator/build/libs/output-comparator-${{ steps.prep.outputs.versionTag }}_cli.jar
   pack-master:
     needs: [ validate-gradle-wrapper ]
     runs-on: ubuntu-latest
@@ -178,21 +182,58 @@ jobs:
         env:
           OUTPUT_BASE: ${{ github.sha }}
       - name: Persist reports
-        if: always()
         uses: actions/upload-artifact@v2
         with:
           name: reports_all
           path: ${{ github.sha }}/output
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@master
+      - name: Set up and authorize Cloud
+        uses: google-github-actions/auth@v0
         with:
-          project_id: ${{ secrets.VALIDATOR_PROJECT_ID }}
-          service_account_key: ${{ secrets.VALIDATOR_SA_KEY }}
-          export_default_credentials: true
+          credentials_json: ${{ secrets.VALIDATOR_SA_KEY }}
       - name: Upload reports to Google Cloud Storage
         id: upload-files
-        if: always()
         uses: google-github-actions/upload-cloud-storage@main
         with:
           path: ${{ github.sha }}/output
           destination: gtfs-validator-reports
+  compare-outputs:
+    needs: [ get-reports ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Download comparator .jar file from previous job
+        uses: actions/download-artifact@v2
+        with:
+          name: comparator-snapshot
+      - name: Retrieve reports from previous job
+        uses: actions/download-artifact@v2
+        with:
+          name: reports_all
+      - name: Retrieve gtfs latest versions from previous job
+        uses: actions/download-artifact@v2
+        with:
+          name: datasets_metadata
+      - name: Generate acceptance report test
+        run: |
+          java -jar output-comparator*.jar --report_directory . --new_error_threshold 1 --reference_report_name reference.json --latest_report_name latest.json --percent_invalid_datasets_threshold 1 --output_base acceptance-test-output --source_urls gtfs_latest_versions.json --percent_corrupted_sources 2
+      - name: Persist acceptance test reports
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: acceptance_test_report
+          path: acceptance-test-output
+      - name: Generate PR comment
+        id: generate-comment
+        if: always()
+        run: |
+          python3 scripts/comment_generator.py -u gtfs_latest_versions.json -a acceptance-test-output/acceptance_report.json -c ${{ github.sha }} -r ${{github.run_id}} -x acceptance-test-output/sources_corruption_report.json > pr_comment.txt
+          PR_COMMENT=$(<pr_comment.txt)
+          echo "PR_COMMENT<<EOF" >> $GITHUB_ENV
+          echo "$PR_COMMENT" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+      - name: Comment Pull Request
+        if: always()
+        uses: thollander/[email protected]
+        with:
+          message: ${{ env.PR_COMMENT }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

README.md

@@ -77,6 +77,9 @@ Instructions to build the project from the command-line using [Gradle](https://g
 # Architecture
 The architecture of the `gtfs-validator` is described on our [Architecture page](/docs/ARCHITECTURE.md).
 
+# Acceptance tests
+In order to avoid sudden changes in the validation output that might declare previously valid datasets invalid, all code changes in pull requests are tested against GTFS datasets in the [MobilityDatabase](http://mobilitydatabase.org/wiki/Main_Page). The acceptance test process is described in [ACCEPTANCE_TESTS.md](docs/ACCEPTANCE_TESTS.md).
+
 # License
 Code licensed under the [Apache 2.0 License](http://www.apache.org/licenses/LICENSE-2.0).
 

core/src/main/java/org/mobilitydata/gtfsvalidator/io/ValidationReportDeserializer.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2021 MobilityData IO
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.mobilitydata.gtfsvalidator.io;
+
+import static org.mobilitydata.gtfsvalidator.notice.Notice.GSON;
+
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.internal.LinkedTreeMap;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.mobilitydata.gtfsvalidator.model.NoticeReport;
+import org.mobilitydata.gtfsvalidator.model.ValidationReport;
+import org.mobilitydata.gtfsvalidator.notice.Notice;
+import org.mobilitydata.gtfsvalidator.notice.NoticeContainer;
+
+/**
+ * Used to (de)serialize a JSON validation report. This represents a validation report as a list of
+ * {@code NoticeReport} which provides information about each notice generated during a GTFS dataset
+ * validation.
+ */
+public class ValidationReportDeserializer implements JsonDeserializer<ValidationReport> {
+
+  private static final Gson GSON =
+      new GsonBuilder().serializeNulls().serializeSpecialFloatingPointValues().create();
+  private static final String NOTICES_MEMBER_NAME = "notices";
+
+  @Override
+  public ValidationReport deserialize(
+      JsonElement json, Type typoOfT, JsonDeserializationContext context) {
+    Set<NoticeReport> notices = new LinkedHashSet<>();
+    JsonObject rootObject = json.getAsJsonObject();
+    JsonArray noticesArray = rootObject.getAsJsonArray(NOTICES_MEMBER_NAME);
+    for (JsonElement childObject : noticesArray) {
+      notices.add(GSON.fromJson(childObject, NoticeReport.class));
+    }
+    return new ValidationReport(notices);
+  }
+
+  public static <T extends Notice> JsonObject serialize(
+      List<T> notices,
+      int maxExportsPerNoticeTypeAndSeverity,
+      Map<String, Integer> noticesCountPerTypeAndSeverity) {
+    Set<NoticeReport> noticeReports = new LinkedHashSet<>();
+    Gson gson = new Gson();
+    Type contextType = new TypeToken<Map<String, Object>>() {}.getType();
+    for (Collection<T> noticesOfType :
+        NoticeContainer.groupNoticesByTypeAndSeverity(notices).asMap().values()) {
+      T firstNotice = noticesOfType.iterator().next();
+      List<LinkedTreeMap<String, Object>> contexts = new ArrayList<>();
+      int i = 0;
+      for (T notice : noticesOfType) {
+        ++i;
+        if (i > maxExportsPerNoticeTypeAndSeverity) {
+          // Do not export too many notices for this type.
+          break;
+        }
+        contexts.add(gson.fromJson(notice.getContext(), contextType));
+      }
+      noticeReports.add(
+          new NoticeReport(
+              firstNotice.getCode(),
+              firstNotice.getSeverityLevel(),
+              noticesCountPerTypeAndSeverity.get(firstNotice.getMappingKey()),
+              contexts));
+    }
+    return GSON.toJsonTree(new ValidationReport(noticeReports)).getAsJsonObject();
+  }
+}

core/src/main/java/org/mobilitydata/gtfsvalidator/model/NoticeReport.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2021 MobilityData IO
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.mobilitydata.gtfsvalidator.model;
+
+import com.google.gson.annotations.Expose;
+import com.google.gson.internal.LinkedTreeMap;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import org.mobilitydata.gtfsvalidator.notice.SeverityLevel;
+
+/**
+ * Used to (de)serialize a JSON validation report. It is used to store information about one type of
+ * notice encountered in a validation report: error code, severity level, the total number of
+ * notices related to the error code and a list of notice contexts (which provides additional
+ * information about each notice.
+ */
+public class NoticeReport {
+
+  @Expose() private final String code;
+  @Expose() private final SeverityLevel severity;
+  @Expose() private final int totalNotices;
+  @Expose() private final List<LinkedTreeMap<String, Object>> sampleNotices;
+
+  public NoticeReport(
+      String code,
+      SeverityLevel severity,
+      int count,
+      List<LinkedTreeMap<String, Object>> sampleNotices) {
+    this.code = code;
+    this.severity = severity;
+    this.totalNotices = count;
+    this.sampleNotices = sampleNotices;
+  }
+
+  public int getTotalNotices() {
+    return totalNotices;
+  }
+
+  public SeverityLevel getSeverity() {
+    return severity;
+  }
+
+  public String getCode() {
+    return code;
+  }
+
+  public List<LinkedTreeMap<String, Object>> getSampleNotices() {
+    return Collections.unmodifiableList(sampleNotices);
+  }
+
+  public boolean isError() {
+    return getSeverity().ordinal() >= SeverityLevel.ERROR.ordinal();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (this == other) {
+      return true;
+    }
+    if (other instanceof NoticeReport) {
+      NoticeReport otherNoticeReport = (NoticeReport) other;
+      return this.getCode().equals(otherNoticeReport.getCode())
+          && this.getSeverity().equals(otherNoticeReport.getSeverity())
+          && this.getTotalNotices() == (otherNoticeReport.getTotalNotices())
+          && getSampleNotices().equals(otherNoticeReport.getSampleNotices());
+    }
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(code, severity);
+  }
+}