Add support for validating models in generator (#6136)

* Add support for validating models in generator

This commit introduces `ModelValidator` which allows us to do validation
of the `IntermediateModel` before generation starts; violations are
emitted to a `validation-report.json` file under
`generated-sources/sdk/models`.

As part of this change, `CodeGenerator` was updated to accept
`IntermediateModel` as a parameter. This is because all validators
operate on the `IntermediateModel`, which already have any SDK specific
customizations applied. Since some validators require referencing
multiple models, (such as when we start validating shared shapes),
having the ability to pass in `IntermediateModel` directly means we only
need to build these models once, upfront.

* Changelog

* Add javadocs

Author: dagnir

.changes/next-release/feature-AWSSDKforJavav2-bd762da.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "Add support for defining service model validators and generating valdiation reports during code generation."
+}

codegen/src/main/java/software/amazon/awssdk/codegen/CodeGenerator.java

@@ -19,20 +19,33 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import java.util.concurrent.ForkJoinTask;
 import software.amazon.awssdk.codegen.emitters.GeneratorTask;
 import software.amazon.awssdk.codegen.emitters.GeneratorTaskParams;
 import software.amazon.awssdk.codegen.emitters.tasks.AwsGeneratorTasks;
 import software.amazon.awssdk.codegen.internal.Jackson;
 import software.amazon.awssdk.codegen.internal.Utils;
 import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
+import software.amazon.awssdk.codegen.validation.ModelValidationContext;
+import software.amazon.awssdk.codegen.validation.ModelValidationReport;
+import software.amazon.awssdk.codegen.validation.ModelValidator;
+import software.amazon.awssdk.codegen.validation.ValidationEntry;
 import software.amazon.awssdk.utils.Logger;
 
 public class CodeGenerator {
     private static final Logger log = Logger.loggerFor(CodeGenerator.class);
     private static final String MODEL_DIR_NAME = "models";
 
-    private final C2jModels models;
+    // TODO: add validators
+    private static final List<ModelValidator> DEFAULT_MODEL_VALIDATORS = Collections.emptyList();
+
+    private final C2jModels c2jModels;
+
+    private final IntermediateModel intermediateModel;
+    private final IntermediateModel shareModelsTarget;
     private final String sourcesDirectory;
     private final String resourcesDirectory;
     private final String testsDirectory;
@@ -42,6 +55,9 @@ public class CodeGenerator {
      */
     private final String fileNamePrefix;
 
+    private final List<ModelValidator> modelValidators;
+    private final boolean emitValidationReport;
+
     static {
         // Make sure ClassName is statically initialized before we do anything in parallel.
         // Parallel static initialization of ClassName and TypeName can result in a deadlock:
@@ -50,12 +66,21 @@ public class CodeGenerator {
     }
 
     public CodeGenerator(Builder builder) {
-        this.models = builder.models;
+        this.c2jModels = builder.models;
+        this.intermediateModel = builder.intermediateModel;
+
+        if (this.c2jModels != null && this.intermediateModel != null) {
+            throw new IllegalArgumentException("Only one of c2jModels and intermediateModel must be specified");
+        }
+
+        this.shareModelsTarget = builder.shareModelsTarget;
         this.sourcesDirectory = builder.sourcesDirectory;
         this.testsDirectory = builder.testsDirectory;
         this.resourcesDirectory = builder.resourcesDirectory != null ? builder.resourcesDirectory
                                                                      : builder.sourcesDirectory;
         this.fileNamePrefix = builder.fileNamePrefix;
+        this.modelValidators = builder.modelValidators == null ? DEFAULT_MODEL_VALIDATORS : builder.modelValidators;
+        this.emitValidationReport = builder.emitValidationReport;
     }
 
     public static File getModelDirectory(String outputDirectory) {
@@ -76,13 +101,31 @@ public static Builder builder() {
      * code.
      */
     public void execute() {
-        try {
-            IntermediateModel intermediateModel = new IntermediateModelBuilder(models).build();
+        ModelValidationReport report = new ModelValidationReport();
+
+        IntermediateModel modelToGenerate;
+        if (c2jModels != null) {
+            modelToGenerate = new IntermediateModelBuilder(c2jModels).build();
+        } else {
+            modelToGenerate = intermediateModel;
+        }
+
+        List<ValidationEntry> validatorEntries = runModelValidators(modelToGenerate);
+        report.setValidationEntries(validatorEntries);
 
+        if (emitValidationReport) {
+            writeValidationReport(report);
+        }
+
+        if (!validatorEntries.isEmpty()) {
+            throw new RuntimeException("Validation failed. See validation report for details.");
+        }
+
+        try {
             if (fileNamePrefix != null) {
-                writeIntermediateModel(intermediateModel);
+                writeIntermediateModel(modelToGenerate);
             }
-            emitCode(intermediateModel);
+            emitCode(modelToGenerate);
 
         } catch (Exception e) {
             log.error(() -> "Failed to generate code. ", e);
@@ -91,7 +134,32 @@ public void execute() {
         }
     }
 
+    private List<ValidationEntry> runModelValidators(IntermediateModel intermediateModel) {
+        ModelValidationContext ctx = ModelValidationContext.builder()
+                                                           .intermediateModel(intermediateModel)
+                                                           .shareModelsTarget(shareModelsTarget)
+                                                           .build();
+
+        List<ValidationEntry> validationEntries = new ArrayList<>();
+
+        modelValidators.forEach(v -> validationEntries.addAll(v.validateModels(ctx)));
+
+        return validationEntries;
+    }
+
+    private void writeValidationReport(ModelValidationReport report) {
+        try {
+            writeModel(report, "validation-report.json");
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     private void writeIntermediateModel(IntermediateModel model) throws IOException {
+        writeModel(model, fileNamePrefix + "-intermediate.json");
+    }
+
+    private void writeModel(Object model, String name) throws IOException {
         File modelDir = getModelDirectory(sourcesDirectory);
         PrintWriter writer = null;
         try {
@@ -100,7 +168,7 @@ private void writeIntermediateModel(IntermediateModel model) throws IOException
                 throw new RuntimeException("Failed to create " + outDir.getAbsolutePath());
             }
 
-            File outputFile = new File(modelDir, fileNamePrefix + "-intermediate.json");
+            File outputFile = new File(modelDir, name);
 
             if (!outputFile.exists() && !outputFile.createNewFile()) {
                 throw new RuntimeException("Error creating file " + outputFile.getAbsolutePath());
@@ -134,10 +202,14 @@ private GeneratorTask createGeneratorTasks(IntermediateModel intermediateModel)
     public static final class Builder {
 
         private C2jModels models;
+        private IntermediateModel intermediateModel;
+        private IntermediateModel shareModelsTarget;
         private String sourcesDirectory;
         private String resourcesDirectory;
         private String testsDirectory;
         private String fileNamePrefix;
+        private List<ModelValidator> modelValidators;
+        private boolean emitValidationReport;
 
         private Builder() {
         }
@@ -147,6 +219,16 @@ public Builder models(C2jModels models) {
             return this;
         }
 
+        public Builder intermediateModel(IntermediateModel intermediateModel) {
+            this.intermediateModel = intermediateModel;
+            return this;
+        }
+
+        public Builder shareModelsTarget(IntermediateModel shareModelsTarget) {
+            this.shareModelsTarget = shareModelsTarget;
+            return this;
+        }
+
         public Builder sourcesDirectory(String sourcesDirectory) {
             this.sourcesDirectory = sourcesDirectory;
             return this;
@@ -167,6 +249,16 @@ public Builder intermediateModelFileNamePrefix(String fileNamePrefix) {
             return this;
         }
 
+        public Builder modelValidators(List<ModelValidator> modelValidators) {
+            this.modelValidators = modelValidators;
+            return this;
+        }
+
+        public Builder emitValidationReport(boolean emitValidationReport) {
+            this.emitValidationReport = emitValidationReport;
+            return this;
+        }
+
         /**
          * @return An immutable {@link CodeGenerator} object.
          */

codegen/src/main/java/software/amazon/awssdk/codegen/poet/rules/EndpointRulesSpecUtils.java

@@ -29,7 +29,12 @@
 import com.squareup.javapoet.TypeName;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
@@ -52,6 +57,7 @@
 import software.amazon.awssdk.utils.internal.CodegenNamingUtils;
 
 public class EndpointRulesSpecUtils {
+    private static final String RULES_ENGINE_RESOURCE_FILES_PREFIX = "software/amazon/awssdk/codegen/rules/";
     private final IntermediateModel intermediateModel;
 
     public EndpointRulesSpecUtils(IntermediateModel intermediateModel) {
@@ -213,16 +219,45 @@ public TypeName resolverReturnType() {
 
     public List<String> rulesEngineResourceFiles() {
         URL currentJarUrl = EndpointRulesSpecUtils.class.getProtectionDomain().getCodeSource().getLocation();
+
+        // This would happen if the classes aren't loaded from a JAR, e.g. when unit testing
+        if (!currentJarUrl.toString().endsWith(".jar")) {
+            return rulesEngineFilesFromDirectory(currentJarUrl);
+        }
+
         try (JarFile jarFile = new JarFile(currentJarUrl.getFile())) {
             return jarFile.stream()
                           .map(ZipEntry::getName)
-                          .filter(e -> e.startsWith("software/amazon/awssdk/codegen/rules/"))
+                          .filter(e -> e.startsWith(RULES_ENGINE_RESOURCE_FILES_PREFIX))
                           .collect(Collectors.toList());
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
     }
 
+    public List<String> rulesEngineFilesFromDirectory(URL location) {
+        URI locationUri;
+        try {
+            locationUri = location.toURI();
+            if (!"file".equals(locationUri.getScheme())) {
+                throw new RuntimeException("Expected location to be a directory");
+            }
+        } catch (URISyntaxException e) {
+            throw new RuntimeException(e);
+        }
+
+        try {
+            Path directory = Paths.get(locationUri);
+            return Files.walk(directory)
+                        // Remove the root directory if the classes, paths are expected to be relative to this directory
+                        .map(f -> directory.relativize(f).toString())
+                        .filter(f -> f.startsWith(RULES_ENGINE_RESOURCE_FILES_PREFIX))
+                        .collect(Collectors.toList());
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
     public List<String> rulesEngineResourceFiles2() {
         URL currentJarUrl = EndpointRulesSpecUtils.class.getProtectionDomain().getCodeSource().getLocation();
         try (JarFile jarFile = new JarFile(currentJarUrl.getFile())) {

codegen/src/main/java/software/amazon/awssdk/codegen/validation/ModelValidationContext.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.codegen.validation;
+
+import java.util.Optional;
+import software.amazon.awssdk.codegen.model.config.customization.ShareModelConfig;
+import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
+
+/**
+ * Context object for {@link ModelValidator}s. This object contains all the information available to the validations in order
+ * for them to perform their tasks.
+ */
+public final class ModelValidationContext {
+    private final IntermediateModel intermediateModel;
+    private final IntermediateModel shareModelsTarget;
+
+    private ModelValidationContext(Builder builder) {
+        this.intermediateModel = builder.intermediateModel;
+        this.shareModelsTarget = builder.shareModelsTarget;
+    }
+
+    /**
+     * The service model for which code is being generated.
+     */
+    public IntermediateModel intermediateModel() {
+        return intermediateModel;
+    }
+
+    /**
+     * The model of the service that the currently generating service shares models with. In other words, this is the service
+     * model for the service defined in {@link ShareModelConfig#getShareModelWith()}.
+     */
+    public Optional<IntermediateModel> shareModelsTarget() {
+        return Optional.ofNullable(shareModelsTarget);
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static class Builder {
+        private IntermediateModel intermediateModel;
+        private IntermediateModel shareModelsTarget;
+
+        /**
+         * The service model for which code is being generated.
+         */
+        public Builder intermediateModel(IntermediateModel intermediateModel) {
+            this.intermediateModel = intermediateModel;
+            return this;
+        }
+
+        /**
+         * The model of the service that the currently generating service shares models with. In other words, this is the service
+         * model for the service defined in {@link ShareModelConfig#getShareModelWith()}.
+         */
+        public Builder shareModelsTarget(IntermediateModel shareModelsTarget) {
+            this.shareModelsTarget = shareModelsTarget;
+            return this;
+        }
+
+        public ModelValidationContext build() {
+            return new ModelValidationContext(this);
+        }
+    }
+}

codegen/src/main/java/software/amazon/awssdk/codegen/validation/ModelValidationReport.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.codegen.validation;
+
+import java.util.Collections;
+import java.util.List;
+
+public class ModelValidationReport {
+    private List<ValidationEntry> validationEntries = Collections.emptyList();
+
+    public List<ValidationEntry> getValidationEntries() {
+        return validationEntries;
+    }
+
+    public void setValidationEntries(List<ValidationEntry> validationEntries) {
+        if (validationEntries != null) {
+            this.validationEntries = validationEntries;
+        } else {
+            this.validationEntries = Collections.emptyList();
+        }
+    }
+
+    public ModelValidationReport withValidationEntries(List<ValidationEntry> validationEntries) {
+        setValidationEntries(validationEntries);
+        return this;
+    }
+}