From 8e242cf2d9944eaef8ae9192394f0308e13779b6 Mon Sep 17 00:00:00 2001 From: omar Date: Wed, 11 Sep 2024 14:58:33 -0500 Subject: [PATCH 1/7] optionally include descriptions in schema --- codegen/collector/executor.go | 18 ++++++++++++------ codegen/proto/schemagen/generator.go | 3 +++ codegen/proto/schemagen/protoc.go | 9 +++++---- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/codegen/collector/executor.go b/codegen/collector/executor.go index 274ebfd1..23f075d3 100644 --- a/codegen/collector/executor.go +++ b/codegen/collector/executor.go @@ -108,14 +108,20 @@ func (o *OpenApiProtocExecutor) Execute(protoFile string, toFile string, imports directoryPath := filepath.Join(o.OutputDir, directoryName) _ = os.Mkdir(directoryPath, os.ModePerm) - cmd.Args = append(cmd.Args, - fmt.Sprintf("--openapi_out=yaml=true,single_file=false,include_description=true,multiline_description=true,enum_as_int_or_string=%v,proto_oneof=true,int_native=true,additional_empty_schema=%v,disable_kube_markers=%v:%s", - o.EnumAsIntOrString, - strings.Join(o.MessagesWithEmptySchema, "+"), - o.DisableKubeMarkers, - directoryPath), + args := fmt.Sprintf("--openapi_out=yaml=true,single_file=false,multiline_description=true,enum_as_int_or_string=%v,proto_oneof=true,int_native=true,additional_empty_schema=%v,disable_kube_markers=%v", + o.EnumAsIntOrString, + strings.Join(o.MessagesWithEmptySchema, "+"), + o.DisableKubeMarkers, ) + if !o.IncludeDescriptionsInSchema { + args += ",include_description=false" + } + + args = fmt.Sprintf("%s:%s", args, directoryPath) + + cmd.Args = append(cmd.Args, args) + cmd.Args = append(cmd.Args, "-o", toFile, diff --git a/codegen/proto/schemagen/generator.go b/codegen/proto/schemagen/generator.go index 85b65c43..98fe2463 100644 --- a/codegen/proto/schemagen/generator.go +++ b/codegen/proto/schemagen/generator.go @@ -8,6 +8,9 @@ const ( ) type ValidationSchemaOptions struct { + // Whether to include descriptions in validation schemas + IncludeDescriptionsInSchema bool + // Whether to assign Enum fields the `x-kubernetes-int-or-string` property // which allows the value to either be an integer or a string // If this is false, only string values are allowed diff --git a/codegen/proto/schemagen/protoc.go b/codegen/proto/schemagen/protoc.go index ebb8a386..c59119aa 100644 --- a/codegen/proto/schemagen/protoc.go +++ b/codegen/proto/schemagen/protoc.go @@ -40,10 +40,11 @@ func (p *protocGenerator) GetJSONSchemas(protoFiles []string, imports []string, // The Executor used to compile protos protocExecutor := &collector.OpenApiProtocExecutor{ - OutputDir: tmpOutputDir, - EnumAsIntOrString: p.validationSchemaOptions.EnumAsIntOrString, - MessagesWithEmptySchema: p.validationSchemaOptions.MessagesWithEmptySchema, - DisableKubeMarkers: p.validationSchemaOptions.DisableKubeMarkers, + OutputDir: tmpOutputDir, + EnumAsIntOrString: p.validationSchemaOptions.EnumAsIntOrString, + MessagesWithEmptySchema: p.validationSchemaOptions.MessagesWithEmptySchema, + DisableKubeMarkers: p.validationSchemaOptions.DisableKubeMarkers, + IncludeDescriptionsInSchema: p.validationSchemaOptions.IncludeDescriptionsInSchema, } // 1. Generate the openApiSchemas for the project, writing them to a temp directory (schemaOutputDir) From a7c35d3a1725409e3d8d5bec90e33bb2063f44da Mon Sep 17 00:00:00 2001 From: omar Date: Wed, 11 Sep 2024 16:16:55 -0500 Subject: [PATCH 2/7] invert logic to maintain default=true behavior --- codegen/collector/executor.go | 4 ++-- codegen/proto/schemagen/generator.go | 2 +- codegen/proto/schemagen/protoc.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/codegen/collector/executor.go b/codegen/collector/executor.go index e28ba46a..89cd0f9f 100644 --- a/codegen/collector/executor.go +++ b/codegen/collector/executor.go @@ -70,7 +70,7 @@ type OpenApiProtocExecutor struct { OutputDir string // Whether to include descriptions in validation schemas - IncludeDescriptionsInSchema bool + ExcludeDescriptionsInSchema bool // Whether to assign Enum fields the `x-kubernetes-int-or-string` property // which allows the value to either be an integer or a string @@ -113,7 +113,7 @@ func (o *OpenApiProtocExecutor) Execute(protoFile string, toFile string, imports baseArgument = fmt.Sprintf("%s,additional_empty_schema=%v", baseArgument, strings.Join(o.MessagesWithEmptySchema, "+")) baseArgument = fmt.Sprintf("%s,disable_kube_markers=%v", baseArgument, o.DisableKubeMarkers) baseArgument = fmt.Sprintf("%s,ignored_kube_marker_substrings=%v", baseArgument, strings.Join(o.IgnoredKubeMarkerSubstrings, "+")) - baseArgument = fmt.Sprintf("%s,include_description=%v", baseArgument, o.IncludeDescriptionsInSchema) + baseArgument = fmt.Sprintf("%s,include_description=%v", baseArgument, !o.ExcludeDescriptionsInSchema) // Create the directory directoryPath := filepath.Join(o.OutputDir, directoryName) diff --git a/codegen/proto/schemagen/generator.go b/codegen/proto/schemagen/generator.go index 4b8d1c4b..9de8a854 100644 --- a/codegen/proto/schemagen/generator.go +++ b/codegen/proto/schemagen/generator.go @@ -9,7 +9,7 @@ const ( type ValidationSchemaOptions struct { // Whether to include descriptions in validation schemas - IncludeDescriptionsInSchema bool + ExcludeDescriptionsInSchema bool // Whether to assign Enum fields the `x-kubernetes-int-or-string` property // which allows the value to either be an integer or a string diff --git a/codegen/proto/schemagen/protoc.go b/codegen/proto/schemagen/protoc.go index 066ec19a..01ff6089 100644 --- a/codegen/proto/schemagen/protoc.go +++ b/codegen/proto/schemagen/protoc.go @@ -44,7 +44,7 @@ func (p *protocGenerator) GetJSONSchemas(protoFiles []string, imports []string, EnumAsIntOrString: p.validationSchemaOptions.EnumAsIntOrString, MessagesWithEmptySchema: p.validationSchemaOptions.MessagesWithEmptySchema, DisableKubeMarkers: p.validationSchemaOptions.DisableKubeMarkers, - IncludeDescriptionsInSchema: p.validationSchemaOptions.IncludeDescriptionsInSchema, + ExcludeDescriptionsInSchema: p.validationSchemaOptions.ExcludeDescriptionsInSchema, IgnoredKubeMarkerSubstrings: p.validationSchemaOptions.IgnoredKubeMarkerSubstrings, } From 18beb2390901c9e98c8b13a5f2a7d095202a29d8 Mon Sep 17 00:00:00 2001 From: omar Date: Wed, 11 Sep 2024 16:16:58 -0500 Subject: [PATCH 3/7] cl --- changelog/v0.41.1/exclude-descriptions-in-schema.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changelog/v0.41.1/exclude-descriptions-in-schema.yaml diff --git a/changelog/v0.41.1/exclude-descriptions-in-schema.yaml b/changelog/v0.41.1/exclude-descriptions-in-schema.yaml new file mode 100644 index 00000000..5c5e44f9 --- /dev/null +++ b/changelog/v0.41.1/exclude-descriptions-in-schema.yaml @@ -0,0 +1,6 @@ +changelog: + - type: NON_USER_FACING + issueLink: https://github.com/solo-io/gloo-mesh-enterprise/issues/18562 + description: > + Add option to exclude descriptions in schema. + skipCI: "false" From 367befc3c0c53d97d2a7a43554adcbd42bb37bdf Mon Sep 17 00:00:00 2001 From: omar Date: Thu, 19 Sep 2024 15:24:03 -0500 Subject: [PATCH 4/7] determine whether to skip generating schema descriptions per group use existing field SkipSchemaDescriptions --- codegen/collector/executor.go | 2 +- codegen/proto/schemagen/generator.go | 3 --- codegen/proto/schemagen/protoc.go | 10 ++++++---- codegen/render/manifests_renderer.go | 4 ++-- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/codegen/collector/executor.go b/codegen/collector/executor.go index 89cd0f9f..8acb41da 100644 --- a/codegen/collector/executor.go +++ b/codegen/collector/executor.go @@ -69,7 +69,7 @@ func (d *DefaultProtocExecutor) Execute(protoFile string, toFile string, imports type OpenApiProtocExecutor struct { OutputDir string - // Whether to include descriptions in validation schemas + // Whether to include descriptions in schemas ExcludeDescriptionsInSchema bool // Whether to assign Enum fields the `x-kubernetes-int-or-string` property diff --git a/codegen/proto/schemagen/generator.go b/codegen/proto/schemagen/generator.go index 9de8a854..24e7c748 100644 --- a/codegen/proto/schemagen/generator.go +++ b/codegen/proto/schemagen/generator.go @@ -8,9 +8,6 @@ const ( ) type ValidationSchemaOptions struct { - // Whether to include descriptions in validation schemas - ExcludeDescriptionsInSchema bool - // Whether to assign Enum fields the `x-kubernetes-int-or-string` property // which allows the value to either be an integer or a string // If this is false, only string values are allowed diff --git a/codegen/proto/schemagen/protoc.go b/codegen/proto/schemagen/protoc.go index 01ff6089..59fcf3ae 100644 --- a/codegen/proto/schemagen/protoc.go +++ b/codegen/proto/schemagen/protoc.go @@ -19,12 +19,14 @@ import ( // Implementation of JsonSchemaGenerator that uses a plugin for the protocol buffer compiler type protocGenerator struct { - validationSchemaOptions *ValidationSchemaOptions + validationSchemaOptions *ValidationSchemaOptions + excludeDescriptionsInSchema bool } -func NewProtocGenerator(validationSchemaOptions *ValidationSchemaOptions) *protocGenerator { +func NewProtocGenerator(validationSchemaOptions *ValidationSchemaOptions, excludeDescriptionsInSchema bool) *protocGenerator { return &protocGenerator{ - validationSchemaOptions: validationSchemaOptions, + validationSchemaOptions: validationSchemaOptions, + excludeDescriptionsInSchema: excludeDescriptionsInSchema, } } @@ -44,7 +46,7 @@ func (p *protocGenerator) GetJSONSchemas(protoFiles []string, imports []string, EnumAsIntOrString: p.validationSchemaOptions.EnumAsIntOrString, MessagesWithEmptySchema: p.validationSchemaOptions.MessagesWithEmptySchema, DisableKubeMarkers: p.validationSchemaOptions.DisableKubeMarkers, - ExcludeDescriptionsInSchema: p.validationSchemaOptions.ExcludeDescriptionsInSchema, + ExcludeDescriptionsInSchema: p.excludeDescriptionsInSchema, IgnoredKubeMarkerSubstrings: p.validationSchemaOptions.IgnoredKubeMarkerSubstrings, } diff --git a/codegen/render/manifests_renderer.go b/codegen/render/manifests_renderer.go index 290973ed..b16d1a18 100644 --- a/codegen/render/manifests_renderer.go +++ b/codegen/render/manifests_renderer.go @@ -139,7 +139,7 @@ func generateOpenApi(grp model.Group, protoDir string, protoOpts protoutil.Optio } // Use protoc-gen-openapi for transpiling protobuf schemas to openapi v3 with k8s structural schema constraints. -func generateOpenApiFromProtocGen(grp model.Group, protoDir string, _ protoutil.Options, groupOptions model.GroupOptions) (model.OpenApiSchemas, error) { +func generateOpenApiFromProtocGen(grp model.Group, _ string, _ protoutil.Options, groupOptions model.GroupOptions) (model.OpenApiSchemas, error) { // Collect all protobuf definitions including transitive dependencies. var imports []string for _, fileDescriptor := range grp.Descriptors { @@ -152,7 +152,7 @@ func generateOpenApiFromProtocGen(grp model.Group, protoDir string, _ protoutil. for i, fileDescriptor := range grp.Descriptors { protoFiles[i] = fileDescriptor.ProtoFilePath } - protoGen := schemagen.NewProtocGenerator(&groupOptions.SchemaValidationOpts) + protoGen := schemagen.NewProtocGenerator(&groupOptions.SchemaValidationOpts, grp.SkipSchemaDescriptions) gvkSchemas, err := protoGen.GetJSONSchemas(protoFiles, imports, grp.GroupVersion) if err != nil { return nil, err From 25081a8ed5a8fbc230f1a70a57821a3df95fb85b Mon Sep 17 00:00:00 2001 From: omar Date: Thu, 19 Sep 2024 15:24:10 -0500 Subject: [PATCH 5/7] lint --- codegen/collector/compiler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codegen/collector/compiler.go b/codegen/collector/compiler.go index 47f385a9..3cae3ce1 100644 --- a/codegen/collector/compiler.go +++ b/codegen/collector/compiler.go @@ -95,7 +95,7 @@ func (p *protoCompiler) CompileDescriptorsFromRoot(root string, skipDirs []strin sem = make(chan struct{}, maxProtocs) limitConcurrency = true } - for _, dir := range append([]string{root}) { + for _, dir := range []string{root} { absoluteDir, err := filepath.Abs(dir) if err != nil { return nil, err From ce47235dc30b8cb3d298bc5e228f3b197e4cd1e1 Mon Sep 17 00:00:00 2001 From: omar Date: Fri, 20 Sep 2024 10:59:04 -0500 Subject: [PATCH 6/7] changelog --- changelog/v0.41.1/exclude-descriptions-in-schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/v0.41.1/exclude-descriptions-in-schema.yaml b/changelog/v0.41.1/exclude-descriptions-in-schema.yaml index 5c5e44f9..f7f8dac7 100644 --- a/changelog/v0.41.1/exclude-descriptions-in-schema.yaml +++ b/changelog/v0.41.1/exclude-descriptions-in-schema.yaml @@ -1,5 +1,5 @@ changelog: - - type: NON_USER_FACING + - type: NEW_FEATURE issueLink: https://github.com/solo-io/gloo-mesh-enterprise/issues/18562 description: > Add option to exclude descriptions in schema. From e5fa8a3f563af8fdfb703549111514f7544fa285 Mon Sep 17 00:00:00 2001 From: omar Date: Fri, 20 Sep 2024 10:59:10 -0500 Subject: [PATCH 7/7] clean up --- codegen/render/manifests_renderer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codegen/render/manifests_renderer.go b/codegen/render/manifests_renderer.go index b16d1a18..64f40b6c 100644 --- a/codegen/render/manifests_renderer.go +++ b/codegen/render/manifests_renderer.go @@ -133,13 +133,13 @@ func (r ManifestsRenderer) RenderManifests(grps []*Group, protoOpts protoutil.Op func generateOpenApi(grp model.Group, protoDir string, protoOpts protoutil.Options, groupOptions model.GroupOptions) (model.OpenApiSchemas, error) { if groupOptions.SchemaGenerator == schemagen.ProtocGenOpenAPI { - return generateOpenApiFromProtocGen(grp, protoDir, protoOpts, groupOptions) + return generateOpenApiFromProtocGen(grp, groupOptions) } return generateOpenApiFromCue(grp, protoDir, protoOpts, groupOptions) } // Use protoc-gen-openapi for transpiling protobuf schemas to openapi v3 with k8s structural schema constraints. -func generateOpenApiFromProtocGen(grp model.Group, _ string, _ protoutil.Options, groupOptions model.GroupOptions) (model.OpenApiSchemas, error) { +func generateOpenApiFromProtocGen(grp model.Group, groupOptions model.GroupOptions) (model.OpenApiSchemas, error) { // Collect all protobuf definitions including transitive dependencies. var imports []string for _, fileDescriptor := range grp.Descriptors {