From 6d4516833f8c4b5b293412e3485a9f813d7584a1 Mon Sep 17 00:00:00 2001 From: Yun Zou Date: Wed, 7 May 2025 13:38:02 -0700 Subject: [PATCH 1/8] add location keyword for generic table --- spec/polaris-catalog-apis/generic-tables-api.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/spec/polaris-catalog-apis/generic-tables-api.yaml b/spec/polaris-catalog-apis/generic-tables-api.yaml index 92ee7f9a59..3569b39e00 100644 --- a/spec/polaris-catalog-apis/generic-tables-api.yaml +++ b/spec/polaris-catalog-apis/generic-tables-api.yaml @@ -199,6 +199,8 @@ components: type: object additionalProperties: type: string + location: + type: string GenericTable: type: object @@ -212,6 +214,8 @@ components: - `properties` properties for the generic table passed on creation - `doc` comment or description for the generic table + + - `location` location for the table required: - name - format @@ -226,6 +230,8 @@ components: type: object additionalProperties: type: string + location: + type: string LoadGenericTableResponse: description: Result used when a table is successfully loaded. From 0fa36fbb329205a44aa0817914ba7a4274968865 Mon Sep 17 00:00:00 2001 From: Yun Zou Date: Mon, 19 May 2025 10:11:22 -0700 Subject: [PATCH 2/8] add change --- .../it/test/PolarisRestCatalogIntegrationTest.java | 11 ++++++----- .../spark/rest/CreateGenericTableRESTRequest.java | 10 ++++++++-- spec/polaris-catalog-apis/generic-tables-api.yaml | 10 +++++----- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java index a859338b97..90d2aa2ea0 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java @@ -1468,11 +1468,12 @@ public void testCreateGenericTableWithReservedProperty() { Map.of("cat", currentCatalogName, "ns", ns)) .post( Entity.json( - new CreateGenericTableRequest( - tableIdentifier.name(), - "format", - "doc", - Map.of("polaris.reserved", "true"))))) { + CreateGenericTableRequest.builder() + .setName(tableIdentifier.name()) + .setFormat("format") + .setDoc("doc") + .setProperties(Map.of("polaris.reserved", "true")) + .build()))) { Assertions.assertThat(res.getStatus()).isEqualTo(Response.Status.BAD_REQUEST.getStatusCode()); Assertions.assertThat(res.readEntity(String.class)).contains("reserved prefix"); } diff --git a/plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/rest/CreateGenericTableRESTRequest.java b/plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/rest/CreateGenericTableRESTRequest.java index 6a9c89a5ff..1d6ab91575 100644 --- a/plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/rest/CreateGenericTableRESTRequest.java +++ b/plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/rest/CreateGenericTableRESTRequest.java @@ -36,13 +36,19 @@ public class CreateGenericTableRESTRequest extends CreateGenericTableRequest public CreateGenericTableRESTRequest( @JsonProperty(value = "name", required = true) String name, @JsonProperty(value = "format", required = true) String format, + @JsonProperty(value = "location") String location, @JsonProperty(value = "doc") String doc, @JsonProperty(value = "properties") Map properties) { - super(name, format, doc, properties); + super(name, format, location, doc, properties); } public CreateGenericTableRESTRequest(CreateGenericTableRequest request) { - this(request.getName(), request.getFormat(), request.getDoc(), request.getProperties()); + this( + request.getName(), + request.getFormat(), + request.getLocation(), + request.getDoc(), + request.getProperties()); } @Override diff --git a/spec/polaris-catalog-apis/generic-tables-api.yaml b/spec/polaris-catalog-apis/generic-tables-api.yaml index 3569b39e00..a6a97c3ad7 100644 --- a/spec/polaris-catalog-apis/generic-tables-api.yaml +++ b/spec/polaris-catalog-apis/generic-tables-api.yaml @@ -193,14 +193,14 @@ components: type: string format: type: string + location: + type: string doc: type: string properties: type: object additionalProperties: type: string - location: - type: string GenericTable: type: object @@ -215,7 +215,7 @@ components: - `doc` comment or description for the generic table - - `location` location for the table + - `location` table root location in URI format. For example: s3:///path/to/table required: - name - format @@ -224,14 +224,14 @@ components: type: string format: type: string + location: + type: string doc: type: string properties: type: object additionalProperties: type: string - location: - type: string LoadGenericTableResponse: description: Result used when a table is successfully loaded. From 30ffffd7fa73eaf46d3f139f053f7a6c81ea58f9 Mon Sep 17 00:00:00 2001 From: Yun Zou Date: Wed, 21 May 2025 09:35:28 -0700 Subject: [PATCH 3/8] fix table --- spec/polaris-catalog-apis/generic-tables-api.yaml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/spec/polaris-catalog-apis/generic-tables-api.yaml b/spec/polaris-catalog-apis/generic-tables-api.yaml index a6a97c3ad7..dac2e5ac69 100644 --- a/spec/polaris-catalog-apis/generic-tables-api.yaml +++ b/spec/polaris-catalog-apis/generic-tables-api.yaml @@ -207,15 +207,19 @@ components: description: > Generic Table information. - - `name` name for the generic table + - `name`(REQUIRED): name for the generic table - - `format` format for the generic table, i.e. "delta", "csv" + - `format`(REQUIRED): format for the generic table, i.e. "delta", "csv" - - `properties` properties for the generic table passed on creation + - `location`(OPTIONAL): table root location in URI format. For example: s3:///path/to/table. + - the table root location is a location that includes all files for the table. + - clients (engines) are responsible to make sure all files are written under the configured location. + - a table with multiple root locations (i.e. contain files that are outside the configured root location) is not compliant with the generic table support of Polaris. + - no two tables are allowed to be under the same location or have overlapped location, otherwise, a BadRequestErrorResponse will thrown. - - `doc` comment or description for the generic table + - `properties`(OPTIONAL): properties for the generic table passed on creation - - `location` table root location in URI format. For example: s3:///path/to/table + - `doc`(OPTIONAL): comment or description for the generic table required: - name - format From b1f0f2aa4f395cb701da73fe3ffb71f5dbc14ce6 Mon Sep 17 00:00:00 2001 From: Yun Zou Date: Wed, 21 May 2025 15:43:31 -0700 Subject: [PATCH 4/8] add doc for location --- spec/polaris-catalog-apis/generic-tables-api.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/spec/polaris-catalog-apis/generic-tables-api.yaml b/spec/polaris-catalog-apis/generic-tables-api.yaml index dac2e5ac69..e50014c03b 100644 --- a/spec/polaris-catalog-apis/generic-tables-api.yaml +++ b/spec/polaris-catalog-apis/generic-tables-api.yaml @@ -212,10 +212,12 @@ components: - `format`(REQUIRED): format for the generic table, i.e. "delta", "csv" - `location`(OPTIONAL): table root location in URI format. For example: s3:///path/to/table. - - the table root location is a location that includes all files for the table. - - clients (engines) are responsible to make sure all files are written under the configured location. - - a table with multiple root locations (i.e. contain files that are outside the configured root location) is not compliant with the generic table support of Polaris. - - no two tables are allowed to be under the same location or have overlapped location, otherwise, a BadRequestErrorResponse will thrown. + - The table root location is a location that includes all files for the table. + - Clients (engines) are responsible to make sure all files are written under the configured location. + - A table with multiple root locations (i.e. containing files that are outside the configured root location) is not compliant with the current generic table support of Polaris. + - No two tables can have the same or overlapped location, otherwise, a ForbiddenException will be thrown on creation. + - If no location is provided, clients (engines) are responsible to manage the location and location related concerns such as path conflict, credential vending etc. + - The location configuration can not be updated once the table is created. - `properties`(OPTIONAL): properties for the generic table passed on creation From 4b9c3614637e90e78afe3fe03ecfa1f6dd7475c4 Mon Sep 17 00:00:00 2001 From: Yun Zou Date: Wed, 21 May 2025 16:35:26 -0700 Subject: [PATCH 5/8] update spec --- spec/polaris-catalog-apis/generic-tables-api.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/polaris-catalog-apis/generic-tables-api.yaml b/spec/polaris-catalog-apis/generic-tables-api.yaml index e50014c03b..3c72e20de6 100644 --- a/spec/polaris-catalog-apis/generic-tables-api.yaml +++ b/spec/polaris-catalog-apis/generic-tables-api.yaml @@ -214,9 +214,9 @@ components: - `location`(OPTIONAL): table root location in URI format. For example: s3:///path/to/table. - The table root location is a location that includes all files for the table. - Clients (engines) are responsible to make sure all files are written under the configured location. - - A table with multiple root locations (i.e. containing files that are outside the configured root location) is not compliant with the current generic table support of Polaris. + - A table with multiple root locations (i.e. containing files that are outside the configured root location) is not compliant with the current generic table support in Polaris. - No two tables can have the same or overlapped location, otherwise, a ForbiddenException will be thrown on creation. - - If no location is provided, clients (engines) are responsible to manage the location and location related concerns such as path conflict, credential vending etc. + - If no location is provided, clients or users are responsible to manage the location and location related concerns such as path conflict etc. - The location configuration can not be updated once the table is created. - `properties`(OPTIONAL): properties for the generic table passed on creation From c8a8a156558db5ca99d720116bdb9345de5fc79d Mon Sep 17 00:00:00 2001 From: Yun Zou Date: Thu, 5 Jun 2025 13:12:34 -0700 Subject: [PATCH 6/8] address feedback --- spec/polaris-catalog-apis/generic-tables-api.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/spec/polaris-catalog-apis/generic-tables-api.yaml b/spec/polaris-catalog-apis/generic-tables-api.yaml index 3c72e20de6..ede0a5206e 100644 --- a/spec/polaris-catalog-apis/generic-tables-api.yaml +++ b/spec/polaris-catalog-apis/generic-tables-api.yaml @@ -215,9 +215,7 @@ components: - The table root location is a location that includes all files for the table. - Clients (engines) are responsible to make sure all files are written under the configured location. - A table with multiple root locations (i.e. containing files that are outside the configured root location) is not compliant with the current generic table support in Polaris. - - No two tables can have the same or overlapped location, otherwise, a ForbiddenException will be thrown on creation. - If no location is provided, clients or users are responsible to manage the location and location related concerns such as path conflict etc. - - The location configuration can not be updated once the table is created. - `properties`(OPTIONAL): properties for the generic table passed on creation From dd39e6dedb3f6409ce07dcccbf006a84b22bc2d4 Mon Sep 17 00:00:00 2001 From: Yun Zou Date: Tue, 10 Jun 2025 14:52:25 -0700 Subject: [PATCH 7/8] fix spec --- .../rest/CreateGenericTableRESTRequest.java | 2 +- .../generic-tables-api.yaml | 21 +++++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/rest/CreateGenericTableRESTRequest.java b/plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/rest/CreateGenericTableRESTRequest.java index 1d6ab91575..705bcaf9ed 100644 --- a/plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/rest/CreateGenericTableRESTRequest.java +++ b/plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/rest/CreateGenericTableRESTRequest.java @@ -46,7 +46,7 @@ public CreateGenericTableRESTRequest(CreateGenericTableRequest request) { this( request.getName(), request.getFormat(), - request.getLocation(), + request.getBaseLocation(), request.getDoc(), request.getProperties()); } diff --git a/spec/polaris-catalog-apis/generic-tables-api.yaml b/spec/polaris-catalog-apis/generic-tables-api.yaml index ede0a5206e..6377c8f99c 100644 --- a/spec/polaris-catalog-apis/generic-tables-api.yaml +++ b/spec/polaris-catalog-apis/generic-tables-api.yaml @@ -193,7 +193,7 @@ components: type: string format: type: string - location: + base-location: type: string doc: type: string @@ -207,19 +207,18 @@ components: description: > Generic Table information. - - `name`(REQUIRED): name for the generic table + - `name` (REQUIRED): name for the generic table - - `format`(REQUIRED): format for the generic table, i.e. "delta", "csv" + - `format` (REQUIRED): format for the generic table, i.e. "delta", "csv" - - `location`(OPTIONAL): table root location in URI format. For example: s3:///path/to/table. - - The table root location is a location that includes all files for the table. - - Clients (engines) are responsible to make sure all files are written under the configured location. - - A table with multiple root locations (i.e. containing files that are outside the configured root location) is not compliant with the current generic table support in Polaris. - - If no location is provided, clients or users are responsible to manage the location and location related concerns such as path conflict etc. + - `base-location` (OPTIONAL): table base location in URI format. For example: s3:///path/to/table. + - The table base location is a location that includes all files for the table. + - A table with multiple disjoint locations (i.e. containing files that are outside the configured base location) is not compliant with the current generic table support in Polaris. + - If no location is provided, clients or users are responsible for managing the location. - - `properties`(OPTIONAL): properties for the generic table passed on creation + - `properties` (OPTIONAL): properties for the generic table passed on creation - - `doc`(OPTIONAL): comment or description for the generic table + - `doc` (OPTIONAL): comment or description for the generic table required: - name - format @@ -228,7 +227,7 @@ components: type: string format: type: string - location: + base-location: type: string doc: type: string From f365865dd0eef54ac278fdbc9c9e17cf81f36d78 Mon Sep 17 00:00:00 2001 From: Yun Zou Date: Tue, 10 Jun 2025 15:31:50 -0700 Subject: [PATCH 8/8] fix error --- api/polaris-catalog-service/build.gradle.kts | 2 ++ .../rest/CreateGenericTableRESTRequest.java | 4 ++-- .../bundled-polaris-catalog-service.yaml | 17 +++++++++++++---- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/api/polaris-catalog-service/build.gradle.kts b/api/polaris-catalog-service/build.gradle.kts index 7b4ea7454c..5263839952 100644 --- a/api/polaris-catalog-service/build.gradle.kts +++ b/api/polaris-catalog-service/build.gradle.kts @@ -122,3 +122,5 @@ sourceSets { tasks.named("javadoc") { dependsOn("jandex") } tasks.named("processResources") { dependsOn("openApiGenerate") } + +tasks.named("openApiGenerate") { outputs.cacheIf { false } } diff --git a/plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/rest/CreateGenericTableRESTRequest.java b/plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/rest/CreateGenericTableRESTRequest.java index 705bcaf9ed..54ff841fc4 100644 --- a/plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/rest/CreateGenericTableRESTRequest.java +++ b/plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/rest/CreateGenericTableRESTRequest.java @@ -36,10 +36,10 @@ public class CreateGenericTableRESTRequest extends CreateGenericTableRequest public CreateGenericTableRESTRequest( @JsonProperty(value = "name", required = true) String name, @JsonProperty(value = "format", required = true) String format, - @JsonProperty(value = "location") String location, + @JsonProperty(value = "base-location") String baseLocation, @JsonProperty(value = "doc") String doc, @JsonProperty(value = "properties") Map properties) { - super(name, format, location, doc, properties); + super(name, format, baseLocation, doc, properties); } public CreateGenericTableRESTRequest(CreateGenericTableRequest request) { diff --git a/spec/generated/bundled-polaris-catalog-service.yaml b/spec/generated/bundled-polaris-catalog-service.yaml index 6c1d9dceb3..7dffb017d9 100644 --- a/spec/generated/bundled-polaris-catalog-service.yaml +++ b/spec/generated/bundled-polaris-catalog-service.yaml @@ -3880,6 +3880,8 @@ components: type: string format: type: string + base-location: + type: string doc: type: string properties: @@ -3890,10 +3892,15 @@ components: type: object description: | Generic Table information. - - `name` name for the generic table - - `format` format for the generic table, i.e. "delta", "csv" - - `properties` properties for the generic table passed on creation - - `doc` comment or description for the generic table + - `name` (REQUIRED): name for the generic table + - `format` (REQUIRED): format for the generic table, i.e. "delta", "csv" + - `base-location` (OPTIONAL): table base location in URI format. For example: s3:///path/to/table. + - The table base location is a location that includes all files for the table. + - A table with multiple disjoint locations (i.e. containing files that are outside the configured base location) is not compliant with the current generic table support in Polaris. + - If no location is provided, clients or users are responsible for managing the location. + + - `properties` (OPTIONAL): properties for the generic table passed on creation + - `doc` (OPTIONAL): comment or description for the generic table required: - name - format @@ -3902,6 +3909,8 @@ components: type: string format: type: string + base-location: + type: string doc: type: string properties: