Skip to content

Commit f4f5466

Browse files
AngersZhuuuuyaooqinn
AngersZhuuuu
authored and
yaooqinn
committed
[KYUUBI #5446][AUTHZ] Support Create/Drop/Show/Reresh index command for Hudi
### _Why are the changes needed?_ To close #5446. Kyuubi authz support Create/Drop/Show/Reresh index based command IndexBasedCommand SQL grammer is in https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlCommon.g4 Below command under https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/IndexCommands.scala - CreateIndexCommand - DropIndexCommand - RefreshIndexCommand - ShowIndexesCommand ### _How was this patch tested?_ - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request ### _Was this patch authored or co-authored using generative AI tooling?_ No Closes #5448 from AngersZhuuuu/KYUUBI-5446. Closes #5446 1be0561 [Angerszhuuuu] Update ObjectType.scala c4ae073 [Angerszhuuuu] follow comment ed1544e [Angerszhuuuu] Update 214bb86 [Angerszhuuuu] Update OperationType.scala 97cb0f8 [Angerszhuuuu] Update PrivilegeObject.scala bc65abd [Angerszhuuuu] update 0e6f6df [Angerszhuuuu] Update 4019f45 [Angerszhuuuu] update dc9188d [Angerszhuuuu] update f9a398c [Angerszhuuuu] Merge branch 'master' into KYUUBI-5446 52f67e9 [Angerszhuuuu] update bed945e [Angerszhuuuu] Merge branch 'master' into KYUUBI-5446 f174ef2 [Angerszhuuuu] Update table_command_spec.json 4764531 [Angerszhuuuu] Merge branch 'master' into KYUUBI-5446 a78425b [Angerszhuuuu] Update HudiCommands.scala 81881db [Angerszhuuuu] Update HudiCommands.scala 544a443 [Angerszhuuuu] Update HudiCommands.scala aed91cd [Angerszhuuuu] Merge branch 'master' into KYUUBI-5446 9632886 [Angerszhuuuu] Update HudiCatalogRangerSparkExtensionSuite.scala c404fd7 [Angerszhuuuu] Update HudiCatalogRangerSparkExtensionSuite.scala e70c8f1 [Angerszhuuuu] Update HudiCatalogRangerSparkExtensionSuite.scala fefb021 [Angerszhuuuu] Update table_command_spec.json ab8b96d [Angerszhuuuu] Merge branch 'master' into KYUUBI-5446 fcb3d45 [Angerszhuuuu] Update HudiCatalogRangerSparkExtensionSuite.scala b1657d9 [Angerszhuuuu] follow comment 0222e02 [Angerszhuuuu] [KYUUBI-5446][AUTHZ] Support Create/Drop/Show/Reresh index command Authored-by: Angerszhuuuu <[email protected]> Signed-off-by: Kent Yao <[email protected]>
1 parent 5cff4fb commit f4f5466

File tree

7 files changed

+153
-7
lines changed

7 files changed

+153
-7
lines changed

Diff for: ‎extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json

+56
Original file line numberDiff line numberDiff line change
@@ -1623,6 +1623,20 @@
16231623
} ],
16241624
"opType" : "CREATETABLE",
16251625
"queryDescs" : [ ]
1626+
}, {
1627+
"classname" : "org.apache.spark.sql.hudi.command.CreateIndexCommand",
1628+
"tableDescs" : [ {
1629+
"fieldName" : "table",
1630+
"fieldExtractor" : "CatalogTableTableExtractor",
1631+
"columnDesc" : null,
1632+
"actionTypeDesc" : null,
1633+
"tableTypeDesc" : null,
1634+
"catalogDesc" : null,
1635+
"isInput" : false,
1636+
"setCurrentDatabaseIfMissing" : false
1637+
} ],
1638+
"opType" : "CREATEINDEX",
1639+
"queryDescs" : [ ]
16261640
}, {
16271641
"classname" : "org.apache.spark.sql.hudi.command.DeleteHoodieTableCommand",
16281642
"tableDescs" : [ {
@@ -1659,6 +1673,20 @@
16591673
} ],
16601674
"opType" : "DROPTABLE",
16611675
"queryDescs" : [ ]
1676+
}, {
1677+
"classname" : "org.apache.spark.sql.hudi.command.DropIndexCommand",
1678+
"tableDescs" : [ {
1679+
"fieldName" : "table",
1680+
"fieldExtractor" : "CatalogTableTableExtractor",
1681+
"columnDesc" : null,
1682+
"actionTypeDesc" : null,
1683+
"tableTypeDesc" : null,
1684+
"catalogDesc" : null,
1685+
"isInput" : false,
1686+
"setCurrentDatabaseIfMissing" : false
1687+
} ],
1688+
"opType" : "DROPINDEX",
1689+
"queryDescs" : [ ]
16621690
}, {
16631691
"classname" : "org.apache.spark.sql.hudi.command.InsertIntoHoodieTableCommand",
16641692
"tableDescs" : [ {
@@ -1701,6 +1729,20 @@
17011729
"fieldName" : "mergeInto",
17021730
"fieldExtractor" : "HudiMergeIntoSourceTableExtractor"
17031731
} ]
1732+
}, {
1733+
"classname" : "org.apache.spark.sql.hudi.command.RefreshIndexCommand",
1734+
"tableDescs" : [ {
1735+
"fieldName" : "table",
1736+
"fieldExtractor" : "CatalogTableTableExtractor",
1737+
"columnDesc" : null,
1738+
"actionTypeDesc" : null,
1739+
"tableTypeDesc" : null,
1740+
"catalogDesc" : null,
1741+
"isInput" : false,
1742+
"setCurrentDatabaseIfMissing" : false
1743+
} ],
1744+
"opType" : "ALTERINDEX_REBUILD",
1745+
"queryDescs" : [ ]
17041746
}, {
17051747
"classname" : "org.apache.spark.sql.hudi.command.RepairHoodieTableCommand",
17061748
"tableDescs" : [ {
@@ -1732,6 +1774,20 @@
17321774
} ],
17331775
"opType" : "SHOWPARTITIONS",
17341776
"queryDescs" : [ ]
1777+
}, {
1778+
"classname" : "org.apache.spark.sql.hudi.command.ShowIndexesCommand",
1779+
"tableDescs" : [ {
1780+
"fieldName" : "table",
1781+
"fieldExtractor" : "CatalogTableTableExtractor",
1782+
"columnDesc" : null,
1783+
"actionTypeDesc" : null,
1784+
"tableTypeDesc" : null,
1785+
"catalogDesc" : null,
1786+
"isInput" : true,
1787+
"setCurrentDatabaseIfMissing" : false
1788+
} ],
1789+
"opType" : "SHOWINDEXES",
1790+
"queryDescs" : [ ]
17351791
}, {
17361792
"classname" : "org.apache.spark.sql.hudi.command.Spark31AlterTableCommand",
17371793
"tableDescs" : [ {

Diff for: ‎extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ObjectType.scala

+2-1
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,12 @@ object ObjectType extends Enumeration {
2323

2424
type ObjectType = Value
2525

26-
val DATABASE, TABLE, VIEW, COLUMN, FUNCTION = Value
26+
val DATABASE, TABLE, VIEW, COLUMN, FUNCTION, INDEX = Value
2727

2828
def apply(obj: PrivilegeObject, opType: OperationType): ObjectType = {
2929
obj.privilegeObjectType match {
3030
case PrivilegeObjectType.DATABASE => DATABASE
31+
case PrivilegeObjectType.TABLE_OR_VIEW if opType.toString.contains("INDEX") => INDEX
3132
case PrivilegeObjectType.TABLE_OR_VIEW if obj.columns.nonEmpty => COLUMN
3233
case PrivilegeObjectType.TABLE_OR_VIEW if opType.toString.contains("VIEW") => VIEW
3334
case PrivilegeObjectType.TABLE_OR_VIEW => TABLE

Diff for: ‎extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/OperationType.scala

+5-2
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,16 @@ package org.apache.kyuubi.plugin.spark.authz
2020
object OperationType extends Enumeration {
2121

2222
type OperationType = Value
23-
23+
// According to https://scalameta.org/scalafmt/docs/known-issues.html
24+
// format: off
2425
val ALTERDATABASE, ALTERDATABASE_LOCATION, ALTERTABLE_ADDCOLS, ALTERTABLE_ADDPARTS,
2526
ALTERTABLE_RENAMECOL, ALTERTABLE_REPLACECOLS, ALTERTABLE_DROPPARTS, ALTERTABLE_RENAMEPART,
2627
ALTERTABLE_RENAME, ALTERTABLE_PROPERTIES, ALTERTABLE_SERDEPROPERTIES, ALTERTABLE_LOCATION,
2728
ALTERVIEW_AS, ALTERVIEW_RENAME, ANALYZE_TABLE, CREATEDATABASE, CREATETABLE,
2829
CREATETABLE_AS_SELECT, CREATEFUNCTION, CREATEVIEW, DESCDATABASE, DESCFUNCTION, DESCTABLE,
2930
DROPDATABASE, DROPFUNCTION, DROPTABLE, DROPVIEW, EXPLAIN, LOAD, MSCK, QUERY, RELOADFUNCTION,
3031
SHOWCONF, SHOW_CREATETABLE, SHOWCOLUMNS, SHOWDATABASES, SHOWFUNCTIONS, SHOWPARTITIONS,
31-
SHOWTABLES, SHOW_TBLPROPERTIES, SWITCHDATABASE, TRUNCATETABLE = Value
32+
SHOWTABLES, SHOW_TBLPROPERTIES, SWITCHDATABASE, TRUNCATETABLE,
33+
CREATEINDEX, DROPINDEX, ALTERINDEX_REBUILD, SHOWINDEXES = Value
34+
// format: on
3235
}

Diff for: ‎extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessResource.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ object AccessResource {
5757
resource.setValue("database", firstLevelResource)
5858
resource.setValue("table", secondLevelResource)
5959
resource.setValue("column", thirdLevelResource)
60-
case TABLE | VIEW => // fixme spark have added index support
60+
case TABLE | VIEW | INDEX =>
6161
resource.setValue("database", firstLevelResource)
6262
resource.setValue("table", secondLevelResource)
6363
}

Diff for: ‎extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessType.scala

+6-3
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ object AccessType extends Enumeration {
2525

2626
type AccessType = Value
2727

28-
val NONE, CREATE, ALTER, DROP, SELECT, UPDATE, USE, READ, WRITE, ALL, ADMIN = Value
28+
val NONE, CREATE, ALTER, DROP, SELECT, UPDATE, USE, READ, WRITE, ALL, ADMIN, INDEX = Value
2929

3030
def apply(obj: PrivilegeObject, opType: OperationType, isInput: Boolean): AccessType = {
3131
obj.actionType match {
@@ -48,14 +48,16 @@ object AccessType extends Enumeration {
4848
ALTERTABLE_REPLACECOLS |
4949
ALTERTABLE_SERDEPROPERTIES |
5050
ALTERVIEW_RENAME |
51-
MSCK => ALTER
51+
MSCK |
52+
ALTERINDEX_REBUILD => ALTER
5253
case ALTERVIEW_AS => if (isInput) SELECT else ALTER
53-
case DROPDATABASE | DROPTABLE | DROPFUNCTION | DROPVIEW => DROP
54+
case DROPDATABASE | DROPTABLE | DROPFUNCTION | DROPVIEW | DROPINDEX => DROP
5455
case LOAD => if (isInput) SELECT else UPDATE
5556
case QUERY |
5657
SHOW_CREATETABLE |
5758
SHOW_TBLPROPERTIES |
5859
SHOWPARTITIONS |
60+
SHOWINDEXES |
5961
ANALYZE_TABLE => SELECT
6062
case SHOWCOLUMNS | DESCTABLE => SELECT
6163
case SHOWDATABASES |
@@ -65,6 +67,7 @@ object AccessType extends Enumeration {
6567
SHOWFUNCTIONS |
6668
DESCFUNCTION => USE
6769
case TRUNCATETABLE => UPDATE
70+
case CREATEINDEX => INDEX
6871
case _ => NONE
6972
}
7073
case PrivilegeObjectActionType.DELETE => DROP

Diff for: ‎extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala

+28
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,30 @@ object HudiCommands {
145145
TableCommandSpec(cmd, Seq(tableDesc), SHOW_TBLPROPERTIES)
146146
}
147147

148+
val CreateIndexCommand = {
149+
val cmd = "org.apache.spark.sql.hudi.command.CreateIndexCommand"
150+
val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor])
151+
TableCommandSpec(cmd, Seq(tableDesc), CREATEINDEX)
152+
}
153+
154+
val DropIndexCommand = {
155+
val cmd = "org.apache.spark.sql.hudi.command.DropIndexCommand"
156+
val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor])
157+
TableCommandSpec(cmd, Seq(tableDesc), DROPINDEX)
158+
}
159+
160+
val ShowIndexCommand = {
161+
val cmd = "org.apache.spark.sql.hudi.command.ShowIndexesCommand"
162+
val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor], isInput = true)
163+
TableCommandSpec(cmd, Seq(tableDesc), SHOWINDEXES)
164+
}
165+
166+
val RefreshIndexCommand = {
167+
val cmd = "org.apache.spark.sql.hudi.command.RefreshIndexCommand"
168+
val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor])
169+
TableCommandSpec(cmd, Seq(tableDesc), ALTERINDEX_REBUILD)
170+
}
171+
148172
val InsertIntoHoodieTableCommand = {
149173
val cmd = "org.apache.spark.sql.hudi.command.InsertIntoHoodieTableCommand"
150174
val tableDesc = TableDesc(
@@ -228,13 +252,17 @@ object HudiCommands {
228252
CreateHoodieTableAsSelectCommand,
229253
CreateHoodieTableCommand,
230254
CreateHoodieTableLikeCommand,
255+
CreateIndexCommand,
231256
CompactionHoodieTableCommand,
232257
CompactionShowHoodieTableCommand,
233258
DeleteHoodieTableCommand,
234259
DropHoodieTableCommand,
260+
DropIndexCommand,
235261
InsertIntoHoodieTableCommand,
236262
MergeIntoHoodieTableCommand,
263+
RefreshIndexCommand,
237264
RepairHoodieTableCommand,
265+
ShowIndexCommand,
238266
TruncateHoodieTableCommand,
239267
ShowHoodieTablePartitionsCommand,
240268
Spark31AlterTableCommand,

Diff for: ‎extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala

+55
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ class HudiCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite {
5353
val table1 = "table1_hoodie"
5454
val table2 = "table2_hoodie"
5555
val outputTable1 = "outputTable_hoodie"
56+
val index1 = "table_hoodie_index1"
5657

5758
override def withFixture(test: NoArgTest): Outcome = {
5859
assume(isSupportedVersion)
@@ -522,4 +523,58 @@ class HudiCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite {
522523
}
523524
}
524525
}
526+
527+
test("IndexBasedCommand") {
528+
assume(
529+
!isSparkV33OrGreater,
530+
"Hudi index creation not supported on Spark 3.3 or greater currently")
531+
withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (namespace1, "database"))) {
532+
doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1"))
533+
doAs(
534+
admin,
535+
sql(
536+
s"""
537+
|CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string)
538+
|USING HUDI
539+
|OPTIONS (
540+
| type = 'cow',
541+
| primaryKey = 'id',
542+
| 'hoodie.datasource.hive_sync.enable' = 'false'
543+
|)
544+
|PARTITIONED BY(city)
545+
|""".stripMargin))
546+
547+
// CreateIndexCommand
548+
val createIndex = s"CREATE INDEX $index1 ON $namespace1.$table1 USING LUCENE (id)"
549+
interceptContains[AccessControlException](
550+
doAs(
551+
someone,
552+
sql(createIndex)))(s"does not have [index] privilege on [$namespace1/$table1]")
553+
doAs(admin, sql(createIndex))
554+
555+
// RefreshIndexCommand
556+
val refreshIndex = s"REFRESH INDEX $index1 ON $namespace1.$table1"
557+
interceptContains[AccessControlException](
558+
doAs(
559+
someone,
560+
sql(refreshIndex)))(s"does not have [alter] privilege on [$namespace1/$table1]")
561+
doAs(admin, sql(refreshIndex))
562+
563+
// ShowIndexesCommand
564+
val showIndex = s"SHOW INDEXES FROM TABLE $namespace1.$table1"
565+
interceptContains[AccessControlException](
566+
doAs(
567+
someone,
568+
sql(showIndex)))(s"does not have [select] privilege on [$namespace1/$table1]")
569+
doAs(admin, sql(showIndex))
570+
571+
// DropIndexCommand
572+
val dropIndex = s"DROP INDEX $index1 ON $namespace1.$table1"
573+
interceptContains[AccessControlException](
574+
doAs(
575+
someone,
576+
sql(dropIndex)))(s"does not have [drop] privilege on [$namespace1/$table1]")
577+
doAs(admin, sql(dropIndex))
578+
}
579+
}
525580
}

0 commit comments

Comments
 (0)