Skip to content

Commit

Permalink
fix tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@wecharyu
wecharyu committed Jun 4, 2024
1 parent 3adee56 commit fb49693
Show file tree
Hide file tree
Showing 7 changed files with 62 additions and 34 deletions.
4 changes: 2 additions & 2 deletions ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,7 @@ public List<String> getTables(String dbName, String tablePattern) throws MetaExc
if (tables == null || tables.size() == 0) {
return tableNames;
}
tablePattern = tablePattern.replaceAll("\\*", ".*");
tablePattern = tablePattern.replaceAll("\\.\\*", "\\*").replaceAll("\\*", ".*");
Pattern pattern = Pattern.compile(tablePattern);
Matcher matcher = pattern.matcher("");
Set<String> combinedTableNames = new HashSet<String>();
Expand Down Expand Up @@ -351,7 +351,7 @@ public List<String> getTables(String dbname, String tablePattern, TableType tabl
if (tables == null || tables.size() == 0) {
return tableNames;
}
tablePattern = tablePattern.replaceAll("\\*", ".*");
tablePattern = tablePattern.replaceAll("\\.\\*", "\\*").replaceAll("\\*", ".*");
Pattern pattern = Pattern.compile(tablePattern);
Matcher matcher = pattern.matcher("");
Set<String> combinedTableNames = new HashSet<String>();
Expand Down
2 changes: 2 additions & 0 deletions ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
Expand Down Expand Up @@ -435,6 +436,7 @@ public void testGetAndDropTables() throws Throwable {
hm.createTable(tbl2);

List<String> fts = hm.getTablesForDb(dbName, ".*");
Collections.sort(fts);
assertEquals(ts, fts);
assertEquals(2, fts.size());

Expand Down
4 changes: 2 additions & 2 deletions ql/src/test/results/clientpositive/llap/authorization_privilege_objects.q.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ PREHOOK: query: SHOW TABLES IN test_auth_obj_db
PREHOOK: type: SHOWTABLES
PREHOOK: Input: database:test_auth_obj_db
filterListCmdObjects
HIVE PRIVILEGE OBJECT { objectName: test_privs type: TABLE_OR_VIEW actionType: OTHER dbName: test_auth_obj_db OWNER: testuser OWNERTYPE: USER}
HIVE PRIVILEGE OBJECT { objectName: test_privs2 type: TABLE_OR_VIEW actionType: OTHER dbName: test_auth_obj_db OWNER: testuser2 OWNERTYPE: USER}
HIVE PRIVILEGE OBJECT { objectName: test_privs type: TABLE_OR_VIEW actionType: OTHER dbName: test_auth_obj_db}
HIVE PRIVILEGE OBJECT { objectName: test_privs2 type: TABLE_OR_VIEW actionType: OTHER dbName: test_auth_obj_db}
POSTHOOK: query: SHOW TABLES IN test_auth_obj_db
POSTHOOK: type: SHOWTABLES
POSTHOOK: Input: database:test_auth_obj_db
Expand Down
41 changes: 35 additions & 6 deletions .../metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2901,13 +2901,42 @@ public List<String> getTables(String catName, String dbName, String tablePattern
tablePattern = ".*";
}
List<String> tables = new ArrayList<>();
String[] patterns = tablePattern.split("\\|");
for (String pattern : patterns) {
pattern = "(?i)" + pattern.replaceAll("\\*", ".*");
String filter = hive_metastoreConstants.HIVE_FILTER_FIELD_TABLE_NAME + " like \"" + pattern + "\"";
tables.addAll(listTableNamesByFilter(catName, dbName, filter, (short) -1));
Database db = null;
try {
db = getDatabase(catName, dbName);
} catch (Exception e) { /* appears exception is not thrown currently if db doesnt exist */ }

if (MetaStoreUtils.isDatabaseRemote(db)) {
// TODO: remote database does not support list table names by pattern yet.
// This branch can be removed once it's supported.
GetProjectionsSpec projectionsSpec = new GetProjectionsSpec();
projectionsSpec.setFieldList(Arrays.asList("dbName", "tableName", "owner", "ownerType"));
GetTablesRequest req = new GetTablesRequest(dbName);
req.setCatName(catName);
req.setCapabilities(version);
req.setTblNames(null);
req.setTablesPattern(tablePattern);
if (processorCapabilities != null)
req.setProcessorCapabilities(new ArrayList<String>(Arrays.asList(processorCapabilities)));
if (processorIdentifier != null)
req.setProcessorIdentifier(processorIdentifier);
req.setProjectionSpec(projectionsSpec);
List<Table> tableObjects = client.get_table_objects_by_name_req(req).getTables();
tableObjects = deepCopyTables(FilterUtils.filterTablesIfEnabled(isClientFilterEnabled, filterHook, tableObjects));
for (Table tbl : tableObjects) {
tables.add(tbl.getTableName());
}
} else {
// This trick handles pattern for both string regex and wildcards ('*' and '|').
// We need unify the pattern definition, see HIVE-28297 for details.
String[] patterns = tablePattern.split("\\|");
for (String pattern : patterns) {
pattern = "(?i)" + pattern.replaceAll("\\.\\*", "\\*").replaceAll("\\*", ".*");
String filter = hive_metastoreConstants.HIVE_FILTER_FIELD_TABLE_NAME + " like \"" + pattern + "\"";
tables.addAll(listTableNamesByFilter(catName, dbName, filter, (short) -1));
}
}
return FilterUtils.filterTableNamesIfEnabled(isClientFilterEnabled, filterHook, catName, dbName, tables);
return tables;
}

@Override
Expand Down
5 changes: 5 additions & 0 deletions ...metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.DatabaseType;
import org.apache.hadoop.hive.metastore.api.WMPoolSchedulingPolicy;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
Expand Down Expand Up @@ -1342,4 +1343,8 @@ public static String convertSqlPatternToRegExp(String likePattern) {
}
return sb.toString();
}

public static boolean isDatabaseRemote(Database db) {
return db != null && db.getType() == DatabaseType.REMOTE;
}
}
35 changes: 14 additions & 21 deletions ...metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1523,7 +1523,7 @@ private void drop_database_core(RawStore ms, DropDatabaseRequest req) throws NoS
try {
ms.openTransaction();
db = ms.getDatabase(req.getCatalogName(), req.getName());
if (db.getType() == DatabaseType.REMOTE) {
if (MetaStoreUtils.isDatabaseRemote(db)) {
success = drop_remote_database_core(ms, db);
return;
}
Expand Down Expand Up @@ -2256,7 +2256,7 @@ private void create_table_core(final RawStore ms, final CreateTableRequest req)
}

Database db = get_database_core(tbl.getCatName(), tbl.getDbName());
if (db != null && db.getType().equals(DatabaseType.REMOTE)) {
if (MetaStoreUtils.isDatabaseRemote(db)) {
// HIVE-24425: Create table in REMOTE db should fail
throw new MetaException("Create table in REMOTE database " + db.getName() + " is not allowed");
}
Expand Down Expand Up @@ -2902,7 +2902,7 @@ private boolean drop_table_core(final RawStore ms, final String catName, final S

// HIVE-25282: Drop/Alter table in REMOTE db should fail
db = ms.getDatabase(catName, dbname);
if (db.getType() == DatabaseType.REMOTE) {
if (MetaStoreUtils.isDatabaseRemote(db)) {
throw new MetaException("Drop table in REMOTE database " + db.getName() + " is not allowed");
}
isReplicated = isDbReplicationTarget(db);
Expand Down Expand Up @@ -3630,15 +3630,13 @@ public Table get_table_core(GetTableRequest getTableRequest) throws MetaExceptio
db = get_database_core(catName, dbName);
} catch (Exception e) { /* appears exception is not thrown currently if db doesnt exist */ }

if (db != null) {
if (db.getType().equals(DatabaseType.REMOTE)) {
t = DataConnectorProviderFactory.getDataConnectorProvider(db).getTable(tblName);
if (t == null) {
throw new NoSuchObjectException(TableName.getQualified(catName, dbName, tblName) + " table not found");
}
t.setDbName(dbName);
return t;
if (MetaStoreUtils.isDatabaseRemote(db)) {
t = DataConnectorProviderFactory.getDataConnectorProvider(db).getTable(tblName);
if (t == null) {
throw new NoSuchObjectException(TableName.getQualified(catName, dbName, tblName) + " table not found");
}
t.setDbName(dbName);
return t;
}

try {
Expand Down Expand Up @@ -5051,13 +5049,10 @@ private boolean isDatabaseRemote(String name) {
try {
String[] parsedDbName = parseDbName(name, conf);
Database db = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]);
if (db != null && db.getType() != null && db.getType() == DatabaseType.REMOTE) {
return true;
}
return MetaStoreUtils.isDatabaseRemote(db);
} catch (Exception e) {
return false;
}
return false;
}

private void deleteParentRecursive(Path parent, int depth, boolean mustPurge, boolean needRecycle)
Expand Down Expand Up @@ -6010,7 +6005,7 @@ private void alter_table_core(String catName, String dbname, String name, Table
// HIVE-25282: Drop/Alter table in REMOTE db should fail
try {
Database db = get_database_core(catName, dbname);
if (db != null && db.getType().equals(DatabaseType.REMOTE)) {
if (MetaStoreUtils.isDatabaseRemote(db)) {
throw new MetaException("Alter table in REMOTE database " + db.getName() + " is not allowed");
}
} catch (NoSuchObjectException e) {
Expand Down Expand Up @@ -6126,10 +6121,8 @@ private List<String> getTablesByTypeCore(final String catName, final String dbna
Database db = null;
try {
db = get_database_core(catName, dbname);
if (db != null) {
if (db.getType().equals(DatabaseType.REMOTE)) {
return DataConnectorProviderFactory.getDataConnectorProvider(db).getTableNames();
}
if (MetaStoreUtils.isDatabaseRemote(db)) {
return DataConnectorProviderFactory.getDataConnectorProvider(db).getTableNames();
}
} catch (Exception e) { /* ignore */ }

Expand Down Expand Up @@ -8446,7 +8439,7 @@ public void create_function(Function func) throws TException {
throw new NoSuchObjectException("The database " + func.getDbName() + " does not exist");
}

if (db.getType() == DatabaseType.REMOTE) {
if (MetaStoreUtils.isDatabaseRemote(db)) {
throw new MetaException("Operation create_function not support for REMOTE database");
}

Expand Down
5 changes: 2 additions & 3 deletions ...etastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesList.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,12 +253,11 @@ public void testListTableNamesByFilterCheckLimit() throws Exception {
|| tableNames.contains(testTables[3].getTableName()));
}

@Test
@Test(expected = UnknownDBException.class)
public void testListTableNamesByFilterCheckNoSuchDatabase() throws Exception {
// No such database
List<String> tableNames = client.listTableNamesByFilter("no_such_database",
client.listTableNamesByFilter("no_such_database",
hive_metastoreConstants.HIVE_FILTER_FIELD_LAST_ACCESS + ">2000", (short)-1);
Assert.assertEquals("Found tables", 0, tableNames.size());
}

@Test(expected = UnknownDBException.class)
Expand Down

0 comments on commit fb49693

Please sign in to comment.