Skip to content

Commit fe19a79

Browse files
wecharyuwecharyu
committed
address comments
1 parent 14de002 commit fe19a79

File tree

4 files changed

+128
-138
lines changed

4 files changed

+128
-138
lines changed

‎standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -679,24 +679,6 @@ public List<String> createInsertValuesStmt(String tblColumns, List<String> rows,
679679
}
680680
}
681681

682-
public String createUpdatePreparedStmt(String tableName, List<String> columnNames,
683-
List<String> conditionKeys) {
684-
StringBuilder sb = new StringBuilder();
685-
sb.append("update " + tableName + " set ");
686-
sb.append(columnNames.stream().map(col -> col + "=?").collect(Collectors.joining(",")));
687-
sb.append(" where " + conditionKeys.stream().map(cond -> cond + "=?").collect(Collectors.joining(" and ")));
688-
return sb.toString();
689-
}
690-
691-
public String createInsertPreparedStmt(String tableName, List<String> columnNames) {
692-
StringBuilder sb = new StringBuilder();
693-
sb.append("insert into " + tableName + "(");
694-
sb.append(columnNames.stream().collect(Collectors.joining(",")));
695-
String placeholder = columnNames.stream().map(col -> "?").collect(Collectors.joining(","));
696-
sb.append(") values (" + placeholder + ")");
697-
return sb.toString();
698-
}
699-
700682
public String addEscapeCharacters(String s) {
701683
if (isMYSQL()) {
702684
return s.replaceAll("\\\\", "\\\\\\\\");

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdate.java renamed to standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdatePart.java

Lines changed: 103 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -80,19 +80,21 @@
8080
* the underlying database. It should use ANSI SQL and be compatible with common databases
8181
* such as MySQL (note that MySQL doesn't use full ANSI mode by default), Postgres, etc.
8282
*
83-
* This class separates out the statistics update part from MetaStoreDirectSql class.
83+
* This class separates out the update part from MetaStoreDirectSql class.
8484
*/
85-
class DirectSqlUpdate {
86-
private static final Logger LOG = LoggerFactory.getLogger(DirectSqlUpdate.class.getName());
87-
PersistenceManager pm;
88-
Configuration conf;
89-
DatabaseProduct dbType;
90-
int maxBatchSize;
91-
SQLGenerator sqlGenerator;
85+
class DirectSqlUpdatePart {
86+
private static final Logger LOG = LoggerFactory.getLogger(DirectSqlUpdatePart.class.getName());
87+
88+
private final PersistenceManager pm;
89+
private final Configuration conf;
90+
private final DatabaseProduct dbType;
91+
private final int maxBatchSize;
92+
private final SQLGenerator sqlGenerator;
93+
9294
private static final ReentrantLock derbyLock = new ReentrantLock(true);
93-
94-
public DirectSqlUpdate(PersistenceManager pm, Configuration conf,
95-
DatabaseProduct dbType, int batchSize) {
95+
96+
public DirectSqlUpdatePart(PersistenceManager pm, Configuration conf,
97+
DatabaseProduct dbType, int batchSize) {
9698
this.pm = pm;
9799
this.conf = conf;
98100
this.dbType = dbType;
@@ -362,9 +364,6 @@ private Map<String, Map<String, String>> updatePartitionParamTable(Connection db
362364
throws SQLException, MetaException {
363365
Map<String, Map<String, String>> result = new HashMap<>();
364366
boolean areTxnStatsSupported = MetastoreConf.getBoolVar(conf, ConfVars.HIVE_TXN_STATS_ENABLED);
365-
PreparedStatement statementInsert = null;
366-
PreparedStatement statementDelete = null;
367-
PreparedStatement statementUpdate = null;
368367
String insert = "INSERT INTO \"PARTITION_PARAMS\" (\"PART_ID\", \"PARAM_KEY\", \"PARAM_VALUE\") "
369368
+ "VALUES( ? , 'COLUMN_STATS_ACCURATE' , ? )";
370369
String delete = "DELETE from \"PARTITION_PARAMS\" "
@@ -384,10 +383,9 @@ private Map<String, Map<String, String>> updatePartitionParamTable(Connection db
384383
// get the old parameters from PARTITION_PARAMS table.
385384
Map<Long, String> partIdToParaMap = getParamValues(dbConn, partIdList);
386385

387-
try {
388-
statementInsert = dbConn.prepareStatement(insert);
389-
statementDelete = dbConn.prepareStatement(delete);
390-
statementUpdate = dbConn.prepareStatement(update);
386+
try (PreparedStatement statementInsert = dbConn.prepareStatement(insert);
387+
PreparedStatement statementDelete = dbConn.prepareStatement(delete);
388+
PreparedStatement statementUpdate = dbConn.prepareStatement(update)) {
391389
for (Map.Entry entry : partitionInfoMap.entrySet()) {
392390
PartitionInfo partitionInfo = (PartitionInfo) entry.getKey();
393391
ColumnStatistics colStats = (ColumnStatistics) entry.getValue();
@@ -472,83 +470,9 @@ private Map<String, Map<String, String>> updatePartitionParamTable(Connection db
472470
updateWriteIdForPartitions(dbConn, writeId, partIdList);
473471
}
474472
return result;
475-
} finally {
476-
closeStmt(statementInsert);
477-
closeStmt(statementUpdate);
478-
closeStmt(statementDelete);
479473
}
480474
}
481475

482-
private static class PartitionInfo {
483-
long partitionId;
484-
long writeId;
485-
String partitionName;
486-
public PartitionInfo(long partitionId, long writeId, String partitionName) {
487-
this.partitionId = partitionId;
488-
this.writeId = writeId;
489-
this.partitionName = partitionName;
490-
}
491-
492-
@Override
493-
public int hashCode() {
494-
return (int)partitionId;
495-
}
496-
497-
@Override
498-
public boolean equals(Object o)
499-
{
500-
if (this == o) {
501-
return true;
502-
}
503-
if (o == null) {
504-
return false;
505-
}
506-
if (!(o instanceof PartitionInfo)) {
507-
return false;
508-
}
509-
PartitionInfo other = (PartitionInfo)o;
510-
if (this.partitionId != other.partitionId) {
511-
return false;
512-
}
513-
return true;
514-
}
515-
}
516-
517-
private static class PartColNameInfo {
518-
long partitionId;
519-
String colName;
520-
public PartColNameInfo(long partitionId, String colName) {
521-
this.partitionId = partitionId;
522-
this.colName = colName;
523-
}
524-
525-
@Override
526-
public int hashCode() {
527-
return (int)partitionId;
528-
}
529-
530-
@Override
531-
public boolean equals(Object o)
532-
{
533-
if (this == o) {
534-
return true;
535-
}
536-
if (o == null) {
537-
return false;
538-
}
539-
if (!(o instanceof PartColNameInfo)) {
540-
return false;
541-
}
542-
PartColNameInfo other = (PartColNameInfo)o;
543-
if (this.partitionId != other.partitionId) {
544-
return false;
545-
}
546-
if (this.colName.equalsIgnoreCase(other.colName)) {
547-
return true;
548-
}
549-
return false;
550-
}
551-
}
552476

553477
private Map<PartitionInfo, ColumnStatistics> getPartitionInfo(Connection dbConn, long tblId,
554478
Map<String, ColumnStatistics> partColStatsMap)
@@ -792,7 +716,7 @@ private void updatePartitionsInBatch(Map<List<String>, Long> partValuesToId,
792716
List<Partition> newParts) throws MetaException {
793717
List<String> columns = Arrays.asList("\"CREATE_TIME\"", "\"LAST_ACCESS_TIME\"", "\"WRITE_ID\"");
794718
List<String> conditionKeys = Arrays.asList("\"PART_ID\"");
795-
String stmt = dbType.createUpdatePreparedStmt("\"PARTITIONS\"", columns, conditionKeys);
719+
String stmt = TxnUtils.createUpdatePreparedStmt("\"PARTITIONS\"", columns, conditionKeys);
796720
int maxRows = dbType.getMaxRows(maxBatchSize, 4);
797721
updateWithStatement(statement -> Batchable.runBatched(maxRows, newParts, new Batchable<Partition, Void>() {
798722
@Override
@@ -916,7 +840,7 @@ private void updateParams(String paramTable, String idColumn,
916840
List<Pair<Long, Pair<String, String>>> updateIdAndParams) throws MetaException {
917841
List<String> columns = Arrays.asList("\"PARAM_VALUE\"");
918842
List<String> conditionKeys = Arrays.asList(idColumn, "\"PARAM_KEY\"");
919-
String stmt = dbType.createUpdatePreparedStmt(paramTable, columns, conditionKeys);
843+
String stmt = TxnUtils.createUpdatePreparedStmt(paramTable, columns, conditionKeys);
920844
int maxRows = dbType.getMaxRows(maxBatchSize, 3);
921845
updateWithStatement(statement -> Batchable.runBatched(maxRows, updateIdAndParams,
922846
new Batchable<Pair<Long, Pair<String, String>>, Object>() {
@@ -938,7 +862,7 @@ public List<Object> run(List<Pair<Long, Pair<String, String>>> input) throws SQL
938862
private void insertParams(String paramTable, String idColumn,
939863
List<Pair<Long, Pair<String, String>>> addIdAndParams) throws MetaException {
940864
List<String> columns = Arrays.asList(idColumn, "\"PARAM_KEY\"", "\"PARAM_VALUE\"");
941-
String query = dbType.createInsertPreparedStmt(paramTable, columns);
865+
String query = TxnUtils.createInsertPreparedStmt(paramTable, columns);
942866
int maxRows = dbType.getMaxRows(maxBatchSize, 3);
943867
updateWithStatement(statement -> Batchable.runBatched(maxRows, addIdAndParams,
944868
new Batchable<Pair<Long, Pair<String, String>>, Void>() {
@@ -1046,7 +970,7 @@ private void updateSDInBatch(List<Long> ids, Map<Long, StorageDescriptor> idToSd
1046970
List<String> columns = Arrays.asList("\"CD_ID\"", "\"INPUT_FORMAT\"", "\"IS_COMPRESSED\"",
1047971
"\"IS_STOREDASSUBDIRECTORIES\"", "\"LOCATION\"", "\"NUM_BUCKETS\"", "\"OUTPUT_FORMAT\"");
1048972
List<String> conditionKeys = Arrays.asList("\"SD_ID\"");
1049-
String stmt = dbType.createUpdatePreparedStmt("\"SDS\"", columns, conditionKeys);
973+
String stmt = TxnUtils.createUpdatePreparedStmt("\"SDS\"", columns, conditionKeys);
1050974
int maxRows = dbType.getMaxRows(maxBatchSize, 8);
1051975
updateWithStatement(statement -> Batchable.runBatched(maxRows, ids,
1052976
new Batchable<Long, Void>() {
@@ -1083,7 +1007,7 @@ public List<Void> run(List<Long> input) throws MetaException {
10831007
}
10841008
});
10851009
List<String> columns = Arrays.asList("\"SD_ID\"", "\"INTEGER_IDX\"", "\"BUCKET_COL_NAME\"");
1086-
String stmt = dbType.createInsertPreparedStmt("\"BUCKETING_COLS\"", columns);
1010+
String stmt = TxnUtils.createInsertPreparedStmt("\"BUCKETING_COLS\"", columns);
10871011
List<Long> idWithBucketCols = filterIdsByNonNullValue(sdIds, sdIdToBucketCols);
10881012
int maxRows = dbType.getMaxRows(maxBatchSize, 3);
10891013
updateWithStatement(statement -> Batchable.runBatched(maxRows, idWithBucketCols, new Batchable<Long, Object>() {
@@ -1117,7 +1041,7 @@ public List<Void> run(List<Long> input) throws MetaException {
11171041
});
11181042

11191043
List<String> columns = Arrays.asList("\"SD_ID\"", "\"INTEGER_IDX\"", "\"COLUMN_NAME\"", "\"ORDER\"");
1120-
String stmt = dbType.createInsertPreparedStmt("\"SORT_COLS\"", columns);
1044+
String stmt = TxnUtils.createInsertPreparedStmt("\"SORT_COLS\"", columns);
11211045
List<Long> idWithSortCols = filterIdsByNonNullValue(sdIds, sdIdToSortCols);
11221046
int maxRows = dbType.getMaxRows(maxBatchSize, 4);
11231047
updateWithStatement(statement -> Batchable.runBatched(maxRows, idWithSortCols, new Batchable<Long, Object>() {
@@ -1229,7 +1153,7 @@ private Long getDataStoreId(Class<?> modelClass) throws MetaException {
12291153
private void insertSkewedColNamesInBatch(Map<Long, List<String>> sdIdToSkewedColNames,
12301154
List<Long> sdIds) throws MetaException {
12311155
List<String> columns = Arrays.asList("\"SD_ID\"", "\"INTEGER_IDX\"", "\"SKEWED_COL_NAME\"");
1232-
String stmt = dbType.createInsertPreparedStmt("\"SKEWED_COL_NAMES\"", columns);
1156+
String stmt = TxnUtils.createInsertPreparedStmt("\"SKEWED_COL_NAMES\"", columns);
12331157
List<Long> idWithSkewedCols = filterIdsByNonNullValue(sdIds, sdIdToSkewedColNames);
12341158
int maxRows = dbType.getMaxRows(maxBatchSize, 3);
12351159
updateWithStatement(statement -> Batchable.runBatched(maxRows, idWithSkewedCols, new Batchable<Long, Object>() {
@@ -1252,7 +1176,7 @@ public List<Object> run(List<Long> input) throws SQLException {
12521176

12531177
private void insertStringListInBatch(List<Long> stringListIds) throws MetaException {
12541178
List<String> columns = Arrays.asList("\"STRING_LIST_ID\"");
1255-
String insertQuery = dbType.createInsertPreparedStmt("\"SKEWED_STRING_LIST\"", columns);
1179+
String insertQuery = TxnUtils.createInsertPreparedStmt("\"SKEWED_STRING_LIST\"", columns);
12561180
int maxRows = dbType.getMaxRows(maxBatchSize, 1);
12571181
updateWithStatement(statement -> Batchable.runBatched(maxRows, stringListIds,
12581182
new Batchable<Long, Void>() {
@@ -1272,7 +1196,7 @@ public List<Void> run(List<Long> input) throws SQLException {
12721196
private void insertStringListValuesInBatch(Map<Long, List<String>> stringListIdToValues,
12731197
List<Long> stringListIds) throws MetaException {
12741198
List<String> columns = Arrays.asList("\"STRING_LIST_ID\"", "\"INTEGER_IDX\"", "\"STRING_LIST_VALUE\"");
1275-
String insertQuery = dbType.createInsertPreparedStmt("\"SKEWED_STRING_LIST_VALUES\"", columns);
1199+
String insertQuery = TxnUtils.createInsertPreparedStmt("\"SKEWED_STRING_LIST_VALUES\"", columns);
12761200
List<Long> idWithStringList = filterIdsByNonNullValue(stringListIds, stringListIdToValues);
12771201
int maxRows = dbType.getMaxRows(maxBatchSize, 3);
12781202
updateWithStatement(statement -> Batchable.runBatched(maxRows, idWithStringList,
@@ -1298,7 +1222,7 @@ public List<Void> run(List<Long> input) throws SQLException {
12981222
private void insertSkewedValuesInBatch(Map<Long, List<Long>> sdIdToStringListId,
12991223
List<Long> sdIds) throws MetaException {
13001224
List<String> columns = Arrays.asList("\"SD_ID_OID\"", "\"INTEGER_IDX\"", "\"STRING_LIST_ID_EID\"");
1301-
String insertQuery = dbType.createInsertPreparedStmt("\"SKEWED_VALUES\"", columns);
1225+
String insertQuery = TxnUtils.createInsertPreparedStmt("\"SKEWED_VALUES\"", columns);
13021226
List<Long> idWithSkewedValues = filterIdsByNonNullValue(sdIds, sdIdToStringListId);
13031227
int maxRows = dbType.getMaxRows(maxBatchSize, 3);
13041228
updateWithStatement(statement -> Batchable.runBatched(maxRows, idWithSkewedValues,
@@ -1324,7 +1248,7 @@ public List<Void> run(List<Long> input) throws Exception {
13241248
private void insertSkewColValueLocInBatch(Map<Long, List<Pair<Long, String>>> sdIdToColValueLoc,
13251249
List<Long> sdIds) throws MetaException {
13261250
List<String> columns = Arrays.asList("\"SD_ID\"", "\"STRING_LIST_ID_KID\"", "\"LOCATION\"");
1327-
String insertQuery = dbType.createInsertPreparedStmt("\"SKEWED_COL_VALUE_LOC_MAP\"", columns);
1251+
String insertQuery = TxnUtils.createInsertPreparedStmt("\"SKEWED_COL_VALUE_LOC_MAP\"", columns);
13281252
List<Long> idWithColValueLoc = filterIdsByNonNullValue(sdIds, sdIdToColValueLoc);
13291253
int maxRows = dbType.getMaxRows(maxBatchSize, 3);
13301254
updateWithStatement(statement -> Batchable.runBatched(maxRows, idWithColValueLoc,
@@ -1415,7 +1339,7 @@ public List<Void> run(List<Long> input) throws Exception {
14151339

14161340
private void insertCDInBatch(List<Long> ids, Map<Long, List<FieldSchema>> idToCols)
14171341
throws MetaException {
1418-
String insertCds = dbType.createInsertPreparedStmt("\"CDS\"", Arrays.asList("\"CD_ID\""));
1342+
String insertCds = TxnUtils.createInsertPreparedStmt("\"CDS\"", Arrays.asList("\"CD_ID\""));
14191343
int maxRows = dbType.getMaxRows(maxBatchSize, 1);
14201344
updateWithStatement(statement -> Batchable.runBatched(maxRows, ids,
14211345
new Batchable<Long, Void>() {
@@ -1432,7 +1356,7 @@ public List<Void> run(List<Long> input) throws SQLException {
14321356

14331357
List<String> columns = Arrays.asList("\"CD_ID\"",
14341358
"\"COMMENT\"", "\"COLUMN_NAME\"", "\"TYPE_NAME\"", "\"INTEGER_IDX\"");
1435-
String insertColumns = dbType.createInsertPreparedStmt("\"COLUMNS_V2\"", columns);
1359+
String insertColumns = TxnUtils.createInsertPreparedStmt("\"COLUMNS_V2\"", columns);
14361360
int maxRowsForCDs = dbType.getMaxRows(maxBatchSize, 5);
14371361
updateWithStatement(statement -> Batchable.runBatched(maxRowsForCDs, ids,
14381362
new Batchable<Long, Void>() {
@@ -1463,8 +1387,8 @@ private void updateKeyConstraintsInBatch(Map<Long, Long> oldCdIdToNewCdId,
14631387
List<String> parentColumns = Arrays.asList("\"PARENT_CD_ID\"", "\"PARENT_INTEGER_IDX\"");
14641388
List<String> childColumns = Arrays.asList("\"CHILD_CD_ID\"", "\"CHILD_INTEGER_IDX\"");
14651389

1466-
String updateParent = dbType.createUpdatePreparedStmt(tableName, parentColumns, parentColumns);
1467-
String updateChild = dbType.createUpdatePreparedStmt(tableName, childColumns, childColumns);
1390+
String updateParent = TxnUtils.createUpdatePreparedStmt(tableName, parentColumns, parentColumns);
1391+
String updateChild = TxnUtils.createUpdatePreparedStmt(tableName, childColumns, childColumns);
14681392
for (String updateStmt : new String[]{updateParent, updateChild}) {
14691393
int maxRows = dbType.getMaxRows(maxBatchSize, 4);
14701394
updateWithStatement(statement -> Batchable.runBatched(maxRows, oldCdIds,
@@ -1519,7 +1443,7 @@ private void updateSerdeInBatch(List<Long> ids, Map<Long, SerDeInfo> idToSerde)
15191443
// Followed the jdo implement to update only NAME and SLIB of SERDES.
15201444
List<String> columns = Arrays.asList("\"NAME\"", "\"SLIB\"");
15211445
List<String> condKeys = Arrays.asList("\"SERDE_ID\"");
1522-
String updateStmt = dbType.createUpdatePreparedStmt("\"SERDES\"", columns, condKeys);
1446+
String updateStmt = TxnUtils.createUpdatePreparedStmt("\"SERDES\"", columns, condKeys);
15231447
List<Long> idWithSerde = filterIdsByNonNullValue(ids, idToSerde);
15241448
int maxRows = dbType.getMaxRows(maxBatchSize, 3);
15251449
updateWithStatement(statement -> Batchable.runBatched(maxRows, idWithSerde,
@@ -1538,4 +1462,75 @@ public List<Void> run(List<Long> input) throws SQLException {
15381462
}
15391463
}), updateStmt);
15401464
}
1465+
1466+
private static final class PartitionInfo {
1467+
long partitionId;
1468+
long writeId;
1469+
String partitionName;
1470+
public PartitionInfo(long partitionId, long writeId, String partitionName) {
1471+
this.partitionId = partitionId;
1472+
this.writeId = writeId;
1473+
this.partitionName = partitionName;
1474+
}
1475+
1476+
@Override
1477+
public int hashCode() {
1478+
return (int)partitionId;
1479+
}
1480+
1481+
@Override
1482+
public boolean equals(Object o)
1483+
{
1484+
if (this == o) {
1485+
return true;
1486+
}
1487+
if (o == null) {
1488+
return false;
1489+
}
1490+
if (!(o instanceof PartitionInfo)) {
1491+
return false;
1492+
}
1493+
PartitionInfo other = (PartitionInfo)o;
1494+
if (this.partitionId != other.partitionId) {
1495+
return false;
1496+
}
1497+
return true;
1498+
}
1499+
}
1500+
1501+
private static final class PartColNameInfo {
1502+
long partitionId;
1503+
String colName;
1504+
public PartColNameInfo(long partitionId, String colName) {
1505+
this.partitionId = partitionId;
1506+
this.colName = colName;
1507+
}
1508+
1509+
@Override
1510+
public int hashCode() {
1511+
return (int)partitionId;
1512+
}
1513+
1514+
@Override
1515+
public boolean equals(Object o)
1516+
{
1517+
if (this == o) {
1518+
return true;
1519+
}
1520+
if (o == null) {
1521+
return false;
1522+
}
1523+
if (!(o instanceof PartColNameInfo)) {
1524+
return false;
1525+
}
1526+
PartColNameInfo other = (PartColNameInfo)o;
1527+
if (this.partitionId != other.partitionId) {
1528+
return false;
1529+
}
1530+
if (this.colName.equalsIgnoreCase(other.colName)) {
1531+
return true;
1532+
}
1533+
return false;
1534+
}
1535+
}
15411536
}

0 commit comments

Comments
 (0)