Skip to content

Commit

Permalink
HIVE-28655: Implement HMS Related Drop Stats Changes
Browse files Browse the repository at this point in the history
  • Loading branch information
Hongdan Zhu committed Dec 13, 2024
1 parent 0e75930 commit e6bfa1f
Show file tree
Hide file tree
Showing 12 changed files with 762 additions and 24 deletions.
25 changes: 25 additions & 0 deletions ...hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -877,6 +877,13 @@ public boolean deleteTableColumnStatistics(String catName, String dbName, String
return objectStore.deleteTableColumnStatistics(catName, dbName, tableName, colName, engine);
}

@Override
public boolean deleteTableMultiColumnStatistics(String catName, String dbName, String tableName,
List<String> colNames, String engine)
throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
return objectStore.deleteTableMultiColumnStatistics(catName, dbName, tableName, colNames, engine);
}

@Override
public boolean deletePartitionColumnStatistics(String catName, String dbName, String tableName,
String partName, List<String> partVals, String colName, String engine)
Expand All @@ -886,6 +893,24 @@ public boolean deletePartitionColumnStatistics(String catName, String dbName, St
partVals, colName, engine);
}

@Override
public boolean deletePartitionMultiColumnStatistics(String catName, String dbName, String tableName,
String partName, List<String> partVals, List<String> colNames, String engine)
throws NoSuchObjectException, MetaException, InvalidObjectException,
InvalidInputException {
return objectStore.deletePartitionMultiColumnStatistics(catName, dbName, tableName, partName,
partVals, colNames, engine);
}

@Override
public boolean deletePartitionColumnStatisticsInBatch(String catName, String dbName, String tableName,
String partName, List<String> partVals, List<String> colNames, String engine)
throws NoSuchObjectException, MetaException, InvalidObjectException,
InvalidInputException {
return objectStore.deletePartitionColumnStatisticsInBatch(catName, dbName, tableName, partName,
partVals, colNames, engine);
}

@Override
public Map<String, String> updateTableColumnStatistics(ColumnStatistics statsObj, String validWriteIds, long writeId)
throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
Expand Down
27 changes: 27 additions & 0 deletions .../metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3658,6 +3658,20 @@ public boolean deletePartitionColumnStatistics(String catName, String dbName, St
tableName, partName, colName, engine);
}

@Override
public boolean deletePartitionMultiColumnStatistics(String dbName, String tableName, String partName,
List<String> colNames, String engine) throws TException {
DeletePartitionColumnStatisticsRequest req = new DeletePartitionColumnStatisticsRequest(getDefaultCatalog(conf),
dbName, tableName, partName, engine);
req.setCol_names(colNames);
return deletePartitionMultiColumnStatistics(req);
}

@Override
public boolean deletePartitionMultiColumnStatistics(DeletePartitionColumnStatisticsRequest req) throws TException {
return client.delete_partition_column_statistics_req(req);
}

@Override
public boolean deleteTableColumnStatistics(String dbName, String tableName, String colName, String engine)
throws TException {
Expand All @@ -3671,6 +3685,19 @@ public boolean deleteTableColumnStatistics(String catName, String dbName, String
tableName, colName, engine);
}

@Override
public boolean deleteTableMultiColumnStatistics(String dbName, String tableName, List<String> colNames, String engine)
throws TException {
DeleteTableColumnStatisticsRequest req = new DeleteTableColumnStatisticsRequest(getDefaultCatalog(conf), dbName, tableName, engine);
req.setCol_names(colNames);
return deleteTableMultiColumnStatistics(req);
}

@Override
public boolean deleteTableMultiColumnStatistics(DeleteTableColumnStatisticsRequest req) throws TException {
return client.delete_table_column_statistics_req(req);
}

@Override
public void updateTransactionalStatistics(UpdateTransactionalStatsRequest req) throws TException {
client.update_transaction_statistics(req);
Expand Down
53 changes: 53 additions & 0 deletions ...ore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.thrift.TException;

import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName;

/**
* Wrapper around hive metastore thrift api
*/
Expand Down Expand Up @@ -2716,6 +2719,33 @@ boolean deletePartitionColumnStatistics(String catName, String dbName, String ta
String partName, String colName, String engine)
throws NoSuchObjectException, MetaException, InvalidObjectException, TException, InvalidInputException;

/**
* Delete partition level column statistics given dbName, tableName, partName and colName, or
* all columns in a partition.
* @param dbName database name.
* @param tableName table name.
* @param partName partition name.
* @param colNames a list of column name, or null for all columns
* @param engine engine, or null for all engines
* @return boolean indicating outcome of the operation
* @throws NoSuchObjectException no such partition exists
* @throws InvalidObjectException error dropping the stats data
* @throws MetaException error accessing the RDBMS
* @throws TException thrift transport error
* @throws InvalidInputException input is invalid or null.
*/
public boolean deletePartitionMultiColumnStatistics(String dbName, String tableName, String partName,
List<String> colNames, String engine) throws TException;

/**
* Delete partition level column statistics given dbName, tableName, partName and colName, or
* all columns in a partition.
* @param req the DeletePartitionColumnStatisticsRequest which including
* catalog name, database name, table name, partition name, column names, and engine name
* @throws TException thrift transport error
*/
public boolean deletePartitionMultiColumnStatistics(DeletePartitionColumnStatisticsRequest req) throws TException;

/**
* Delete table level column statistics given dbName, tableName and colName, or all columns in
* a table. This should be used for non-partitioned tables.
Expand Down Expand Up @@ -2751,6 +2781,29 @@ boolean deleteTableColumnStatistics(String dbName, String tableName, String colN
boolean deleteTableColumnStatistics(String catName, String dbName, String tableName, String colName, String engine)
throws NoSuchObjectException, MetaException, InvalidObjectException, TException, InvalidInputException;

/**
* Delete table level column statistics given dbName, tableName and colName, or all columns in
* a table. This should be used for non-partitioned tables.
* @param dbName database name
* @param tableName table name
* @param colNames a list of column names, or null to drop stats for all columns
* @param engine engine, or null for all engines
* @return boolean indicating the outcome of the operation
* @throws TException thrift transport error
*/
boolean deleteTableMultiColumnStatistics(String dbName, String tableName, List<String> colNames, String engine)
throws TException;

/**
* Delete table level column statistics given dbName, tableName and colName, or all columns in
* a table. This should be used for non-partitioned tables.
* @param req the DeleteTableColumnStatisticsRequest which including
* catalog name, database name, table name, column names, and engine name
* @return boolean indicating the outcome of the operation
* @throws TException thrift transport error
*/
public boolean deleteTableMultiColumnStatistics(DeleteTableColumnStatisticsRequest req) throws TException;

void updateTransactionalStatistics(UpdateTransactionalStatsRequest req) throws TException;

/**
Expand Down
23 changes: 23 additions & 0 deletions standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2510,6 +2510,23 @@ struct GetAllWriteEventInfoRequest {
3: optional string tableName
}

struct DeletePartitionColumnStatisticsRequest {
1: required string cat_name,
2: required string db_name,
3: required string tbl_name,
4: required string part_name,
5: optional list<string> col_names,
6: required string engine
}

struct DeleteTableColumnStatisticsRequest {
1: required string cat_name,
2: required string db_name,
3: required string tbl_name,
4: optional list<string> col_names,
5: required string engine
}

// Exceptions.

exception MetaException {
Expand Down Expand Up @@ -3019,9 +3036,15 @@ PartitionsResponse get_partitions_req(1:PartitionsRequest req)
bool delete_partition_column_statistics(1:string db_name, 2:string tbl_name, 3:string part_name, 4:string col_name, 5:string engine) throws
(1:NoSuchObjectException o1, 2:MetaException o2, 3:InvalidObjectException o3,
4:InvalidInputException o4)
bool delete_partition_column_statistics_req(1:DeletePartitionColumnStatisticsRequest req) throws
(1:NoSuchObjectException o1, 2:MetaException o2, 3:InvalidObjectException o3,
4:InvalidInputException o4)
bool delete_table_column_statistics(1:string db_name, 2:string tbl_name, 3:string col_name, 4:string engine) throws
(1:NoSuchObjectException o1, 2:MetaException o2, 3:InvalidObjectException o3,
4:InvalidInputException o4)
bool delete_table_column_statistics_req(1: DeleteTableColumnStatisticsRequest req) throws
(1:NoSuchObjectException o1, 2:MetaException o2, 3:InvalidObjectException o3,
4:InvalidInputException o4)

//
// user-defined functions
Expand Down
120 changes: 120 additions & 0 deletions ...metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7420,6 +7420,64 @@ public boolean delete_partition_column_statistics(String dbName, String tableNam
return ret;
}

@Override
public boolean delete_partition_column_statistics_req(DeletePartitionColumnStatisticsRequest req) throws TException {
String dbName = req.getDb_name();
String tableName = req.getTbl_name();
List<String> colNames = req.getCol_names();
String partName = req.getPart_name();
String engine = req.getEngine();
dbName = dbName.toLowerCase();
String[] parsedDbName = parseDbName(dbName, conf);
tableName = tableName.toLowerCase();
if (colNames != null) {
for (String colName : colNames) {
colName = colName.toLowerCase();
startFunction("delete_column_statistics_by_partition",": table=" +
TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName) +
" partition=" + partName + " column=" + colName);
}
}
boolean ret = false, committed = false;

getMS().openTransaction();
try {
List<String> partVals = getPartValsFromName(getMS(), parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, partName);
Table table = getMS().getTable(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName);
// This API looks unused; if it were used we'd need to update stats state and write ID.
// We cannot just randomly nuke some txn stats.
if (TxnUtils.isTransactionalTable(table)) {
throw new MetaException("Cannot delete stats via this API for a transactional table");
}

ret = getMS().deletePartitionMultiColumnStatistics(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,
partName, partVals, colNames, engine);
if (ret && colNames != null) {
for (String colName : colNames) {
if (transactionalListeners != null && !transactionalListeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.DELETE_PARTITION_COLUMN_STAT,
new DeletePartitionColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,
partName, partVals, colName, engine, this));
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.DELETE_PARTITION_COLUMN_STAT,
new DeletePartitionColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,
partName, partVals, colName, engine, this));
}
}
}
committed = getMS().commitTransaction();
} finally {
if (!committed) {
getMS().rollbackTransaction();
}
endFunction("delete_column_statistics_by_partition", ret != false, null, tableName);
}
return ret;
}

@Override
public boolean delete_table_column_statistics(String dbName, String tableName, String colName, String engine)
throws TException {
Expand Down Expand Up @@ -7461,6 +7519,68 @@ public boolean delete_table_column_statistics(String dbName, String tableName, S
tableName, colName, engine, this));
}
}
boolean isPartitioned = table.isSetPartitionKeys() && table.getPartitionKeysSize() > 0;
if (isPartitioned) {
delete_partition_column_statistics(dbName, tableName, null, colName, engine);
}
committed = getMS().commitTransaction();
} finally {
if (!committed) {
getMS().rollbackTransaction();
}
endFunction("delete_column_statistics_by_table", ret != false, null, tableName);
}
return ret;
}

@Override
public boolean delete_table_column_statistics_req(DeleteTableColumnStatisticsRequest req)
throws TException {
String dbName = req.getDb_name();
String tableName = req.getTbl_name();
List<String> colNames = req.getCol_names();
String engine = req.getEngine();
dbName = dbName.toLowerCase();
tableName = tableName.toLowerCase();

String[] parsedDbName = parseDbName(dbName, conf);

if (colNames != null) {
for (String colName : colNames) {
colName = colName.toLowerCase();
startFunction("delete_column_statistics_by_table", ": table=" +
TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName) + " column=" +
colName);
}
}

boolean ret = false, committed = false;
getMS().openTransaction();
try {
Table table = getMS().getTable(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName);
// This API looks unused; if it were used we'd need to update stats state and write ID.
// We cannot just randomly nuke some txn stats.
if (TxnUtils.isTransactionalTable(table)) {
throw new MetaException("Cannot delete stats via this API for a transactional table");
}

ret = getMS().deleteTableMultiColumnStatistics(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, colNames, engine);
if (ret && colNames != null) {
for (String colName : colNames) {
if (transactionalListeners != null && !transactionalListeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.DELETE_TABLE_COLUMN_STAT,
new DeleteTableColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tableName, colName, engine, this));
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.DELETE_TABLE_COLUMN_STAT,
new DeleteTableColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tableName, colName, engine, this));
}
}
}
committed = getMS().commitTransaction();
} finally {
if (!committed) {
Expand Down
Loading

0 comments on commit e6bfa1f

Please sign in to comment.