This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 2e6618efbe5 HIVE-27780: Implement direct SQL for get_all_functions
(#4786). (zhangbutao, reviewed by Ayush Saxena)
2e6618efbe5 is described below
commit 2e6618efbe5c870873ec06cf8c4d4c9702e31f0c
Author: Butao Zhang <[email protected]>
AuthorDate: Mon Oct 30 04:09:33 2023 +0800
HIVE-27780: Implement direct SQL for get_all_functions (#4786).
(zhangbutao, reviewed by Ayush Saxena)
---
.../hadoop/hive/metastore/MetaStoreDirectSql.java | 105 ++++++++++++++++++++-
.../hive/metastore/MetastoreDirectSqlUtils.java | 19 +++-
.../apache/hadoop/hive/metastore/ObjectStore.java | 27 ++++++
.../hive/metastore/client/TestFunctions.java | 8 ++
4 files changed, 155 insertions(+), 4 deletions(-)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 92865954bcd..6f04fd03720 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -68,6 +68,8 @@ import
org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.DatabaseType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.FunctionType;
import org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec;
import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
@@ -92,6 +94,7 @@ import
org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.apache.hadoop.hive.metastore.model.MConstraint;
import org.apache.hadoop.hive.metastore.model.MCreationMetadata;
import org.apache.hadoop.hive.metastore.model.MDatabase;
+import org.apache.hadoop.hive.metastore.model.MFunction;
import org.apache.hadoop.hive.metastore.model.MNotificationLog;
import org.apache.hadoop.hive.metastore.model.MNotificationNextId;
import org.apache.hadoop.hive.metastore.model.MPartition;
@@ -182,7 +185,7 @@ class MetaStoreDirectSql {
SDS, SERDES, SKEWED_STRING_LIST_VALUES, SKEWED_VALUES, BUCKETING_COLS,
SKEWED_COL_NAMES,
SKEWED_COL_VALUE_LOC_MAP, COLUMNS_V2, PARTITION_KEYS, SERDE_PARAMS,
PART_COL_STATS, KEY_CONSTRAINTS,
TAB_COL_STATS, PARTITION_KEY_VALS, PART_PRIVS, PART_COL_PRIVS,
SKEWED_STRING_LIST, CDS,
- TBL_COL_PRIVS;
+ TBL_COL_PRIVS, FUNCS, FUNC_RU;
public MetaStoreDirectSql(PersistenceManager pm, Configuration conf, String
schema) {
this.pm = pm;
@@ -3064,4 +3067,104 @@ class MetaStoreDirectSql {
long csId = updateStat.getNextCSIdForMPartitionColumnStatistics(numStats);
return updateStat.updatePartitionColumnStatistics(partColStatsMap, tbl,
csId, validWriteIds, writeId, listeners);
}
+
+ public List<Function> getFunctions(String catName) throws MetaException {
+ List<Long> funcIds = getFunctionIds(catName);
+ // Get full objects. For Oracle/etc. do it in batches.
+ return Batchable.runBatched(batchSize, funcIds, new Batchable<Long,
Function>() {
+ @Override
+ public List<Function> run(List<Long> input) throws MetaException {
+ return getFunctionsFromFunctionIds(input, catName);
+ }
+ });
+ }
+
+ private List<Function> getFunctionsFromFunctionIds(List<Long> funcIdList,
String catName) throws MetaException {
+ String funcIds = getIdListForIn(funcIdList);
+ final int funcIdIndex = 0;
+ final int funcNameIndex = 1;
+ final int dbNameIndex = 2;
+ final int funcClassNameIndex = 3;
+ final int funcOwnerNameIndex = 4;
+ final int funcOwnerTypeIndex = 5;
+ final int funcCreateTimeIndex = 6;
+ final int funcTypeIndex = 7;
+
+ String queryText = "SELECT " +
+ FUNCS + ".\"FUNC_ID\", " +
+ FUNCS + ".\"FUNC_NAME\", " +
+ DBS + ".\"NAME\", " +
+ FUNCS + ".\"CLASS_NAME\", " +
+ FUNCS + ".\"OWNER_NAME\", " +
+ FUNCS + ".\"OWNER_TYPE\", " +
+ FUNCS + ".\"CREATE_TIME\", " +
+ FUNCS + ".\"FUNC_TYPE\"" +
+ " FROM " + FUNCS +
+ " LEFT JOIN " + DBS + " ON " + FUNCS + ".\"DB_ID\" = " + DBS +
".\"DB_ID\"" +
+ " where " + FUNCS +".\"FUNC_ID\" in (" + funcIds + ") order by " +
FUNCS + ".\"FUNC_NAME\" asc";
+
+ List<Function> results = new ArrayList<>();
+ TreeMap<Long, Function> funcs = new TreeMap<>();
+ final boolean doTrace = LOG.isDebugEnabled();
+ long start = doTrace ? System.nanoTime() : 0;
+ try (QueryWrapper query = new
QueryWrapper(pm.newQuery("javax.jdo.query.SQL", queryText))) {
+ List<Object[]> queryResult = executeWithArray(query.getInnerQuery(),
null, queryText);
+ long end = doTrace ? System.nanoTime() : 0;
+ MetastoreDirectSqlUtils.timingTrace(doTrace, queryText, start, end);
+
+ for (Object[] function : queryResult) {
+ Long funcId =
MetastoreDirectSqlUtils.extractSqlLong(function[funcIdIndex]);
+ String funcName =
MetastoreDirectSqlUtils.extractSqlString(function[funcNameIndex]);
+ String dbName =
MetastoreDirectSqlUtils.extractSqlString(function[dbNameIndex]);
+ String funcClassName =
MetastoreDirectSqlUtils.extractSqlString(function[funcClassNameIndex]);
+ String funcOwnerName =
MetastoreDirectSqlUtils.extractSqlString(function[funcOwnerNameIndex]);
+ String funcOwnerType =
MetastoreDirectSqlUtils.extractSqlString(function[funcOwnerTypeIndex]);
+ int funcCreateTime =
MetastoreDirectSqlUtils.extractSqlInt(function[funcCreateTimeIndex]);
+ int funcType =
MetastoreDirectSqlUtils.extractSqlInt(function[funcTypeIndex]);
+
+ Function func = new Function();
+ func.setFunctionName(funcName);
+ func.setDbName(dbName);
+ func.setCatName(catName);
+ func.setClassName(funcClassName);
+ func.setOwnerName(funcOwnerName);
+ func.setOwnerType(PrincipalType.valueOf(funcOwnerType));
+ func.setCreateTime(funcCreateTime);
+ func.setFunctionType(FunctionType.findByValue(funcType));
+ func.setResourceUris(new ArrayList<>());
+
+ results.add(func);
+ funcs.put(funcId, func);
+ }
+ }
+ MetastoreDirectSqlUtils.setFunctionResourceUris(FUNC_RU, pm, funcIds,
funcs);
+ return results;
+ }
+
+ private List<Long> getFunctionIds(String catName) throws MetaException {
+ boolean doTrace = LOG.isDebugEnabled();
+
+ String queryText = "select " + FUNCS + ".\"FUNC_ID\" from " + FUNCS +
+ " LEFT JOIN " + DBS + " ON " + FUNCS + ".\"DB_ID\" = " + DBS +
".\"DB_ID\"" +
+ " where " + DBS + ".\"CTLG_NAME\" = ? ";
+
+ long start = doTrace ? System.nanoTime() : 0;
+ Object[] params = new Object[1];
+ params[0] = catName;
+ try (QueryWrapper query = new
QueryWrapper(pm.newQuery("javax.jdo.query.SQL", queryText))) {
+ List<Object> sqlResult = executeWithArray(query.getInnerQuery(), params,
queryText);
+ long queryTime = doTrace ? System.nanoTime() : 0;
+ MetastoreDirectSqlUtils.timingTrace(doTrace, queryText, start,
queryTime);
+ final List<Long> result;
+ if (sqlResult.isEmpty()) {
+ result = Collections.emptyList();
+ } else {
+ result = new ArrayList<>(sqlResult.size());
+ for (Object fields : sqlResult) {
+ result.add(MetastoreDirectSqlUtils.extractSqlLong(fields));
+ }
+ }
+ return result;
+ }
+ }
}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDirectSqlUtils.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDirectSqlUtils.java
index e26ea2ee1a8..b3099977312 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDirectSqlUtils.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDirectSqlUtils.java
@@ -19,17 +19,18 @@
package org.apache.hadoop.hive.metastore;
-import com.google.common.base.Joiner;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Function;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.ResourceType;
+import org.apache.hadoop.hive.metastore.api.ResourceUri;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
-import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,7 +47,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
-import java.util.stream.Collectors;
/**
* Helper utilities used by DirectSQL code in HiveMetastore.
@@ -514,6 +514,19 @@ class MetastoreDirectSqlUtils {
}
}
+ static void setFunctionResourceUris(String FUNC_RU, PersistenceManager pm,
String funcIds,
+ TreeMap<Long, Function> functions)
+ throws MetaException {
+ String queryText;
+ queryText = "select \"FUNC_ID\", \"RESOURCE_TYPE\", \"RESOURCE_URI\" from
" + FUNC_RU
+ + " where \"FUNC_ID\" in (" + funcIds + ")"
+ + " order by \"FUNC_ID\" asc, \"INTEGER_IDX\" asc";
+ loopJoinOrderedResult(pm, functions, queryText, 0, (t, fields) -> {
+ ResourceUri resourceUri = new
ResourceUri(ResourceType.findByValue((int)fields[1]), (String) fields[2]);
+ t.getResourceUris().add(resourceUri);
+ });
+ }
+
/**
* Convert a boolean value returned from the RDBMS to a Java Boolean object.
* MySQL has booleans, but e.g. Derby uses 'Y'/'N' mapping and Oracle DB
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index e8996cb2498..b3e8bbb82c2 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -11541,6 +11541,33 @@ public class ObjectStore implements RawStore,
Configurable {
@Override
public List<Function> getAllFunctions(String catName) throws MetaException {
+ try {
+ return getFunctionsInternal(catName);
+ } catch (NoSuchObjectException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected List<Function> getFunctionsInternal(String catalogName)
+ throws MetaException, NoSuchObjectException {
+ return new GetListHelper<Function>(catalogName, "", "", true, true) {
+ @Override
+ protected List<Function> getSqlResult(GetHelper<List<Function>> ctx)
throws MetaException {
+ return directSql.getFunctions(catalogName);
+ }
+ @Override
+ protected List<Function> getJdoResult(GetHelper<List<Function>> ctx)
throws MetaException {
+ try {
+ return getAllFunctionsViaJDO(catalogName);
+ } catch (Exception e) {
+ LOG.error("Failed to convert to functions", e);
+ throw new MetaException(e.getMessage());
+ }
+ }
+ }.run(false);
+ }
+
+ private List<Function> getAllFunctionsViaJDO (String catName) {
boolean commited = false;
Query query = null;
try {
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestFunctions.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestFunctions.java
index aafbdd1bf51..efa57840f1c 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestFunctions.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestFunctions.java
@@ -377,12 +377,20 @@ public class TestFunctions extends MetaStoreClientTest {
for(Function function : allFunctions) {
if (function.getDbName().equals(OTHER_DATABASE)) {
Assert.assertEquals("Comparing functions", testFunctions[3], function);
+ Assert.assertEquals("Checking function's resourceUris",
+ testFunctions[3].getResourceUris().toString(),
function.getResourceUris().toString());
} else if (function.getFunctionName().equals("test_function_hidden_1")) {
Assert.assertEquals("Comparing functions", testFunctions[2], function);
+ Assert.assertEquals("Checking function's resourceUris",
+ testFunctions[2].getResourceUris().toString(),
function.getResourceUris().toString());
} else if (function.getFunctionName().equals("test_function_to_find_2"))
{
Assert.assertEquals("Comparing functions", testFunctions[1], function);
+ Assert.assertEquals("Checking function's resourceUris",
+ testFunctions[1].getResourceUris().toString(),
function.getResourceUris().toString());
} else {
Assert.assertEquals("Comparing functions", testFunctions[0], function);
+ Assert.assertEquals("Checking function's resourceUris",
+ testFunctions[0].getResourceUris().toString(),
function.getResourceUris().toString());
}
}
