deniskuzZ commented on code in PR #6379:
URL: https://github.com/apache/hive/pull/6379#discussion_r3046089682
##########
ql/src/java/org/apache/hadoop/hive/llap/ProactiveEviction.java:
##########
@@ -152,77 +152,81 @@ public void run() {
}
/**
- * Holds information on entities: DB name(s), table name(s), partitions.
+ * Holds information on entities: catalog name(s), DB name(s), table
name(s), partitions.
*/
public static final class Request {
- // Holds a hierarchical structure of DBs, tables and partitions such as:
- // { testdb : { testtab0 : [], testtab1 : [ {pk0 : p0v0, pk1 : p0v1}, {pk0
: p1v0, pk1 : p1v1} ] }, testdb2 : {} }
- private final Map<String, Map<String, Set<LinkedHashMap<String, String>>>>
entities;
+ public record PartitionSpec(Map<String, String> spec) {}
+ // Holds a hierarchical structure of catalogs, DBs, tables and partitions
such as:
+ // { testcatalog : { testdb : { testtab0 : [], testtab1 : [ {pk0 : p0v0,
pk1 : p0v1} ] }, testdb2 : {} } }
+ // The catalog key defaults to Warehouse.DEFAULT_CATALOG_NAME ("hive")
when no explicit catalog is given.
+ private final Map<String, Map<String, Map<String, Set<PartitionSpec>>>>
entities;
Review Comment:
````
Subject: [PATCH] patch
---
Index:
llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
(revision 5e82676aa447c5066283bead8553dfc682053297)
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
(date 1775575270784)
@@ -324,11 +324,10 @@
if (LOG.isDebugEnabled()) {
StringBuilder sb = new StringBuilder();
sb.append(markedBytes).append(" bytes marked for eviction from LLAP
cache buffers that belong to table(s): ");
- request.getEntities().forEach((catalog, dbs) ->
- dbs.forEach((db, tables) ->
- tables.forEach((table, partitions) ->
- sb.append(catalog + "." + db + "." + table).append(" "))
- )
+ request.getEntities().forEach((catalogDb, tables) ->
+ tables.forEach((table, partitions) ->
+
sb.append(catalogDb.catalog()).append(".").append(catalogDb.database())
+ .append(".").append(table).append(" "))
);
sb.append(" Duration: ").append(time).append(" ms");
LOG.debug(sb.toString());
Index: ql/src/java/org/apache/hadoop/hive/llap/ProactiveEviction.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/ql/src/java/org/apache/hadoop/hive/llap/ProactiveEviction.java
b/ql/src/java/org/apache/hadoop/hive/llap/ProactiveEviction.java
--- a/ql/src/java/org/apache/hadoop/hive/llap/ProactiveEviction.java
(revision 5e82676aa447c5066283bead8553dfc682053297)
+++ b/ql/src/java/org/apache/hadoop/hive/llap/ProactiveEviction.java (date
1775575734238)
@@ -157,16 +157,23 @@
public static final class Request {
public record PartitionSpec(Map<String, String> spec) {}
- // Holds a hierarchical structure of catalogs, DBs, tables and
partitions such as:
- // { testcatalog : { testdb : { testtab0 : [], testtab1 : [ {pk0 :
p0v0, pk1 : p0v1} ] }, testdb2 : {} } }
- // The catalog key defaults to Warehouse.DEFAULT_CATALOG_NAME ("hive")
when no explicit catalog is given.
- private final Map<String, Map<String, Map<String, Set<PartitionSpec>>>>
entities;
+
+ public record CatalogDb(String catalog, String database) {
+ public static CatalogDb of(String catalog, String database) {
+ return new CatalogDb(catalog, database);
+ }
+ }
- private Request(Map<String, Map<String, Map<String,
Set<PartitionSpec>>>> entities) {
+ // Holds a hierarchical structure of (catalog, DB) -> tables ->
partitions such as:
+ // { (testcatalog, testdb) : { testtab0 : [], testtab1 : [ {pk0 : p0v0,
pk1 : p0v1} ] } }
+ // The catalog defaults to Warehouse.DEFAULT_CATALOG_NAME ("hive") when
no explicit catalog is given.
+ private final Map<CatalogDb, Map<String, Set<PartitionSpec>>> entities;
+
+ private Request(Map<CatalogDb, Map<String, Set<PartitionSpec>>>
entities) {
this.entities = entities;
}
- public Map<String, Map<String, Map<String, Set<PartitionSpec>>>>
getEntities() {
+ public Map<CatalogDb, Map<String, Set<PartitionSpec>>> getEntities() {
return entities;
}
@@ -174,57 +181,44 @@
return entities.isEmpty();
}
- public boolean hasCatalogName(String catalogName) {
- return entities.containsKey(catalogName);
- }
-
- public boolean hasDatabaseName(String catalogName, String dbName) {
- return hasCatalogName(catalogName) &&
entities.get(catalogName).containsKey(dbName);
- }
-
/**
* Translate to Protobuf requests.
* @return list of request instances ready to be sent over protobuf.
*/
public List<LlapDaemonProtocolProtos.EvictEntityRequestProto>
toProtoRequests() {
return entities.entrySet().stream()
- .flatMap(catalogEntry -> {
- String catalogName = catalogEntry.getKey();
- Map<String, Map<String, Set<PartitionSpec>>> dbs =
catalogEntry.getValue();
-
- return dbs.entrySet().stream().map(dbEntry -> {
- String dbName = dbEntry.getKey();
- Map<String, Set<PartitionSpec>> tables = dbEntry.getValue();
+ .map(entry -> {
+ CatalogDb catalogDb = entry.getKey();
+ Map<String, Set<PartitionSpec>> tables = entry.getValue();
- LlapDaemonProtocolProtos.EvictEntityRequestProto.Builder
requestBuilder =
-
LlapDaemonProtocolProtos.EvictEntityRequestProto.newBuilder();
+ LlapDaemonProtocolProtos.EvictEntityRequestProto.Builder
requestBuilder =
+
LlapDaemonProtocolProtos.EvictEntityRequestProto.newBuilder();
- requestBuilder.setCatalogName(catalogName.toLowerCase());
- requestBuilder.setDbName(dbName.toLowerCase());
+
requestBuilder.setCatalogName(catalogDb.catalog().toLowerCase());
+ requestBuilder.setDbName(catalogDb.database().toLowerCase());
- tables.forEach((tableName, partitions) -> {
- LlapDaemonProtocolProtos.TableProto.Builder tableBuilder =
- LlapDaemonProtocolProtos.TableProto.newBuilder();
+ tables.forEach((tableName, partitions) -> {
+ LlapDaemonProtocolProtos.TableProto.Builder tableBuilder =
+ LlapDaemonProtocolProtos.TableProto.newBuilder();
- tableBuilder.setTableName(tableName.toLowerCase());
+ tableBuilder.setTableName(tableName.toLowerCase());
- Set<String> partitionKeys = null;
+ Set<String> partitionKeys = null;
- for (PartitionSpec partitionSpec : partitions) {
- if (partitionKeys == null) {
- partitionKeys = new
LinkedHashSet<>(partitionSpec.spec().keySet());
- tableBuilder.addAllPartKey(partitionKeys);
- }
- for (String partKey : tableBuilder.getPartKeyList()) {
-
tableBuilder.addPartVal(partitionSpec.spec().get(partKey));
- }
- }
- // For a given table the set of partition columns (keys)
should not change.
- requestBuilder.addTable(tableBuilder.build());
- });
+ for (PartitionSpec partitionSpec : partitions) {
+ if (partitionKeys == null) {
+ partitionKeys = new
LinkedHashSet<>(partitionSpec.spec().keySet());
+ tableBuilder.addAllPartKey(partitionKeys);
+ }
+ for (String partKey : tableBuilder.getPartKeyList()) {
+
tableBuilder.addPartVal(partitionSpec.spec().get(partKey));
+ }
+ }
+ // For a given table the set of partition columns (keys)
should not change.
+ requestBuilder.addTable(tableBuilder.build());
+ });
- return requestBuilder.build();
- });
+ return requestBuilder.build();
})
.toList();
}
@@ -253,18 +247,12 @@
// getTableName() returns "catalog.db.table"; TableName.fromString
handles 3-part names.
TableName tagTableName =
TableName.fromString(cacheTag.getTableName(), null, null);
- // Check that the tag's catalog is present in the eviction request.
- if (!entities.containsKey(catalog)) {
+ CatalogDb key = CatalogDb.of(catalog, db);
+ if (!entities.containsKey(key)) {
return false;
}
- // Check that the tag's DB is present in the eviction request for
this catalog.
- Map<String, Map<String, Set<PartitionSpec>>> catalogEntities =
entities.getOrDefault(catalog, Map.of());
- if (!catalogEntities.containsKey(db)) {
- return false;
- }
-
- Map<String, Set<PartitionSpec>> tables =
catalogEntities.getOrDefault(db, Map.of());
+ Map<String, Set<PartitionSpec>> tables = entities.getOrDefault(key,
Map.of());
// If true, must be a drop DB event and this cacheTag matches.
if (tables.isEmpty()) {
@@ -311,7 +299,7 @@
*/
public static final class Builder {
- private final Map<String, Map<String, Map<String,
Set<PartitionSpec>>>> entities;
+ private final Map<CatalogDb, Map<String, Set<PartitionSpec>>>
entities;
private Builder() {
this.entities = new HashMap<>();
@@ -327,7 +315,7 @@
public Builder addPartitionOfATable(String catalog, String db, String
tableName,
Map<String, String> partSpec) {
ensureTable(catalog, db, tableName);
- entities.get(catalog).get(db).get(tableName).add(new
PartitionSpec(partSpec));
+ entities.get(CatalogDb.of(catalog, db)).get(tableName).add(new
PartitionSpec(partSpec));
return this;
}
@@ -342,7 +330,7 @@
* Add a database scoped to the given catalog.
*/
public Builder addDb(String catalog, String db) {
- ensureDb(catalog, db);
+ entities.computeIfAbsent(CatalogDb.of(catalog, db), k -> new
HashMap<>());
return this;
}
@@ -372,18 +360,10 @@
return new Request(entities);
}
- private void ensureCatalog(String catalogName) {
- entities.computeIfAbsent(catalogName, k -> new HashMap<>());
- }
-
- private void ensureDb(String catalogName, String dbName) {
- ensureCatalog(catalogName);
- entities.get(catalogName).computeIfAbsent(dbName, k -> new
HashMap<>());
- }
-
private void ensureTable(String catalogName, String dbName, String
tableName) {
- ensureDb(catalogName, dbName);
- entities.get(catalogName).get(dbName).computeIfAbsent(tableName, k
-> new HashSet<>());
+ CatalogDb key = CatalogDb.of(catalogName, dbName);
+ entities.computeIfAbsent(key, k -> new HashMap<>())
+ .computeIfAbsent(tableName, k -> new HashSet<>());
}
/**
@@ -396,6 +376,7 @@
String catalogName = protoRequest.getCatalogName().toLowerCase();
String dbName = protoRequest.getDbName().toLowerCase();
+ CatalogDb key = CatalogDb.of(catalogName, dbName);
Map<String, Set<PartitionSpec>> entitiesInDb = new HashMap<>();
List<LlapDaemonProtocolProtos.TableProto> tables =
protoRequest.getTableList();
@@ -425,9 +406,7 @@
entitiesInDb.put(dbAndTableName, partitions);
}
}
- Map<String, Map<String, Set<PartitionSpec>>> entitiesInCatalog =
new HashMap<>();
- entitiesInCatalog.put(dbName, entitiesInDb);
- entities.put(catalogName, entitiesInCatalog);
+ entities.put(key, entitiesInDb);
return this;
}
}
````
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
