This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new db62bf92cea Call getDeviceIdList / getAttributeColumnNameList /
getAttributeValueList only once to reduce unnecessary cost & Added log for
table fetch update (#13922)
db62bf92cea is described below
commit db62bf92ceab81c5d94f9c90e8eee034bb29f4d3
Author: Caideyipi <[email protected]>
AuthorDate: Mon Oct 28 16:29:10 2024 +0800
Call getDeviceIdList / getAttributeColumnNameList / getAttributeValueList
only once to reduce unnecessary cost & Added log for table fetch update (#13922)
---
.../fetcher/TableDeviceSchemaValidator.java | 62 +++++++++++++++-------
.../db/schemaengine/table/DataNodeTableCache.java | 6 +++
2 files changed, 48 insertions(+), 20 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java
index b1761329990..27b97a80af3 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java
@@ -72,24 +72,44 @@ public class TableDeviceSchemaValidator {
public void validateDeviceSchema(
final ITableDeviceSchemaValidation schemaValidation, final
MPPQueryContext context) {
- ValidateResult validateResult =
validateDeviceSchemaInCache(schemaValidation);
+ // High-cost operations, shall only be called once
+ final List<Object[]> deviceIdList = schemaValidation.getDeviceIdList();
+ final List<String> attributeKeyList =
schemaValidation.getAttributeColumnNameList();
+ final List<Object[]> attributeValueList =
schemaValidation.getAttributeValueList();
+
+ ValidateResult validateResult =
+ validateDeviceSchemaInCache(
+ schemaValidation, deviceIdList, attributeKeyList,
attributeValueList);
if (!validateResult.missingDeviceIndexList.isEmpty()) {
- validateResult = fetchAndValidateDeviceSchema(schemaValidation,
validateResult, context);
+ validateResult =
+ fetchAndValidateDeviceSchema(
+ schemaValidation,
+ validateResult,
+ context,
+ deviceIdList,
+ attributeKeyList,
+ attributeValueList);
}
if (!validateResult.missingDeviceIndexList.isEmpty()
|| !validateResult.attributeUpdateDeviceIndexList.isEmpty()) {
- autoCreateOrUpdateDeviceSchema(schemaValidation, validateResult,
context);
+ autoCreateOrUpdateDeviceSchema(
+ schemaValidation,
+ validateResult,
+ context,
+ deviceIdList,
+ attributeKeyList,
+ attributeValueList);
}
}
private ValidateResult validateDeviceSchemaInCache(
- final ITableDeviceSchemaValidation schemaValidation) {
+ final ITableDeviceSchemaValidation schemaValidation,
+ final List<Object[]> deviceIdList,
+ final List<String> attributeKeyList,
+ final List<Object[]> attributeValueList) {
final ValidateResult result = new ValidateResult();
- final List<Object[]> deviceIdList = schemaValidation.getDeviceIdList();
- final List<String> attributeKeyList =
schemaValidation.getAttributeColumnNameList();
- final List<Object[]> attributeValueList =
schemaValidation.getAttributeValueList();
for (int i = 0, size = deviceIdList.size(); i < size; i++) {
final Map<String, String> attributeMap =
@@ -116,27 +136,26 @@ public class TableDeviceSchemaValidator {
private ValidateResult fetchAndValidateDeviceSchema(
final ITableDeviceSchemaValidation schemaValidation,
final ValidateResult previousValidateResult,
- final MPPQueryContext context) {
+ final MPPQueryContext context,
+ final List<Object[]> deviceIdList,
+ final List<String> attributeKeyList,
+ final List<Object[]> attributeValueList) {
final Map<IDeviceID, Map<String, String>> fetchedDeviceSchema =
fetcher.fetchMissingDeviceSchemaForDataInsertion(
new FetchDevice(
schemaValidation.getDatabase(),
schemaValidation.getTableName(),
previousValidateResult.missingDeviceIndexList.stream()
- .map(index ->
schemaValidation.getDeviceIdList().get(index))
+ .map(deviceIdList::get)
.collect(Collectors.toList())),
context);
- final List<String> attributeKeyList =
schemaValidation.getAttributeColumnNameList();
- final List<Object[]> attributeValueList =
schemaValidation.getAttributeValueList();
-
final ValidateResult result = new ValidateResult();
for (final int index : previousValidateResult.missingDeviceIndexList) {
final Map<String, String> attributeMap =
fetchedDeviceSchema.get(
convertIdValuesToDeviceID(
- schemaValidation.getTableName(),
- (String[]) schemaValidation.getDeviceIdList().get(index)));
+ schemaValidation.getTableName(), (String[])
deviceIdList.get(index)));
if (attributeMap == null) {
result.missingDeviceIndexList.add(index);
} else {
@@ -174,7 +193,10 @@ public class TableDeviceSchemaValidator {
private void autoCreateOrUpdateDeviceSchema(
final ITableDeviceSchemaValidation schemaValidation,
final ValidateResult previousValidateResult,
- final MPPQueryContext context) {
+ final MPPQueryContext context,
+ final List<Object[]> inputDeviceIdList,
+ final List<String> attributeKeyList,
+ final List<Object[]> intPutAttributeValueList) {
final int size =
previousValidateResult.missingDeviceIndexList.size()
+ previousValidateResult.attributeUpdateDeviceIndexList.size();
@@ -183,14 +205,14 @@ public class TableDeviceSchemaValidator {
previousValidateResult.missingDeviceIndexList.forEach(
index -> {
- deviceIdList.add(schemaValidation.getDeviceIdList().get(index));
-
attributeValueList.add(schemaValidation.getAttributeValueList().get(index));
+ deviceIdList.add(inputDeviceIdList.get(index));
+ attributeValueList.add(intPutAttributeValueList.get(index));
});
previousValidateResult.attributeUpdateDeviceIndexList.forEach(
index -> {
- deviceIdList.add(schemaValidation.getDeviceIdList().get(index));
-
attributeValueList.add(schemaValidation.getAttributeValueList().get(index));
+ deviceIdList.add(inputDeviceIdList.get(index));
+ attributeValueList.add(intPutAttributeValueList.get(index));
});
final ExecutionResult executionResult =
@@ -199,7 +221,7 @@ public class TableDeviceSchemaValidator {
schemaValidation.getDatabase(),
schemaValidation.getTableName(),
deviceIdList,
- schemaValidation.getAttributeColumnNameList(),
+ attributeKeyList,
attributeValueList),
relationSqlParser,
SessionManager.getInstance().getCurrSession(),
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java
index 4ada1b6b95e..7c77c052bbc 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java
@@ -302,6 +302,12 @@ public class DataNodeTableCache implements ITableCache {
previousVersions.get(database).get(tableName))) {
return;
}
+ LOGGER.info(
+ "Update table {}.{} by table fetch, table in
preUpdateMap: {}, new table: {}",
+ database,
+ existingPair.getLeft().getTableName(),
+ existingPair.getLeft(),
+ tsTable);
existingPair.setLeft(null);
if (Objects.nonNull(tsTable)) {
databaseTableMap
