This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new b72b5b1d01c Fix construct of PartialPath in Compaction (#11579)
b72b5b1d01c is described below
commit b72b5b1d01c3fb2900c82adbadd69cc4cd5041f1
Author: shuwenwei <[email protected]>
AuthorDate: Tue Nov 21 10:52:40 2023 +0800
Fix construct of PartialPath in Compaction (#11579)
---
.../impl/ReadChunkCompactionPerformer.java | 5 ++-
.../execute/utils/CompactionPathUtils.java | 40 ++++++++++++++++++++++
.../execute/utils/MultiTsFileDeviceIterator.java | 7 ++--
.../fast/AlignedSeriesCompactionExecutor.java | 21 +++++++-----
.../fast/NonAlignedSeriesCompactionExecutor.java | 12 +++----
5 files changed, 62 insertions(+), 23 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
index 56d32172bc4..d7f1ef14dac 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.db.exception.StorageEngineException;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionTargetFileCountExceededException;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ISeqCompactionPerformer;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CompactionTaskSummary;
+import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionPathUtils;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.MultiTsFileDeviceIterator;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.readchunk.AlignedSeriesCompactionExecutor;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.readchunk.SingleSeriesCompactionExecutor;
@@ -34,7 +35,6 @@ import
org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFi
import
org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
@@ -169,8 +169,7 @@ public class ReadChunkCompactionPerformer implements
ISeqCompactionPerformer {
while (seriesIterator.hasNextSeries()) {
checkThreadInterrupted();
// TODO: we can provide a configuration item to enable concurrent
between each series
- String pathStr = device + TsFileConstant.PATH_SEPARATOR +
seriesIterator.nextSeries();
- PartialPath p = new PartialPath(pathStr.split("\\."));
+ PartialPath p = CompactionPathUtils.getPath(device,
seriesIterator.nextSeries());
// TODO: seriesIterator needs to be refactor.
// This statement must be called before next hasNextSeries() called, or
it may be trapped in a
// dead-loop.
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java
new file mode 100644
index 00000000000..7fb292c1e5d
--- /dev/null
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils;
+
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.PartialPath;
+import
org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache;
+import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
+
+public class CompactionPathUtils {
+
+ private CompactionPathUtils() {}
+
+ public static PartialPath getPath(String device, String measurement) throws
IllegalPathException {
+ PartialPath path;
+ if (device.contains(TsFileConstant.BACK_QUOTE_STRING)) {
+ path = DataNodeDevicePathCache.getInstance().getPartialPath(device);
+ } else {
+ path = new PartialPath(device.split("\\."));
+ }
+ return path.concatNode(measurement);
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
index 7be323e1000..2aef8c8094c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
@@ -28,7 +28,6 @@ import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFil
import
org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.utils.ModificationUtils;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
@@ -393,7 +392,8 @@ public class MultiTsFileDeviceIterator implements
AutoCloseable {
valueSeriesPaths.add(
valueChunkMetadata == null
? null
- : new PartialPath(currentDevice.left,
valueChunkMetadata.getMeasurementUid()));
+ : CompactionPathUtils.getPath(
+ currentDevice.left, valueChunkMetadata.getMeasurementUid()));
}
for (Modification modification : modifications) {
@@ -548,8 +548,7 @@ public class MultiTsFileDeviceIterator implements
AutoCloseable {
LinkedList<Pair<TsFileSequenceReader, List<ChunkMetadata>>>
readerAndChunkMetadataForThisSeries = new LinkedList<>();
- String pathStr = device + TsFileConstant.PATH_SEPARATOR +
currentCompactingSeries;
- PartialPath path = new PartialPath(pathStr.split("\\."));
+ PartialPath path = CompactionPathUtils.getPath(device,
currentCompactingSeries);
for (TsFileResource resource : tsFileResourcesSortedByAsc) {
TsFileSequenceReader reader = readerMap.get(resource);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/AlignedSeriesCompactionExecutor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/AlignedSeriesCompactionExecutor.java
index 68895d3de84..93fece19121 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/AlignedSeriesCompactionExecutor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/AlignedSeriesCompactionExecutor.java
@@ -20,9 +20,9 @@
package
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast;
import org.apache.iotdb.commons.exception.IllegalPathException;
-import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.exception.WriteProcessException;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary;
+import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionPathUtils;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.element.ChunkMetadataElement;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.element.FileElement;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.element.PageElement;
@@ -31,7 +31,6 @@ import
org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFi
import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.utils.ModificationUtils;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.exception.write.PageException;
import org.apache.iotdb.tsfile.file.MetaMarker;
import org.apache.iotdb.tsfile.file.header.ChunkHeader;
@@ -201,13 +200,17 @@ public class AlignedSeriesCompactionExecutor extends
SeriesCompactionExecutor {
.getValueChunkMetadataList()
.forEach(
x -> {
- if (x == null) {
- valueModifications.add(null);
- } else {
- String pathStr =
- deviceId + TsFileConstant.PATH_SEPARATOR +
x.getMeasurementUid();
- valueModifications.add(
- getModificationsFromCache(resource, new
PartialPath(pathStr.split("\\."))));
+ try {
+ if (x == null) {
+ valueModifications.add(null);
+ } else {
+ valueModifications.add(
+ getModificationsFromCache(
+ resource,
+ CompactionPathUtils.getPath(deviceId,
x.getMeasurementUid())));
+ }
+ } catch (IllegalPathException e) {
+ throw new RuntimeException(e);
}
});
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/NonAlignedSeriesCompactionExecutor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/NonAlignedSeriesCompactionExecutor.java
index a4f47eed5de..2a1c44ff36d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/NonAlignedSeriesCompactionExecutor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/NonAlignedSeriesCompactionExecutor.java
@@ -20,9 +20,9 @@
package
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast;
import org.apache.iotdb.commons.exception.IllegalPathException;
-import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.exception.WriteProcessException;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary;
+import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionPathUtils;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.element.ChunkMetadataElement;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.element.FileElement;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.element.PageElement;
@@ -30,7 +30,6 @@ import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.wri
import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.utils.ModificationUtils;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.exception.write.PageException;
import org.apache.iotdb.tsfile.file.MetaMarker;
import org.apache.iotdb.tsfile.file.header.ChunkHeader;
@@ -139,13 +138,12 @@ public class NonAlignedSeriesCompactionExecutor extends
SeriesCompactionExecutor
if (!iChunkMetadataList.isEmpty()) {
// modify chunk metadatas
- String pathStr =
- deviceId
- + TsFileConstant.PATH_SEPARATOR
- + iChunkMetadataList.get(0).getMeasurementUid();
ModificationUtils.modifyChunkMetaData(
iChunkMetadataList,
- getModificationsFromCache(resource, new
PartialPath(pathStr.split("\\."))));
+ getModificationsFromCache(
+ resource,
+ CompactionPathUtils.getPath(
+ deviceId, iChunkMetadataList.get(0).getMeasurementUid())));
if (iChunkMetadataList.isEmpty()) {
// all chunks has been deleted in this file, just remove it
removeFile(fileElement);
