This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch load_v2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/load_v2 by this push:
new 3eee45f67c0 speed up tests
3eee45f67c0 is described below
commit 3eee45f67c00167bd9eac09b01e54015aeadaf84
Author: Tian Jiang <[email protected]>
AuthorDate: Tue Sep 5 09:54:02 2023 +0800
speed up tests
---
.../execution/load/MergedTsFileSplitterTest.java | 3 +-
.../db/queryengine/execution/load/TestBase.java | 65 ++++++++++++----------
2 files changed, 37 insertions(+), 31 deletions(-)
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/load/MergedTsFileSplitterTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/load/MergedTsFileSplitterTest.java
index 857a5a9a4e7..6461ce04585 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/load/MergedTsFileSplitterTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/load/MergedTsFileSplitterTest.java
@@ -42,7 +42,7 @@ public class MergedTsFileSplitterTest extends TestBase {
try {
splitter.splitTsFileByDataPartition();
for (TsFileData tsFileData : resultSet) {
- // System.out.println(tsFileData);
+// System.out.println(tsFileData);
}
} finally {
splitter.close();
@@ -55,7 +55,6 @@ public class MergedTsFileSplitterTest extends TestBase {
resultSet.add(data);
if (resultSet.size() % 1000 == 0) {
System.out.printf("%d chunks split\n", resultSet.size());
- System.out.printf("Maxmem: %d, freemem: %d\n",
Runtime.getRuntime().maxMemory(), Runtime.getRuntime().freeMemory());
}
return true;
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/load/TestBase.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/load/TestBase.java
index 9002b3e6168..cbbab9bd7c3 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/load/TestBase.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/load/TestBase.java
@@ -23,6 +23,7 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.stream.IntStream;
import org.apache.iotdb.db.utils.TimePartitionUtils;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -48,7 +49,7 @@ public class TestBase {
protected int fileNum = 100;
// series number of each file, sn non-aligned series and 1 aligned series
with sn measurements
- protected int seriesNum = 1000;
+ protected int seriesNum = 100;
// number of chunks of each series in a file, each series has only one chunk
in a file
protected double chunkTimeRangeRatio = 0.3;
// the interval between two consecutive points of a series
@@ -68,42 +69,48 @@ public class TestBase {
}
}
- public void setupFiles() throws IOException, WriteProcessException {
+ public void setupFiles() {
- for (int i = 0; i < fileNum; i++) {
- File file = new File(getTestTsFilePath("root.sg1", 0, 0, i));
- files.add(file);
-
- try (TsFileWriter writer = new TsFileWriter(file)) {
- // 3 non-aligned series under d1 and 1 aligned series with 3
measurements under d2
- for (int sn = 0; sn < seriesNum; sn++) {
- writer.registerTimeseries(
- new Path("d1"), new MeasurementSchema("s" + sn,
TSDataType.DOUBLE));
- }
- List<MeasurementSchema> alignedSchemas = new ArrayList<>();
- for (int sn = 0; sn < seriesNum; sn++) {
- alignedSchemas.add(new MeasurementSchema("s" + sn,
TSDataType.DOUBLE));
+ IntStream.range(0, fileNum).parallel().forEach(i -> {
+ try {
+ File file = new File(getTestTsFilePath("root.sg1", 0, 0, i));
+ synchronized (files) {
+ files.add(file);
}
- writer.registerAlignedTimeseries(new Path("d2"), alignedSchemas);
-
- long timePartitionInterval =
TimePartitionUtils.getTimePartitionInterval();
- long chunkTimeRange = (long) (timePartitionInterval *
chunkTimeRangeRatio);
- int chunkPointNum = (int) (chunkTimeRange / pointInterval);
- for (int pn = 0; pn < chunkPointNum; pn++) {
- long currTime = chunkTimeRange * fileNum + pointInterval * pn;
- TSRecord record = new TSRecord(currTime, "d1");
+ try (TsFileWriter writer = new TsFileWriter(file)) {
+ // 3 non-aligned series under d1 and 1 aligned series with 3
measurements under d2
+ for (int sn = 0; sn < seriesNum; sn++) {
+ writer.registerTimeseries(
+ new Path("d1"), new MeasurementSchema("s" + sn,
TSDataType.DOUBLE));
+ }
+ List<MeasurementSchema> alignedSchemas = new ArrayList<>();
for (int sn = 0; sn < seriesNum; sn++) {
- record.addTuple(new DoubleDataPoint("s" + sn, pn * 1.0));
+ alignedSchemas.add(new MeasurementSchema("s" + sn,
TSDataType.DOUBLE));
}
- writer.write(record);
+ writer.registerAlignedTimeseries(new Path("d2"), alignedSchemas);
- record.deviceId = "d2";
- writer.writeAligned(record);
+ long timePartitionInterval =
TimePartitionUtils.getTimePartitionInterval();
+ long chunkTimeRange = (long) (timePartitionInterval *
chunkTimeRangeRatio);
+ int chunkPointNum = (int) (chunkTimeRange / pointInterval);
+
+ for (int pn = 0; pn < chunkPointNum; pn++) {
+ long currTime = chunkTimeRange * i + pointInterval * pn;
+ TSRecord record = new TSRecord(currTime, "d1");
+ for (int sn = 0; sn < seriesNum; sn++) {
+ record.addTuple(new DoubleDataPoint("s" + sn, pn * 1.0));
+ }
+ writer.write(record);
+
+ record.deviceId = "d2";
+ writer.writeAligned(record);
+ }
+ writer.flushAllChunkGroups();
}
- writer.flushAllChunkGroups();
+ } catch (IOException | WriteProcessException e) {
+ throw new RuntimeException(e);
}
- }
+ });
}
public static String getTestTsFilePath(
