This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch snapshot/2.0.0-250109 in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit ccc50a1583625b7ae0eab044efb6408522630e3b Author: JackieTien97 <[email protected]> AuthorDate: Thu Jan 9 14:37:55 2025 +0800 Revert "Modify tablet usage (#358)" This reverts commit 12bbf5642ab32b6cc8821a299df295705a199835. --- .../tsfile/TsFileWriteAlignedWithTablet.java | 5 +- .../org/apache/tsfile/TsFileWriteWithTablet.java | 3 +- .../read/query/dataset/AbstractResultSet.java | 2 +- .../apache/tsfile/utils/TsFileGeneratorUtils.java | 5 +- .../write/chunk/AlignedChunkGroupWriterImpl.java | 23 +++--- .../chunk/NonAlignedChunkGroupWriterImpl.java | 31 +++---- .../org/apache/tsfile/write/record/Tablet.java | 93 +++------------------ .../apache/tsfile/tableview/PerformanceTest.java | 15 ++-- .../tsfile/write/DefaultSchemaTemplateTest.java | 5 +- .../tsfile/write/MetadataIndexConstructorTest.java | 5 +- .../apache/tsfile/write/TsFileWriteApiTest.java | 96 ++++++++-------------- .../org/apache/tsfile/write/TsFileWriterTest.java | 13 +-- .../org/apache/tsfile/write/record/TabletTest.java | 20 ++--- 13 files changed, 115 insertions(+), 201 deletions(-) diff --git a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java index ca13fdb3..784df63b 100644 --- a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java +++ b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java @@ -133,6 +133,8 @@ public class TsFileWriteAlignedWithTablet { measurementSchemas.add(new MeasurementSchema(SENSOR_1, TSDataType.INT64, TSEncoding.RLE)); measurementSchemas.add(new MeasurementSchema(SENSOR_2, TSDataType.INT64, TSEncoding.RLE)); Tablet tablet = new Tablet(DEVICE_2, measurementSchemas); + long[] timestamps = tablet.timestamps; + Object[] values = tablet.values; int rowNum = 100; int sensorNum = measurementSchemas.size(); long timestamp = 1; @@ -141,7 +143,8 @@ public class TsFileWriteAlignedWithTablet { int row = tablet.getRowSize(); tablet.addTimestamp(row, timestamp++); for (int i = 0; i < sensorNum; i++) { - tablet.addValue(row, i, value); + long[] sensor = (long[]) values[i]; + sensor[row] = value; } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { diff --git a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java index 05446ce8..bb0293a5 100644 --- a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java +++ b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java @@ -88,11 +88,12 @@ public class TsFileWriteWithTablet { long startValue) throws IOException, WriteProcessException { Tablet tablet = new Tablet(deviceId, schemas); + long[] timestamps = tablet.timestamps; long sensorNum = schemas.size(); for (long r = 0; r < rowNum; r++, startValue++) { int row = tablet.getRowSize(); - tablet.addTimestamp(row, startTime++); + timestamps[row] = startTime++; for (int i = 0; i < sensorNum; i++) { tablet.addValue( schemas.get(i).getMeasurementName(), diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/AbstractResultSet.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/AbstractResultSet.java index 10352790..5b07dd46 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/AbstractResultSet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/AbstractResultSet.java @@ -168,7 +168,7 @@ public abstract class AbstractResultSet implements ResultSet { } protected Field getField(int columnIndex) { - if (columnIndex > this.columnNameToColumnIndexMap.size() || columnIndex <= 0) { + if (columnIndex > this.columnNameToColumnIndexMap.size()) { throw new IndexOutOfBoundsException("column index " + columnIndex + " out of bound"); } Field field; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java index e4b8abcd..6b87f6e1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java @@ -117,13 +117,16 @@ public class TsFileGeneratorUtils { boolean isAligned) throws IOException, WriteProcessException { Tablet tablet = new Tablet(deviceId, schemas); + long[] timestamps = tablet.timestamps; + Object[] values = tablet.values; long sensorNum = schemas.size(); for (long r = 0; r < rowNum; r++, startValue++) { int row = tablet.getRowSize(); tablet.addTimestamp(row, startTime++); for (int i = 0; i < sensorNum; i++) { - tablet.addValue(row, i, startValue); + long[] sensor = (long[]) values[i]; + sensor[row] = startValue; } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java index c051a702..0b53b75a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java @@ -233,7 +233,7 @@ public class AlignedChunkGroupWriterImpl implements IChunkGroupWriter { // TODO: changing to a column-first style by calculating the remaining page space of each // column firsts for (int row = startRowIndex; row < endRowIndex; row++) { - long time = tablet.getTimestamps()[row]; + long time = tablet.timestamps[row]; checkIsHistoryData(time); for (int columnIndex = 0; columnIndex < tablet.getSchemas().size(); columnIndex++) { if (tablet.getColumnTypes() != null @@ -242,19 +242,18 @@ public class AlignedChunkGroupWriterImpl implements IChunkGroupWriter { } boolean isNull = - tablet.getBitMaps() != null - && tablet.getBitMaps()[columnIndex] != null - && tablet.getBitMaps()[columnIndex].isMarked(row); + tablet.bitMaps != null + && tablet.bitMaps[columnIndex] != null + && tablet.bitMaps[columnIndex].isMarked(row); // check isNull by bitMap in tablet ValueChunkWriter valueChunkWriter = tryToAddSeriesWriterInternal(measurementSchemas.get(columnIndex)); switch (measurementSchemas.get(columnIndex).getType()) { case BOOLEAN: - valueChunkWriter.write( - time, ((boolean[]) tablet.getValues()[columnIndex])[row], isNull); + valueChunkWriter.write(time, ((boolean[]) tablet.values[columnIndex])[row], isNull); break; case INT32: - valueChunkWriter.write(time, ((int[]) tablet.getValues()[columnIndex])[row], isNull); + valueChunkWriter.write(time, ((int[]) tablet.values[columnIndex])[row], isNull); break; case DATE: valueChunkWriter.write( @@ -262,23 +261,23 @@ public class AlignedChunkGroupWriterImpl implements IChunkGroupWriter { isNull ? 0 : DateUtils.parseDateExpressionToInt( - ((LocalDate[]) tablet.getValues()[columnIndex])[row]), + ((LocalDate[]) tablet.values[columnIndex])[row]), isNull); break; case INT64: case TIMESTAMP: - valueChunkWriter.write(time, ((long[]) tablet.getValues()[columnIndex])[row], isNull); + valueChunkWriter.write(time, ((long[]) tablet.values[columnIndex])[row], isNull); break; case FLOAT: - valueChunkWriter.write(time, ((float[]) tablet.getValues()[columnIndex])[row], isNull); + valueChunkWriter.write(time, ((float[]) tablet.values[columnIndex])[row], isNull); break; case DOUBLE: - valueChunkWriter.write(time, ((double[]) tablet.getValues()[columnIndex])[row], isNull); + valueChunkWriter.write(time, ((double[]) tablet.values[columnIndex])[row], isNull); break; case TEXT: case BLOB: case STRING: - valueChunkWriter.write(time, ((Binary[]) tablet.getValues()[columnIndex])[row], isNull); + valueChunkWriter.write(time, ((Binary[]) tablet.values[columnIndex])[row], isNull); break; default: throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java index cc20b252..3f4d8aa0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java @@ -121,51 +121,42 @@ public class NonAlignedChunkGroupWriterImpl implements IChunkGroupWriter { pointCount = 0; for (int row = startRowIndex; row < endRowIndex; row++) { // check isNull in tablet - if (tablet.getBitMaps() != null - && tablet.getBitMaps()[column] != null - && tablet.getBitMaps()[column].isMarked(row)) { + if (tablet.bitMaps != null + && tablet.bitMaps[column] != null + && tablet.bitMaps[column].isMarked(row)) { continue; } - long time = tablet.getTimestamps()[row]; + long time = tablet.timestamps[row]; checkIsHistoryData(measurementId, time); pointCount++; switch (tsDataType) { case INT32: - chunkWriters.get(measurementId).write(time, ((int[]) tablet.getValues()[column])[row]); + chunkWriters.get(measurementId).write(time, ((int[]) tablet.values[column])[row]); break; case DATE: chunkWriters .get(measurementId) .write( time, - DateUtils.parseDateExpressionToInt( - ((LocalDate[]) tablet.getValues()[column])[row])); + DateUtils.parseDateExpressionToInt(((LocalDate[]) tablet.values[column])[row])); break; case INT64: case TIMESTAMP: - chunkWriters.get(measurementId).write(time, ((long[]) tablet.getValues()[column])[row]); + chunkWriters.get(measurementId).write(time, ((long[]) tablet.values[column])[row]); break; case FLOAT: - chunkWriters - .get(measurementId) - .write(time, ((float[]) tablet.getValues()[column])[row]); + chunkWriters.get(measurementId).write(time, ((float[]) tablet.values[column])[row]); break; case DOUBLE: - chunkWriters - .get(measurementId) - .write(time, ((double[]) tablet.getValues()[column])[row]); + chunkWriters.get(measurementId).write(time, ((double[]) tablet.values[column])[row]); break; case BOOLEAN: - chunkWriters - .get(measurementId) - .write(time, ((boolean[]) tablet.getValues()[column])[row]); + chunkWriters.get(measurementId).write(time, ((boolean[]) tablet.values[column])[row]); break; case TEXT: case BLOB: case STRING: - chunkWriters - .get(measurementId) - .write(time, ((Binary[]) tablet.getValues()[column])[row]); + chunkWriters.get(measurementId).write(time, ((Binary[]) tablet.values[column])[row]); break; default: throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 93e3f1e9..2580a2a4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -80,11 +80,14 @@ public class Tablet { /** MeasurementId->indexOf({@link MeasurementSchema}) */ private final Map<String, Integer> measurementIndex; - private long[] timestamps; + /** Timestamps in this {@link Tablet} */ + public long[] timestamps; - private Object[] values; + /** Each object is a primitive type array, which represents values of one measurement */ + public Object[] values; - private BitMap[] bitMaps; + /** Each {@link BitMap} represents the existence of each value in the current column. */ + public BitMap[] bitMaps; /** * For compatibility with the usage of directly modifying Tablet content through public fields. @@ -289,7 +292,6 @@ public class Tablet { public void addTimestamp(int rowIndex, long timestamp) { timestamps[rowIndex] = timestamp; this.rowSize = Math.max(this.rowSize, rowIndex + 1); - initBitMapsWithApiUsage(); } public void addValue(final String measurementId, final int rowIndex, final Object value) { @@ -411,10 +413,6 @@ public class Tablet { @TsFileApi public void addValue(int rowIndex, int columnIndex, int val) { - if (!(values[columnIndex] instanceof int[])) { - throw new IllegalArgumentException( - "The data type of column index " + columnIndex + " is not INT32"); - } final int[] sensor = (int[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); @@ -428,10 +426,6 @@ public class Tablet { @TsFileApi public void addValue(int rowIndex, int columnIndex, long val) { - if (!(values[columnIndex] instanceof long[])) { - throw new IllegalArgumentException( - "The data type of column index " + columnIndex + " is not INT64/TIMESTAMP"); - } final long[] sensor = (long[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); @@ -445,10 +439,6 @@ public class Tablet { @TsFileApi public void addValue(int rowIndex, int columnIndex, float val) { - if (!(values[columnIndex] instanceof float[])) { - throw new IllegalArgumentException( - "The data type of column index " + columnIndex + " is not FLOAT"); - } final float[] sensor = (float[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); @@ -462,10 +452,6 @@ public class Tablet { @TsFileApi public void addValue(int rowIndex, int columnIndex, double val) { - if (!(values[columnIndex] instanceof double[])) { - throw new IllegalArgumentException( - "The data type of column index " + columnIndex + " is not DOUBLE"); - } final double[] sensor = (double[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); @@ -479,10 +465,6 @@ public class Tablet { @TsFileApi public void addValue(int rowIndex, int columnIndex, boolean val) { - if (!(values[columnIndex] instanceof boolean[])) { - throw new IllegalArgumentException( - "The data type of column index " + columnIndex + " is not BOOLEAN"); - } final boolean[] sensor = (boolean[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); @@ -496,10 +478,6 @@ public class Tablet { @TsFileApi public void addValue(int rowIndex, int columnIndex, String val) { - if (!(values[columnIndex] instanceof Binary[])) { - throw new IllegalArgumentException( - "The data type of column index " + columnIndex + " is not TEXT/STRING/BLOB"); - } final Binary[] sensor = (Binary[]) values[columnIndex]; sensor[rowIndex] = new Binary(val, TSFileConfig.STRING_CHARSET); updateBitMap(rowIndex, columnIndex, false); @@ -513,10 +491,6 @@ public class Tablet { @TsFileApi public void addValue(int rowIndex, int columnIndex, byte[] val) { - if (!(values[columnIndex] instanceof Binary[])) { - throw new IllegalArgumentException( - "The data type of column index " + columnIndex + " is not TEXT/STRING/BLOB"); - } final Binary[] sensor = (Binary[]) values[columnIndex]; sensor[rowIndex] = new Binary(val); updateBitMap(rowIndex, columnIndex, false); @@ -530,10 +504,6 @@ public class Tablet { @TsFileApi public void addValue(int rowIndex, int columnIndex, LocalDate val) { - if (!(values[columnIndex] instanceof LocalDate[])) { - throw new IllegalArgumentException( - "The data type of column index " + columnIndex + " is not DATE"); - } final LocalDate[] sensor = (LocalDate[]) values[columnIndex]; sensor[rowIndex] = val; updateBitMap(rowIndex, columnIndex, false); @@ -551,15 +521,6 @@ public class Tablet { } private void updateBitMap(int rowIndex, int columnIndex, boolean mark) { - initBitMapsWithApiUsage(); - if (mark) { - bitMaps[columnIndex].mark(rowIndex); - } else { - bitMaps[columnIndex].unmark(rowIndex); - } - } - - private void initBitMapsWithApiUsage() { if (bitMaps == null) { initBitMaps(); } @@ -569,6 +530,11 @@ public class Tablet { bitMap.markAll(); } } + if (mark) { + bitMaps[columnIndex].mark(rowIndex); + } else { + bitMaps[columnIndex].unmark(rowIndex); + } } public List<IMeasurementSchema> getSchemas() { @@ -1229,34 +1195,6 @@ public class Tablet { this.rowSize = rowSize; } - public long getTimestamp(int i) { - return timestamps[i]; - } - - public long[] getTimestamps() { - return timestamps; - } - - public void setTimestamps(long[] timestamps) { - this.timestamps = timestamps; - } - - public Object[] getValues() { - return values; - } - - public void setValues(Object[] values) { - this.values = values; - } - - public BitMap[] getBitMaps() { - return bitMaps; - } - - public void setBitMaps(BitMap[] bitMaps) { - this.bitMaps = bitMaps; - } - public enum ColumnCategory { TAG, FIELD, @@ -1309,13 +1247,4 @@ public class Tablet { public List<ColumnCategory> getColumnTypes() { return columnCategories; } - - public boolean isSorted() { - for (int i = 1; i < rowSize; i++) { - if (timestamps[i] < timestamps[i - 1]) { - return false; - } - } - return true; - } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java b/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java index 9f454d4e..1cef0a40 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java @@ -274,13 +274,15 @@ public class PerformanceTest { private void fillTreeTablet(Tablet tablet, int tableNum, int deviceNum, int tabletNum) { tablet.setDeviceId(genTreeDeviceId(tableNum, deviceNum).toString()); for (int i = 0; i < measurementSchemaCnt; i++) { + long[] values = (long[]) tablet.values[i]; for (int valNum = 0; valNum < pointPerSeries; valNum++) { - tablet.addValue(valNum, i, tabletNum * pointPerSeries + valNum); + values[valNum] = (long) tabletNum * pointPerSeries + valNum; } } for (int valNum = 0; valNum < pointPerSeries; valNum++) { - tablet.addTimestamp(valNum, tabletNum * pointPerSeries + valNum); + tablet.timestamps[valNum] = (long) tabletNum * pointPerSeries + valNum; } + tablet.setRowSize(pointPerSeries); } private Tablet initTableTablet() { @@ -301,18 +303,21 @@ public class PerformanceTest { IDeviceID deviceID = genTableDeviceId(tableNum, deviceNum); tablet.setTableName(deviceID.segment(0).toString()); for (int i = 0; i < idSchemaCnt; i++) { + String[] strings = ((String[]) tablet.values[i]); for (int rowNum = 0; rowNum < pointPerSeries; rowNum++) { - tablet.addValue(rowNum, i, deviceID.segment(i + 1).toString()); + strings[rowNum] = deviceID.segment(i + 1).toString(); } } for (int i = 0; i < measurementSchemaCnt; i++) { + long[] values = (long[]) tablet.values[i + idSchemaCnt]; for (int valNum = 0; valNum < pointPerSeries; valNum++) { - tablet.addValue(valNum, i + idSchemaCnt, tabletNum * pointPerSeries + valNum); + values[valNum] = (long) tabletNum * pointPerSeries + valNum; } } for (int valNum = 0; valNum < pointPerSeries; valNum++) { - tablet.addTimestamp(valNum, tabletNum * pointPerSeries + valNum); + tablet.timestamps[valNum] = (long) tabletNum * pointPerSeries + valNum; } + tablet.setRowSize(pointPerSeries); } private void registerTree(TsFileWriter writer) throws WriteProcessException { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/DefaultSchemaTemplateTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/DefaultSchemaTemplateTest.java index 8a9351a0..3ff19a1b 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/DefaultSchemaTemplateTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/DefaultSchemaTemplateTest.java @@ -66,6 +66,8 @@ public class DefaultSchemaTemplateTest { writer.registerDevice("d1", "defaultTemplate"); Tablet tablet = new Tablet("d1", schemaList); + long[] timestamps = tablet.timestamps; + Object[] values = tablet.values; long timestamp = 1; long value = 1L; @@ -74,7 +76,8 @@ public class DefaultSchemaTemplateTest { int row = tablet.getRowSize(); tablet.addTimestamp(row, timestamp++); for (int i = 0; i < 2; i++) { - tablet.addValue(row, i, value); + long[] sensor = (long[]) values[i]; + sensor[row] = value; } // write Tablet to TsFile if (tablet.getRowSize() == tablet.getMaxRowNumber()) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java index 79908270..32cbb6e3 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java @@ -477,13 +477,16 @@ public class MetadataIndexConstructorTest { // add measurements into TSFileWriter // construct the tablet Tablet tablet = new Tablet(device.toString(), tabletSchema); + long[] timestamps = tablet.timestamps; + Object[] values = tablet.values; long timestamp = 1; long value = 1000000L; for (int r = 0; r < rowNum; r++, value++) { int row = tablet.getRowSize(); tablet.addTimestamp(row, timestamp++); for (int j = 0; j < measurementNum; j++) { - tablet.addValue(row, j, value); + long[] sensor = (long[]) values[j]; + sensor[row] = value; } // write Tablet to TsFile if (tablet.getRowSize() == tablet.getMaxRowNumber()) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java index c5ecab75..914e25b3 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java @@ -18,9 +18,9 @@ */ package org.apache.tsfile.write; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.exception.read.ReadProcessException; import org.apache.tsfile.exception.write.WriteProcessException; import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; @@ -37,9 +37,7 @@ import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.read.common.Path; import org.apache.tsfile.read.expression.QueryExpression; import org.apache.tsfile.read.query.dataset.QueryDataSet; -import org.apache.tsfile.read.query.dataset.ResultSet; -import org.apache.tsfile.read.v4.ITsFileReader; -import org.apache.tsfile.read.v4.TsFileReaderBuilder; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsFileGeneratorUtils; import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; import org.apache.tsfile.write.chunk.ChunkWriterImpl; @@ -49,8 +47,6 @@ import org.apache.tsfile.write.record.datapoint.DateDataPoint; import org.apache.tsfile.write.record.datapoint.StringDataPoint; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; -import org.apache.tsfile.write.v4.ITsFileWriter; -import org.apache.tsfile.write.v4.TsFileWriterBuilder; import org.apache.tsfile.write.writer.TsFileIOWriter; import org.junit.After; @@ -408,6 +404,9 @@ public class TsFileWriteApiTest { tsFileWriter.registerTimeseries(new Path(deviceId), measurementSchemas); Tablet tablet = new Tablet(deviceId, measurementSchemas); + long[] timestamps = tablet.timestamps; + Object[] values = tablet.values; + tablet.initBitMaps(); int sensorNum = measurementSchemas.size(); long startTime = 0; for (long r = 0; r < 10000; r++) { @@ -415,15 +414,17 @@ public class TsFileWriteApiTest { tablet.addTimestamp(row, startTime++); for (int i = 0; i < sensorNum - 1; i++) { if (i == 1 && r > 1000) { - tablet.getBitMaps()[i].mark((int) r % tablet.getMaxRowNumber()); + tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber()); continue; } - tablet.addValue(row, i, "testString........."); + Binary[] textSensor = (Binary[]) values[i]; + textSensor[row] = new Binary("testString.........", TSFileConfig.STRING_CHARSET); } if (r > 1000) { - tablet.getBitMaps()[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); + tablet.bitMaps[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); } else { - tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1)); + LocalDate[] dateSensor = (LocalDate[]) values[sensorNum - 1]; + dateSensor[row] = LocalDate.of(2024, 4, 1); } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { @@ -456,6 +457,8 @@ public class TsFileWriteApiTest { tsFileWriter.registerTimeseries(new Path(deviceId), measurementSchemas); Tablet tablet = new Tablet(deviceId, measurementSchemas); + long[] timestamps = tablet.timestamps; + Object[] values = tablet.values; tablet.initBitMaps(); int sensorNum = measurementSchemas.size(); long startTime = -100; @@ -464,15 +467,17 @@ public class TsFileWriteApiTest { tablet.addTimestamp(row, startTime++); for (int i = 0; i < sensorNum - 1; i++) { if (i == 1 && r > 1000) { - tablet.getBitMaps()[i].mark((int) r % tablet.getMaxRowNumber()); + tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber()); continue; } - tablet.addValue(row, i, "testString........."); + Binary[] textSensor = (Binary[]) values[i]; + textSensor[row] = new Binary("testString.........", TSFileConfig.STRING_CHARSET); } if (r > 1000) { - tablet.getBitMaps()[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); + tablet.bitMaps[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); } else { - tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1)); + LocalDate[] dateSensor = (LocalDate[]) values[sensorNum - 1]; + dateSensor[row] = LocalDate.of(2024, 4, 1); } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { @@ -505,23 +510,28 @@ public class TsFileWriteApiTest { tsFileWriter.registerAlignedTimeseries(new Path(deviceId), measurementSchemas); Tablet tablet = new Tablet(deviceId, measurementSchemas); + long[] timestamps = tablet.timestamps; + Object[] values = tablet.values; tablet.initBitMaps(); int sensorNum = measurementSchemas.size(); long startTime = 0; for (long r = 0; r < 10000; r++) { int row = tablet.getRowSize(); tablet.addTimestamp(row, startTime++); + timestamps[row] = startTime++; for (int i = 0; i < sensorNum - 1; i++) { if (i == 1 && r > 1000) { - tablet.getBitMaps()[i].mark((int) r % tablet.getMaxRowNumber()); + tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber()); continue; } - tablet.addValue(row, i, "testString........."); + Binary[] textSensor = (Binary[]) values[i]; + textSensor[row] = new Binary("testString.........", TSFileConfig.STRING_CHARSET); } if (r > 1000) { - tablet.getBitMaps()[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); + tablet.bitMaps[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); } else { - tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1)); + LocalDate[] dateSensor = (LocalDate[]) values[sensorNum - 1]; + dateSensor[row] = LocalDate.of(2024, 4, 1); } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { @@ -579,6 +589,8 @@ public class TsFileWriteApiTest { tsFileWriter.registerAlignedTimeseries(new Path(deviceId), measurementSchemas); Tablet tablet = new Tablet(deviceId, measurementSchemas); + long[] timestamps = tablet.timestamps; + Object[] values = tablet.values; tablet.initBitMaps(); int sensorNum = measurementSchemas.size(); long startTime = -1000; @@ -587,15 +599,17 @@ public class TsFileWriteApiTest { tablet.addTimestamp(row, startTime++); for (int i = 0; i < sensorNum - 1; i++) { if (i == 1 && r > 1000) { - tablet.getBitMaps()[i].mark((int) r % tablet.getMaxRowNumber()); + tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber()); continue; } - tablet.addValue(row, i, "testString........."); + Binary[] textSensor = (Binary[]) values[i]; + textSensor[row] = new Binary("testString.........", TSFileConfig.STRING_CHARSET); } if (r > 1000) { - tablet.getBitMaps()[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); + tablet.bitMaps[sensorNum - 1].mark((int) r % tablet.getMaxRowNumber()); } else { - tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1)); + LocalDate[] dateSensor = (LocalDate[]) values[sensorNum - 1]; + dateSensor[row] = LocalDate.of(2024, 4, 1); } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { @@ -933,42 +947,4 @@ public class TsFileWriteApiTest { Assert.assertTrue(reader.getAllMeasurements().containsKey("measurementcolumn")); } } - - @Test - public void writeAllNullValueTablet() - throws IOException, WriteProcessException, ReadProcessException { - setEnv(100 * 1024 * 1024, 10 * 1024); - Tablet tablet = - new Tablet( - "table1", - Arrays.asList("tag1", "field1"), - Arrays.asList(TSDataType.STRING, TSDataType.BOOLEAN), - Arrays.asList(Tablet.ColumnCategory.TAG, Tablet.ColumnCategory.FIELD)); - tablet.addTimestamp(0, 0); - tablet.addTimestamp(1, 1); - TableSchema tableSchema = - new TableSchema( - "Table1", - Arrays.asList( - new ColumnSchema("tag1", TSDataType.STRING, Tablet.ColumnCategory.TAG), - new ColumnSchema("field1", TSDataType.BOOLEAN, Tablet.ColumnCategory.FIELD))); - Assert.assertEquals("table1", tableSchema.getTableName()); - try (ITsFileWriter writer = - new TsFileWriterBuilder().file(f).tableSchema(tableSchema).build()) { - writer.write(tablet); - } - try (ITsFileReader reader = new TsFileReaderBuilder().file(f).build(); - ResultSet resultSet = - reader.query( - "table1", Arrays.asList("tag1", "field1"), Long.MIN_VALUE, Long.MAX_VALUE)) { - Assert.assertTrue(resultSet.next()); - Assert.assertEquals(0, resultSet.getLong(1)); - Assert.assertTrue(resultSet.isNull(2)); - Assert.assertTrue(resultSet.isNull(3)); - Assert.assertTrue(resultSet.next()); - Assert.assertEquals(1, resultSet.getLong(1)); - Assert.assertTrue(resultSet.isNull(2)); - Assert.assertTrue(resultSet.isNull(3)); - } - } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java index baf979a7..676aecfc 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java @@ -244,9 +244,10 @@ public class TsFileWriterTest { "s1", TSDataType.FLOAT, TSEncoding.RLE, CompressionType.SNAPPY), new MeasurementSchema( "s2", TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY))); - tablet.addTimestamp(0, 10000); - tablet.addValue(0, 0, 5.0f); - tablet.addValue(0, 1, 5); + tablet.timestamps[0] = 10000; + ((float[]) tablet.values[0])[0] = 5.0f; + ((int[]) tablet.values[1])[0] = 5; + tablet.setRowSize(1); writer.writeTree(tablet); closeFile(); readOneRow(); @@ -262,9 +263,9 @@ public class TsFileWriterTest { "s1", TSDataType.FLOAT, TSEncoding.RLE, CompressionType.SNAPPY), new MeasurementSchema( "s2", TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY))); - tablet.addTimestamp(0, 10000); - tablet.addValue(0, 0, 5.0f); - tablet.addValue(0, 1, 0); + tablet.timestamps[0] = 10000; + ((float[]) tablet.values[0])[0] = 5.0f; + tablet.setRowSize(1); writer.writeTree(tablet); closeFile(); // in this case, the value of s2 = 0 at time 10000. diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java index 07aedad4..a73e22f9 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java @@ -66,21 +66,21 @@ public class TabletTest { Assert.assertFalse((Boolean) tablet.getValue(1, 0)); Assert.assertTrue((Boolean) tablet.getValue(1, 1)); Assert.assertTrue((Boolean) tablet.getValue(2, 0)); - Assert.assertFalse(tablet.getBitMaps()[0].isMarked(0)); - Assert.assertFalse(tablet.getBitMaps()[0].isMarked(1)); - Assert.assertFalse(tablet.getBitMaps()[0].isMarked(2)); - Assert.assertFalse(tablet.getBitMaps()[1].isMarked(0)); - Assert.assertFalse(tablet.getBitMaps()[1].isMarked(1)); - Assert.assertTrue(tablet.getBitMaps()[1].isMarked(2)); + Assert.assertFalse(tablet.bitMaps[0].isMarked(0)); + Assert.assertFalse(tablet.bitMaps[0].isMarked(1)); + Assert.assertFalse(tablet.bitMaps[0].isMarked(2)); + Assert.assertFalse(tablet.bitMaps[1].isMarked(0)); + Assert.assertFalse(tablet.bitMaps[1].isMarked(1)); + Assert.assertTrue(tablet.bitMaps[1].isMarked(2)); tablet.addTimestamp(9, 9); Assert.assertEquals(10, tablet.getRowSize()); tablet.reset(); Assert.assertEquals(0, tablet.getRowSize()); - Assert.assertTrue(tablet.getBitMaps()[0].isAllMarked()); - Assert.assertTrue(tablet.getBitMaps()[0].isAllMarked()); - Assert.assertTrue(tablet.getBitMaps()[0].isAllMarked()); + Assert.assertTrue(tablet.bitMaps[0].isAllMarked()); + Assert.assertTrue(tablet.bitMaps[0].isAllMarked()); + Assert.assertTrue(tablet.bitMaps[0].isAllMarked()); } @Test @@ -163,7 +163,7 @@ public class TabletTest { i, LocalDate.of(2000 + i, i / 100 + 1, i / 100 + 1)); - tablet.getBitMaps()[i % measurementSchemas.size()].mark(i); + tablet.bitMaps[i % measurementSchemas.size()].mark(i); } // Test add null
