This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new afd6ec4a Modify tablet usage (#358)
afd6ec4a is described below
commit afd6ec4a70341ec00e3567fdbc19d88cf940830e
Author: shuwenwei <[email protected]>
AuthorDate: Thu Jan 9 11:08:28 2025 +0800
Modify tablet usage (#358)
* modify tablet usage
* modify tablet
* modify tablet
* modify tablet
* revert pom
* add ut
* modify exception message
---
.../tsfile/TsFileWriteAlignedWithTablet.java | 5 +-
.../org/apache/tsfile/TsFileWriteWithTablet.java | 3 +-
.../read/query/dataset/AbstractResultSet.java | 2 +-
.../apache/tsfile/utils/TsFileGeneratorUtils.java | 5 +-
.../write/chunk/AlignedChunkGroupWriterImpl.java | 23 +++---
.../chunk/NonAlignedChunkGroupWriterImpl.java | 31 ++++---
.../org/apache/tsfile/write/record/Tablet.java | 93 ++++++++++++++++++---
.../apache/tsfile/tableview/PerformanceTest.java | 15 ++--
.../tsfile/write/DefaultSchemaTemplateTest.java | 5 +-
.../tsfile/write/MetadataIndexConstructorTest.java | 5 +-
.../apache/tsfile/write/TsFileWriteApiTest.java | 96 ++++++++++++++--------
.../org/apache/tsfile/write/TsFileWriterTest.java | 13 ++-
.../org/apache/tsfile/write/record/TabletTest.java | 20 ++---
13 files changed, 201 insertions(+), 115 deletions(-)
diff --git
a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java
b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java
index 784df63b..ca13fdb3 100644
---
a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java
+++
b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java
@@ -133,8 +133,6 @@ public class TsFileWriteAlignedWithTablet {
measurementSchemas.add(new MeasurementSchema(SENSOR_1, TSDataType.INT64,
TSEncoding.RLE));
measurementSchemas.add(new MeasurementSchema(SENSOR_2, TSDataType.INT64,
TSEncoding.RLE));
Tablet tablet = new Tablet(DEVICE_2, measurementSchemas);
- long[] timestamps = tablet.timestamps;
- Object[] values = tablet.values;
int rowNum = 100;
int sensorNum = measurementSchemas.size();
long timestamp = 1;
@@ -143,8 +141,7 @@ public class TsFileWriteAlignedWithTablet {
int row = tablet.getRowSize();
tablet.addTimestamp(row, timestamp++);
for (int i = 0; i < sensorNum; i++) {
- long[] sensor = (long[]) values[i];
- sensor[row] = value;
+ tablet.addValue(row, i, value);
}
// write
if (tablet.getRowSize() == tablet.getMaxRowNumber()) {
diff --git
a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java
b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java
index bb0293a5..05446ce8 100644
--- a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java
+++ b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java
@@ -88,12 +88,11 @@ public class TsFileWriteWithTablet {
long startValue)
throws IOException, WriteProcessException {
Tablet tablet = new Tablet(deviceId, schemas);
- long[] timestamps = tablet.timestamps;
long sensorNum = schemas.size();
for (long r = 0; r < rowNum; r++, startValue++) {
int row = tablet.getRowSize();
- timestamps[row] = startTime++;
+ tablet.addTimestamp(row, startTime++);
for (int i = 0; i < sensorNum; i++) {
tablet.addValue(
schemas.get(i).getMeasurementName(),
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/AbstractResultSet.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/AbstractResultSet.java
index 5b07dd46..10352790 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/AbstractResultSet.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/AbstractResultSet.java
@@ -168,7 +168,7 @@ public abstract class AbstractResultSet implements
ResultSet {
}
protected Field getField(int columnIndex) {
- if (columnIndex > this.columnNameToColumnIndexMap.size()) {
+ if (columnIndex > this.columnNameToColumnIndexMap.size() || columnIndex <=
0) {
throw new IndexOutOfBoundsException("column index " + columnIndex + "
out of bound");
}
Field field;
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java
b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java
index 6b87f6e1..e4b8abcd 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java
@@ -117,16 +117,13 @@ public class TsFileGeneratorUtils {
boolean isAligned)
throws IOException, WriteProcessException {
Tablet tablet = new Tablet(deviceId, schemas);
- long[] timestamps = tablet.timestamps;
- Object[] values = tablet.values;
long sensorNum = schemas.size();
for (long r = 0; r < rowNum; r++, startValue++) {
int row = tablet.getRowSize();
tablet.addTimestamp(row, startTime++);
for (int i = 0; i < sensorNum; i++) {
- long[] sensor = (long[]) values[i];
- sensor[row] = startValue;
+ tablet.addValue(row, i, startValue);
}
// write
if (tablet.getRowSize() == tablet.getMaxRowNumber()) {
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
index 0b53b75a..c051a702 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
@@ -233,7 +233,7 @@ public class AlignedChunkGroupWriterImpl implements
IChunkGroupWriter {
// TODO: changing to a column-first style by calculating the remaining
page space of each
// column firsts
for (int row = startRowIndex; row < endRowIndex; row++) {
- long time = tablet.timestamps[row];
+ long time = tablet.getTimestamps()[row];
checkIsHistoryData(time);
for (int columnIndex = 0; columnIndex < tablet.getSchemas().size();
columnIndex++) {
if (tablet.getColumnTypes() != null
@@ -242,18 +242,19 @@ public class AlignedChunkGroupWriterImpl implements
IChunkGroupWriter {
}
boolean isNull =
- tablet.bitMaps != null
- && tablet.bitMaps[columnIndex] != null
- && tablet.bitMaps[columnIndex].isMarked(row);
+ tablet.getBitMaps() != null
+ && tablet.getBitMaps()[columnIndex] != null
+ && tablet.getBitMaps()[columnIndex].isMarked(row);
// check isNull by bitMap in tablet
ValueChunkWriter valueChunkWriter =
tryToAddSeriesWriterInternal(measurementSchemas.get(columnIndex));
switch (measurementSchemas.get(columnIndex).getType()) {
case BOOLEAN:
- valueChunkWriter.write(time, ((boolean[])
tablet.values[columnIndex])[row], isNull);
+ valueChunkWriter.write(
+ time, ((boolean[]) tablet.getValues()[columnIndex])[row],
isNull);
break;
case INT32:
- valueChunkWriter.write(time, ((int[])
tablet.values[columnIndex])[row], isNull);
+ valueChunkWriter.write(time, ((int[])
tablet.getValues()[columnIndex])[row], isNull);
break;
case DATE:
valueChunkWriter.write(
@@ -261,23 +262,23 @@ public class AlignedChunkGroupWriterImpl implements
IChunkGroupWriter {
isNull
? 0
: DateUtils.parseDateExpressionToInt(
- ((LocalDate[]) tablet.values[columnIndex])[row]),
+ ((LocalDate[]) tablet.getValues()[columnIndex])[row]),
isNull);
break;
case INT64:
case TIMESTAMP:
- valueChunkWriter.write(time, ((long[])
tablet.values[columnIndex])[row], isNull);
+ valueChunkWriter.write(time, ((long[])
tablet.getValues()[columnIndex])[row], isNull);
break;
case FLOAT:
- valueChunkWriter.write(time, ((float[])
tablet.values[columnIndex])[row], isNull);
+ valueChunkWriter.write(time, ((float[])
tablet.getValues()[columnIndex])[row], isNull);
break;
case DOUBLE:
- valueChunkWriter.write(time, ((double[])
tablet.values[columnIndex])[row], isNull);
+ valueChunkWriter.write(time, ((double[])
tablet.getValues()[columnIndex])[row], isNull);
break;
case TEXT:
case BLOB:
case STRING:
- valueChunkWriter.write(time, ((Binary[])
tablet.values[columnIndex])[row], isNull);
+ valueChunkWriter.write(time, ((Binary[])
tablet.getValues()[columnIndex])[row], isNull);
break;
default:
throw new UnSupportedDataTypeException(
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java
index 3f4d8aa0..cc20b252 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java
@@ -121,42 +121,51 @@ public class NonAlignedChunkGroupWriterImpl implements
IChunkGroupWriter {
pointCount = 0;
for (int row = startRowIndex; row < endRowIndex; row++) {
// check isNull in tablet
- if (tablet.bitMaps != null
- && tablet.bitMaps[column] != null
- && tablet.bitMaps[column].isMarked(row)) {
+ if (tablet.getBitMaps() != null
+ && tablet.getBitMaps()[column] != null
+ && tablet.getBitMaps()[column].isMarked(row)) {
continue;
}
- long time = tablet.timestamps[row];
+ long time = tablet.getTimestamps()[row];
checkIsHistoryData(measurementId, time);
pointCount++;
switch (tsDataType) {
case INT32:
- chunkWriters.get(measurementId).write(time, ((int[])
tablet.values[column])[row]);
+ chunkWriters.get(measurementId).write(time, ((int[])
tablet.getValues()[column])[row]);
break;
case DATE:
chunkWriters
.get(measurementId)
.write(
time,
- DateUtils.parseDateExpressionToInt(((LocalDate[])
tablet.values[column])[row]));
+ DateUtils.parseDateExpressionToInt(
+ ((LocalDate[]) tablet.getValues()[column])[row]));
break;
case INT64:
case TIMESTAMP:
- chunkWriters.get(measurementId).write(time, ((long[])
tablet.values[column])[row]);
+ chunkWriters.get(measurementId).write(time, ((long[])
tablet.getValues()[column])[row]);
break;
case FLOAT:
- chunkWriters.get(measurementId).write(time, ((float[])
tablet.values[column])[row]);
+ chunkWriters
+ .get(measurementId)
+ .write(time, ((float[]) tablet.getValues()[column])[row]);
break;
case DOUBLE:
- chunkWriters.get(measurementId).write(time, ((double[])
tablet.values[column])[row]);
+ chunkWriters
+ .get(measurementId)
+ .write(time, ((double[]) tablet.getValues()[column])[row]);
break;
case BOOLEAN:
- chunkWriters.get(measurementId).write(time, ((boolean[])
tablet.values[column])[row]);
+ chunkWriters
+ .get(measurementId)
+ .write(time, ((boolean[]) tablet.getValues()[column])[row]);
break;
case TEXT:
case BLOB:
case STRING:
- chunkWriters.get(measurementId).write(time, ((Binary[])
tablet.values[column])[row]);
+ chunkWriters
+ .get(measurementId)
+ .write(time, ((Binary[]) tablet.getValues()[column])[row]);
break;
default:
throw new UnSupportedDataTypeException(
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
index 2580a2a4..93e3f1e9 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
@@ -80,14 +80,11 @@ public class Tablet {
/** MeasurementId->indexOf({@link MeasurementSchema}) */
private final Map<String, Integer> measurementIndex;
- /** Timestamps in this {@link Tablet} */
- public long[] timestamps;
+ private long[] timestamps;
- /** Each object is a primitive type array, which represents values of one
measurement */
- public Object[] values;
+ private Object[] values;
- /** Each {@link BitMap} represents the existence of each value in the
current column. */
- public BitMap[] bitMaps;
+ private BitMap[] bitMaps;
/**
* For compatibility with the usage of directly modifying Tablet content
through public fields.
@@ -292,6 +289,7 @@ public class Tablet {
public void addTimestamp(int rowIndex, long timestamp) {
timestamps[rowIndex] = timestamp;
this.rowSize = Math.max(this.rowSize, rowIndex + 1);
+ initBitMapsWithApiUsage();
}
public void addValue(final String measurementId, final int rowIndex, final
Object value) {
@@ -413,6 +411,10 @@ public class Tablet {
@TsFileApi
public void addValue(int rowIndex, int columnIndex, int val) {
+ if (!(values[columnIndex] instanceof int[])) {
+ throw new IllegalArgumentException(
+ "The data type of column index " + columnIndex + " is not INT32");
+ }
final int[] sensor = (int[]) values[columnIndex];
sensor[rowIndex] = val;
updateBitMap(rowIndex, columnIndex, false);
@@ -426,6 +428,10 @@ public class Tablet {
@TsFileApi
public void addValue(int rowIndex, int columnIndex, long val) {
+ if (!(values[columnIndex] instanceof long[])) {
+ throw new IllegalArgumentException(
+ "The data type of column index " + columnIndex + " is not
INT64/TIMESTAMP");
+ }
final long[] sensor = (long[]) values[columnIndex];
sensor[rowIndex] = val;
updateBitMap(rowIndex, columnIndex, false);
@@ -439,6 +445,10 @@ public class Tablet {
@TsFileApi
public void addValue(int rowIndex, int columnIndex, float val) {
+ if (!(values[columnIndex] instanceof float[])) {
+ throw new IllegalArgumentException(
+ "The data type of column index " + columnIndex + " is not FLOAT");
+ }
final float[] sensor = (float[]) values[columnIndex];
sensor[rowIndex] = val;
updateBitMap(rowIndex, columnIndex, false);
@@ -452,6 +462,10 @@ public class Tablet {
@TsFileApi
public void addValue(int rowIndex, int columnIndex, double val) {
+ if (!(values[columnIndex] instanceof double[])) {
+ throw new IllegalArgumentException(
+ "The data type of column index " + columnIndex + " is not DOUBLE");
+ }
final double[] sensor = (double[]) values[columnIndex];
sensor[rowIndex] = val;
updateBitMap(rowIndex, columnIndex, false);
@@ -465,6 +479,10 @@ public class Tablet {
@TsFileApi
public void addValue(int rowIndex, int columnIndex, boolean val) {
+ if (!(values[columnIndex] instanceof boolean[])) {
+ throw new IllegalArgumentException(
+ "The data type of column index " + columnIndex + " is not BOOLEAN");
+ }
final boolean[] sensor = (boolean[]) values[columnIndex];
sensor[rowIndex] = val;
updateBitMap(rowIndex, columnIndex, false);
@@ -478,6 +496,10 @@ public class Tablet {
@TsFileApi
public void addValue(int rowIndex, int columnIndex, String val) {
+ if (!(values[columnIndex] instanceof Binary[])) {
+ throw new IllegalArgumentException(
+ "The data type of column index " + columnIndex + " is not
TEXT/STRING/BLOB");
+ }
final Binary[] sensor = (Binary[]) values[columnIndex];
sensor[rowIndex] = new Binary(val, TSFileConfig.STRING_CHARSET);
updateBitMap(rowIndex, columnIndex, false);
@@ -491,6 +513,10 @@ public class Tablet {
@TsFileApi
public void addValue(int rowIndex, int columnIndex, byte[] val) {
+ if (!(values[columnIndex] instanceof Binary[])) {
+ throw new IllegalArgumentException(
+ "The data type of column index " + columnIndex + " is not
TEXT/STRING/BLOB");
+ }
final Binary[] sensor = (Binary[]) values[columnIndex];
sensor[rowIndex] = new Binary(val);
updateBitMap(rowIndex, columnIndex, false);
@@ -504,6 +530,10 @@ public class Tablet {
@TsFileApi
public void addValue(int rowIndex, int columnIndex, LocalDate val) {
+ if (!(values[columnIndex] instanceof LocalDate[])) {
+ throw new IllegalArgumentException(
+ "The data type of column index " + columnIndex + " is not DATE");
+ }
final LocalDate[] sensor = (LocalDate[]) values[columnIndex];
sensor[rowIndex] = val;
updateBitMap(rowIndex, columnIndex, false);
@@ -521,6 +551,15 @@ public class Tablet {
}
private void updateBitMap(int rowIndex, int columnIndex, boolean mark) {
+ initBitMapsWithApiUsage();
+ if (mark) {
+ bitMaps[columnIndex].mark(rowIndex);
+ } else {
+ bitMaps[columnIndex].unmark(rowIndex);
+ }
+ }
+
+ private void initBitMapsWithApiUsage() {
if (bitMaps == null) {
initBitMaps();
}
@@ -530,11 +569,6 @@ public class Tablet {
bitMap.markAll();
}
}
- if (mark) {
- bitMaps[columnIndex].mark(rowIndex);
- } else {
- bitMaps[columnIndex].unmark(rowIndex);
- }
}
public List<IMeasurementSchema> getSchemas() {
@@ -1195,6 +1229,34 @@ public class Tablet {
this.rowSize = rowSize;
}
+ public long getTimestamp(int i) {
+ return timestamps[i];
+ }
+
+ public long[] getTimestamps() {
+ return timestamps;
+ }
+
+ public void setTimestamps(long[] timestamps) {
+ this.timestamps = timestamps;
+ }
+
+ public Object[] getValues() {
+ return values;
+ }
+
+ public void setValues(Object[] values) {
+ this.values = values;
+ }
+
+ public BitMap[] getBitMaps() {
+ return bitMaps;
+ }
+
+ public void setBitMaps(BitMap[] bitMaps) {
+ this.bitMaps = bitMaps;
+ }
+
public enum ColumnCategory {
TAG,
FIELD,
@@ -1247,4 +1309,13 @@ public class Tablet {
public List<ColumnCategory> getColumnTypes() {
return columnCategories;
}
+
+ public boolean isSorted() {
+ for (int i = 1; i < rowSize; i++) {
+ if (timestamps[i] < timestamps[i - 1]) {
+ return false;
+ }
+ }
+ return true;
+ }
}
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java
index 1cef0a40..9f454d4e 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java
@@ -274,15 +274,13 @@ public class PerformanceTest {
private void fillTreeTablet(Tablet tablet, int tableNum, int deviceNum, int
tabletNum) {
tablet.setDeviceId(genTreeDeviceId(tableNum, deviceNum).toString());
for (int i = 0; i < measurementSchemaCnt; i++) {
- long[] values = (long[]) tablet.values[i];
for (int valNum = 0; valNum < pointPerSeries; valNum++) {
- values[valNum] = (long) tabletNum * pointPerSeries + valNum;
+ tablet.addValue(valNum, i, tabletNum * pointPerSeries + valNum);
}
}
for (int valNum = 0; valNum < pointPerSeries; valNum++) {
- tablet.timestamps[valNum] = (long) tabletNum * pointPerSeries + valNum;
+ tablet.addTimestamp(valNum, tabletNum * pointPerSeries + valNum);
}
- tablet.setRowSize(pointPerSeries);
}
private Tablet initTableTablet() {
@@ -303,21 +301,18 @@ public class PerformanceTest {
IDeviceID deviceID = genTableDeviceId(tableNum, deviceNum);
tablet.setTableName(deviceID.segment(0).toString());
for (int i = 0; i < idSchemaCnt; i++) {
- String[] strings = ((String[]) tablet.values[i]);
for (int rowNum = 0; rowNum < pointPerSeries; rowNum++) {
- strings[rowNum] = deviceID.segment(i + 1).toString();
+ tablet.addValue(rowNum, i, deviceID.segment(i + 1).toString());
}
}
for (int i = 0; i < measurementSchemaCnt; i++) {
- long[] values = (long[]) tablet.values[i + idSchemaCnt];
for (int valNum = 0; valNum < pointPerSeries; valNum++) {
- values[valNum] = (long) tabletNum * pointPerSeries + valNum;
+ tablet.addValue(valNum, i + idSchemaCnt, tabletNum * pointPerSeries +
valNum);
}
}
for (int valNum = 0; valNum < pointPerSeries; valNum++) {
- tablet.timestamps[valNum] = (long) tabletNum * pointPerSeries + valNum;
+ tablet.addTimestamp(valNum, tabletNum * pointPerSeries + valNum);
}
- tablet.setRowSize(pointPerSeries);
}
private void registerTree(TsFileWriter writer) throws WriteProcessException {
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/write/DefaultSchemaTemplateTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/write/DefaultSchemaTemplateTest.java
index 3ff19a1b..8a9351a0 100644
---
a/java/tsfile/src/test/java/org/apache/tsfile/write/DefaultSchemaTemplateTest.java
+++
b/java/tsfile/src/test/java/org/apache/tsfile/write/DefaultSchemaTemplateTest.java
@@ -66,8 +66,6 @@ public class DefaultSchemaTemplateTest {
writer.registerDevice("d1", "defaultTemplate");
Tablet tablet = new Tablet("d1", schemaList);
- long[] timestamps = tablet.timestamps;
- Object[] values = tablet.values;
long timestamp = 1;
long value = 1L;
@@ -76,8 +74,7 @@ public class DefaultSchemaTemplateTest {
int row = tablet.getRowSize();
tablet.addTimestamp(row, timestamp++);
for (int i = 0; i < 2; i++) {
- long[] sensor = (long[]) values[i];
- sensor[row] = value;
+ tablet.addValue(row, i, value);
}
// write Tablet to TsFile
if (tablet.getRowSize() == tablet.getMaxRowNumber()) {
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java
index 32cbb6e3..79908270 100644
---
a/java/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java
+++
b/java/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java
@@ -477,16 +477,13 @@ public class MetadataIndexConstructorTest {
// add measurements into TSFileWriter
// construct the tablet
Tablet tablet = new Tablet(device.toString(), tabletSchema);
- long[] timestamps = tablet.timestamps;
- Object[] values = tablet.values;
long timestamp = 1;
long value = 1000000L;
for (int r = 0; r < rowNum; r++, value++) {
int row = tablet.getRowSize();
tablet.addTimestamp(row, timestamp++);
for (int j = 0; j < measurementNum; j++) {
- long[] sensor = (long[]) values[j];
- sensor[row] = value;
+ tablet.addValue(row, j, value);
}
// write Tablet to TsFile
if (tablet.getRowSize() == tablet.getMaxRowNumber()) {
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
index 0a4d92f1..160866e4 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
@@ -18,9 +18,9 @@
*/
package org.apache.tsfile.write;
-import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.read.ReadProcessException;
import org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.file.MetaMarker;
import org.apache.tsfile.file.header.ChunkHeader;
@@ -37,7 +37,9 @@ import org.apache.tsfile.read.common.Chunk;
import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.read.expression.QueryExpression;
import org.apache.tsfile.read.query.dataset.QueryDataSet;
-import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.read.query.dataset.ResultSet;
+import org.apache.tsfile.read.v4.ITsFileReader;
+import org.apache.tsfile.read.v4.TsFileReaderBuilder;
import org.apache.tsfile.utils.TsFileGeneratorUtils;
import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl;
import org.apache.tsfile.write.chunk.ChunkWriterImpl;
@@ -47,6 +49,8 @@ import org.apache.tsfile.write.record.datapoint.DateDataPoint;
import org.apache.tsfile.write.record.datapoint.StringDataPoint;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.apache.tsfile.write.v4.ITsFileWriter;
+import org.apache.tsfile.write.v4.TsFileWriterBuilder;
import org.apache.tsfile.write.writer.TsFileIOWriter;
import org.junit.After;
@@ -404,9 +408,6 @@ public class TsFileWriteApiTest {
tsFileWriter.registerTimeseries(new Path(deviceId), measurementSchemas);
Tablet tablet = new Tablet(deviceId, measurementSchemas);
- long[] timestamps = tablet.timestamps;
- Object[] values = tablet.values;
- tablet.initBitMaps();
int sensorNum = measurementSchemas.size();
long startTime = 0;
for (long r = 0; r < 10000; r++) {
@@ -414,17 +415,15 @@ public class TsFileWriteApiTest {
tablet.addTimestamp(row, startTime++);
for (int i = 0; i < sensorNum - 1; i++) {
if (i == 1 && r > 1000) {
- tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber());
+ tablet.getBitMaps()[i].mark((int) r % tablet.getMaxRowNumber());
continue;
}
- Binary[] textSensor = (Binary[]) values[i];
- textSensor[row] = new Binary("testString.........",
TSFileConfig.STRING_CHARSET);
+ tablet.addValue(row, i, "testString.........");
}
if (r > 1000) {
- tablet.bitMaps[sensorNum - 1].mark((int) r %
tablet.getMaxRowNumber());
+ tablet.getBitMaps()[sensorNum - 1].mark((int) r %
tablet.getMaxRowNumber());
} else {
- LocalDate[] dateSensor = (LocalDate[]) values[sensorNum - 1];
- dateSensor[row] = LocalDate.of(2024, 4, 1);
+ tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1));
}
// write
if (tablet.getRowSize() == tablet.getMaxRowNumber()) {
@@ -457,8 +456,6 @@ public class TsFileWriteApiTest {
tsFileWriter.registerTimeseries(new Path(deviceId), measurementSchemas);
Tablet tablet = new Tablet(deviceId, measurementSchemas);
- long[] timestamps = tablet.timestamps;
- Object[] values = tablet.values;
tablet.initBitMaps();
int sensorNum = measurementSchemas.size();
long startTime = -100;
@@ -467,17 +464,15 @@ public class TsFileWriteApiTest {
tablet.addTimestamp(row, startTime++);
for (int i = 0; i < sensorNum - 1; i++) {
if (i == 1 && r > 1000) {
- tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber());
+ tablet.getBitMaps()[i].mark((int) r % tablet.getMaxRowNumber());
continue;
}
- Binary[] textSensor = (Binary[]) values[i];
- textSensor[row] = new Binary("testString.........",
TSFileConfig.STRING_CHARSET);
+ tablet.addValue(row, i, "testString.........");
}
if (r > 1000) {
- tablet.bitMaps[sensorNum - 1].mark((int) r %
tablet.getMaxRowNumber());
+ tablet.getBitMaps()[sensorNum - 1].mark((int) r %
tablet.getMaxRowNumber());
} else {
- LocalDate[] dateSensor = (LocalDate[]) values[sensorNum - 1];
- dateSensor[row] = LocalDate.of(2024, 4, 1);
+ tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1));
}
// write
if (tablet.getRowSize() == tablet.getMaxRowNumber()) {
@@ -510,28 +505,23 @@ public class TsFileWriteApiTest {
tsFileWriter.registerAlignedTimeseries(new Path(deviceId),
measurementSchemas);
Tablet tablet = new Tablet(deviceId, measurementSchemas);
- long[] timestamps = tablet.timestamps;
- Object[] values = tablet.values;
tablet.initBitMaps();
int sensorNum = measurementSchemas.size();
long startTime = 0;
for (long r = 0; r < 10000; r++) {
int row = tablet.getRowSize();
tablet.addTimestamp(row, startTime++);
- timestamps[row] = startTime++;
for (int i = 0; i < sensorNum - 1; i++) {
if (i == 1 && r > 1000) {
- tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber());
+ tablet.getBitMaps()[i].mark((int) r % tablet.getMaxRowNumber());
continue;
}
- Binary[] textSensor = (Binary[]) values[i];
- textSensor[row] = new Binary("testString.........",
TSFileConfig.STRING_CHARSET);
+ tablet.addValue(row, i, "testString.........");
}
if (r > 1000) {
- tablet.bitMaps[sensorNum - 1].mark((int) r %
tablet.getMaxRowNumber());
+ tablet.getBitMaps()[sensorNum - 1].mark((int) r %
tablet.getMaxRowNumber());
} else {
- LocalDate[] dateSensor = (LocalDate[]) values[sensorNum - 1];
- dateSensor[row] = LocalDate.of(2024, 4, 1);
+ tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1));
}
// write
if (tablet.getRowSize() == tablet.getMaxRowNumber()) {
@@ -589,8 +579,6 @@ public class TsFileWriteApiTest {
tsFileWriter.registerAlignedTimeseries(new Path(deviceId),
measurementSchemas);
Tablet tablet = new Tablet(deviceId, measurementSchemas);
- long[] timestamps = tablet.timestamps;
- Object[] values = tablet.values;
tablet.initBitMaps();
int sensorNum = measurementSchemas.size();
long startTime = -1000;
@@ -599,17 +587,15 @@ public class TsFileWriteApiTest {
tablet.addTimestamp(row, startTime++);
for (int i = 0; i < sensorNum - 1; i++) {
if (i == 1 && r > 1000) {
- tablet.bitMaps[i].mark((int) r % tablet.getMaxRowNumber());
+ tablet.getBitMaps()[i].mark((int) r % tablet.getMaxRowNumber());
continue;
}
- Binary[] textSensor = (Binary[]) values[i];
- textSensor[row] = new Binary("testString.........",
TSFileConfig.STRING_CHARSET);
+ tablet.addValue(row, i, "testString.........");
}
if (r > 1000) {
- tablet.bitMaps[sensorNum - 1].mark((int) r %
tablet.getMaxRowNumber());
+ tablet.getBitMaps()[sensorNum - 1].mark((int) r %
tablet.getMaxRowNumber());
} else {
- LocalDate[] dateSensor = (LocalDate[]) values[sensorNum - 1];
- dateSensor[row] = LocalDate.of(2024, 4, 1);
+ tablet.addValue(row, sensorNum - 1, LocalDate.of(2024, 4, 1));
}
// write
if (tablet.getRowSize() == tablet.getMaxRowNumber()) {
@@ -947,4 +933,42 @@ public class TsFileWriteApiTest {
Assert.assertTrue(reader.getAllMeasurements().containsKey("measurementcolumn"));
}
}
+
+ @Test
+ public void writeAllNullValueTablet()
+ throws IOException, WriteProcessException, ReadProcessException {
+ setEnv(100 * 1024 * 1024, 10 * 1024);
+ Tablet tablet =
+ new Tablet(
+ "table1",
+ Arrays.asList("tag1", "field1"),
+ Arrays.asList(TSDataType.STRING, TSDataType.BOOLEAN),
+ Arrays.asList(Tablet.ColumnCategory.TAG,
Tablet.ColumnCategory.FIELD));
+ tablet.addTimestamp(0, 0);
+ tablet.addTimestamp(1, 1);
+ TableSchema tableSchema =
+ new TableSchema(
+ "Table1",
+ Arrays.asList(
+ new ColumnSchema("tag1", TSDataType.STRING,
Tablet.ColumnCategory.TAG),
+ new ColumnSchema("field1", TSDataType.BOOLEAN,
Tablet.ColumnCategory.FIELD)));
+ Assert.assertEquals("table1", tableSchema.getTableName());
+ try (ITsFileWriter writer =
+ new TsFileWriterBuilder().file(f).tableSchema(tableSchema).build()) {
+ writer.write(tablet);
+ }
+ try (ITsFileReader reader = new TsFileReaderBuilder().file(f).build();
+ ResultSet resultSet =
+ reader.query(
+ "table1", Arrays.asList("tag1", "field1"), Long.MIN_VALUE,
Long.MAX_VALUE)) {
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(0, resultSet.getLong(1));
+ Assert.assertTrue(resultSet.isNull(2));
+ Assert.assertTrue(resultSet.isNull(3));
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(1, resultSet.getLong(1));
+ Assert.assertTrue(resultSet.isNull(2));
+ Assert.assertTrue(resultSet.isNull(3));
+ }
+ }
}
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java
index 676aecfc..baf979a7 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java
@@ -244,10 +244,9 @@ public class TsFileWriterTest {
"s1", TSDataType.FLOAT, TSEncoding.RLE,
CompressionType.SNAPPY),
new MeasurementSchema(
"s2", TSDataType.INT32, TSEncoding.RLE,
CompressionType.SNAPPY)));
- tablet.timestamps[0] = 10000;
- ((float[]) tablet.values[0])[0] = 5.0f;
- ((int[]) tablet.values[1])[0] = 5;
- tablet.setRowSize(1);
+ tablet.addTimestamp(0, 10000);
+ tablet.addValue(0, 0, 5.0f);
+ tablet.addValue(0, 1, 5);
writer.writeTree(tablet);
closeFile();
readOneRow();
@@ -263,9 +262,9 @@ public class TsFileWriterTest {
"s1", TSDataType.FLOAT, TSEncoding.RLE,
CompressionType.SNAPPY),
new MeasurementSchema(
"s2", TSDataType.INT32, TSEncoding.RLE,
CompressionType.SNAPPY)));
- tablet.timestamps[0] = 10000;
- ((float[]) tablet.values[0])[0] = 5.0f;
- tablet.setRowSize(1);
+ tablet.addTimestamp(0, 10000);
+ tablet.addValue(0, 0, 5.0f);
+ tablet.addValue(0, 1, 0);
writer.writeTree(tablet);
closeFile();
// in this case, the value of s2 = 0 at time 10000.
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java
index a73e22f9..07aedad4 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java
@@ -66,21 +66,21 @@ public class TabletTest {
Assert.assertFalse((Boolean) tablet.getValue(1, 0));
Assert.assertTrue((Boolean) tablet.getValue(1, 1));
Assert.assertTrue((Boolean) tablet.getValue(2, 0));
- Assert.assertFalse(tablet.bitMaps[0].isMarked(0));
- Assert.assertFalse(tablet.bitMaps[0].isMarked(1));
- Assert.assertFalse(tablet.bitMaps[0].isMarked(2));
- Assert.assertFalse(tablet.bitMaps[1].isMarked(0));
- Assert.assertFalse(tablet.bitMaps[1].isMarked(1));
- Assert.assertTrue(tablet.bitMaps[1].isMarked(2));
+ Assert.assertFalse(tablet.getBitMaps()[0].isMarked(0));
+ Assert.assertFalse(tablet.getBitMaps()[0].isMarked(1));
+ Assert.assertFalse(tablet.getBitMaps()[0].isMarked(2));
+ Assert.assertFalse(tablet.getBitMaps()[1].isMarked(0));
+ Assert.assertFalse(tablet.getBitMaps()[1].isMarked(1));
+ Assert.assertTrue(tablet.getBitMaps()[1].isMarked(2));
tablet.addTimestamp(9, 9);
Assert.assertEquals(10, tablet.getRowSize());
tablet.reset();
Assert.assertEquals(0, tablet.getRowSize());
- Assert.assertTrue(tablet.bitMaps[0].isAllMarked());
- Assert.assertTrue(tablet.bitMaps[0].isAllMarked());
- Assert.assertTrue(tablet.bitMaps[0].isAllMarked());
+ Assert.assertTrue(tablet.getBitMaps()[0].isAllMarked());
+ Assert.assertTrue(tablet.getBitMaps()[0].isAllMarked());
+ Assert.assertTrue(tablet.getBitMaps()[0].isAllMarked());
}
@Test
@@ -163,7 +163,7 @@ public class TabletTest {
i,
LocalDate.of(2000 + i, i / 100 + 1, i / 100 + 1));
- tablet.bitMaps[i % measurementSchemas.size()].mark(i);
+ tablet.getBitMaps()[i % measurementSchemas.size()].mark(i);
}
// Test add null
