This is an automated email from the ASF dual-hosted git repository. jt2594838 pushed a commit to branch add_property_read_write_interface in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 395e57a19c3a6c71be055ced50d6ebbec5d16aa5 Author: Tian Jiang <[email protected]> AuthorDate: Wed Jul 29 17:46:57 2026 +0800 May record table point count --- .../apache/tsfile/write/writer/TsFileIOWriter.java | 32 ++++++++++++ .../apache/tsfile/write/TsFileWriteApiTest.java | 61 ++++++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java index a379e4314..79034b28e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java @@ -126,6 +126,7 @@ public class TsFileIOWriter implements AutoCloseable { protected LinkedList<Long> endPosInCMTForDevice = new LinkedList<>(); private volatile int chunkMetadataCount = 0; public static final String CHUNK_METADATA_TEMP_FILE_SUFFIX = ".meta"; + public static final String TABLE_POINT_COUNT_PROPERTY_PREFIX = "tablePointCount."; private boolean generateTableSchema = false; @@ -144,6 +145,8 @@ public class TsFileIOWriter implements AutoCloseable { protected Map<String, Long> tableSizeMap = new HashMap<>(); private final Map<String, String> tsFileProperties = new HashMap<>(); + private final Map<String, Long> tablePointCountMap = new HashMap<>(); + private boolean recordTablePointCount; /** empty construct function. */ protected TsFileIOWriter() { @@ -165,6 +168,15 @@ public class TsFileIOWriter implements AutoCloseable { this(file, TS_FILE_CONFIG); } + /** + * Creates a writer and optionally records the number of non-null field points for each table in + * the TsFile properties. + */ + public TsFileIOWriter(File file, boolean recordTablePointCount) throws IOException { + this(file); + this.recordTablePointCount = recordTablePointCount; + } + public TsFileIOWriter(File file, EncryptParameter param) throws IOException { this(file, TS_FILE_CONFIG, param); } @@ -368,6 +380,7 @@ public class TsFileIOWriter implements AutoCloseable { chunkHeader.getCompressionType(), out.getPosition(), chunkMetadata.getStatistics()); + currentChunkMetadata.setMask(chunkMetadata.getMask()); chunkHeader.serializeTo(out.wrapAsStream()); out.write(chunk.getData()); endCurrentChunk(); @@ -419,6 +432,10 @@ public class TsFileIOWriter implements AutoCloseable { chunkHeader.getCompressionType(), out.getPosition(), chunk.getChunkStatistic()); + currentChunkMetadata.setMask( + (byte) + (chunkHeader.getChunkType() + & (TsFileConstant.TIME_COLUMN_MASK | TsFileConstant.VALUE_COLUMN_MASK))); chunkHeader.serializeTo(out.wrapAsStream()); out.write(chunk.getData()); endCurrentChunk(); @@ -428,6 +445,15 @@ public class TsFileIOWriter implements AutoCloseable { public void endCurrentChunk() { this.currentChunkMetadataSize += currentChunkMetadata.getRetainedSizeInBytes(); chunkMetadataCount++; + if (recordTablePointCount + && currentChunkGroupDeviceId != null + && currentChunkGroupDeviceId.isTableModel() + && (currentChunkMetadata.getMask() & TsFileConstant.TIME_COLUMN_MASK) == 0) { + tablePointCountMap.merge( + currentChunkGroupDeviceId.getTableName(), + currentChunkMetadata.getNumOfPoints(), + Long::sum); + } chunkMetadataList.add(currentChunkMetadata); currentChunkMetadata = null; } @@ -598,6 +624,12 @@ public class TsFileIOWriter implements AutoCloseable { tsFileMetadata.setMetaOffset(metaOffset); tsFileMetadata.setBloomFilter(filter); tsFileProperties.forEach(tsFileMetadata::addProperty); + if (recordTablePointCount) { + tablePointCountMap.forEach( + (tableName, pointCount) -> + tsFileMetadata.addProperty( + TABLE_POINT_COUNT_PROPERTY_PREFIX + tableName, Long.toString(pointCount))); + } tsFileMetadata.addProperty("encryptLevel", encryptLevel); tsFileMetadata.addProperty("encryptType", encryptType); tsFileMetadata.addProperty("encryptKey", encryptKey); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java index fcd761d3f..c8592afe1 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java @@ -1237,6 +1237,67 @@ public class TsFileWriteApiTest { } } + /** + * Verifies that table point counting records only non-null FIELD values in TsFile properties. + * Table1 contains three s1 values and two non-null s2 values, while table2 contains two s1 + * values. Timestamps, TAG values, and the null s2 value are excluded, so the expected counts are + * five and two respectively. + */ + @Test + public void recordTablePointCountInProperties() throws IOException, WriteProcessException { + TableSchema tableSchema1 = + new TableSchema( + "table1", + Arrays.asList( + new ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + new ColumnSchema("s1", TSDataType.INT32, ColumnCategory.FIELD), + new ColumnSchema("s2", TSDataType.INT32, ColumnCategory.FIELD))); + TableSchema tableSchema2 = + new TableSchema( + "table2", + Arrays.asList( + new ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + new ColumnSchema("s1", TSDataType.INT32, ColumnCategory.FIELD))); + Tablet tablet1 = + new Tablet( + "table1", + IMeasurementSchema.getMeasurementNameList(tableSchema1.getColumnSchemas()), + IMeasurementSchema.getDataTypeList(tableSchema1.getColumnSchemas()), + tableSchema1.getColumnTypes()); + for (int row = 0; row < 3; row++) { + tablet1.addTimestamp(row, row); + tablet1.addValue("device", row, "d1"); + tablet1.addValue("s1", row, row); + tablet1.addValue("s2", row, row == 2 ? null : row); + } + Tablet tablet2 = + new Tablet( + "table2", + IMeasurementSchema.getMeasurementNameList(tableSchema2.getColumnSchemas()), + IMeasurementSchema.getDataTypeList(tableSchema2.getColumnSchemas()), + tableSchema2.getColumnTypes()); + for (int row = 0; row < 2; row++) { + tablet2.addTimestamp(row, row); + tablet2.addValue("device", row, "d1"); + tablet2.addValue("s1", row, row); + } + + try (TsFileWriter writer = new TsFileWriter(new TsFileIOWriter(f, true))) { + writer.registerTableSchema(tableSchema1); + writer.registerTableSchema(tableSchema2); + writer.writeTable(tablet1); + writer.writeTable(tablet2); + } + + try (TsFileSequenceReader reader = new TsFileSequenceReader(f.getAbsolutePath())) { + Map<String, String> properties = reader.getTsFileProperties(); + Assert.assertEquals( + "5", properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table1")); + Assert.assertEquals( + "2", properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table2")); + } + } + @Test public void calculateTableSize() throws IOException, WriteProcessException { TableSchema tableSchema1 =
