This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new ea0945935 Support TsFile properties and optional table point counting
(#882)
ea0945935 is described below
commit ea0945935a9bee5154072a8349bd4df7c9a04143
Author: Jiang Tian <[email protected]>
AuthorDate: Tue Aug 4 12:19:14 2026 +0800
Support TsFile properties and optional table point counting (#882)
* add property read-write interface
* May record table point count
* Add perf test
* Add table point count repair tool
* add point-count tool
* add point-count tool
* supplement usage for other os
Co-authored-by: Copilot Autofix powered by AI
<[email protected]>
* supplement usage for other os
Co-authored-by: Copilot Autofix powered by AI
<[email protected]>
* use byte[] for properties
* use byte[8] for table point count
---------
Co-authored-by: Copilot Autofix powered by AI
<[email protected]>
---
.../org/apache/tsfile/i18n/messages.properties | 7 +
.../org/apache/tsfile/i18n/messages_zh.properties | 7 +
java/tools/README-zh.md | 21 ++
java/tools/README.md | 26 ++
.../resources/tools/tsfile-table-point-count.bat | 49 ++++
.../resources/tools/tsfile-table-point-count.sh | 50 ++++
java/tools/src/assembly/tools.xml | 9 +
.../tsfile/file/metadata/TsFileMetadata.java | 74 +++--
.../apache/tsfile/read/TsFileSequenceReader.java | 5 +
.../tsfile/read/v4/DeviceTableModelReader.java | 6 +
.../org/apache/tsfile/read/v4/ITsFileReader.java | 4 +
.../tsfile/utils/TsFileTablePointCountTool.java | 318 +++++++++++++++++++++
.../write/v4/AbstractTableModelTsFileWriter.java | 6 +
.../org/apache/tsfile/write/v4/ITsFileWriter.java | 3 +
.../apache/tsfile/write/writer/TsFileIOWriter.java | 48 +++-
.../tsfile/file/metadata/TsFileMetadataTest.java | 11 +-
.../apache/tsfile/file/metadata/utils/Utils.java | 21 +-
.../read/TsFileV4ReadWriteInterfacesTest.java | 39 +++
.../utils/TsFileTablePointCountToolTest.java | 141 +++++++++
.../write/TablePointCountPerformanceTest.java | 269 +++++++++++++++++
.../apache/tsfile/write/TsFileWriteApiTest.java | 64 +++++
21 files changed, 1142 insertions(+), 36 deletions(-)
diff --git
a/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties
b/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties
index 98909f7a6..9f6f35857 100644
--- a/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties
+++ b/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties
@@ -1548,3 +1548,10 @@ log.tools.csv_read_error = Error reading CSV file: %1$s
# CsvSourceReader — error closing CSV reader
log.tools.csv_close_reader_error = Error closing CSV reader
+
+# TsFileTablePointCountTool
+error.utils.table_point_count_tool_usage = Usage: TsFileTablePointCountTool
<file.tsfile>
+error.utils.table_point_count_tool_file_not_found = TsFile does not exist: %1$s
+error.utils.table_point_count_tool_incomplete_file = TsFile is incomplete: %1$s
+error.utils.table_point_count_tool_copy_failed = Failed to copy the TsFile
data and metadata prefix
+info.utils.table_point_count_tool_result = Processed %1$s: %2$s
diff --git
a/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties
b/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties
index a1a437cfb..17c341af2 100644
---
a/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties
+++
b/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties
@@ -1548,3 +1548,10 @@ log.tools.csv_read_error = 读取 CSV 文件出错: %1$s
# CsvSourceReader — error closing CSV reader
log.tools.csv_close_reader_error = 关闭 CSV reader 出错
+
+# TsFileTablePointCountTool
+error.utils.table_point_count_tool_usage = 用法：TsFileTablePointCountTool
<file.tsfile>
+error.utils.table_point_count_tool_file_not_found = TsFile 不存在：%1$s
+error.utils.table_point_count_tool_incomplete_file = TsFile 不完整：%1$s
+error.utils.table_point_count_tool_copy_failed = 复制 TsFile 数据和元数据前缀失败
+info.utils.table_point_count_tool_result = 已处理 %1$s：%2$s
diff --git a/java/tools/README-zh.md b/java/tools/README-zh.md
index 46c615ed0..7ff5ef6e4 100644
--- a/java/tools/README-zh.md
+++ b/java/tools/README-zh.md
@@ -229,3 +229,24 @@ arrow2tsfile.bat --source .\data\arrow --target .\output
--fail_dir .\failed
- 多批次：`{源文件名}_1.tsfile`、`{源文件名}_2.tsfile`、...
- 表名与输出文件名相互独立——表名来自 schema 或 `--table_name`，文件名来自源文件。
+## 表点数统计工具
+
+`tools/tsfile-table-point-count.sh`（Linux/macOS）和
`tools\tsfile-table-point-count.bat`（Windows）用于检查完整 TsFile
是否包含表级点数统计属性，并在属性缺失时补写。每张表的点数是其所有 FIELD 列中非空值的总数，不统计 TAG 列和时间列。
+
+设置 `JAVA_HOME` 后，传入且仅传入一个 TsFile 路径：
+
+ # Linux/macOS
+ tools/tsfile-table-point-count.sh /data/example.tsfile
+
+ :: Windows
+ tools\tsfile-table-point-count.bat C:\data\example.tsfile
+
+工具会输出以下状态之一：
+
+| 状态 | 说明 |
+|------|------|
+| `UPDATED` | 已计算缺失的点数统计属性并写回文件。 |
+| `ALREADY_PRESENT` | 每张表都已有有效的点数统计属性，未修改文件。 |
+| `NO_TABLE` | 文件不包含表 Schema，例如仅含树模型数据的 TsFile，未修改文件。 |
+
+仅当状态为 `UPDATED`
时，工具才会原地修改输入文件。工具先在源文件所在目录生成完整的临时文件，确保重写后的元数据落盘后再替换源文件。运行前请确保目录可写、磁盘空间足以存放临时副本，并备份重要文件。不存在的文件和未写完整的
TsFile 会被拒绝处理。
diff --git a/java/tools/README.md b/java/tools/README.md
index ad1f40215..18c5cd780 100644
--- a/java/tools/README.md
+++ b/java/tools/README.md
@@ -228,3 +228,29 @@ arrow2tsfile.bat --source .\data\arrow --target .\output
--fail_dir .\failed
- Multiple batches: `{source_basename}_1.tsfile`,
`{source_basename}_2.tsfile`, ...
- Table name and output filename are independent — table name comes from
schema or `--table_name`, filename comes from source file.
+## Table Point Count Tool
+
+`tools/tsfile-table-point-count.sh` (Linux/macOS) and
`tools\tsfile-table-point-count.bat` (Windows) check whether a complete TsFile
contains table-level
+point-count properties and backfills them when they are missing. For each
table, the point count is
+the total number of non-null values in its FIELD columns; TAG and time columns
are not counted.
+
+Set `JAVA_HOME`, then run the tool with exactly one TsFile path:
+
+ # Linux/macOS
+ tools/tsfile-table-point-count.sh /data/example.tsfile
+
+ :: Windows
+ tools\tsfile-table-point-count.bat C:\data\example.tsfile
+
+The tool prints one of the following statuses:
+
+| Status | Description |
+|--------|-------------|
+| `UPDATED` | Missing point-count properties were calculated and written to
the file. |
+| `ALREADY_PRESENT` | Every table already had a valid point-count property;
the file was not modified. |
+| `NO_TABLE` | The file contains no table schema, for example a
tree-model-only TsFile; the file was not modified. |
+
+The tool modifies the input file in place only for `UPDATED`. It writes a
complete temporary copy
+next to the source file and replaces the source after the rewritten metadata
has been flushed. Make
+sure the directory is writable, leave enough free space for a temporary copy,
and back up important
+files before running the tool. Missing files and incomplete TsFiles are
rejected.
diff --git
a/java/tools/src/assembly/resources/tools/tsfile-table-point-count.bat
b/java/tools/src/assembly/resources/tools/tsfile-table-point-count.bat
new file mode 100644
index 000000000..a89d6c13d
--- /dev/null
+++ b/java/tools/src/assembly/resources/tools/tsfile-table-point-count.bat
@@ -0,0 +1,49 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+@echo off
+setlocal enabledelayedexpansion
+
+if "%OS%" == "Windows_NT" setlocal
+
+pushd %~dp0..
+if NOT DEFINED TSFILE_HOME set TSFILE_HOME=%CD%
+popd
+
+set JAVA_OPTS=-ea^
+ -DTSFILE_HOME="%TSFILE_HOME%"
+
+if NOT DEFINED JAVA_HOME goto :err
+
+echo ------------------------------------------
+echo Starting TsFile Table Point Count Tool
+echo ------------------------------------------
+
+set CLASSPATH="%TSFILE_HOME%\lib\*"
+set MAIN_CLASS=org.apache.tsfile.utils.TsFileTablePointCountTool
+
+@REM The argument is the TsFile to inspect. The tool rewrites its properties
only when table-level
+@REM point-count statistics are missing.
+"%JAVA_HOME%\bin\java" -DTSFILE_HOME=!TSFILE_HOME! !JAVA_OPTS! -cp !CLASSPATH!
!MAIN_CLASS! %*
+exit /b %ERRORLEVEL%
+
+:err
+echo JAVA_HOME environment variable must be set!
+set ret_code=1
+exit /b %ret_code%
diff --git
a/java/tools/src/assembly/resources/tools/tsfile-table-point-count.sh
b/java/tools/src/assembly/resources/tools/tsfile-table-point-count.sh
new file mode 100644
index 000000000..34c902628
--- /dev/null
+++ b/java/tools/src/assembly/resources/tools/tsfile-table-point-count.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+echo ------------------------------------------
+echo Starting TsFile Table Point Count Tool
+echo ------------------------------------------
+
+if [ -z "${TSFILE_HOME}" ]; then
+ export TSFILE_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+if [ -n "$JAVA_HOME" ]; then
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+else
+ JAVA=java
+fi
+
+if [ -z $JAVA ] ; then
+ echo Unable to find java executable. Check JAVA_HOME and PATH environment
variables. > /dev/stderr
+ exit 1;
+fi
+
+CLASSPATH=${TSFILE_HOME}/lib/*
+MAIN_CLASS=org.apache.tsfile.utils.TsFileTablePointCountTool
+
+# The argument is the TsFile to inspect. The tool rewrites its properties only
when table-level
+# point-count statistics are missing.
+exec "$JAVA" -DTSFILE_HOME=${TSFILE_HOME} -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
diff --git a/java/tools/src/assembly/tools.xml
b/java/tools/src/assembly/tools.xml
index 31e9c81a0..b58e8b8e2 100644
--- a/java/tools/src/assembly/tools.xml
+++ b/java/tools/src/assembly/tools.xml
@@ -69,5 +69,14 @@
<source>${maven.multiModuleProjectDirectory}/java/tools/src/assembly/resources/tools/arrow2tsfile.bat</source>
<destName>tools/arrow2tsfile.bat</destName>
</file>
+ <file>
+
<source>${maven.multiModuleProjectDirectory}/java/tools/src/assembly/resources/tools/tsfile-table-point-count.sh</source>
+ <destName>tools/tsfile-table-point-count.sh</destName>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+
<source>${maven.multiModuleProjectDirectory}/java/tools/src/assembly/resources/tools/tsfile-table-point-count.bat</source>
+ <destName>tools/tsfile-table-point-count.bat</destName>
+ </file>
</files>
</assembly>
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
index 318728553..7caae06b2 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
@@ -19,6 +19,7 @@
package org.apache.tsfile.file.metadata;
+import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.compatibility.DeserializeConfig;
import org.apache.tsfile.encrypt.EncryptUtils;
import org.apache.tsfile.exception.encrypt.EncryptException;
@@ -45,7 +46,7 @@ public class TsFileMetadata {
private Map<String, MetadataIndexNode> tableMetadataIndexNodeMap;
private Map<String, TableSchema> tableSchemaMap;
private boolean hasTableSchemaMapCache;
- private Map<String, String> tsFileProperties;
+ private Map<String, byte[]> tsFileProperties;
// offset of MetaMarker.SEPARATOR
private long metaOffset;
@@ -117,21 +118,27 @@ public class TsFileMetadata {
if (buffer.hasRemaining()) {
int propertiesSize = ReadWriteForEncodingUtils.readVarInt(buffer);
- Map<String, String> propertiesMap = new HashMap<>();
+ Map<String, byte[]> propertiesMap = new HashMap<>();
for (int i = 0; i < propertiesSize; i++) {
String key = ReadWriteIOUtils.readVarIntString(buffer);
- String value = ReadWriteIOUtils.readVarIntString(buffer);
+ int valueSize = ReadWriteForEncodingUtils.readVarInt(buffer);
+ byte[] value = null;
+ if (valueSize >= 0) {
+ value = new byte[valueSize];
+ buffer.get(value);
+ }
propertiesMap.put(key, value);
}
+ String encryptLevel = getPropertyAsString(propertiesMap, "encryptLevel");
// if the file is not encrypted, set the default value(for compatible
reason)
- if (!propertiesMap.containsKey("encryptLevel") ||
propertiesMap.get("encryptLevel") == null) {
- propertiesMap.put("encryptLevel", "0");
- propertiesMap.put("encryptType",
"org.apache.tsfile.encrypt.UNENCRYPTED");
- propertiesMap.put("encryptKey", "");
- } else if (propertiesMap.get("encryptLevel").equals("0")) {
- propertiesMap.put("encryptType",
"org.apache.tsfile.encrypt.UNENCRYPTED");
- propertiesMap.put("encryptKey", "");
- } else if (propertiesMap.get("encryptLevel").equals("1")) {
+ if (!propertiesMap.containsKey("encryptLevel") || encryptLevel == null) {
+ propertiesMap.put("encryptLevel", stringToBytes("0"));
+ propertiesMap.put("encryptType",
stringToBytes("org.apache.tsfile.encrypt.UNENCRYPTED"));
+ propertiesMap.put("encryptKey", stringToBytes(""));
+ } else if (encryptLevel.equals("0")) {
+ propertiesMap.put("encryptType",
stringToBytes("org.apache.tsfile.encrypt.UNENCRYPTED"));
+ propertiesMap.put("encryptKey", stringToBytes(""));
+ } else if (encryptLevel.equals("1")) {
if (!propertiesMap.containsKey("encryptType")) {
throw new EncryptException(
Messages.format("error.file.tsfile_metadata_no_encrypt_type",
1));
@@ -140,15 +147,15 @@ public class TsFileMetadata {
throw new EncryptException(
Messages.format("error.file.tsfile_metadata_no_encrypt_key", 1));
}
- if (propertiesMap.get("encryptKey") == null ||
propertiesMap.get("encryptKey").isEmpty()) {
+ String encryptKey = getPropertyAsString(propertiesMap, "encryptKey");
+ if (encryptKey == null || encryptKey.isEmpty()) {
throw new EncryptException(
Messages.format("error.file.tsfile_metadata_null_encrypt_key",
1));
}
- String str = propertiesMap.get("encryptKey");
fileMetaData.encryptLevel = 1;
- fileMetaData.secondKey = EncryptUtils.getSecondKeyFromStr(str);
- fileMetaData.encryptType = propertiesMap.get("encryptType");
- } else if (propertiesMap.get("encryptLevel").equals("2")) {
+ fileMetaData.secondKey = EncryptUtils.getSecondKeyFromStr(encryptKey);
+ fileMetaData.encryptType = getPropertyAsString(propertiesMap,
"encryptType");
+ } else if (encryptLevel.equals("2")) {
if (!propertiesMap.containsKey("encryptType")) {
throw new EncryptException(
Messages.format("error.file.tsfile_metadata_no_encrypt_type",
2));
@@ -157,19 +164,17 @@ public class TsFileMetadata {
throw new EncryptException(
Messages.format("error.file.tsfile_metadata_no_encrypt_key", 2));
}
- if (propertiesMap.get("encryptKey") == null ||
propertiesMap.get("encryptKey").isEmpty()) {
+ String encryptKey = getPropertyAsString(propertiesMap, "encryptKey");
+ if (encryptKey == null || encryptKey.isEmpty()) {
throw new EncryptException(
Messages.format("error.file.tsfile_metadata_null_encrypt_key",
2));
}
fileMetaData.encryptLevel = 2;
- String str = propertiesMap.get("encryptKey");
- fileMetaData.secondKey = EncryptUtils.getSecondKeyFromStr(str);
- fileMetaData.encryptType = propertiesMap.get("encryptType");
+ fileMetaData.secondKey = EncryptUtils.getSecondKeyFromStr(encryptKey);
+ fileMetaData.encryptType = getPropertyAsString(propertiesMap,
"encryptType");
} else {
throw new EncryptException(
- Messages.format(
- "error.file.tsfile_metadata_unsupported_encrypt_level",
- propertiesMap.get("encryptLevel")));
+
Messages.format("error.file.tsfile_metadata_unsupported_encrypt_level",
encryptLevel));
}
fileMetaData.tsFileProperties = propertiesMap;
}
@@ -177,7 +182,7 @@ public class TsFileMetadata {
return fileMetaData;
}
- public void addProperty(String key, String value) {
+ public void addProperty(String key, byte[] value) {
if (tsFileProperties == null) {
tsFileProperties = new HashMap<>();
}
@@ -248,9 +253,15 @@ public class TsFileMetadata {
ReadWriteForEncodingUtils.writeVarInt(
tsFileProperties != null ? tsFileProperties.size() : 0,
outputStream);
if (tsFileProperties != null) {
- for (Entry<String, String> entry : tsFileProperties.entrySet()) {
+ for (Entry<String, byte[]> entry : tsFileProperties.entrySet()) {
byteLen += ReadWriteIOUtils.writeVar(entry.getKey(), outputStream);
- byteLen += ReadWriteIOUtils.writeVar(entry.getValue(), outputStream);
+ byte[] value = entry.getValue();
+ byteLen +=
+ ReadWriteForEncodingUtils.writeVarInt(value == null ? -1 :
value.length, outputStream);
+ if (value != null) {
+ outputStream.write(value);
+ byteLen += value.length;
+ }
}
}
@@ -295,7 +306,16 @@ public class TsFileMetadata {
return tableSchemaMap;
}
- public Map<String, String> getTsFileProperties() {
+ public Map<String, byte[]> getTsFileProperties() {
return tsFileProperties;
}
+
+ private static String getPropertyAsString(Map<String, byte[]> properties,
String key) {
+ byte[] value = properties.get(key);
+ return value == null ? null : new String(value,
TSFileConfig.STRING_CHARSET);
+ }
+
+ private static byte[] stringToBytes(String value) {
+ return value.getBytes(TSFileConfig.STRING_CHARSET);
+ }
}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
index 1fcabdab6..9407eda65 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
@@ -607,6 +607,11 @@ public class TsFileSequenceReader implements AutoCloseable
{
return getTableSchemaMap(null);
}
+ /** Get the properties stored in the TsFile metadata. */
+ public Map<String, byte[]> getTsFileProperties() throws IOException {
+ return readFileMetadata().getTsFileProperties();
+ }
+
public Map<String, TableSchema> getTableSchemaMap(LongConsumer
ioSizeRecorder)
throws IOException {
if (tsFileMetaData != null && tsFileMetaData.hasTableSchemaMapCache()) {
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
index e7749d42b..a3d125a1d 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
@@ -78,6 +78,12 @@ public class DeviceTableModelReader implements ITsFileReader
{
return Optional.ofNullable(tableSchemaMap.get(tableName.toLowerCase()));
}
+ @Override
+ @TsFileApi
+ public Map<String, byte[]> getTsFileProperties() throws IOException {
+ return fileReader.getTsFileProperties();
+ }
+
@TsFileApi
public ResultSet query(String tableName, List<String> columnNames, long
startTime, long endTime)
throws IOException, NoTableException, NoMeasurementException,
ReadProcessException {
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java
index 995f73f64..dd0c188e1 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java
@@ -29,6 +29,7 @@ import org.apache.tsfile.read.query.dataset.ResultSet;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
public interface ITsFileReader extends AutoCloseable {
@@ -48,6 +49,9 @@ public interface ITsFileReader extends AutoCloseable {
@TsFileApi
List<TableSchema> getAllTableSchema() throws IOException;
+ @TsFileApi
+ Map<String, byte[]> getTsFileProperties() throws IOException;
+
@TsFileApi
void close();
}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java
b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java
new file mode 100644
index 000000000..973c0215e
--- /dev/null
+++
b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tsfile.utils;
+
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.enums.ColumnCategory;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.file.metadata.TimeseriesMetadata;
+import org.apache.tsfile.file.metadata.TsFileMetadata;
+import org.apache.tsfile.i18n.Messages;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.writer.TsFileIOWriter;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/** Detects and backfills table-level point-count properties in a complete
TsFile. */
+public final class TsFileTablePointCountTool {
+
+ private TsFileTablePointCountTool() {}
+
+ public enum UpdateStatus {
+ UPDATED,
+ ALREADY_PRESENT,
+ NO_TABLE
+ }
+
+ public static void main(String[] args) throws IOException {
+ if (args.length != 1) {
+
System.err.println(Messages.get("error.utils.table_point_count_tool_usage"));
+ return;
+ }
+ File file = new File(args[0]);
+ UpdateStatus status = updateTablePointCountIfMissing(file);
+ System.out.println(
+ Messages.format(
+ "info.utils.table_point_count_tool_result",
file.getAbsolutePath(), status));
+ }
+
+ /**
+ * Returns whether every table in the file has a valid non-negative
point-count property.
+ * Tree-model-only files return {@code false}.
+ */
+ public static boolean containsTablePointCount(File file) throws IOException {
+ try (TsFileSequenceReader reader = openCompleteFile(file)) {
+ Map<String, TableSchema> tableSchemas = reader.getTableSchemaMap();
+ return !tableSchemas.isEmpty()
+ && hasAllTablePointCountProperties(reader.getTsFileProperties(),
tableSchemas.keySet());
+ }
+ }
+
+ /**
+ * Adds table point-count properties when they are missing. Existing
complete properties are left
+ * untouched. The source file is replaced only after a complete rewritten
copy has been forced to
+ * disk.
+ */
+ public static UpdateStatus updateTablePointCountIfMissing(File file) throws
IOException {
+ TsFileMetadata metadata;
+ long fileMetadataPosition;
+ Map<String, Long> tablePointCounts;
+ try (TsFileSequenceReader reader = openCompleteFile(file)) {
+ reader.setEnableCacheTableSchemaMap();
+ metadata = reader.readFileMetadata();
+ Map<String, TableSchema> tableSchemas = metadata.getTableSchemaMap();
+ if (tableSchemas.isEmpty()) {
+ return UpdateStatus.NO_TABLE;
+ }
+ if (hasAllTablePointCountProperties(metadata.getTsFileProperties(),
tableSchemas.keySet())) {
+ return UpdateStatus.ALREADY_PRESENT;
+ }
+ tablePointCounts = scanTablePointCounts(reader, tableSchemas);
+ fileMetadataPosition = reader.getFileMetadataPos();
+ }
+
+ tablePointCounts.forEach(
+ (tableName, pointCount) ->
+ metadata.addProperty(
+ TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + tableName,
+ BytesUtils.longToBytes(pointCount)));
+ rewriteFileMetadata(file.toPath(), fileMetadataPosition, metadata);
+ return UpdateStatus.UPDATED;
+ }
+
+ static Map<String, Long> scanTablePointCounts(
+ TsFileSequenceReader reader, Map<String, TableSchema> tableSchemas)
throws IOException {
+ Map<String, Set<String>> fieldNamesByTable = new HashMap<>();
+ Map<String, Long> tablePointCounts = new HashMap<>();
+ for (Map.Entry<String, TableSchema> tableEntry : tableSchemas.entrySet()) {
+ List<IMeasurementSchema> columns =
tableEntry.getValue().getColumnSchemas();
+ List<ColumnCategory> columnCategories =
tableEntry.getValue().getColumnTypes();
+ Set<String> fieldNames = new HashSet<>();
+ for (int i = 0; i < columns.size(); i++) {
+ if (columnCategories.get(i) == ColumnCategory.FIELD) {
+ fieldNames.add(columns.get(i).getMeasurementName());
+ }
+ }
+ fieldNamesByTable.put(tableEntry.getKey(), fieldNames);
+ tablePointCounts.put(tableEntry.getKey(), 0L);
+ }
+
+ Iterator<Pair<IDeviceID, List<TimeseriesMetadata>>> metadataIterator =
+ reader.iterAllTimeseriesMetadata(false, false);
+ while (metadataIterator.hasNext()) {
+ Pair<IDeviceID, List<TimeseriesMetadata>> deviceMetadata =
metadataIterator.next();
+ IDeviceID device = deviceMetadata.left;
+ String tableName = device.getTableName();
+ Set<String> fieldNames = fieldNamesByTable.getOrDefault(tableName,
Collections.emptySet());
+ if (fieldNames.isEmpty()) {
+ continue;
+ }
+ for (TimeseriesMetadata timeseriesMetadata : deviceMetadata.right) {
+ if (fieldNames.contains(timeseriesMetadata.getMeasurementId())) {
+ tablePointCounts.merge(
+ tableName, (long) timeseriesMetadata.getStatistics().getCount(),
Long::sum);
+ }
+ }
+ }
+ return tablePointCounts;
+ }
+
+ private static TsFileSequenceReader openCompleteFile(File file) throws
IOException {
+ if (!file.isFile()) {
+ throw new IOException(
+ Messages.format("error.utils.table_point_count_tool_file_not_found",
file));
+ }
+ TsFileSequenceReader reader = new
TsFileSequenceReader(file.getAbsolutePath());
+ try {
+ if (!reader.isComplete()) {
+ throw new IOException(
+
Messages.format("error.utils.table_point_count_tool_incomplete_file", file));
+ }
+ return reader;
+ } catch (IOException | RuntimeException e) {
+ try {
+ reader.close();
+ } catch (IOException closeException) {
+ e.addSuppressed(closeException);
+ }
+ throw e;
+ }
+ }
+
+ private static boolean hasAllTablePointCountProperties(
+ Map<String, byte[]> properties, Set<String> tableNames) {
+ if (properties == null) {
+ return false;
+ }
+ for (String tableName : tableNames) {
+ byte[] value =
properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + tableName);
+ try {
+ if (value == null || value.length != Long.BYTES ||
BytesUtils.bytesToLong(value) < 0) {
+ return false;
+ }
+ } catch (RuntimeException e) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static void rewriteFileMetadata(
+ Path sourceFile, long fileMetadataPosition, TsFileMetadata metadata)
throws IOException {
+ Path absoluteSource = sourceFile.toAbsolutePath();
+ // Keep the temporary file beside the source so reflink and atomic move
can stay on the same
+ // file system. The source is not replaced until the rewritten temporary
file has been forced.
+ Path temporaryFile =
+ Files.createTempFile(
+ absoluteSource.getParent(),
+ absoluteSource.getFileName().toString(),
+ ".point-count.tmp");
+ try {
+ if (tryCreateReflink(absoluteSource, temporaryFile)) {
+ // A reflink shares unchanged data blocks with the source. Truncating
and rewriting only the
+ // TsFileMetadata tail therefore triggers copy-on-write for the
affected tail blocks.
+ try (FileChannel targetChannel =
+ FileChannel.open(temporaryFile, StandardOpenOption.WRITE)) {
+ targetChannel.truncate(fileMetadataPosition);
+ writeFileMetadata(targetChannel, fileMetadataPosition, metadata);
+ }
+ } else {
+ // Reflink is an optional optimization. Copy the immutable data/index
prefix when the
+ // command or file system does not support copy-on-write cloning.
+ try (FileChannel sourceChannel = FileChannel.open(absoluteSource,
StandardOpenOption.READ);
+ FileChannel targetChannel =
+ FileChannel.open(
+ temporaryFile,
+ StandardOpenOption.WRITE,
+ StandardOpenOption.TRUNCATE_EXISTING)) {
+ copyPrefix(sourceChannel, targetChannel, fileMetadataPosition);
+ writeFileMetadata(targetChannel, fileMetadataPosition, metadata);
+ }
+ }
+ replaceSourceFile(temporaryFile, absoluteSource);
+ } finally {
+ Files.deleteIfExists(temporaryFile);
+ }
+ }
+
+ /**
+ * Attempts a same-file-system copy-on-write clone using the native platform
command. GNU/Linux
+ * uses {@code cp --reflink=always}, while macOS uses {@code cp -c}.
Unsupported platforms,
+ * missing commands, unsupported file systems, invalid clone results, and
command failures return
+ * {@code false} so the caller can safely fall back to a full prefix copy.
+ */
+ private static boolean tryCreateReflink(Path sourceFile, Path targetFile)
throws IOException {
+ String operatingSystem = System.getProperty("os.name",
"").toLowerCase(Locale.ROOT);
+ List<String> command;
+ if (operatingSystem.contains("linux")) {
+ command =
+ List.of("cp", "--reflink=always", "--", sourceFile.toString(),
targetFile.toString());
+ } else if (operatingSystem.contains("mac")) {
+ command = List.of("cp", "-c", sourceFile.toString(),
targetFile.toString());
+ } else {
+ return false;
+ }
+
+ try {
+ Process process =
+ new ProcessBuilder(command)
+ .redirectErrorStream(true)
+ .redirectOutput(ProcessBuilder.Redirect.DISCARD)
+ .start();
+ // A reflink should complete quickly regardless of file size. Bound the
external command so
+ // an unavailable or unhealthy file system cannot block the repair
indefinitely.
+ if (!process.waitFor(30, TimeUnit.SECONDS)) {
+ process.destroyForcibly();
+ // Wait for termination before the fallback path opens and rewrites
the same temporary file.
+ process.waitFor();
+ return false;
+ }
+ // Do not trust the exit code alone: verify that the clone produced a
complete file before
+ // truncating its metadata tail.
+ return process.exitValue() == 0
+ && Files.isRegularFile(targetFile)
+ && Files.size(targetFile) == Files.size(sourceFile);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException(e);
+ } catch (IOException | SecurityException e) {
+ return false;
+ }
+ }
+
+ /** Writes the replacement TsFileMetadata, its serialized size, and the
trailing TsFile magic. */
+ private static void writeFileMetadata(
+ FileChannel targetChannel, long fileMetadataPosition, TsFileMetadata
metadata)
+ throws IOException {
+ targetChannel.position(fileMetadataPosition);
+ OutputStream output = Channels.newOutputStream(targetChannel);
+ int metadataSize = metadata.serializeTo(output);
+ ReadWriteIOUtils.write(metadataSize, output);
+
output.write(TSFileConfig.MAGIC_STRING.getBytes(TSFileConfig.STRING_CHARSET));
+ output.flush();
+ targetChannel.force(true);
+ }
+
+ /** Copies the immutable data and metadata-index prefix used by the
non-reflink fallback. */
+ private static void copyPrefix(
+ FileChannel sourceChannel, FileChannel targetChannel, long prefixLength)
throws IOException {
+ long copied = 0;
+ while (copied < prefixLength) {
+ long currentCopied = sourceChannel.transferTo(copied, prefixLength -
copied, targetChannel);
+ if (currentCopied <= 0) {
+ throw new
IOException(Messages.get("error.utils.table_point_count_tool_copy_failed"));
+ }
+ copied += currentCopied;
+ }
+ }
+
+ private static void replaceSourceFile(Path temporaryFile, Path sourceFile)
throws IOException {
+ try {
+ // Prefer an atomic replacement so readers never observe a partially
rewritten TsFile.
+ Files.move(
+ temporaryFile,
+ sourceFile,
+ StandardCopyOption.ATOMIC_MOVE,
+ StandardCopyOption.REPLACE_EXISTING);
+ } catch (AtomicMoveNotSupportedException e) {
+ // Some file systems do not provide atomic move even within one
directory.
+ Files.move(temporaryFile, sourceFile,
StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
index 94be1fdc1..108de0051 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
@@ -259,6 +259,12 @@ abstract class AbstractTableModelTsFileWriter implements
ITsFileWriter {
return fileWriter.getSchema();
}
+ @Override
+ @TsFileApi
+ public void addTsFileProperty(String key, byte[] value) {
+ fileWriter.addTsFileProperty(key, value);
+ }
+
/**
* calling this method to write the last data remaining in memory and close
the normal and error
* OutputStream.
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java
index 25ccb031b..d20c432c5 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java
@@ -36,4 +36,7 @@ public interface ITsFileWriter extends AutoCloseable {
@TsFileApi
void write(TSRecord record) throws IOException, WriteProcessException;
+
+ @TsFileApi
+ void addTsFileProperty(String key, byte[] value);
}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
index df6d99b20..93ddec569 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
@@ -126,6 +126,7 @@ public class TsFileIOWriter implements AutoCloseable {
protected LinkedList<Long> endPosInCMTForDevice = new LinkedList<>();
private volatile int chunkMetadataCount = 0;
public static final String CHUNK_METADATA_TEMP_FILE_SUFFIX = ".meta";
+ public static final String TABLE_POINT_COUNT_PROPERTY_PREFIX =
"tablePointCount.";
private boolean generateTableSchema = false;
@@ -143,6 +144,10 @@ public class TsFileIOWriter implements AutoCloseable {
protected Map<String, Long> tableSizeMap = new HashMap<>();
+ private final Map<String, byte[]> tsFileProperties = new HashMap<>();
+ private final Map<String, Long> tablePointCountMap = new HashMap<>();
+ private boolean recordTablePointCount;
+
/** empty construct function. */
protected TsFileIOWriter() {
setEncryptParam(
@@ -163,6 +168,15 @@ public class TsFileIOWriter implements AutoCloseable {
this(file, TS_FILE_CONFIG);
}
+ /**
+ * Creates a writer and optionally records the number of non-null field
points for each table in
+ * the TsFile properties.
+ */
+ public TsFileIOWriter(File file, boolean recordTablePointCount) throws
IOException {
+ this(file);
+ this.recordTablePointCount = recordTablePointCount;
+ }
+
public TsFileIOWriter(File file, EncryptParameter param) throws IOException {
this(file, TS_FILE_CONFIG, param);
}
@@ -246,6 +260,11 @@ public class TsFileIOWriter implements AutoCloseable {
}
}
+ /** Add a custom property to the TsFile metadata. */
+ public void addTsFileProperty(String key, byte[] value) {
+ tsFileProperties.put(key, value);
+ }
+
public void addFlushListener(FlushChunkMetadataListener listener) {
flushListeners.add(listener);
}
@@ -361,6 +380,7 @@ public class TsFileIOWriter implements AutoCloseable {
chunkHeader.getCompressionType(),
out.getPosition(),
chunkMetadata.getStatistics());
+ currentChunkMetadata.setMask(chunkMetadata.getMask());
chunkHeader.serializeTo(out.wrapAsStream());
out.write(chunk.getData());
endCurrentChunk();
@@ -412,6 +432,10 @@ public class TsFileIOWriter implements AutoCloseable {
chunkHeader.getCompressionType(),
out.getPosition(),
chunk.getChunkStatistic());
+ currentChunkMetadata.setMask(
+ (byte)
+ (chunkHeader.getChunkType()
+ & (TsFileConstant.TIME_COLUMN_MASK |
TsFileConstant.VALUE_COLUMN_MASK)));
chunkHeader.serializeTo(out.wrapAsStream());
out.write(chunk.getData());
endCurrentChunk();
@@ -421,6 +445,15 @@ public class TsFileIOWriter implements AutoCloseable {
public void endCurrentChunk() {
this.currentChunkMetadataSize +=
currentChunkMetadata.getRetainedSizeInBytes();
chunkMetadataCount++;
+ if (recordTablePointCount
+ && currentChunkGroupDeviceId != null
+ && currentChunkGroupDeviceId.isTableModel()
+ && (currentChunkMetadata.getMask() & TsFileConstant.TIME_COLUMN_MASK)
== 0) {
+ tablePointCountMap.merge(
+ currentChunkGroupDeviceId.getTableName(),
+ currentChunkMetadata.getNumOfPoints(),
+ Long::sum);
+ }
chunkMetadataList.add(currentChunkMetadata);
currentChunkMetadata = null;
}
@@ -590,9 +623,18 @@ public class TsFileIOWriter implements AutoCloseable {
tsFileMetadata.setTableSchemaMap(schema.getTableSchemaMap());
tsFileMetadata.setMetaOffset(metaOffset);
tsFileMetadata.setBloomFilter(filter);
- tsFileMetadata.addProperty("encryptLevel", encryptLevel);
- tsFileMetadata.addProperty("encryptType", encryptType);
- tsFileMetadata.addProperty("encryptKey", encryptKey);
+ tsFileProperties.forEach(tsFileMetadata::addProperty);
+ if (recordTablePointCount) {
+ tablePointCountMap.forEach(
+ (tableName, pointCount) ->
+ tsFileMetadata.addProperty(
+ TABLE_POINT_COUNT_PROPERTY_PREFIX + tableName,
+ BytesUtils.longToBytes(pointCount)));
+ }
+ tsFileMetadata.addProperty("encryptLevel",
encryptLevel.getBytes(TSFileConfig.STRING_CHARSET));
+ tsFileMetadata.addProperty("encryptType",
encryptType.getBytes(TSFileConfig.STRING_CHARSET));
+ tsFileMetadata.addProperty(
+ "encryptKey", encryptKey == null ? null :
encryptKey.getBytes(TSFileConfig.STRING_CHARSET));
int size = tsFileMetadata.serializeTo(out.wrapAsStream());
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/TsFileMetadataTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/TsFileMetadataTest.java
index dcf343918..6488acde9 100644
---
a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/TsFileMetadataTest.java
+++
b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/TsFileMetadataTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.tsfile.file.metadata;
+import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.compatibility.DeserializeConfig;
import org.apache.tsfile.constant.TestConstant;
import org.apache.tsfile.file.metadata.utils.TestHelper;
@@ -53,10 +54,12 @@ public class TsFileMetadataTest {
@Test
public void testWriteFileMetaData() {
TsFileMetadata tsfMetaData = TestHelper.createSimpleFileMetaData();
- tsfMetaData.addProperty("encryptLevel", "0");
- tsfMetaData.addProperty("encryptType",
"org.apache.tsfile.encrypt.UNENCRYPTED");
- tsfMetaData.addProperty("encryptKey", "");
- tsfMetaData.addProperty("d", "1");
+ tsfMetaData.addProperty("encryptLevel",
"0".getBytes(TSFileConfig.STRING_CHARSET));
+ tsfMetaData.addProperty(
+ "encryptType",
+
"org.apache.tsfile.encrypt.UNENCRYPTED".getBytes(TSFileConfig.STRING_CHARSET));
+ tsfMetaData.addProperty("encryptKey",
"".getBytes(TSFileConfig.STRING_CHARSET));
+ tsfMetaData.addProperty("d", new byte[] {0, 1, -1});
serialized(tsfMetaData);
TsFileMetadata readMetaData = deSerialized();
Assert.assertTrue(Utils.isFileMetaDataEqual(tsfMetaData, readMetaData));
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/utils/Utils.java
b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/utils/Utils.java
index dc37e00f9..27ec1ab24 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/utils/Utils.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/utils/Utils.java
@@ -25,7 +25,8 @@ import
org.apache.tsfile.file.metadata.statistics.BooleanStatistics;
import org.apache.tsfile.file.metadata.statistics.IntegerStatistics;
import org.apache.tsfile.file.metadata.statistics.Statistics;
-import java.util.Objects;
+import java.util.Arrays;
+import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -61,13 +62,29 @@ public class Utils {
MetadataIndexNode metaDataIndex2 =
metadata2.getTableMetadataIndexNode(TestHelper.TEST_TABLE_NAME);
- return Objects.equals(metadata1.getTsFileProperties(),
metadata2.getTsFileProperties())
+ return arePropertiesEqual(metadata1.getTsFileProperties(),
metadata2.getTsFileProperties())
&& metaDataIndex1.getChildren().size() ==
metaDataIndex2.getChildren().size();
}
}
return false;
}
+ private static boolean arePropertiesEqual(
+ Map<String, byte[]> properties1, Map<String, byte[]> properties2) {
+ if (properties1 == null || properties2 == null) {
+ return properties1 == properties2;
+ }
+ if (!properties1.keySet().equals(properties2.keySet())) {
+ return false;
+ }
+ for (String key : properties1.keySet()) {
+ if (!Arrays.equals(properties1.get(key), properties2.get(key))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
public static void isPageHeaderEqual(PageHeader header1, PageHeader header2)
{
if (Utils.isTwoObjectsNotNULL(header1, header2, "PageHeader")) {
assertEquals(header1.getUncompressedSize(),
header2.getUncompressedSize());
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java
index 51aa64ce6..bbcc3278b 100644
---
a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java
+++
b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java
@@ -27,6 +27,7 @@ import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.StringArrayDeviceID;
import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.read.v4.DeviceTableModelReader;
+import org.apache.tsfile.read.v4.ITsFileReader;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.utils.TsFileGeneratorForTest;
import org.apache.tsfile.utils.TsFileGeneratorUtils;
@@ -41,6 +42,7 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
@@ -49,6 +51,43 @@ import java.util.List;
public class TsFileV4ReadWriteInterfacesTest {
+ @Test
+ public void testTsFileProperties() throws IOException {
+ String filePath = TsFileGeneratorForTest.getTestTsFilePath("properties",
0, 0, 0);
+ File file = new File(filePath);
+ TableSchema tableSchema =
+ new TableSchema(
+ "t1",
+ Arrays.asList(new MeasurementSchema("s1", TSDataType.INT32)),
+ Arrays.asList(ColumnCategory.FIELD));
+
+ try {
+ try (ITsFileWriter writer =
+ new
TsFileWriterBuilder().file(file).tableSchema(tableSchema).build()) {
+ writer.addTsFileProperty(
+ "creator",
"TsFileV4ReadWriteInterfacesTest".getBytes(StandardCharsets.UTF_8));
+ writer.addTsFileProperty("version",
"1".getBytes(StandardCharsets.UTF_8));
+ }
+
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) {
+ Assert.assertArrayEquals(
+ "TsFileV4ReadWriteInterfacesTest".getBytes(StandardCharsets.UTF_8),
+ reader.getTsFileProperties().get("creator"));
+ Assert.assertArrayEquals(
+ "1".getBytes(StandardCharsets.UTF_8),
reader.getTsFileProperties().get("version"));
+ }
+ try (ITsFileReader reader = new DeviceTableModelReader(file)) {
+ Assert.assertArrayEquals(
+ "TsFileV4ReadWriteInterfacesTest".getBytes(StandardCharsets.UTF_8),
+ reader.getTsFileProperties().get("creator"));
+ Assert.assertArrayEquals(
+ "1".getBytes(StandardCharsets.UTF_8),
reader.getTsFileProperties().get("version"));
+ }
+ } finally {
+ Files.deleteIfExists(file.toPath());
+ }
+ }
+
@Test
public void testWriteSomeColumns() throws IOException, WriteProcessException
{
String filePath = TsFileGeneratorForTest.getTestTsFilePath("db", 0, 0, 0);
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java
new file mode 100644
index 000000000..4133f031b
--- /dev/null
+++
b/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tsfile.utils;
+
+import org.apache.tsfile.enums.ColumnCategory;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.ChunkMetadata;
+import org.apache.tsfile.file.metadata.ColumnSchema;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.writer.TsFileIOWriter;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+public class TsFileTablePointCountToolTest {
+
+ /**
+ * Creates a two-table file without point-count properties. The repair tool
must traverse the
+ * metadata index tree and count only non-null FIELD values, preserve
unrelated properties and
+ * file data, and leave the file byte-for-byte unchanged when run again.
+ */
+ @Test
+ public void backfillMissingTablePointCountProperties() throws Exception {
+ File file = File.createTempFile("table-point-count-tool", ".tsfile");
+ try {
+ TableSchema tableSchema1 =
+ new TableSchema(
+ "table1",
+ Arrays.asList(
+ new ColumnSchema("device", TSDataType.STRING,
ColumnCategory.TAG),
+ new ColumnSchema("s1", TSDataType.INT32,
ColumnCategory.FIELD),
+ new ColumnSchema("s2", TSDataType.INT32,
ColumnCategory.FIELD)));
+ TableSchema tableSchema2 =
+ new TableSchema(
+ "table2",
+ Arrays.asList(
+ new ColumnSchema("device", TSDataType.STRING,
ColumnCategory.TAG),
+ new ColumnSchema("s1", TSDataType.INT32,
ColumnCategory.FIELD)));
+ try (TsFileIOWriter ioWriter = new TsFileIOWriter(file);
+ TsFileWriter writer = new TsFileWriter(ioWriter)) {
+ ioWriter.addTsFileProperty("custom.property",
"preserved".getBytes(StandardCharsets.UTF_8));
+ writer.registerTableSchema(tableSchema1);
+ writer.registerTableSchema(tableSchema2);
+ writer.writeTable(createTablet(tableSchema1, 3, true));
+ writer.writeTable(createTablet(tableSchema2, 2, false));
+ }
+
+ try (TsFileSequenceReader reader =
+ new TsFileSequenceReader(file.getAbsolutePath()) {
+ @Override
+ public List<IDeviceID> getAllDevices() {
+ throw new AssertionError("The tool must not enumerate all
devices");
+ }
+
+ @Override
+ public Map<String, List<ChunkMetadata>>
readChunkMetadataInDevice(IDeviceID device) {
+ throw new AssertionError("The tool must not query chunk metadata
by device");
+ }
+ }) {
+ Map<String, Long> pointCounts =
+ TsFileTablePointCountTool.scanTablePointCounts(reader,
reader.getTableSchemaMap());
+ Assert.assertEquals(Long.valueOf(5), pointCounts.get("table1"));
+ Assert.assertEquals(Long.valueOf(2), pointCounts.get("table2"));
+ }
+
+
Assert.assertFalse(TsFileTablePointCountTool.containsTablePointCount(file));
+ Assert.assertEquals(
+ TsFileTablePointCountTool.UpdateStatus.UPDATED,
+ TsFileTablePointCountTool.updateTablePointCountIfMissing(file));
+
Assert.assertTrue(TsFileTablePointCountTool.containsTablePointCount(file));
+
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getAbsolutePath())) {
+ Assert.assertTrue(reader.isComplete());
+ Map<String, byte[]> properties = reader.getTsFileProperties();
+ Assert.assertArrayEquals(
+ "preserved".getBytes(StandardCharsets.UTF_8),
properties.get("custom.property"));
+ Assert.assertArrayEquals(
+ BytesUtils.longToBytes(5),
+ properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX +
"table1"));
+ Assert.assertArrayEquals(
+ BytesUtils.longToBytes(2),
+ properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX +
"table2"));
+ Assert.assertEquals(2, reader.getAllDevices().size());
+ }
+
+ byte[] repairedFile = Files.readAllBytes(file.toPath());
+ Assert.assertEquals(
+ TsFileTablePointCountTool.UpdateStatus.ALREADY_PRESENT,
+ TsFileTablePointCountTool.updateTablePointCountIfMissing(file));
+ Assert.assertArrayEquals(repairedFile,
Files.readAllBytes(file.toPath()));
+ } finally {
+ Files.deleteIfExists(file.toPath());
+ }
+ }
+
+ private Tablet createTablet(TableSchema tableSchema, int rowCount, boolean
nullableSecondField) {
+ Tablet tablet =
+ new Tablet(
+ tableSchema.getTableName(),
+
IMeasurementSchema.getMeasurementNameList(tableSchema.getColumnSchemas()),
+ IMeasurementSchema.getDataTypeList(tableSchema.getColumnSchemas()),
+ tableSchema.getColumnTypes());
+ for (int row = 0; row < rowCount; row++) {
+ tablet.addTimestamp(row, row);
+ tablet.addValue("device", row, "d1");
+ tablet.addValue("s1", row, row);
+ if (nullableSecondField) {
+ tablet.addValue("s2", row, row == rowCount - 1 ? null : row);
+ }
+ }
+ return tablet;
+ }
+}
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java
new file mode 100644
index 000000000..786b36e13
--- /dev/null
+++
b/java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tsfile.write;
+
+import org.apache.tsfile.enums.ColumnCategory;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.file.metadata.ColumnSchema;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.utils.BytesUtils;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.writer.TsFileIOWriter;
+
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class TablePointCountPerformanceTest {
+
+ private static final String MODE_PROPERTY = "tablePointCountEnabled";
+ private static final int TABLE1_ROW_COUNT = 200_000;
+ private static final int TABLE2_ROW_COUNT = 100_000;
+ private static final int BATCH_SIZE = 100;
+ private static final long TABLE1_POINT_COUNT = 333_333;
+ private static final long TABLE2_POINT_COUNT = 100_000;
+
+ /**
+ * Measures one table point-count mode per JVM. Invoke this test twice with
{@code
+ * -DtablePointCountEnabled=false} and {@code true}; each JVM performs its
own warm-up and writes
+ * a mode-specific metrics file. Once both files exist, the test generates a
comparison report
+ * containing the two JVM process IDs, file sizes, and measured write
durations.
+ */
+ @Test
+ public void measureTablePointCountInDedicatedJvm() throws IOException,
WriteProcessException {
+ String configuredMode = System.getProperty(MODE_PROPERTY);
+ Assume.assumeTrue(
+ "Performance test requires -D" + MODE_PROPERTY + "=true|false",
configuredMode != null);
+ Assert.assertTrue(
+ "Invalid -D" + MODE_PROPERTY + " value: " + configuredMode,
+ "true".equals(configuredMode) || "false".equals(configuredMode));
+ boolean enabled = Boolean.parseBoolean(configuredMode);
+
+ TableSchema tableSchema1 =
+ new TableSchema(
+ "table1",
+ Arrays.asList(
+ new ColumnSchema("device", TSDataType.STRING,
ColumnCategory.TAG),
+ new ColumnSchema("s1", TSDataType.INT32, ColumnCategory.FIELD),
+ new ColumnSchema("s2", TSDataType.INT32,
ColumnCategory.FIELD)));
+ TableSchema tableSchema2 =
+ new TableSchema(
+ "table2",
+ Arrays.asList(
+ new ColumnSchema("device", TSDataType.STRING,
ColumnCategory.TAG),
+ new ColumnSchema("s1", TSDataType.INT32,
ColumnCategory.FIELD)));
+ List<Tablet> table1Tablets = createTablets(tableSchema1, TABLE1_ROW_COUNT,
true);
+ List<Tablet> table2Tablets = createTablets(tableSchema2, TABLE2_ROW_COUNT,
false);
+
+ File targetDirectory = new File("target");
+ File warmUpFile = new File(targetDirectory, "table-point-count-warm-up-" +
enabled + ".tsfile");
+ File measuredFile = new File(targetDirectory, "table-point-count-" +
enabled + ".tsfile");
+ try {
+ writeFile(warmUpFile, enabled, tableSchema1, tableSchema2,
table1Tablets, table2Tablets);
+ Assert.assertTrue(warmUpFile.delete());
+
+ long startNanos = System.nanoTime();
+ writeFile(measuredFile, enabled, tableSchema1, tableSchema2,
table1Tablets, table2Tablets);
+ long elapsedNanos = System.nanoTime() - startNanos;
+ Assert.assertTrue(elapsedNanos > 0);
+
+ verifyProperties(measuredFile, enabled);
+ long pid = ProcessHandle.current().pid();
+ writeMetrics(targetDirectory, enabled, pid, measuredFile.length(),
elapsedNanos);
+ generateReportIfBothModesExist(targetDirectory);
+ System.out.printf(
+ "Table point count enabled=%s: pid=%d, size=%d bytes, write=%d ns
(%.3f ms)%n",
+ enabled, pid, measuredFile.length(), elapsedNanos, elapsedNanos /
1_000_000.0);
+ } finally {
+ warmUpFile.delete();
+ measuredFile.delete();
+ }
+ }
+
+ private void writeFile(
+ File file,
+ boolean enabled,
+ TableSchema tableSchema1,
+ TableSchema tableSchema2,
+ List<Tablet> table1Tablets,
+ List<Tablet> table2Tablets)
+ throws IOException, WriteProcessException {
+ try (TsFileWriter writer = new TsFileWriter(new TsFileIOWriter(file,
enabled))) {
+ writer.registerTableSchema(tableSchema1);
+ writer.registerTableSchema(tableSchema2);
+ for (Tablet tablet : table1Tablets) {
+ writer.writeTable(tablet);
+ }
+ for (Tablet tablet : table2Tablets) {
+ writer.writeTable(tablet);
+ }
+ }
+ }
+
+ private List<Tablet> createTablets(
+ TableSchema tableSchema, int rowCount, boolean nullableSecondField) {
+ List<String> columnNames =
+
IMeasurementSchema.getMeasurementNameList(tableSchema.getColumnSchemas());
+ List<TSDataType> dataTypes =
IMeasurementSchema.getDataTypeList(tableSchema.getColumnSchemas());
+ List<Tablet> tablets = new ArrayList<>((rowCount + BATCH_SIZE - 1) /
BATCH_SIZE);
+ for (int startRow = 0; startRow < rowCount; startRow += BATCH_SIZE) {
+ int currentBatchSize = Math.min(BATCH_SIZE, rowCount - startRow);
+ Tablet tablet =
+ new Tablet(
+ tableSchema.getTableName(),
+ columnNames,
+ dataTypes,
+ tableSchema.getColumnTypes(),
+ currentBatchSize);
+ for (int row = 0; row < currentBatchSize; row++) {
+ int globalRow = startRow + row;
+ tablet.addTimestamp(row, globalRow);
+ tablet.addValue("device", row, "d1");
+ tablet.addValue("s1", row, globalRow);
+ if (nullableSecondField) {
+ tablet.addValue("s2", row, globalRow % 3 == 0 ? null : globalRow);
+ }
+ }
+ tablets.add(tablet);
+ }
+ return tablets;
+ }
+
+ private void verifyProperties(File file, boolean enabled) throws IOException
{
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getAbsolutePath())) {
+ Map<String, byte[]> properties = reader.getTsFileProperties();
+ String table1Key = TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX +
"table1";
+ String table2Key = TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX +
"table2";
+ if (enabled) {
+ Assert.assertArrayEquals(
+ BytesUtils.longToBytes(TABLE1_POINT_COUNT),
properties.get(table1Key));
+ Assert.assertArrayEquals(
+ BytesUtils.longToBytes(TABLE2_POINT_COUNT),
properties.get(table2Key));
+ } else {
+ Assert.assertFalse(properties.containsKey(table1Key));
+ Assert.assertFalse(properties.containsKey(table2Key));
+ }
+ }
+ }
+
+ private void writeMetrics(
+ File targetDirectory, boolean enabled, long pid, long fileSize, long
elapsedNanos)
+ throws IOException {
+ Properties metrics = new Properties();
+ metrics.setProperty("enabled", Boolean.toString(enabled));
+ metrics.setProperty("pid", Long.toString(pid));
+ metrics.setProperty("fileSize", Long.toString(fileSize));
+ metrics.setProperty("elapsedNanos", Long.toString(elapsedNanos));
+ metrics.setProperty("rows", Integer.toString(TABLE1_ROW_COUNT +
TABLE2_ROW_COUNT));
+ metrics.setProperty("points", Long.toString(TABLE1_POINT_COUNT +
TABLE2_POINT_COUNT));
+ try (FileOutputStream output = new
FileOutputStream(metricsFile(targetDirectory, enabled))) {
+ metrics.store(output, "Table point count performance metrics");
+ }
+ }
+
+ private void generateReportIfBothModesExist(File targetDirectory) throws
IOException {
+ File disabledMetricsFile = metricsFile(targetDirectory, false);
+ File enabledMetricsFile = metricsFile(targetDirectory, true);
+ if (!disabledMetricsFile.isFile() || !enabledMetricsFile.isFile()) {
+ return;
+ }
+ Properties disabled = loadMetrics(disabledMetricsFile);
+ Properties enabled = loadMetrics(enabledMetricsFile);
+ long disabledPid = Long.parseLong(disabled.getProperty("pid"));
+ long enabledPid = Long.parseLong(enabled.getProperty("pid"));
+ Assert.assertNotEquals("Modes must run in separate JVMs", disabledPid,
enabledPid);
+ long disabledSize = Long.parseLong(disabled.getProperty("fileSize"));
+ long enabledSize = Long.parseLong(enabled.getProperty("fileSize"));
+ long disabledNanos = Long.parseLong(disabled.getProperty("elapsedNanos"));
+ long enabledNanos = Long.parseLong(enabled.getProperty("elapsedNanos"));
+ Assert.assertTrue(enabledSize > disabledSize);
+
+ String report =
+ String.format(
+ "# Table Point Count Performance Report%n%n"
+ + "## Dataset%n%n"
+ + "- table1: %,d rows, %,d non-null FIELD points%n"
+ + "- table2: %,d rows, %,d non-null FIELD points%n"
+ + "- Batch size: %,d rows%n"
+ + "- Total: %,d rows, %,d non-null FIELD points%n%n"
+ + "## Results%n%n"
+ + "| Point counting | JVM PID | File size (bytes) | Write time
(ns) | Write time (ms) |%n"
+ + "|---|---:|---:|---:|---:|%n"
+ + "| Disabled | %d | %,d | %,d | %.3f |%n"
+ + "| Enabled | %d | %,d | %,d | %.3f |%n%n"
+ + "- File-size delta: %,d bytes%n"
+ + "- File-size overhead: %.3f%%%n"
+ + "- Write-time delta: %,d ns (%.3f ms)%n"
+ + "- Write-time overhead: %.3f%%%n%n"
+ + "> Each mode ran in a separate JVM and performed an
independent warm-up. Write "
+ + "time includes writer construction, schema registration,
batched writes, "
+ + "metadata serialization, force, and close. Timing is
diagnostic and is not used "
+ + "as a CI pass/fail threshold.%n",
+ TABLE1_ROW_COUNT,
+ TABLE1_POINT_COUNT,
+ TABLE2_ROW_COUNT,
+ TABLE2_POINT_COUNT,
+ BATCH_SIZE,
+ TABLE1_ROW_COUNT + TABLE2_ROW_COUNT,
+ TABLE1_POINT_COUNT + TABLE2_POINT_COUNT,
+ disabledPid,
+ disabledSize,
+ disabledNanos,
+ disabledNanos / 1_000_000.0,
+ enabledPid,
+ enabledSize,
+ enabledNanos,
+ enabledNanos / 1_000_000.0,
+ enabledSize - disabledSize,
+ (enabledSize - disabledSize) * 100.0 / disabledSize,
+ enabledNanos - disabledNanos,
+ (enabledNanos - disabledNanos) / 1_000_000.0,
+ (enabledNanos - disabledNanos) * 100.0 / disabledNanos);
+ File reportFile = new File(targetDirectory,
"table-point-count-performance-report.md");
+ Files.write(reportFile.toPath(), report.getBytes(StandardCharsets.UTF_8));
+ System.out.printf("Table point-count report: %s%n",
reportFile.getAbsolutePath());
+ }
+
+ private Properties loadMetrics(File file) throws IOException {
+ Properties metrics = new Properties();
+ try (FileInputStream input = new FileInputStream(file)) {
+ metrics.load(input);
+ }
+ return metrics;
+ }
+
+ private File metricsFile(File targetDirectory, boolean enabled) {
+ return new File(targetDirectory, "table-point-count-" + enabled +
".properties");
+ }
+}
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
index fcd761d3f..4341da538 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
@@ -44,6 +44,7 @@ import org.apache.tsfile.read.query.dataset.QueryDataSet;
import org.apache.tsfile.read.query.dataset.ResultSet;
import org.apache.tsfile.read.v4.ITsFileReader;
import org.apache.tsfile.read.v4.TsFileReaderBuilder;
+import org.apache.tsfile.utils.BytesUtils;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.utils.TsFileGeneratorUtils;
import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl;
@@ -1237,6 +1238,69 @@ public class TsFileWriteApiTest {
}
}
+ /**
+ * Verifies that table point counting records only non-null FIELD values in
TsFile properties.
+ * Table1 contains three s1 values and two non-null s2 values, while table2
contains two s1
+ * values. Timestamps, TAG values, and the null s2 value are excluded, so
the expected counts are
+ * five and two respectively.
+ */
+ @Test
+ public void recordTablePointCountInProperties() throws IOException,
WriteProcessException {
+ TableSchema tableSchema1 =
+ new TableSchema(
+ "table1",
+ Arrays.asList(
+ new ColumnSchema("device", TSDataType.STRING,
ColumnCategory.TAG),
+ new ColumnSchema("s1", TSDataType.INT32, ColumnCategory.FIELD),
+ new ColumnSchema("s2", TSDataType.INT32,
ColumnCategory.FIELD)));
+ TableSchema tableSchema2 =
+ new TableSchema(
+ "table2",
+ Arrays.asList(
+ new ColumnSchema("device", TSDataType.STRING,
ColumnCategory.TAG),
+ new ColumnSchema("s1", TSDataType.INT32,
ColumnCategory.FIELD)));
+ Tablet tablet1 =
+ new Tablet(
+ "table1",
+
IMeasurementSchema.getMeasurementNameList(tableSchema1.getColumnSchemas()),
+
IMeasurementSchema.getDataTypeList(tableSchema1.getColumnSchemas()),
+ tableSchema1.getColumnTypes());
+ for (int row = 0; row < 3; row++) {
+ tablet1.addTimestamp(row, row);
+ tablet1.addValue("device", row, "d1");
+ tablet1.addValue("s1", row, row);
+ tablet1.addValue("s2", row, row == 2 ? null : row);
+ }
+ Tablet tablet2 =
+ new Tablet(
+ "table2",
+
IMeasurementSchema.getMeasurementNameList(tableSchema2.getColumnSchemas()),
+
IMeasurementSchema.getDataTypeList(tableSchema2.getColumnSchemas()),
+ tableSchema2.getColumnTypes());
+ for (int row = 0; row < 2; row++) {
+ tablet2.addTimestamp(row, row);
+ tablet2.addValue("device", row, "d1");
+ tablet2.addValue("s1", row, row);
+ }
+
+ try (TsFileWriter writer = new TsFileWriter(new TsFileIOWriter(f, true))) {
+ writer.registerTableSchema(tableSchema1);
+ writer.registerTableSchema(tableSchema2);
+ writer.writeTable(tablet1);
+ writer.writeTable(tablet2);
+ }
+
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(f.getAbsolutePath())) {
+ Map<String, byte[]> properties = reader.getTsFileProperties();
+ Assert.assertArrayEquals(
+ BytesUtils.longToBytes(5),
+ properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX +
"table1"));
+ Assert.assertArrayEquals(
+ BytesUtils.longToBytes(2),
+ properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX +
"table2"));
+ }
+ }
+
@Test
public void calculateTableSize() throws IOException, WriteProcessException {
TableSchema tableSchema1 =
