jixuan1989 opened a new issue, #1017:
URL: https://github.com/apache/tsfile/issues/1017
## English
### Problem and shared input
For the same measurement schema, Java serializes and reads back its
properties successfully. Current C++ reports successful serialization but omits
the property key/value bytes. Its output cannot be read by either
implementation.
Both programs use this input:
```json
{
"measurement": "temperature",
"type": "FLOAT",
"encoding": "TS_2DIFF",
"compression": "UNCOMPRESSED",
"properties": {"max_point_number": "0"}
}
```
### Java reproduction code
Save the following complete source as `SchemaPropertiesRepro.java`. With no
arguments, it serializes the shared input and checks the decoded properties.
`--empty`, `--write` and `--read` support the control and cross-language cases.
```java
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.write.schema.MeasurementSchema;
import java.io.ByteArrayOutputStream;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map;
public class SchemaPropertiesRepro {
public static void main(String[] args) throws Exception {
boolean withProperties = true;
Path readPath = null;
Path writePath = null;
for (int i = 0; i < args.length; i++) {
if (args[i].equals("--empty")) withProperties = false;
else if (args[i].equals("--read")) readPath = Path.of(args[++i]);
else if (args[i].equals("--write")) writePath = Path.of(args[++i]);
}
Map<String, String> properties = withProperties ?
Map.of("max_point_number", "0") : Map.of();
MeasurementSchema original = new MeasurementSchema(
"temperature", TSDataType.FLOAT, TSEncoding.TS_2DIFF,
CompressionType.UNCOMPRESSED, properties);
System.out.println("language=Java");
System.out.println("input.measurement=temperature");
System.out.println("input.type=FLOAT encoding=TS_2DIFF
compression=UNCOMPRESSED");
System.out.println("input.properties=" + properties);
byte[] bytes;
if (readPath == null) {
ByteArrayOutputStream output = new ByteArrayOutputStream();
int written = original.serializeTo(output);
bytes = output.toByteArray();
System.out.println("serialize_return=" + written + " (byte count)");
if (writePath != null) Files.write(writePath, bytes);
} else {
bytes = Files.readAllBytes(readPath);
}
System.out.println("serialized_bytes=" + bytes.length);
try {
MeasurementSchema decoded =
MeasurementSchema.deserializeFrom(ByteBuffer.wrap(bytes));
Map<String, String> actual = decoded.getProps() == null ? Map.of() :
decoded.getProps();
boolean valid = actual.equals(properties);
System.out.println("deserialize=SUCCESS");
System.out.println("output.properties=" + actual);
System.out.println("result=" + (valid ? "PASS" : "FAIL"));
if (!valid) System.exit(1);
} catch (RuntimeException error) {
System.out.println("deserialize=FAIL exception=" +
error.getClass().getSimpleName());
System.out.println("output.properties=<unavailable: deserialization
failed>");
System.out.println("result=FAIL");
System.exit(1);
}
}
}
```
### Actual Java input and output
The Java reproducer creates a `MeasurementSchema` with this input, calls
`serializeTo(ByteArrayOutputStream)`, then calls `deserializeFrom(ByteBuffer)`
on the resulting bytes. Actual stdout:
```text
language=Java
input.measurement=temperature
input.type=FLOAT encoding=TS_2DIFF compression=UNCOMPRESSED
input.properties={max_point_number=0}
serialize_return=47 (byte count)
serialized_bytes=47
deserialize=SUCCESS
output.properties={max_point_number=0}
result=PASS
```
Process exit code: `0`.
### C++ reproduction code
Save the following complete source as `schema_properties_repro.cc`. It uses
the same input and supports the same options.
```cpp
#include <fstream>
#include <iostream>
#include <iterator>
#include <vector>
#include "common/schema.h"
#include "reader/tsfile_reader.h"
int main(int argc, char** argv) {
storage::libtsfile_init();
storage::MeasurementSchema original(
"temperature", common::FLOAT, common::TS_2DIFF,
common::UNCOMPRESSED);
bool with_properties = true;
std::string read_path, write_path;
for (int i = 1; i < argc; ++i) {
std::string arg(argv[i]);
if (arg == "--empty") with_properties = false;
if (arg == "--read" && i + 1 < argc) read_path = argv[++i];
if (arg == "--write" && i + 1 < argc) write_path = argv[++i];
}
if (with_properties) original.props_["max_point_number"] = "0";
common::ByteStream bytes(1024, common::MOD_DEFAULT);
int serialize_result = common::E_OK;
std::vector<char> raw_bytes;
if (read_path.empty()) {
serialize_result = original.serialize_to(bytes);
raw_bytes.resize(bytes.total_size());
common::copy_bs_to_buf(bytes, raw_bytes.data(), raw_bytes.size());
if (!write_path.empty()) {
std::ofstream file(write_path, std::ios::binary);
file.write(raw_bytes.data(), raw_bytes.size());
if (!file.good()) return 2;
}
} else {
std::ifstream file(read_path, std::ios::binary);
if (!file) return 2;
raw_bytes.assign(std::istreambuf_iterator<char>(file), {});
bytes.wrap_from(raw_bytes.data(), raw_bytes.size());
}
storage::MeasurementSchema decoded;
int deserialize_result = decoded.deserialize_from(bytes);
std::cout << "language=C++\n"
<< "input.measurement=temperature\n"
<< "input.type=FLOAT encoding=TS_2DIFF
compression=UNCOMPRESSED\n"
<< "input.properties=" << (with_properties ?
"{max_point_number=0}" : "{}") << '\n';
if (read_path.empty()) std::cout << "serialize_return=" <<
serialize_result << " (E_OK=0)\n";
std::cout << "serialized_bytes=" << raw_bytes.size() << '\n'
<< "deserialize_return=" << deserialize_result << " (E_OK=0,
E_PARTIAL_READ=6)\n"
<< "output.properties={";
bool first = true;
for (const auto& item : decoded.props_) {
if (!first) std::cout << ", ";
std::cout << item.first << '=' << item.second;
first = false;
}
std::cout << "}\n";
bool valid = serialize_result == common::E_OK &&
deserialize_result == common::E_OK &&
decoded.props_ == original.props_;
std::cout << "result=" << (valid ? "PASS" : "FAIL") << '\n';
storage::libtsfile_destroy();
return valid ? 0 : 1;
}
```
### Actual current C++ input and output
The C++ reproducer creates the same schema, sets `props_["max_point_number"]
= "0"`, calls `serialize_to(ByteStream&)`, then calls
`deserialize_from(ByteStream&)`. Actual stdout:
```text
language=C++
input.measurement=temperature
input.type=FLOAT encoding=TS_2DIFF compression=UNCOMPRESSED
input.properties={max_point_number=0}
serialize_return=0 (E_OK=0)
serialized_bytes=22
deserialize_return=6 (E_OK=0, E_PARTIAL_READ=6)
output.properties={}
result=FAIL
```
Process exit code: `1`. Java returns a byte count from serialization; C++
returns a status code. The defect is the incomplete bytes and failed read. The
C++ empty property map above is the state after a failed read, not a successful
decode.
### Cross-language and control results
| Check | Actual result |
|---|---|
| Java writes, C++ reads | Succeeds; preserves `max_point_number=0` |
| C++ writes, Java reads | Fails with `BufferUnderflowException` |
| Empty-property control | Both write identical 22-byte output and read it
successfully |
With non-empty properties, C++ outputs exactly the first 22 bytes of Java's
47-byte output. The missing 25 bytes are the property payload: 4-byte key
length + 16-byte `max_point_number` + 4-byte value length + 1-byte `0`. These
are serialized schema bytes, not complete TsFile files.
### Reproduction
Both implementations use Apache TsFile `develop` commit
`12e39fe9d9dace63cc0ad39664aa0fbf5e29f52f` (`2.5.1-SNAPSHOT`). Tested on
Windows x64, MSVC 14.50, Release static library and JDK 24.0.2. Optional codecs
are disabled; this schema-only case does not depend on compression.
With JDK 17+, Maven, CMake and Visual Studio C++ build tools available, save
the two source files above and the two build files below in one directory. Run
the following in PowerShell from that directory against a clean checkout of the
specified commit. No attachment is required:
```powershell
.\reproduce.ps1 -TsFileSource D:\code\tsfile
```
<details>
<summary>Complete build files: CMakeLists.txt and reproduce.ps1</summary>
Save as `CMakeLists.txt`:
```cmake
cmake_minimum_required(VERSION 3.15)
project(tsfile_schema_properties_repro LANGUAGES C CXX)
set(TSFILE_SOURCE "" CACHE PATH "Apache TsFile source checkout")
set(TSFILE_LIBRARY "" CACHE FILEPATH "Built static TsFile library")
if(NOT EXISTS "${TSFILE_SOURCE}/cpp/CMakeLists.txt")
message(FATAL_ERROR "Set TSFILE_SOURCE to an Apache TsFile source
checkout")
endif()
if(NOT EXISTS "${TSFILE_LIBRARY}")
message(FATAL_ERROR "Set TSFILE_LIBRARY to the built static TsFile
library")
endif()
add_library(tsfile_repro_lib STATIC IMPORTED)
set_target_properties(tsfile_repro_lib PROPERTIES IMPORTED_LOCATION
"${TSFILE_LIBRARY}")
add_executable(schema_properties_repro schema_properties_repro.cc)
target_compile_features(schema_properties_repro PRIVATE cxx_std_14)
target_compile_definitions(schema_properties_repro PRIVATE TSFILE_STATIC
NOMINMAX ENABLE_MEM_STAT ENABLE_THREADS)
target_include_directories(schema_properties_repro PRIVATE
"${TSFILE_SOURCE}/cpp/src")
target_link_libraries(schema_properties_repro PRIVATE tsfile_repro_lib)
if(MSVC)
target_compile_options(schema_properties_repro PRIVATE /utf-8 /EHsc
/bigobj)
endif()
```
Save as `reproduce.ps1`:
```powershell
param(
[Parameter(Mandatory = $true)] [string] $TsFileSource,
[string] $BuildRoot = (Join-Path $PSScriptRoot 'build'),
[string] $JavaClasspath = ''
)
$ErrorActionPreference = 'Stop'
$PSNativeCommandUseErrorActionPreference = $false
$sourcePath = (Resolve-Path -LiteralPath $TsFileSource).Path
$libraryBuild = Join-Path $BuildRoot 'library'
$programBuild = Join-Path $BuildRoot 'program'
$cmakeOptions = @(
'-DTSFILE_BUILD_SHARED=OFF', '-DBUILD_TEST=OFF', '-DBUILD_TOOLS=OFF',
'-DENABLE_ANTLR4=OFF', '-DENABLE_SNAPPY=OFF', '-DENABLE_LZ4=OFF',
'-DENABLE_LZOKAY=OFF', '-DENABLE_ZLIB=OFF', '-DENABLE_ZSTD=OFF',
'-DENABLE_LZMA2=OFF', '-DENABLE_SIMD=OFF', '-DENABLE_THREADS=ON'
)
& cmake -S (Join-Path $sourcePath 'cpp') -B $libraryBuild @cmakeOptions
if ($LASTEXITCODE -ne 0) { throw 'TsFile CMake configuration failed' }
& cmake --build $libraryBuild --config Release --target tsfile --parallel 6
if ($LASTEXITCODE -ne 0) { throw 'TsFile build failed' }
$staticLibrary = Join-Path $libraryBuild 'lib\Release\tsfile.lib'
if (-not (Test-Path -LiteralPath $staticLibrary)) {
throw "Static library not found: $staticLibrary. This script uses the
Windows MSVC build layout."
}
& cmake -S $PSScriptRoot -B $programBuild "-DTSFILE_SOURCE=$sourcePath"
"-DTSFILE_LIBRARY=$staticLibrary"
if ($LASTEXITCODE -ne 0) { throw 'Reproducer CMake configuration failed' }
& cmake --build $programBuild --config Release --target
schema_properties_repro
if ($LASTEXITCODE -ne 0) { throw 'Reproducer build failed' }
if ([string]::IsNullOrWhiteSpace($JavaClasspath)) {
& mvn -f (Join-Path $sourcePath 'pom.xml') -P with-java -pl java/tsfile
-am '-Dtest=CompatibilityTest' '-Dsurefire.failIfNoSpecifiedTests=false'
'-Dspotless.skip=true' test
if ($LASTEXITCODE -ne 0) { throw 'Java build failed' }
$javaReport = Join-Path $sourcePath
'java\tsfile\target\surefire-reports\TEST-org.apache.tsfile.compatibility.CompatibilityTest.xml'
[xml]$javaXml = Get-Content -LiteralPath $javaReport -Raw
$JavaClasspath = ($javaXml.testsuite.properties.property | Where-Object
{ $_.name -eq 'java.class.path' }).value
if ([string]::IsNullOrWhiteSpace($JavaClasspath)) { throw 'Java
classpath was not found in the test report' }
}
$javaClasses = Join-Path $BuildRoot 'java-classes'
New-Item -ItemType Directory -Force -Path $javaClasses | Out-Null
& javac -cp $JavaClasspath -d $javaClasses (Join-Path $PSScriptRoot
'SchemaPropertiesRepro.java')
if ($LASTEXITCODE -ne 0) { throw 'Java reproducer compilation failed' }
$runtimeClasspath = $JavaClasspath + ';' + $javaClasses
$program = Join-Path $programBuild 'Release\schema_properties_repro.exe'
$javaBytes = Join-Path $BuildRoot 'java.schema.bin'
$cppBytes = Join-Path $BuildRoot 'cpp.schema.bin'
Write-Output 'JAVA: same non-empty schema input'
& java -cp $runtimeClasspath SchemaPropertiesRepro --write $javaBytes
if ($LASTEXITCODE -ne 0) { throw 'Java reference round-trip failed' }
Write-Output 'C++: same non-empty schema input'
& $program --write $cppBytes
if ($LASTEXITCODE -ne 1) { throw 'The reported C++ failure did not reproduce
as expected' }
Write-Output 'JAVA CONTROL: empty properties'
& java -cp $runtimeClasspath SchemaPropertiesRepro --empty
if ($LASTEXITCODE -ne 0) { throw 'Java empty-property control failed' }
Write-Output 'C++ CONTROL: empty properties'
& $program --empty
if ($LASTEXITCODE -ne 0) { throw 'C++ empty-property control failed' }
Write-Output 'CROSS READ: C++ reads Java output'
& $program --read $javaBytes
if ($LASTEXITCODE -ne 0) { throw 'C++ could not read the Java reference
schema' }
Write-Output 'CROSS READ: Java reads current C++ output'
& java -cp $runtimeClasspath SchemaPropertiesRepro --read $cppBytes
if ($LASTEXITCODE -ne 1) { throw 'The expected cross-language failure did
not reproduce' }
Write-Output 'Same-input Java/C++ difference and both cross-read results
reproduced.'
exit 0
```
</details>
The script builds both implementations and runs all six cases: both round
trips, both empty-property controls and both cross-language reads. Individual
failing cases exit `1`; the script exits `0` only if all six match the reported
reproduction results. The attachment also contains these files, raw logs and
output bytes.
[Download the reproduction
package](https://raw.githubusercontent.com/jixuan1989/tsfile-apache/127813705508a62efe78362e1e141b60a13b2db0/issue-evidence/measurement-schema-properties/reproduction.zip)
### Expected behavior and cause
C++ should write every property after the count, producing the same 47
schema bytes as Java for this input. Both implementations should read the
result and preserve the property. Write errors should propagate to the caller.
In
[`cpp/src/common/schema.h`](https://github.com/apache/tsfile/blob/12e39fe9d9dace63cc0ad39664aa0fbf5e29f52f/cpp/src/common/schema.h#L103),
the property loop is inside the failure branch of the property-count write. It
is skipped when that write succeeds. The [Java
serializer](https://github.com/apache/tsfile/blob/12e39fe9d9dace63cc0ad39664aa0fbf5e29f52f/java/tsfile/src/main/java/org/apache/tsfile/write/schema/MeasurementSchema.java#L295)
writes the count followed by the key/value pairs.
Scope: fix schema property serialization. Applying properties to encoder
precision settings is a separate task.
### Screenshot
The screenshot displays actual captured stdout from both programs in a
browser log viewer.

## 中文翻译
### 问题及相同输入
同一个测点 Schema,Java 可以成功序列化并读回属性;当前 C++ 返回序列化成功,却漏写了属性键值,导致其输出无法被 C++ 自身或 Java
正常读取。
两边使用完全相同的输入,以下 JSON 保持原样:
```json
{
"measurement": "temperature",
"type": "FLOAT",
"encoding": "TS_2DIFF",
"compression": "UNCOMPRESSED",
"properties": {"max_point_number": "0"}
}
```
### Java 复现代码
将下面的完整源码保存为
`SchemaPropertiesRepro.java`。不带参数时,程序序列化上述输入,再检查读回的属性。`--empty`、`--write` 和
`--read` 用于对照和交叉读取案例。
```java
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.write.schema.MeasurementSchema;
import java.io.ByteArrayOutputStream;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map;
public class SchemaPropertiesRepro {
public static void main(String[] args) throws Exception {
boolean withProperties = true;
Path readPath = null;
Path writePath = null;
for (int i = 0; i < args.length; i++) {
if (args[i].equals("--empty")) withProperties = false;
else if (args[i].equals("--read")) readPath = Path.of(args[++i]);
else if (args[i].equals("--write")) writePath = Path.of(args[++i]);
}
Map<String, String> properties = withProperties ?
Map.of("max_point_number", "0") : Map.of();
MeasurementSchema original = new MeasurementSchema(
"temperature", TSDataType.FLOAT, TSEncoding.TS_2DIFF,
CompressionType.UNCOMPRESSED, properties);
System.out.println("language=Java");
System.out.println("input.measurement=temperature");
System.out.println("input.type=FLOAT encoding=TS_2DIFF
compression=UNCOMPRESSED");
System.out.println("input.properties=" + properties);
byte[] bytes;
if (readPath == null) {
ByteArrayOutputStream output = new ByteArrayOutputStream();
int written = original.serializeTo(output);
bytes = output.toByteArray();
System.out.println("serialize_return=" + written + " (byte count)");
if (writePath != null) Files.write(writePath, bytes);
} else {
bytes = Files.readAllBytes(readPath);
}
System.out.println("serialized_bytes=" + bytes.length);
try {
MeasurementSchema decoded =
MeasurementSchema.deserializeFrom(ByteBuffer.wrap(bytes));
Map<String, String> actual = decoded.getProps() == null ? Map.of() :
decoded.getProps();
boolean valid = actual.equals(properties);
System.out.println("deserialize=SUCCESS");
System.out.println("output.properties=" + actual);
System.out.println("result=" + (valid ? "PASS" : "FAIL"));
if (!valid) System.exit(1);
} catch (RuntimeException error) {
System.out.println("deserialize=FAIL exception=" +
error.getClass().getSimpleName());
System.out.println("output.properties=<unavailable: deserialization
failed>");
System.out.println("result=FAIL");
System.exit(1);
}
}
}
```
### Java 的实际输入和输出
Java 使用上述输入创建 `MeasurementSchema`,调用
`serializeTo(ByteArrayOutputStream)`,再对输出字节调用
`deserializeFrom(ByteBuffer)`。实际标准输出如下,保持原样:
```text
language=Java
input.measurement=temperature
input.type=FLOAT encoding=TS_2DIFF compression=UNCOMPRESSED
input.properties={max_point_number=0}
serialize_return=47 (byte count)
serialized_bytes=47
deserialize=SUCCESS
output.properties={max_point_number=0}
result=PASS
```
进程退出码为 `0`。
### C++ 复现代码
将下面的完整源码保存为 `schema_properties_repro.cc`。它使用相同输入,支持相同的命令行选项。
```cpp
#include <fstream>
#include <iostream>
#include <iterator>
#include <vector>
#include "common/schema.h"
#include "reader/tsfile_reader.h"
int main(int argc, char** argv) {
storage::libtsfile_init();
storage::MeasurementSchema original(
"temperature", common::FLOAT, common::TS_2DIFF,
common::UNCOMPRESSED);
bool with_properties = true;
std::string read_path, write_path;
for (int i = 1; i < argc; ++i) {
std::string arg(argv[i]);
if (arg == "--empty") with_properties = false;
if (arg == "--read" && i + 1 < argc) read_path = argv[++i];
if (arg == "--write" && i + 1 < argc) write_path = argv[++i];
}
if (with_properties) original.props_["max_point_number"] = "0";
common::ByteStream bytes(1024, common::MOD_DEFAULT);
int serialize_result = common::E_OK;
std::vector<char> raw_bytes;
if (read_path.empty()) {
serialize_result = original.serialize_to(bytes);
raw_bytes.resize(bytes.total_size());
common::copy_bs_to_buf(bytes, raw_bytes.data(), raw_bytes.size());
if (!write_path.empty()) {
std::ofstream file(write_path, std::ios::binary);
file.write(raw_bytes.data(), raw_bytes.size());
if (!file.good()) return 2;
}
} else {
std::ifstream file(read_path, std::ios::binary);
if (!file) return 2;
raw_bytes.assign(std::istreambuf_iterator<char>(file), {});
bytes.wrap_from(raw_bytes.data(), raw_bytes.size());
}
storage::MeasurementSchema decoded;
int deserialize_result = decoded.deserialize_from(bytes);
std::cout << "language=C++\n"
<< "input.measurement=temperature\n"
<< "input.type=FLOAT encoding=TS_2DIFF
compression=UNCOMPRESSED\n"
<< "input.properties=" << (with_properties ?
"{max_point_number=0}" : "{}") << '\n';
if (read_path.empty()) std::cout << "serialize_return=" <<
serialize_result << " (E_OK=0)\n";
std::cout << "serialized_bytes=" << raw_bytes.size() << '\n'
<< "deserialize_return=" << deserialize_result << " (E_OK=0,
E_PARTIAL_READ=6)\n"
<< "output.properties={";
bool first = true;
for (const auto& item : decoded.props_) {
if (!first) std::cout << ", ";
std::cout << item.first << '=' << item.second;
first = false;
}
std::cout << "}\n";
bool valid = serialize_result == common::E_OK &&
deserialize_result == common::E_OK &&
decoded.props_ == original.props_;
std::cout << "result=" << (valid ? "PASS" : "FAIL") << '\n';
storage::libtsfile_destroy();
return valid ? 0 : 1;
}
```
### 当前 C++ 的实际输入和输出
C++ 创建同样的 Schema,设置 `props_["max_point_number"] = "0"`,调用
`serialize_to(ByteStream&)`,再调用 `deserialize_from(ByteStream&)`。实际标准输出如下,保持原样:
```text
language=C++
input.measurement=temperature
input.type=FLOAT encoding=TS_2DIFF compression=UNCOMPRESSED
input.properties={max_point_number=0}
serialize_return=0 (E_OK=0)
serialized_bytes=22
deserialize_return=6 (E_OK=0, E_PARTIAL_READ=6)
output.properties={}
result=FAIL
```
进程退出码为 `1`。
Java 的序列化返回值是字节数,C++ 的返回值是状态码。问题在于 C++ 输出不完整、随后读取失败。日志中的 C++
空属性表是失败后的对象状态,不能理解为成功读出了空 Schema。
### 交叉读取及对照结果
| 检查 | 实际结果 |
|---|---|
| Java 写、C++ 读 | 成功,保留 `max_point_number=0` |
| C++ 写、Java 读 | 失败,抛出 `BufferUnderflowException` |
| 无属性对照组 | 两边输出完全相同的 22 字节,并且均能成功读回 |
带属性时,C++ 的 22 字节恰好等于 Java 输出的前 22 字节。缺少的 25 字节正是属性内容:键长度 4 字节 +
`max_point_number` 的 16 字节 + 值长度 4 字节 + `0` 的 1 字节。这里比较的是 Schema 序列化字节,不是完整
TsFile 文件。
### 复现方法
两个实现均使用 Apache TsFile `develop` 提交
`12e39fe9d9dace63cc0ad39664aa0fbf5e29f52f`,版本 `2.5.1-SNAPSHOT`。验证环境为 Windows
x64、MSVC 14.50、Release 静态库和 JDK 24.0.2。关闭可选压缩算法;本例不依赖压缩。
准备对应提交的干净源码,并安装 JDK 17+、Maven、CMake 和 Visual Studio C++
编译工具。将上面的两个源码文件及下面的两个构建文件保存在同一目录。从该目录运行以下 PowerShell 命令,将 `D:\code\tsfile`
替换成自己的路径。无需下载附件:
```powershell
.\reproduce.ps1 -TsFileSource D:\code\tsfile
```
<details>
<summary>完整构建文件:CMakeLists.txt 和 reproduce.ps1</summary>
保存为 `CMakeLists.txt`:
```cmake
cmake_minimum_required(VERSION 3.15)
project(tsfile_schema_properties_repro LANGUAGES C CXX)
set(TSFILE_SOURCE "" CACHE PATH "Apache TsFile source checkout")
set(TSFILE_LIBRARY "" CACHE FILEPATH "Built static TsFile library")
if(NOT EXISTS "${TSFILE_SOURCE}/cpp/CMakeLists.txt")
message(FATAL_ERROR "Set TSFILE_SOURCE to an Apache TsFile source
checkout")
endif()
if(NOT EXISTS "${TSFILE_LIBRARY}")
message(FATAL_ERROR "Set TSFILE_LIBRARY to the built static TsFile
library")
endif()
add_library(tsfile_repro_lib STATIC IMPORTED)
set_target_properties(tsfile_repro_lib PROPERTIES IMPORTED_LOCATION
"${TSFILE_LIBRARY}")
add_executable(schema_properties_repro schema_properties_repro.cc)
target_compile_features(schema_properties_repro PRIVATE cxx_std_14)
target_compile_definitions(schema_properties_repro PRIVATE TSFILE_STATIC
NOMINMAX ENABLE_MEM_STAT ENABLE_THREADS)
target_include_directories(schema_properties_repro PRIVATE
"${TSFILE_SOURCE}/cpp/src")
target_link_libraries(schema_properties_repro PRIVATE tsfile_repro_lib)
if(MSVC)
target_compile_options(schema_properties_repro PRIVATE /utf-8 /EHsc
/bigobj)
endif()
```
保存为 `reproduce.ps1`:
```powershell
param(
[Parameter(Mandatory = $true)] [string] $TsFileSource,
[string] $BuildRoot = (Join-Path $PSScriptRoot 'build'),
[string] $JavaClasspath = ''
)
$ErrorActionPreference = 'Stop'
$PSNativeCommandUseErrorActionPreference = $false
$sourcePath = (Resolve-Path -LiteralPath $TsFileSource).Path
$libraryBuild = Join-Path $BuildRoot 'library'
$programBuild = Join-Path $BuildRoot 'program'
$cmakeOptions = @(
'-DTSFILE_BUILD_SHARED=OFF', '-DBUILD_TEST=OFF', '-DBUILD_TOOLS=OFF',
'-DENABLE_ANTLR4=OFF', '-DENABLE_SNAPPY=OFF', '-DENABLE_LZ4=OFF',
'-DENABLE_LZOKAY=OFF', '-DENABLE_ZLIB=OFF', '-DENABLE_ZSTD=OFF',
'-DENABLE_LZMA2=OFF', '-DENABLE_SIMD=OFF', '-DENABLE_THREADS=ON'
)
& cmake -S (Join-Path $sourcePath 'cpp') -B $libraryBuild @cmakeOptions
if ($LASTEXITCODE -ne 0) { throw 'TsFile CMake configuration failed' }
& cmake --build $libraryBuild --config Release --target tsfile --parallel 6
if ($LASTEXITCODE -ne 0) { throw 'TsFile build failed' }
$staticLibrary = Join-Path $libraryBuild 'lib\Release\tsfile.lib'
if (-not (Test-Path -LiteralPath $staticLibrary)) {
throw "Static library not found: $staticLibrary. This script uses the
Windows MSVC build layout."
}
& cmake -S $PSScriptRoot -B $programBuild "-DTSFILE_SOURCE=$sourcePath"
"-DTSFILE_LIBRARY=$staticLibrary"
if ($LASTEXITCODE -ne 0) { throw 'Reproducer CMake configuration failed' }
& cmake --build $programBuild --config Release --target
schema_properties_repro
if ($LASTEXITCODE -ne 0) { throw 'Reproducer build failed' }
if ([string]::IsNullOrWhiteSpace($JavaClasspath)) {
& mvn -f (Join-Path $sourcePath 'pom.xml') -P with-java -pl java/tsfile
-am '-Dtest=CompatibilityTest' '-Dsurefire.failIfNoSpecifiedTests=false'
'-Dspotless.skip=true' test
if ($LASTEXITCODE -ne 0) { throw 'Java build failed' }
$javaReport = Join-Path $sourcePath
'java\tsfile\target\surefire-reports\TEST-org.apache.tsfile.compatibility.CompatibilityTest.xml'
[xml]$javaXml = Get-Content -LiteralPath $javaReport -Raw
$JavaClasspath = ($javaXml.testsuite.properties.property | Where-Object
{ $_.name -eq 'java.class.path' }).value
if ([string]::IsNullOrWhiteSpace($JavaClasspath)) { throw 'Java
classpath was not found in the test report' }
}
$javaClasses = Join-Path $BuildRoot 'java-classes'
New-Item -ItemType Directory -Force -Path $javaClasses | Out-Null
& javac -cp $JavaClasspath -d $javaClasses (Join-Path $PSScriptRoot
'SchemaPropertiesRepro.java')
if ($LASTEXITCODE -ne 0) { throw 'Java reproducer compilation failed' }
$runtimeClasspath = $JavaClasspath + ';' + $javaClasses
$program = Join-Path $programBuild 'Release\schema_properties_repro.exe'
$javaBytes = Join-Path $BuildRoot 'java.schema.bin'
$cppBytes = Join-Path $BuildRoot 'cpp.schema.bin'
Write-Output 'JAVA: same non-empty schema input'
& java -cp $runtimeClasspath SchemaPropertiesRepro --write $javaBytes
if ($LASTEXITCODE -ne 0) { throw 'Java reference round-trip failed' }
Write-Output 'C++: same non-empty schema input'
& $program --write $cppBytes
if ($LASTEXITCODE -ne 1) { throw 'The reported C++ failure did not reproduce
as expected' }
Write-Output 'JAVA CONTROL: empty properties'
& java -cp $runtimeClasspath SchemaPropertiesRepro --empty
if ($LASTEXITCODE -ne 0) { throw 'Java empty-property control failed' }
Write-Output 'C++ CONTROL: empty properties'
& $program --empty
if ($LASTEXITCODE -ne 0) { throw 'C++ empty-property control failed' }
Write-Output 'CROSS READ: C++ reads Java output'
& $program --read $javaBytes
if ($LASTEXITCODE -ne 0) { throw 'C++ could not read the Java reference
schema' }
Write-Output 'CROSS READ: Java reads current C++ output'
& java -cp $runtimeClasspath SchemaPropertiesRepro --read $cppBytes
if ($LASTEXITCODE -ne 1) { throw 'The expected cross-language failure did
not reproduce' }
Write-Output 'Same-input Java/C++ difference and both cross-read results
reproduced.'
exit 0
```
</details>
脚本构建两个实现,运行各自读写、两个无属性对照组和两个方向的交叉读取,共六项。缺陷案例的程序退出码为 `1`;只有六项都符合报告的复现结果,脚本才退出
`0`。附件同时提供这些文件、原始日志和输出字节。
[下载复现包](https://raw.githubusercontent.com/jixuan1989/tsfile-apache/127813705508a62efe78362e1e141b60a13b2db0/issue-evidence/measurement-schema-properties/reproduction.zip)
### 预期行为及原因
C++ 应在属性数量后写入全部键值对。本例应得到与 Java 相同的 47 字节,两个实现都能读回并保留属性;写入失败时应返回错误。
`cpp/src/common/schema.h` 将属性循环放进了“写入属性数量失败”的分支,数量写入成功时反而不执行循环。Java
会写入数量及后续键值对,源码链接见英文部分。
本项只修复 Schema 属性序列化。将属性应用到编码器精度设置,单独处理。
### 截图说明
上方截图来自浏览器日志预览页面,展示两个程序真实运行时捕获的标准输出。
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]