This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch support_schema_evolution in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 235c54935cf738128d4bd0375cb6eeccd0268b6b Author: Tian Jiang <[email protected]> AuthorDate: Tue Nov 11 17:00:09 2025 +0800 temp save --- .../org/apache/tsfile/file/metadata/IDeviceID.java | 4 + .../apache/tsfile/file/metadata/PlainDeviceID.java | 11 ++ .../tsfile/file/metadata/StringArrayDeviceID.java | 11 ++ .../apache/tsfile/file/metadata/TableSchema.java | 67 +------ .../tsfile/file/metadata/TableSchemaMap.java | 53 ------ .../tsfile/file/metadata/TsFileMetadata.java | 34 +++- .../file/metadata/evolution/ColumnRename.java | 15 +- .../file/metadata/evolution/EvolvedSchema.java | 70 ++++++++ .../file/metadata/evolution/SchemaEvolution.java | 8 +- .../file/metadata/evolution/TableRename.java | 23 +-- .../apache/tsfile/read/TsFileSequenceReader.java | 15 +- .../tsfile/read/v4/DeviceTableModelReader.java | 12 +- .../org/apache/tsfile/write/record/Tablet.java | 9 + .../tsfile/write/schema/MeasurementSchema.java | 16 -- .../write/v4/AbstractTableModelTsFileWriter.java | 9 + .../tsfile/write/v4/DeviceTableModelWriter.java | 15 +- .../apache/tsfile/write/writer/TsFileIOWriter.java | 1 - .../evolution/TsFileSchemaEvolutionTest.java | 190 ++++++++++++++++++++ .../evolution/TsFileSchemaRewriterTest.java | 197 +++++++++++++++++++++ 19 files changed, 584 insertions(+), 176 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java index 17faa274..07d7dde8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java @@ -55,6 +55,8 @@ public interface IDeviceID extends Comparable<IDeviceID>, Accountable, Serializa */ String getTableName(); + void setTableName(String tableName); + /** * @return how many segments this DeviceId consists of. For a path-DeviceId, like "root.a.b.c.d", * it is 5; fot a tuple-DeviceId, like "(table1, beijing, turbine)", it is 3. @@ -146,6 +148,8 @@ public interface IDeviceID extends Comparable<IDeviceID>, Accountable, Serializa return startWith(databaseName, true); } + IDeviceID clone(); + interface Deserializer { IDeviceID deserializeFrom(ByteBuffer byteBuffer); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java index ccc8ce2c..f84a373f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java @@ -178,6 +178,11 @@ public class PlainDeviceID implements IDeviceID { return tableName; } + @Override + public void setTableName(String tableName) { + throw new UnsupportedOperationException(); + } + @Override public int segmentNum() { if (segments != null) { @@ -195,6 +200,12 @@ public class PlainDeviceID implements IDeviceID { return segments[i]; } + @SuppressWarnings("MethodDoesntCallSuperMethod") + @Override + public IDeviceID clone() { + return new PlainDeviceID(deviceID); + } + public static class Factory implements IDeviceID.Factory { @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java index 895b2870..c39942e3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java @@ -236,6 +236,11 @@ public class StringArrayDeviceID implements IDeviceID { return segments[0]; } + @Override + public void setTableName(String tableName) { + segments[0] = tableName; + } + @Override public int segmentNum() { return segments.length; @@ -250,6 +255,12 @@ public class StringArrayDeviceID implements IDeviceID { return segments[i]; } + @SuppressWarnings("MethodDoesntCallSuperMethod") + @Override + public IDeviceID clone() { + return new StringArrayDeviceID(Arrays.copyOf(segments, segments.length)); + } + @Override public int compareTo(IDeviceID o) { if (this == o) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java index 51ec72b5..3f77f3fa 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java @@ -45,8 +45,6 @@ public class TableSchema { // the tableName is not serialized since the TableSchema is always stored in a Map, from whose // key the tableName can be known protected String tableName; - // the last name after renames - protected String finalTableName; protected List<MeasurementSchema> measurementSchemas; protected List<ColumnCategory> columnCategories; protected boolean updatable = false; @@ -61,7 +59,6 @@ public class TableSchema { public TableSchema(String tableName) { this.tableName = tableName.toLowerCase(); - this.finalTableName = tableName; this.measurementSchemas = new ArrayList<>(); this.columnCategories = new ArrayList<>(); this.updatable = true; @@ -90,7 +87,6 @@ public class TableSchema { List<IMeasurementSchema> columnSchemas, List<ColumnCategory> columnCategories) { this.tableName = tableName.toLowerCase(); - this.finalTableName = tableName; this.measurementSchemas = new ArrayList<>(columnSchemas.size()); this.columnPosIndex = new HashMap<>(columnSchemas.size()); for (int i = 0; i < columnSchemas.size(); i++) { @@ -119,7 +115,6 @@ public class TableSchema { List<TSDataType> dataTypeList, List<ColumnCategory> categoryList) { this.tableName = tableName.toLowerCase(); - this.finalTableName = tableName; this.measurementSchemas = new ArrayList<>(columnNameList.size()); this.columnPosIndex = new HashMap<>(columnNameList.size()); for (int i = 0; i < columnNameList.size(); i++) { @@ -138,7 +133,6 @@ public class TableSchema { @TsFileApi public TableSchema(String tableName, List<ColumnSchema> columnSchemaList) { this.tableName = tableName.toLowerCase(); - this.finalTableName = tableName; this.measurementSchemas = new ArrayList<>(columnSchemaList.size()); this.columnCategories = new ArrayList<>(columnSchemaList.size()); this.columnPosIndex = new HashMap<>(columnSchemaList.size()); @@ -173,7 +167,7 @@ public class TableSchema { } for (int i = 0; i < measurementSchemas.size(); i++) { MeasurementSchema currentColumnSchema = measurementSchemas.get(i); - columnPosIndex.putIfAbsent(currentColumnSchema.getFinalMeasurementName(), i); + columnPosIndex.putIfAbsent(currentColumnSchema.getMeasurementName(), i); } return columnPosIndex; } @@ -195,7 +189,8 @@ public class TableSchema { lowerCaseColumnName, colName -> { for (int i = 0; i < measurementSchemas.size(); i++) { - if (measurementSchemas.get(i).getFinalMeasurementName().equals(lowerCaseColumnName)) { + if (measurementSchemas.get(i).getMeasurementName() + .equals(lowerCaseColumnName)) { return i; } } @@ -215,7 +210,7 @@ public class TableSchema { colName -> { int columnOrder = 0; for (int i = 0; i < measurementSchemas.size(); i++) { - if (measurementSchemas.get(i).getFinalMeasurementName().equals(lowerCaseColumnName) + if (measurementSchemas.get(i).getMeasurementName().equals(lowerCaseColumnName) && columnCategories.get(i) == ColumnCategory.TAG) { return columnOrder; } else if (columnCategories.get(i) == ColumnCategory.TAG) { @@ -305,17 +300,8 @@ public class TableSchema { return tableName; } - public String getFinalTableName() { - return finalTableName; - } - public void setTableName(String tableName) { this.tableName = tableName.toLowerCase(); - this.finalTableName = tableName; - } - - public void setFinalTableName(String finalTableName) { - this.finalTableName = finalTableName; } @Override @@ -323,7 +309,6 @@ public class TableSchema { return "TableSchema{" + "tableName='" + tableName - + '\'' + (!Objects.equals(finalTableName, tableName) ? "(" + finalTableName + ")" : "") + ", columnSchemas=" + measurementSchemas + ", columnTypes=" @@ -357,48 +342,4 @@ public class TableSchema { tagColumnCnt = (int) columnCategories.stream().filter(c -> c == ColumnCategory.TAG).count(); return tagColumnCnt; } - - public void renameColumn(String nameBefore, String nameAfter) { - nameBefore = nameBefore.toLowerCase(); - nameAfter = nameAfter.toLowerCase(); - int beforeNameIndx = findColumnIndex(nameBefore); - if (beforeNameIndx == -1) { - return; - } - int afterNameIndex = findColumnIndex(nameAfter); - - measurementSchemas.get(beforeNameIndx).setFinalMeasurementName(nameAfter); - // update the columnPosIndex map - if (columnPosIndex != null) { - columnPosIndex.remove(nameBefore); - columnPosIndex.put(nameAfter, beforeNameIndx); - } - - if (tagColumnOrder != null) { - if (tagColumnOrder.containsKey(nameBefore)) { - int order = tagColumnOrder.remove(nameBefore); - tagColumnOrder.put(nameAfter, order); - } - } - - // if the renamed column already exists, then it must be removed previously - if (afterNameIndex != -1) { - measurementSchemas.remove(afterNameIndex).setDeleted(true); - columnCategories.remove(afterNameIndex); - // need to rebuild the columnPosIndex map - columnPosIndex = null; - buildColumnPosIndex(); - // need to rebuild the tagColumnOrder map - tagColumnOrder = null; - getTagColumnOrder(); - } - } - - public boolean isDeleted() { - return deleted; - } - - public void setDeleted(boolean deleted) { - this.deleted = deleted; - } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchemaMap.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchemaMap.java deleted file mode 100644 index 3ada1137..00000000 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchemaMap.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.tsfile.file.metadata; - -import java.util.HashMap; -import java.util.Map; -import org.apache.tsfile.file.metadata.evolution.SchemaEvolution; -import org.apache.tsfile.file.metadata.evolution.SchemaEvolution.Builder; - -/** - * A map from table names to their corresponding TableSchema objects. - */ -public class TableSchemaMap extends HashMap<String,TableSchema> { - - public TableSchemaMap() { - } - - public TableSchemaMap(Map<String, TableSchema> tableSchemaMap) { - this.putAll(tableSchemaMap); - } - - /** - * Update this TableSchemaMap according to the given TSFile properties. - * - * @param tsFileProperties the TSFile properties - */ - public void update(Map<String, String> tsFileProperties) { - Builder schemaEvolutionBuilder = new Builder(); - tsFileProperties.entrySet().forEach(entry -> { - SchemaEvolution evolution = schemaEvolutionBuilder.fromProperty(entry); - if (evolution != null) { - evolution.applyTo(this); - } - }); - } -} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java index 12474bd0..21f89194 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java @@ -19,10 +19,14 @@ package org.apache.tsfile.file.metadata; +import java.util.HashMap; import java.util.LinkedHashMap; import org.apache.tsfile.compatibility.DeserializeConfig; import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.exception.encrypt.EncryptException; +import org.apache.tsfile.file.metadata.evolution.EvolvedSchema; +import org.apache.tsfile.file.metadata.evolution.SchemaEvolution; +import org.apache.tsfile.file.metadata.evolution.SchemaEvolution.Builder; import org.apache.tsfile.utils.BloomFilter; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -42,7 +46,7 @@ public class TsFileMetadata { // List of <name, offset, childMetadataIndexType> private Map<String, MetadataIndexNode> tableMetadataIndexNodeMap; - private TableSchemaMap tableSchemaMap; + private Map<String, TableSchema> tableSchemaMap; private boolean hasTableSchemaMapCache; private Map<String, String> tsFileProperties; @@ -57,6 +61,8 @@ public class TsFileMetadata { private String encryptType; + private EvolvedSchema evolvedSchema; + public static TsFileMetadata deserializeAndCacheTableSchemaMap( ByteBuffer buffer, DeserializeConfig context) { return deserializeFrom(buffer, context, true); @@ -91,7 +97,7 @@ public class TsFileMetadata { // tableSchemas int tableSchemaNum = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer); - TableSchemaMap tableSchemaMap = new TableSchemaMap(); + Map<String, TableSchema> tableSchemaMap = new HashMap<>(); for (int i = 0; i < tableSchemaNum; i++) { String tableName = ReadWriteIOUtils.readVarIntString(buffer); TableSchema tableSchema = context.tableSchemaBufferDeserializer.deserialize(buffer, context); @@ -164,14 +170,23 @@ public class TsFileMetadata { } fileMetaData.tsFileProperties = propertiesMap; - if (needTableSchemaMap){ - tableSchemaMap.update(propertiesMap); - } + fileMetaData.update(propertiesMap); } return fileMetaData; } + // update the TsFileMetadata according to schema evolutions defined in properties + private void update(Map<String, String> propertiesMap) { + Builder schemaEvolutionBuilder = new Builder(); + tsFileProperties.entrySet().forEach(entry -> { + SchemaEvolution evolution = schemaEvolutionBuilder.fromProperty(entry); + if (evolution != null) { + evolution.applyTo(this); + } + }); + } + public void addProperty(String key, String value) { if (tsFileProperties == null) { tsFileProperties = new LinkedHashMap<>(); @@ -265,7 +280,7 @@ public class TsFileMetadata { this.tableMetadataIndexNodeMap = tableMetadataIndexNodeMap; } - public void setTableSchemaMap(TableSchemaMap tableSchemaMap) { + public void setTableSchemaMap(Map<String, TableSchema> tableSchemaMap) { this.tableSchemaMap = tableSchemaMap; this.hasTableSchemaMapCache = true; } @@ -297,4 +312,11 @@ public class TsFileMetadata { public int getPropertiesOffset() { return propertiesOffset; } + + public EvolvedSchema getEvolvedSchema(boolean mayCreate) { + if (evolvedSchema == null && mayCreate) { + evolvedSchema = new EvolvedSchema(); + } + return evolvedSchema; + } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/ColumnRename.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/ColumnRename.java index 8cdcdb0b..8272164b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/ColumnRename.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/ColumnRename.java @@ -21,14 +21,14 @@ package org.apache.tsfile.file.metadata.evolution; import java.util.Map; import org.apache.tsfile.file.metadata.TableSchema; -import org.apache.tsfile.file.metadata.TableSchemaMap; +import org.apache.tsfile.file.metadata.TsFileMetadata; /** * A schema evolution operation that renames a column in a table schema. */ public class ColumnRename implements SchemaEvolution { - static final String KEY_PREFIX = "ColumnRename:"; + static final String KEY_PREFIX = SchemaEvolution.KEY_PREFIX + "ColumnRename:"; private final String tableName; private final String nameBefore; @@ -41,13 +41,8 @@ public class ColumnRename implements SchemaEvolution { } @Override - public void applyTo(TableSchemaMap schemaMap) { - TableSchema tableSchema = schemaMap.get(tableName); - if (tableSchema == null) { - return; - } - - tableSchema.renameColumn(nameBefore, nameAfter); + public void applyTo(TsFileMetadata metadata) { + metadata.getEvolvedSchema(true).renameColumn(tableName, nameBefore, nameAfter); } /** @@ -55,7 +50,7 @@ public class ColumnRename implements SchemaEvolution { */ @Override public String propertyKey() { - return SchemaEvolution.KEY_PREFIX + KEY_PREFIX + tableName.length() + "," + nameBefore.length() + "," + tableName + "," + nameBefore; + return KEY_PREFIX + tableName.length() + "," + nameBefore.length() + "," + tableName + "," + nameBefore; } /** diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/EvolvedSchema.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/EvolvedSchema.java new file mode 100644 index 00000000..2373e74a --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/EvolvedSchema.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.file.metadata.evolution; + +import java.util.HashMap; +import java.util.Map; + +public class EvolvedSchema { + // the evolved table names after applying all schema evolution operations + private final Map<String, String> originalTableNames = new HashMap<>(); + /** + * the first key is the evolved table name, the second key is the evolved column name, + * and the value is the original column name before any schema evolution. + */ + private final Map<String, Map<String, String>> originalColumnNames = new HashMap<>(); + + public void renameTable(String oldTableName, String newTableName) { + if (!originalTableNames.containsKey(oldTableName)) { + originalTableNames.put(newTableName, oldTableName); + } else { + String originalName = originalTableNames.remove(oldTableName); + originalTableNames.put(newTableName, originalName); + } + + if (originalColumnNames.containsKey(oldTableName)) { + Map<String, String> columnMap = originalColumnNames.remove(oldTableName); + originalColumnNames.put(newTableName, columnMap); + } + } + + public void renameColumn(String tableName, String oldColumnName, String newColumnName) { + Map<String, String> columnNameMap = originalColumnNames.computeIfAbsent(tableName, + t -> new HashMap<>()); + if (!columnNameMap.containsKey(oldColumnName)) { + columnNameMap.put(newColumnName, oldColumnName); + } else { + String originalName = columnNameMap.remove(oldColumnName); + columnNameMap.put(newColumnName, originalName); + } + } + + public String getOriginalTableName(String evolvedTableName) { + return originalTableNames.getOrDefault(evolvedTableName, evolvedTableName); + } + + public String getOriginalColumnName(String tableName, String evolvedColumnName) { + Map<String, String> columnNameMap = originalColumnNames.get(tableName); + if (columnNameMap == null) { + return evolvedColumnName; + } + return columnNameMap.getOrDefault(evolvedColumnName, evolvedColumnName); + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/SchemaEvolution.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/SchemaEvolution.java index 68b17815..74b2b77d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/SchemaEvolution.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/SchemaEvolution.java @@ -20,7 +20,7 @@ package org.apache.tsfile.file.metadata.evolution; import java.util.Map.Entry; -import org.apache.tsfile.file.metadata.TableSchemaMap; +import org.apache.tsfile.file.metadata.TsFileMetadata; /** * A schema evolution operation that can be applied to a TableSchemaMap. @@ -29,11 +29,11 @@ public interface SchemaEvolution { String KEY_PREFIX = "SEV:"; /** - * Apply this schema evolution operation to the given schema map. + * Apply this schema evolution operation to the given metadata. * - * @param schemaMap the schema map to apply the operation to + * @param metadata the metadata to apply the operation to */ - void applyTo(TableSchemaMap schemaMap); + void applyTo(TsFileMetadata metadata); /** * Get the property key representing this schema evolution operation. diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/TableRename.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/TableRename.java index 4f556564..3f108358 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/TableRename.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/TableRename.java @@ -20,15 +20,16 @@ package org.apache.tsfile.file.metadata.evolution; import java.util.Map; +import org.apache.tsfile.file.metadata.MetadataIndexNode; import org.apache.tsfile.file.metadata.TableSchema; -import org.apache.tsfile.file.metadata.TableSchemaMap; +import org.apache.tsfile.file.metadata.TsFileMetadata; /** * A schema evolution operation that renames a table in a schema map. */ public class TableRename implements SchemaEvolution{ - static final String KEY_PREFIX = "TableRename:"; + static final String KEY_PREFIX = SchemaEvolution.KEY_PREFIX + "TableRename:"; private final String nameBefore; private final String nameAfter; @@ -39,19 +40,13 @@ public class TableRename implements SchemaEvolution{ } @Override - public void applyTo(TableSchemaMap schemaMap) { - TableSchema schema = schemaMap.remove(nameBefore); - if (schema == null) { - return; + public void applyTo(TsFileMetadata tsFileMetadata) { + MetadataIndexNode indexNode = tsFileMetadata.getTableMetadataIndexNodeMap().remove(nameBefore); + if (indexNode != null) { + tsFileMetadata.getTableMetadataIndexNodeMap().put(nameAfter, indexNode); } - schema.setFinalTableName(nameAfter); - // if the renamed table already exists, then it must be removed previously - TableSchema deletedSchema = schemaMap.remove(nameAfter); - if (deletedSchema != null) { - deletedSchema.setDeleted(true); - } - schemaMap.put(nameAfter, schema); + tsFileMetadata.getEvolvedSchema(true).renameTable(nameBefore, nameAfter); } /** @@ -59,7 +54,7 @@ public class TableRename implements SchemaEvolution{ */ @Override public String propertyKey() { - return SchemaEvolution.KEY_PREFIX + KEY_PREFIX + nameBefore; + return KEY_PREFIX + nameBefore; } /** diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 7a572b9f..5cb79954 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -60,6 +60,7 @@ import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.EncryptionType; import org.apache.tsfile.file.metadata.enums.MetadataIndexNodeType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.evolution.EvolvedSchema; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.fileSystem.FSFactoryProducer; import org.apache.tsfile.read.common.BatchData; @@ -639,8 +640,20 @@ public class TsFileSequenceReader implements AutoCloseable { readFileMetadata(ioSizeConsumer); MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getTableMetadataIndexNode(device.getTableName()); + IDeviceID deviceInIndex = device; + + EvolvedSchema evolvedSchema = tsFileMetaData.getEvolvedSchema(false); + if (evolvedSchema != null) { + evolvedSchema.getOriginalTableName(device.getTableName()); + if (!tableSchema.getTableName().equals(device.getTableName())) { + // the table has been renamed, use the original table name to get deviceMetadataIndexNode + deviceInIndex = device.clone(); + deviceInIndex.setTableName(tableSchema.getTableName()); + } + } + Pair<IMetadataIndexEntry, Long> metadataIndexPair = - getMetadataAndEndOffsetOfDeviceNode(deviceMetadataIndexNode, device, true, ioSizeConsumer); + getMetadataAndEndOffsetOfDeviceNode(deviceMetadataIndexNode, deviceInIndex, true, ioSizeConsumer); if (metadataIndexPair == null) { if (ignoreNotExistDevice) { return null; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java index 927ac895..25a89316 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java @@ -37,6 +37,7 @@ import org.apache.tsfile.read.query.dataset.TableResultSet; import org.apache.tsfile.read.query.executor.TableQueryExecutor; import org.apache.tsfile.read.reader.block.TsBlockReader; +import org.apache.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -92,16 +93,19 @@ public class DeviceTableModelReader implements ITsFileReader { if (tableSchema == null) { throw new NoTableException(tableName); } + List<TSDataType> dataTypeList = new ArrayList<>(columnNames.size()); List<String> lowerCaseColumnNames = new ArrayList<>(columnNames.size()); + Map<String, Integer> column2IndexMap = tableSchema.buildColumnPosIndex(); for (String columnName : columnNames) { - Map<String, Integer> column2IndexMap = tableSchema.buildColumnPosIndex(); - Integer columnIndex = column2IndexMap.get(columnName.toLowerCase()); + String lowerCaseColumnName = columnName.toLowerCase(); + Integer columnIndex = column2IndexMap.get(lowerCaseColumnName); if (columnIndex == null) { throw new NoMeasurementException(columnName); } - lowerCaseColumnNames.add(columnName.toLowerCase()); - dataTypeList.add(tableSchema.getColumnSchemas().get(columnIndex).getType()); + MeasurementSchema measurementSchema = (MeasurementSchema) tableSchema.getColumnSchemas().get(columnIndex); + dataTypeList.add(measurementSchema.getType()); + lowerCaseColumnNames.add(lowerCaseColumnName); } TsBlockReader tsBlockReader = queryExecutor.query( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index c984b422..330ff844 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -19,6 +19,7 @@ package org.apache.tsfile.write.record; +import java.util.stream.Collectors; import org.apache.tsfile.annotations.TableModel; import org.apache.tsfile.annotations.TreeModel; import org.apache.tsfile.annotations.TsFileApi; @@ -27,6 +28,7 @@ import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.StringArrayDeviceID; +import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.BytesUtils; @@ -175,6 +177,13 @@ public class Tablet { this(tableName, columnNameList, dataTypeList, columnCategoryList, DEFAULT_SIZE); } + @TableModel + public Tablet(TableSchema tableSchema) { + this(tableSchema.getTableName(), tableSchema.getColumnSchemas().stream().map(IMeasurementSchema::getMeasurementName).collect( + Collectors.toList()), tableSchema.getColumnSchemas().stream().map(IMeasurementSchema::getType).collect( + Collectors.toList()), tableSchema.getColumnTypes()); + } + @TableModel public Tablet( String tableName, diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/schema/MeasurementSchema.java b/java/tsfile/src/main/java/org/apache/tsfile/write/schema/MeasurementSchema.java index fbb78594..c70c84cc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/schema/MeasurementSchema.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/schema/MeasurementSchema.java @@ -53,8 +53,6 @@ public class MeasurementSchema + RamUsageEstimator.shallowSizeOfInstance(TSEncodingBuilder.class); private String measurementName; - // name after renames - private String finalMeasurementName; private TSDataType dataType; private TSEncoding encoding; private CompressionType compressionType; @@ -108,7 +106,6 @@ public class MeasurementSchema Map<String, String> props) { this.dataType = dataType; this.measurementName = measurementName; - this.finalMeasurementName = measurementName; this.encoding = encoding; this.props = props; this.compressionType = compressionType; @@ -122,7 +119,6 @@ public class MeasurementSchema Map<String, String> props) { this.dataType = TSDataType.getTsDataType(type); this.measurementName = measurementName; - this.finalMeasurementName = measurementName; this.encoding = TSEncoding.deserialize(encoding); this.props = props; this.compressionType = CompressionType.deserialize(compressor); @@ -133,7 +129,6 @@ public class MeasurementSchema MeasurementSchema measurementSchema = new MeasurementSchema(); measurementSchema.measurementName = ReadWriteIOUtils.readString(inputStream); - measurementSchema.finalMeasurementName = measurementSchema.measurementName; measurementSchema.dataType = TSDataType.deserializeFrom(inputStream); @@ -162,7 +157,6 @@ public class MeasurementSchema MeasurementSchema measurementSchema = new MeasurementSchema(); measurementSchema.measurementName = ReadWriteIOUtils.readString(buffer); - measurementSchema.finalMeasurementName = measurementSchema.measurementName; measurementSchema.dataType = TSDataType.deserializeFrom(buffer); @@ -190,7 +184,6 @@ public class MeasurementSchema MeasurementSchema measurementSchema = new MeasurementSchema(); measurementSchema.measurementName = ReadWriteIOUtils.readString(buffer); - measurementSchema.finalMeasurementName = measurementSchema.measurementName; measurementSchema.dataType = TSDataType.deserializeFrom(buffer); @@ -214,15 +207,6 @@ public class MeasurementSchema public void setMeasurementName(String measurementName) { this.measurementName = measurementName; - this.finalMeasurementName = measurementName; - } - - public String getFinalMeasurementName() { - return finalMeasurementName; - } - - public void setFinalMeasurementName(String finalMeasurementName) { - this.finalMeasurementName = finalMeasurementName; } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java index 260d4264..2354d58f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java @@ -26,6 +26,7 @@ import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.encrypt.IEncryptor; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.write.chunk.AlignedChunkGroupWriterImpl; import org.apache.tsfile.write.chunk.IChunkGroupWriter; import org.apache.tsfile.write.chunk.NonAlignedChunkGroupWriterImpl; @@ -88,6 +89,14 @@ abstract class AbstractTableModelTsFileWriter implements ITsFileWriter { new EncryptParameter(config.getEncryptType(), config.getEncryptKey())); } + protected AbstractTableModelTsFileWriter(File file) + throws IOException { + this( + file, + TSFileDescriptor.getInstance().getConfig().getGroupSizeInByte(), + new EncryptParameter(config.getEncryptType(), config.getEncryptKey())); + } + @TsFileApi protected AbstractTableModelTsFileWriter( File file, long chunkGroupSizeThreshold, EncryptParameter firstEncryptParam) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java index b2ae6b1f..c240186f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java @@ -52,6 +52,11 @@ public class DeviceTableModelWriter extends AbstractTableModelTsFileWriter { registerTableSchema(tableSchema); } + public DeviceTableModelWriter(File file) + throws IOException { + super(file); + } + /** * Write the tablet in to the TsFile with the table-view. The method will try to split the tablet * by device. @@ -122,11 +127,13 @@ public class DeviceTableModelWriter extends AbstractTableModelTsFileWriter { private void checkIsTableExistAndSetColumnCategoryList(Tablet tablet) throws WriteProcessException { String tabletTableName = tablet.getTableName(); - if (tabletTableName != null && !this.tableName.equals(tabletTableName)) { + if (tabletTableName == null) { + tabletTableName = this.tableName; + } + if (!getSchema().getTableSchemaMap().containsKey(tabletTableName)) { throw new NoTableException(tabletTableName); } - tablet.setTableName(this.tableName); - final TableSchema tableSchema = getSchema().getTableSchemaMap().get(tableName); + final TableSchema tableSchema = getSchema().getTableSchemaMap().get(tabletTableName); if (tableSchema == null) { throw new NoTableException(tabletTableName); } @@ -148,7 +155,7 @@ public class DeviceTableModelWriter extends AbstractTableModelTsFileWriter { tablet.setColumnCategories(columnCategoryListForTablet); } - private void registerTableSchema(TableSchema tableSchema) { + public void registerTableSchema(TableSchema tableSchema) { this.tableName = tableSchema.getTableName(); this.tableSchema = tableSchema; getSchema().registerTableSchema(tableSchema); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java index 7137d2f5..4019d235 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java @@ -35,7 +35,6 @@ import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.MeasurementMetadataIndexEntry; import org.apache.tsfile.file.metadata.MetadataIndexNode; -import org.apache.tsfile.file.metadata.TableSchemaMap; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.file.metadata.TsFileMetadata; import org.apache.tsfile.file.metadata.enums.CompressionType; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaEvolutionTest.java b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaEvolutionTest.java new file mode 100644 index 00000000..798844dd --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaEvolutionTest.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.file.metadata.evolution; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Optional; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.read.ReadProcessException; +import org.apache.tsfile.exception.write.NoMeasurementException; +import org.apache.tsfile.exception.write.NoTableException; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.query.dataset.ResultSet; +import org.apache.tsfile.read.v4.ITsFileReader; +import org.apache.tsfile.read.v4.TsFileReaderBuilder; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.v4.DeviceTableModelWriter; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +public class TsFileSchemaEvolutionTest { + + private static final String TEST_FILE_PATH = "target/schema_evolution_test.tsfile"; + private File tsFile; + + @Before + public void setUp() throws Exception { + tsFile = new File(TEST_FILE_PATH); + if (tsFile.exists()) { + assertTrue(tsFile.delete()); + } + + List<TableSchema> tableSchemas = Arrays.asList( + new TableSchema("t1", Arrays.asList( + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("tag2", TSDataType.STRING), + new MeasurementSchema("f1", TSDataType.INT32), + new MeasurementSchema("f2", TSDataType.DOUBLE) + ), Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD + )), + new TableSchema("t2", Arrays.asList( + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("tag2", TSDataType.STRING), + new MeasurementSchema("f1", TSDataType.INT32), + new MeasurementSchema("f2", TSDataType.DOUBLE) + ), Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD + )), + new TableSchema("t3", Arrays.asList( + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("tag2", TSDataType.STRING), + new MeasurementSchema("f1", TSDataType.INT32), + new MeasurementSchema("f2", TSDataType.DOUBLE) + ), Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD + )) + ); + + try (DeviceTableModelWriter writer = new DeviceTableModelWriter(tsFile)) { + tableSchemas.forEach(writer::registerTableSchema); + + for (int i = 0; i < tableSchemas.size(); i++) { + TableSchema tableSchema = tableSchemas.get(i); + Tablet tablet = new Tablet(tableSchema); + tablet.addTimestamp(0, 0); + tablet.addValue(0, 0, "t" + (i + 1) + "-tag1"); + tablet.addValue(0, 1, "t" + (i + 1) + "-tag2"); + tablet.addValue(0, 2, i * 10 + 3); + tablet.addValue(0, 3, i * 10 + 4.0); + writer.write(tablet); + } + } + } + + @After + public void tearDown() { + if (tsFile != null && tsFile.exists()) { + assertTrue(tsFile.delete()); + } + } + + @Test + public void testTableRename() + throws IOException, ReadProcessException, NoTableException, NoMeasurementException { + // rename t1 -> t4 + TableRename tableRename = new TableRename("t1", "t4"); + TsFileSchemaRewriter rewriter = new TsFileSchemaRewriter(TEST_FILE_PATH); + rewriter.appendProperties(Collections.singletonMap(tableRename.propertyKey(), tableRename.propertyValue())); + + // Verify the table has been renamed + try (ITsFileReader reader = new TsFileReaderBuilder().file(new File(TEST_FILE_PATH)).build()) { + assertFalse(reader.getTableSchemas("t1").isPresent()); + Optional<TableSchema> t = reader.getTableSchemas("t4"); + assertTrue(t.isPresent()); + TableSchema tableSchema = t.get(); + assertEquals("t1", tableSchema.getTableName()); + + assertThrows(NoTableException.class, () -> reader.query("t1", Arrays.asList("f1", "f2"), 0, 10)); + ResultSet resultSet = reader.query("t4", Arrays.asList("f1", "f2"), 0, 10); + assertTrue(resultSet.next()); + assertEquals(3, resultSet.getInt("f1")); + assertEquals(4.0, resultSet.getDouble("f2"), 0.0001); + assertFalse(resultSet.next()); + } + + // rename t2 -> t1 + tableRename = new TableRename("t2", "t1"); + rewriter.appendProperties(Collections.singletonMap(tableRename.propertyKey(), tableRename.propertyValue())); + + // Verify the table has been renamed + try (ITsFileReader reader = new TsFileReaderBuilder().file(new File(TEST_FILE_PATH)).build()) { + assertFalse(reader.getTableSchemas("t2").isPresent()); + Optional<TableSchema> t = reader.getTableSchemas("t1"); + assertTrue(t.isPresent()); + TableSchema tableSchema = t.get(); + assertEquals("t2", tableSchema.getTableName()); + + assertThrows(NoTableException.class, () -> reader.query("t2", Arrays.asList("f1", "f2"), 0, 10)); + ResultSet resultSet = reader.query("t1", Arrays.asList("f1", "f2"), 0, 10); + assertTrue(resultSet.next()); + assertEquals(13, resultSet.getInt("f1")); + assertEquals(14.0, resultSet.getDouble("f2"), 0.0001); + assertFalse(resultSet.next()); + } + + // t3 is not affected + try (ITsFileReader reader = new TsFileReaderBuilder().file(new File(TEST_FILE_PATH)).build()) { + Optional<TableSchema> t1 = reader.getTableSchemas("t3"); + assertTrue(t1.isPresent()); + TableSchema tableSchema = t1.get(); + assertEquals("t3", tableSchema.getTableName()); + + ResultSet resultSet = reader.query("t3", Arrays.asList("f1", "f2"), 0, 10); + assertTrue(resultSet.next()); + assertEquals(23, resultSet.getInt("f1")); + assertEquals(24.0, resultSet.getDouble("f2"), 0.0001); + assertFalse(resultSet.next()); + } + + // test read timeseries metadata + try (TsFileSequenceReader sequenceReader = new TsFileSequenceReader(TEST_FILE_PATH)) { + TimeseriesMetadata timeseriesMetadata = sequenceReader.readTimeseriesMetadata( + Factory.DEFAULT_FACTORY.create(new String[]{"t4", "t1-tag1", "t1-tag2"}), "f1", false); + assertEquals(3, timeseriesMetadata.getStatistics().getMaxValue()); + } + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaRewriterTest.java b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaRewriterTest.java new file mode 100644 index 00000000..ee47fd1f --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaRewriterTest.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.file.metadata.evolution; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.TsFileMetadata; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.IntDataPoint; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class TsFileSchemaRewriterTest { + private static final String TEST_FILE_PATH = "target/test.tsfile"; + private File tsFile; + + @Before + public void setUp() throws Exception { + tsFile = new File(TEST_FILE_PATH); + if (tsFile.exists()) { + assertTrue(tsFile.delete()); + } + + // Create a test TsFile with some data + try (TsFileWriter writer = new TsFileWriter(tsFile)) { + // Register a timeseries + writer.registerTimeseries( + "d1", + new MeasurementSchema("s1", TSDataType.INT32)); + + // Write some data points + TSRecord record = new TSRecord("d1", 1L); + record.addTuple(new IntDataPoint("s1", 123)); + writer.writeRecord(record); + + record = new TSRecord("d1", 2L); + record.addTuple(new IntDataPoint("s1", 456)); + writer.writeRecord(record); + } + } + + @After + public void tearDown() { + if (tsFile != null && tsFile.exists()) { + assertTrue(tsFile.delete()); + } + } + + @Test + public void testAppendSingleProperty() throws IOException { + // Create rewriter and append a single property + TsFileSchemaRewriter rewriter = new TsFileSchemaRewriter(TEST_FILE_PATH); + Map<String, String> newProperties = new HashMap<>(); + newProperties.put("test_key", "test_value"); + rewriter.appendProperties(newProperties); + + // Verify the property was added + try (TsFileSequenceReader reader = new TsFileSequenceReader(TEST_FILE_PATH)) { + TsFileMetadata metadata = reader.readFileMetadata(); + assertEquals("test_value", metadata.getTsFileProperties().get("test_key")); + } + } + + @Test + public void testAppendMultipleProperties() throws IOException { + // Create rewriter and append multiple properties + TsFileSchemaRewriter rewriter = new TsFileSchemaRewriter(TEST_FILE_PATH); + Map<String, String> newProperties = new HashMap<>(); + newProperties.put("key1", "value1"); + newProperties.put("key2", "value2"); + newProperties.put("key3", "value3"); + rewriter.appendProperties(newProperties); + + // Verify all properties were added + try (TsFileSequenceReader reader = new TsFileSequenceReader(TEST_FILE_PATH)) { + TsFileMetadata metadata = reader.readFileMetadata(); + Map<String, String> properties = metadata.getTsFileProperties(); + assertEquals("value1", properties.get("key1")); + assertEquals("value2", properties.get("key2")); + assertEquals("value3", properties.get("key3")); + } + } + + @Test + public void testAppendPropertiesMultipleTimes() throws IOException { + TsFileSchemaRewriter rewriter = new TsFileSchemaRewriter(TEST_FILE_PATH); + + // First append + Map<String, String> firstProperties = new HashMap<>(); + firstProperties.put("key1", "value1"); + rewriter.appendProperties(firstProperties); + + // Second append + Map<String, String> secondProperties = new HashMap<>(); + secondProperties.put("key2", "value2"); + rewriter.appendProperties(secondProperties); + + // Third append with update to existing property + Map<String, String> thirdProperties = new HashMap<>(); + thirdProperties.put("key1", "new_value1"); + thirdProperties.put("key3", "value3"); + rewriter.appendProperties(thirdProperties); + + // Verify final state + try (TsFileSequenceReader reader = new TsFileSequenceReader(TEST_FILE_PATH)) { + TsFileMetadata metadata = reader.readFileMetadata(); + Map<String, String> properties = metadata.getTsFileProperties(); + assertEquals("new_value1", properties.get("key1")); // Updated value + assertEquals("value2", properties.get("key2")); // Unchanged value + assertEquals("value3", properties.get("key3")); // New value + } + } + + @Test + public void testAppendEmptyProperties() throws IOException { + TsFileSchemaRewriter rewriter = new TsFileSchemaRewriter(TEST_FILE_PATH); + Map<String, String> emptyProperties = new HashMap<>(); + rewriter.appendProperties(emptyProperties); + + // Verify file is still valid and only encryption-related properties were added + try (TsFileSequenceReader reader = new TsFileSequenceReader(TEST_FILE_PATH)) { + TsFileMetadata metadata = reader.readFileMetadata(); + assertEquals(3, metadata.getTsFileProperties().size()); + assertEquals("0", metadata.getTsFileProperties().get("encryptLevel")); + assertEquals("org.apache.tsfile.encrypt.UNENCRYPTED", metadata.getTsFileProperties().get("encryptType")); + assertEquals("", metadata.getTsFileProperties().get("encryptKey")); + } + } + + @Test(expected = IOException.class) + public void testAppendPropertiesToNonExistentFile() throws IOException { + String nonExistentFile = "non_existent.tsfile"; + TsFileSchemaRewriter rewriter = new TsFileSchemaRewriter(nonExistentFile); + Map<String, String> properties = new HashMap<>(); + properties.put("key", "value"); + rewriter.appendProperties(properties); + } + + @Test + public void testAppendLargeProperties() throws IOException { + TsFileSchemaRewriter rewriter = new TsFileSchemaRewriter(TEST_FILE_PATH); + Map<String, String> largeProperties = new HashMap<>(); + + // Add many properties with relatively large values + for (int i = 0; i < 1000; i++) { + StringBuilder value = new StringBuilder(); + for (int j = 0; j < 100; j++) { + value.append("value").append(i).append("_").append(j); + } + largeProperties.put("key" + i, value.toString()); + } + + rewriter.appendProperties(largeProperties); + + // Verify all properties were written correctly + try (TsFileSequenceReader reader = new TsFileSequenceReader(TEST_FILE_PATH)) { + TsFileMetadata metadata = reader.readFileMetadata(); + Map<String, String> properties = metadata.getTsFileProperties(); + + for (int i = 0; i < 1000; i++) { + StringBuilder expectedValue = new StringBuilder(); + for (int j = 0; j < 100; j++) { + expectedValue.append("value").append(i).append("_").append(j); + } + assertEquals(expectedValue.toString(), properties.get("key" + i)); + } + } + } +}
