This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch force_ci/alter_column_datatype
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/force_ci/alter_column_datatype
by this push:
new c85cfa87584 add test case
c85cfa87584 is described below
commit c85cfa87584149fbdff5fe4d8fc48989f3afd0e2
Author: Tian Jiang <[email protected]>
AuthorDate: Fri Jan 10 11:22:29 2025 +0800
add test case
---
.../it/schema/IoTDBAlterColumnTypeIT.java | 182 +++++++++++++++++++--
.../iotdb/confignode/manager/ProcedureManager.java | 6 +-
.../execution/operator/source/SeriesScanUtil.java | 35 +++-
.../db/utils/datastructure/AlignedTVList.java | 31 +++-
.../db/relational/grammar/sql/RelationalSql.g4 | 2 +-
pom.xml | 2 +-
6 files changed, 226 insertions(+), 32 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java
index 019d218934e..737b9c46af5 100644
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java
@@ -30,14 +30,22 @@ import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.RowRecord;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.record.Tablet.ColumnCategory;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
import static
org.apache.iotdb.relational.it.session.IoTDBSessionRelationalIT.genValue;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
@RunWith(IoTDBTestRunner.class)
@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
@@ -58,8 +66,13 @@ public class IoTDBAlterColumnTypeIT {
@Test
public void testWriteAndAlter() throws IoTDBConnectionException,
StatementExecutionException {
- for (TSDataType from : TSDataType.values()) {
- for (TSDataType to : TSDataType.values()) {
+ Set<TSDataType> typesToTest = new HashSet<>();
+ Collections.addAll(typesToTest, TSDataType.values());
+ typesToTest.remove(TSDataType.VECTOR);
+ typesToTest.remove(TSDataType.UNKNOWN);
+
+ for (TSDataType from : typesToTest) {
+ for (TSDataType to : typesToTest) {
System.out.printf("testing %s to %s%n", from, to);
doWriteAndAlter(from, to, false);
doWriteAndAlter(from, to, true);
@@ -75,10 +88,16 @@ public class IoTDBAlterColumnTypeIT {
"CREATE TABLE IF NOT EXISTS write_and_alter_column_type (s1 " + from
+ ")");
// write a point of "from"
- session.executeNonQueryStatement(
- "INSERT INTO write_and_alter_column_type (time, s1) VALUES (1, "
- + genValue(from, 1)
- + ")");
+ Tablet tablet =
+ new Tablet(
+ "write_and_alter_column_type",
+ Collections.singletonList("s1"),
+ Collections.singletonList(from),
+ Collections.singletonList(ColumnCategory.FIELD));
+ tablet.addTimestamp(0, 1);
+ tablet.addValue("s1", 0, genValue(from, 1));
+ session.insert(tablet);
+ tablet.reset();
if (flush) {
session.executeNonQueryStatement("FLUSH");
@@ -94,33 +113,45 @@ public class IoTDBAlterColumnTypeIT {
session.executeNonQueryStatement(
"ALTER TABLE write_and_alter_column_type ALTER COLUMN s1 SET
DATA TYPE " + to);
} catch (StatementExecutionException e) {
- assertEquals("", e.getMessage());
+ assertEquals(
+ "701: New type " + to + " is not compatible with the existing
one " + from,
+ e.getMessage());
}
}
SessionDataSet dataSet =
session.executeQueryStatement("select * from
write_and_alter_column_type order by time");
RowRecord rec = dataSet.next();
+ TSDataType newType = isCompatible ? to : from;
assertEquals(1, rec.getFields().get(0).getLongV());
- if (to == TSDataType.BLOB) {
- assertEquals(genValue(to, 1), rec.getFields().get(1).getBinaryV());
- } else if (to == TSDataType.DATE) {
- assertEquals(genValue(to, 1), rec.getFields().get(1).getDateV());
+ if (newType == TSDataType.BLOB) {
+ assertEquals(genValue(newType, 1),
rec.getFields().get(1).getBinaryV());
+ } else if (newType == TSDataType.DATE) {
+ assertEquals(genValue(newType, 1), rec.getFields().get(1).getDateV());
} else {
- assertEquals(genValue(to, 1).toString(),
rec.getFields().get(1).toString());
+ assertEquals(genValue(newType, 1).toString(),
rec.getFields().get(1).toString());
}
// write a point
- session.executeNonQueryStatement(
- "INSERT INTO write_and_alter_column_type (time, s1) VALUES (2, "
- + genValue(isCompatible ? to : from, 2)
- + ")");
+ tablet =
+ new Tablet(
+ "write_and_alter_column_type",
+ Collections.singletonList("s1"),
+ Collections.singletonList(newType),
+ Collections.singletonList(ColumnCategory.FIELD));
+ tablet.addTimestamp(0, 2);
+ tablet.addValue("s1", 0, genValue(newType, 2));
+ session.insert(tablet);
+ tablet.reset();
+
+ if (flush) {
+ session.executeNonQueryStatement("FLUSH");
+ }
dataSet =
session.executeQueryStatement("select * from
write_and_alter_column_type order by time");
rec = dataSet.next();
assertEquals(1, rec.getFields().get(0).getLongV());
- TSDataType newType = isCompatible ? to : from;
if (newType == TSDataType.BLOB) {
assertEquals(genValue(newType, 1),
rec.getFields().get(1).getBinaryV());
} else if (newType == TSDataType.DATE) {
@@ -142,4 +173,121 @@ public class IoTDBAlterColumnTypeIT {
session.executeNonQueryStatement("DROP TABLE
write_and_alter_column_type");
}
}
+
+ @Test
+ public void testAlterWithoutWrite() throws IoTDBConnectionException,
StatementExecutionException {
+ Set<TSDataType> typesToTest = new HashSet<>();
+ Collections.addAll(typesToTest, TSDataType.values());
+ typesToTest.remove(TSDataType.VECTOR);
+ typesToTest.remove(TSDataType.UNKNOWN);
+
+ for (TSDataType from : typesToTest) {
+ for (TSDataType to : typesToTest) {
+ System.out.printf("testing %s to %s%n", from, to);
+ doAlterWithoutWrite(from, to, false);
+ doAlterWithoutWrite(from, to, true);
+ }
+ }
+ }
+
+ private void doAlterWithoutWrite(TSDataType from, TSDataType to, boolean
flush)
+ throws IoTDBConnectionException, StatementExecutionException {
+ try (ITableSession session =
EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) {
+ // create a table with type of "from"
+ session.executeNonQueryStatement(
+ "CREATE TABLE IF NOT EXISTS just_alter_column_type (s1 " + from +
")");
+
+ // alter the type to "to"
+ boolean isCompatible = to.isCompatible(from);
+ if (isCompatible) {
+ session.executeNonQueryStatement(
+ "ALTER TABLE just_alter_column_type ALTER COLUMN s1 SET DATA TYPE
" + to);
+ } else {
+ try {
+ session.executeNonQueryStatement(
+ "ALTER TABLE just_alter_column_type ALTER COLUMN s1 SET DATA
TYPE " + to);
+ } catch (StatementExecutionException e) {
+ assertEquals(
+ "701: New type " + to + " is not compatible with the existing
one " + from,
+ e.getMessage());
+ }
+ }
+
+ TSDataType newType = isCompatible ? to : from;
+
+ // write a point
+ Tablet tablet =
+ new Tablet(
+ "just_alter_column_type",
+ Collections.singletonList("s1"),
+ Collections.singletonList(newType),
+ Collections.singletonList(ColumnCategory.FIELD));
+ tablet.addTimestamp(0, 1);
+ tablet.addValue("s1", 0, genValue(newType, 1));
+ session.insert(tablet);
+ tablet.reset();
+
+ if (flush) {
+ session.executeNonQueryStatement("FLUSH");
+ }
+
+ SessionDataSet dataSet =
+ session.executeQueryStatement("select * from just_alter_column_type
order by time");
+ RowRecord rec = dataSet.next();
+ assertEquals(1, rec.getFields().get(0).getLongV());
+ if (newType == TSDataType.BLOB) {
+ assertEquals(genValue(newType, 1),
rec.getFields().get(1).getBinaryV());
+ } else if (newType == TSDataType.DATE) {
+ assertEquals(genValue(newType, 1), rec.getFields().get(1).getDateV());
+ } else {
+ assertEquals(genValue(newType, 1).toString(),
rec.getFields().get(1).toString());
+ }
+
+ assertFalse(dataSet.hasNext());
+
+ session.executeNonQueryStatement("DROP TABLE just_alter_column_type");
+ }
+ }
+
+ @Test
+ public void testAlterNonExist() throws IoTDBConnectionException,
StatementExecutionException {
+ try (ITableSession session =
EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) {
+ try {
+ session.executeNonQueryStatement(
+ "ALTER TABLE non_exist ALTER COLUMN s1 SET DATA TYPE INT64");
+ fail("Should throw exception");
+ } catch (StatementExecutionException e) {
+ assertEquals("550: Table 'test.non_exist' does not exist",
e.getMessage());
+ }
+ session.executeNonQueryStatement(
+ "ALTER TABLE IF EXISTS non_exist ALTER COLUMN s1 SET DATA TYPE
INT64");
+
+ session.executeNonQueryStatement("CREATE TABLE IF NOT EXISTS non_exist
(s1 int32)");
+
+ try {
+ session.executeNonQueryStatement(
+ "ALTER TABLE non_exist ALTER COLUMN s2 SET DATA TYPE INT64");
+ fail("Should throw exception");
+ } catch (StatementExecutionException e) {
+ assertEquals("616: Column s2 in table 'test.non_exist' does not
exist.", e.getMessage());
+ }
+ session.executeNonQueryStatement(
+ "ALTER TABLE non_exist ALTER COLUMN IF EXISTS s2 SET DATA TYPE
INT64");
+ }
+ }
+
+ @Test
+ public void testAlterWrongType() throws IoTDBConnectionException,
StatementExecutionException {
+ try (ITableSession session =
EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) {
+ session.executeNonQueryStatement("CREATE TABLE IF NOT EXISTS wrong_type
(s1 int32)");
+
+ try {
+ session.executeNonQueryStatement(
+ "ALTER TABLE non_exist ALTER COLUMN s1 SET DATA TYPE VECTOR");
+ fail("Should throw exception");
+ } catch (StatementExecutionException e) {
+ assertEquals("701: Unknown type: VECTOR", e.getMessage());
+ }
+ }
+ }
}
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
index f79cff952a9..36c3a3c097e 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
@@ -91,6 +91,7 @@ import
org.apache.iotdb.confignode.procedure.impl.schema.table.AlterTableColumnD
import
org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure;
import
org.apache.iotdb.confignode.procedure.impl.schema.table.DeleteDevicesProcedure;
import
org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure;
+import
org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure;
import
org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableColumnProcedure;
import
org.apache.iotdb.confignode.procedure.impl.schema.table.SetTablePropertiesProcedure;
import
org.apache.iotdb.confignode.procedure.impl.subscription.consumer.CreateConsumerProcedure;
@@ -1561,9 +1562,8 @@ public class ProcedureManager {
null,
req.tableName,
req.queryId,
- ProcedureType.DROP_TABLE_COLUMN_PROCEDURE,
- new DropTableColumnProcedure(
- req.database, req.tableName, req.queryId,
ReadWriteIOUtils.readString(req.updateInfo)));
+ ProcedureType.DROP_TABLE_PROCEDURE,
+ new DropTableProcedure(req.database, req.tableName, req.queryId));
}
public TDeleteTableDeviceResp deleteDevices(final TDeleteTableDeviceReq req)
{
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
index cf7e633ec26..00f16bdcf29 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
@@ -36,10 +36,12 @@ import
org.apache.iotdb.db.storageengine.dataregion.read.reader.common.PriorityM
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.IMetadata;
import org.apache.tsfile.file.metadata.ITimeSeriesMetadata;
+import org.apache.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.tsfile.file.metadata.statistics.Statistics;
import org.apache.tsfile.read.TimeValuePair;
import org.apache.tsfile.read.common.block.TsBlock;
@@ -1174,7 +1176,7 @@ public class SeriesScanUtil implements Accountable {
ITimeSeriesMetadata timeseriesMetadata =
loadTimeSeriesMetadata(orderUtils.getNextSeqFileResource(true), true);
// skip if data type is mismatched which may be caused by delete
- if (timeseriesMetadata != null &&
timeseriesMetadata.typeMatch(getTsDataTypeList())) {
+ if (timeseriesMetadata != null && typeCompatible(timeseriesMetadata)) {
timeseriesMetadata.setSeq(true);
seqTimeSeriesMetadata.add(timeseriesMetadata);
return Optional.of(timeseriesMetadata);
@@ -1183,11 +1185,40 @@ public class SeriesScanUtil implements Accountable {
}
}
+ private boolean typeCompatible(ITimeSeriesMetadata timeseriesMetadata) {
+ if (timeseriesMetadata instanceof TimeseriesMetadata) {
+ return getTsDataTypeList()
+ .get(0)
+ .isCompatible(((TimeseriesMetadata)
timeseriesMetadata).getTsDataType());
+ } else {
+ List<TimeseriesMetadata> valueTimeseriesMetadataList =
+ ((AbstractAlignedTimeSeriesMetadata)
timeseriesMetadata).getValueTimeseriesMetadataList();
+ if (getTsDataTypeList().isEmpty()) {
+ return true;
+ }
+ if (valueTimeseriesMetadataList != null) {
+ int incompactibleCount = 0;
+ for (int i = 0, size = getTsDataTypeList().size(); i < size; i++) {
+ TimeseriesMetadata valueTimeSeriesMetadata =
valueTimeseriesMetadataList.get(i);
+ if (valueTimeSeriesMetadata != null
+ && !getTsDataTypeList()
+ .get(i)
+ .isCompatible(valueTimeSeriesMetadata.getTsDataType())) {
+ valueTimeseriesMetadataList.set(i, null);
+ incompactibleCount++;
+ }
+ }
+ return incompactibleCount != getTsDataTypeList().size();
+ }
+ return true;
+ }
+ }
+
private void unpackUnseqTsFileResource() throws IOException {
ITimeSeriesMetadata timeseriesMetadata =
loadTimeSeriesMetadata(orderUtils.getNextUnseqFileResource(true),
false);
// skip if data type is mismatched which may be caused by delete
- if (timeseriesMetadata != null &&
timeseriesMetadata.typeMatch(getTsDataTypeList())) {
+ if (timeseriesMetadata != null && typeCompatible(timeseriesMetadata)) {
timeseriesMetadata.setSeq(false);
unSeqTimeSeriesMetadata.add(timeseriesMetadata);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index b146b25c2ab..f6160b5e833 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -48,6 +48,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
+import java.util.stream.Collectors;
import static
org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager.ARRAY_SIZE;
import static
org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager.TVLIST_SORT_ALGORITHM;
@@ -112,25 +113,39 @@ public abstract class AlignedTVList extends TVList {
}
}
+ private List<Object> convertToType(TSDataType to, TSDataType from,
List<Object> originalValues) {
+ if (!to.isCompatible(from)) {
+ return null;
+ }
+ return originalValues.stream().map(o -> to.castFromArray(from,
o)).collect(Collectors.toList());
+ }
+
@Override
public TVList getTvListByColumnIndex(
- List<Integer> columnIndex, List<TSDataType> dataTypeList, boolean
ignoreAllNullRows) {
+ List<Integer> columnIndexList, List<TSDataType> dataTypeList, boolean
ignoreAllNullRows) {
List<List<Object>> values = new ArrayList<>();
List<List<BitMap>> bitMaps = null;
- for (int i = 0; i < columnIndex.size(); i++) {
+ for (int i = 0; i < columnIndexList.size(); i++) {
// columnIndex == -1 means querying a non-exist column, add null column
here
- if (columnIndex.get(i) == -1) {
+ Integer columnIndex = columnIndexList.get(i);
+ if (columnIndex == -1) {
values.add(null);
} else {
- values.add(this.values.get(columnIndex.get(i)));
- if (this.bitMaps != null && this.bitMaps.get(columnIndex.get(i)) !=
null) {
+ List<Object> column = this.values.get(columnIndex);
+ if (dataTypeList.get(i) == this.dataTypes.get(columnIndex)) {
+ values.add(column);
+ } else {
+ values.add(convertToType(dataTypeList.get(i),
this.dataTypes.get(columnIndex), column));
+ }
+
+ if (this.bitMaps != null && this.bitMaps.get(columnIndex) != null) {
if (bitMaps == null) {
- bitMaps = new ArrayList<>(columnIndex.size());
- for (int j = 0; j < columnIndex.size(); j++) {
+ bitMaps = new ArrayList<>(columnIndexList.size());
+ for (int j = 0; j < columnIndexList.size(); j++) {
bitMaps.add(null);
}
}
- bitMaps.set(i, this.bitMaps.get(columnIndex.get(i)));
+ bitMaps.set(i, this.bitMaps.get(columnIndex));
}
}
}
diff --git
a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
index 794dff013a2..9908843cec4 100644
---
a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
+++
b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
@@ -197,7 +197,7 @@ alterTableStatement
| ALTER TABLE (IF EXISTS)? tableName=qualifiedName DROP COLUMN (IF
EXISTS)? column=identifier #dropColumn
// set TTL can use this
| ALTER TABLE (IF EXISTS)? tableName=qualifiedName SET PROPERTIES
propertyAssignments #setTableProperties
- | ALTER TABLE (IF EXISTS)? tableName=qualifiedName ALTER COLUMN
column=identifier (IF EXISTS)? SET DATA TYPE new_type=type #alterColumnDataType
+ | ALTER TABLE (IF EXISTS)? tableName=qualifiedName ALTER COLUMN (IF
EXISTS)? column=identifier SET DATA TYPE new_type=type #alterColumnDataType
;
diff --git a/pom.xml b/pom.xml
index 28f72ddc9bc..ee9194757a7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -167,7 +167,7 @@
<thrift.version>0.14.1</thrift.version>
<xz.version>1.9</xz.version>
<zstd-jni.version>1.5.6-3</zstd-jni.version>
- <tsfile.version>1.2.0-241224-SNAPSHOT</tsfile.version>
+ <tsfile.version>2.0.0-typeconv-SNAPSHOT</tsfile.version>
</properties>
<!--
if we claim dependencies in dependencyManagement, then we do not claim
