This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch fix_table_name_sensitivity in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5071e7e675f7434c072045b1fbecd531c871aa81 Author: Tian Jiang <[email protected]> AuthorDate: Mon Aug 19 12:16:53 2024 +0800 Fix database and tablename case sensitivity in tablet model --- .../relational/it/db/it/IoTDBInsertTableIT.java | 182 ++++++++++++++++++++- .../iotdb/session/it/IoTDBSessionRelationalIT.java | 3 +- .../plan/analyze/schema/SchemaValidator.java | 2 +- .../relational/sql/ast/WrappedInsertStatement.java | 4 +- .../plan/relational/sql/parser/AstBuilder.java | 3 +- .../plan/statement/crud/InsertBaseStatement.java | 14 +- .../crud/InsertMultiTabletsStatement.java | 4 +- .../plan/statement/crud/InsertRowStatement.java | 2 +- .../crud/InsertRowsOfOneDeviceStatement.java | 4 +- .../plan/statement/crud/InsertRowsStatement.java | 4 +- .../plan/statement/crud/InsertTabletStatement.java | 2 +- .../org/apache/iotdb/commons/path/PartialPath.java | 9 + 12 files changed, 218 insertions(+), 15 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java index b56726f333b..c1c5f52d3a2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java @@ -34,6 +34,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.record.Tablet.ColumnType; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.AfterClass; @@ -57,6 +58,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static org.apache.iotdb.db.it.utils.TestUtils.assertTableNonQueryTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest; @@ -593,7 +595,9 @@ public class IoTDBInsertTableIT { } @Test - public void testInsertCaseSensitivity() throws SQLException { + public void testInsertCaseSensitivity() + throws SQLException, IoTDBConnectionException, StatementExecutionException { + // column case sensitivity try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement st1 = connection.createStatement()) { st1.execute("use \"test\""); @@ -618,6 +622,182 @@ public class IoTDBInsertTableIT { } assertFalse(rs1.next()); } + + // table case sensitivity with record and auto creation + try (ISession session = EnvFactory.getEnv().getSessionConnection(BaseEnv.TABLE_SQL_DIALECT)) { + session.executeNonQueryStatement("USE \"test\""); + + List<IMeasurementSchema> schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("id1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List<ColumnType> columnTypes = + Arrays.asList(ColumnType.ID, ColumnType.ATTRIBUTE, ColumnType.MEASUREMENT); + List<String> measurementIds = + schemaList.stream() + .map(IMeasurementSchema::getMeasurementId) + .collect(Collectors.toList()); + List<TSDataType> dataTypes = + schemaList.stream().map(IMeasurementSchema::getType).collect(Collectors.toList()); + + long timestamp = 0; + + for (long row = 0; row < 15; row++) { + Object[] values = new Object[] {"id:" + row, "attr:" + row, row * 1.0}; + session.insertRelationalRecord( + "TaBle19_2", timestamp + row, measurementIds, dataTypes, columnTypes, values); + } + + int cnt = 0; + SessionDataSet dataSet = + session.executeQueryStatement("select * from table19_2 order by time"); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("id:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(15, cnt); + } + + // table case sensitivity with record and no auto creation + try (ISession session = EnvFactory.getEnv().getSessionConnection(BaseEnv.TABLE_SQL_DIALECT)) { + session.executeNonQueryStatement("USE \"test\""); + session.executeNonQueryStatement( + "CREATE TABLE tAbLE19_3 (id1 string id, attr1 string attribute, " + + "m1 double " + + "measurement)"); + + List<IMeasurementSchema> schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("id1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List<ColumnType> columnTypes = + Arrays.asList(ColumnType.ID, ColumnType.ATTRIBUTE, ColumnType.MEASUREMENT); + List<String> measurementIds = + schemaList.stream() + .map(IMeasurementSchema::getMeasurementId) + .collect(Collectors.toList()); + List<TSDataType> dataTypes = + schemaList.stream().map(IMeasurementSchema::getType).collect(Collectors.toList()); + + long timestamp = 0; + + for (long row = 0; row < 15; row++) { + Object[] values = new Object[] {"id:" + row, "attr:" + row, row * 1.0}; + session.insertRelationalRecord( + "TaBle19_3", timestamp + row, measurementIds, dataTypes, columnTypes, values); + } + + int cnt = 0; + SessionDataSet dataSet = + session.executeQueryStatement("select * from table19_3 order by time"); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("id:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(15, cnt); + } + + // table case sensitivity with tablet and no auto creation + try (ISession session = EnvFactory.getEnv().getSessionConnection(BaseEnv.TABLE_SQL_DIALECT)) { + session.executeNonQueryStatement("USE \"test\""); + + List<IMeasurementSchema> schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("id1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List<ColumnType> columnTypes = + Arrays.asList(ColumnType.ID, ColumnType.ATTRIBUTE, ColumnType.MEASUREMENT); + + long timestamp = 0; + Tablet tablet = new Tablet("TaBle19_4", schemaList, columnTypes, 15); + + for (long row = 0; row < 15; row++) { + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("id1", rowIndex, "id:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + if (tablet.rowSize == tablet.getMaxRowNumber()) { + session.insertRelationalTablet(tablet, true); + tablet.reset(); + } + } + + if (tablet.rowSize != 0) { + session.insertRelationalTablet(tablet); + tablet.reset(); + } + + int cnt = 0; + SessionDataSet dataSet = + session.executeQueryStatement("select * from table19_4 order by time"); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("id:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(15, cnt); + } + + // table case sensitivity with tablet and auto creation + try (ISession session = EnvFactory.getEnv().getSessionConnection(BaseEnv.TABLE_SQL_DIALECT)) { + session.executeNonQueryStatement("USE \"test\""); + session.executeNonQueryStatement( + "CREATE TABLE tAbLE19_5 (id1 string id, attr1 string attribute, " + + "m1 double " + + "measurement)"); + + List<IMeasurementSchema> schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("id1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List<ColumnType> columnTypes = + Arrays.asList(ColumnType.ID, ColumnType.ATTRIBUTE, ColumnType.MEASUREMENT); + + long timestamp = 0; + Tablet tablet = new Tablet("TaBle19_5", schemaList, columnTypes, 15); + + for (long row = 0; row < 15; row++) { + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("id1", rowIndex, "id:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + if (tablet.rowSize == tablet.getMaxRowNumber()) { + session.insertRelationalTablet(tablet, true); + tablet.reset(); + } + } + + if (tablet.rowSize != 0) { + session.insertRelationalTablet(tablet); + tablet.reset(); + } + + int cnt = 0; + SessionDataSet dataSet = + session.executeQueryStatement("select * from table19_5 order by time"); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals("id:" + timestamp, rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals("attr:" + timestamp, rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(15, cnt); + } } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java index aedf0a8631a..8d01cdb15ab 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java @@ -922,9 +922,10 @@ public class IoTDBSessionRelationalIT { session.executeNonQueryStatement("FLush"); for (long row = 15; row < 30; row++) { + // check case sensitivity session.executeNonQueryStatement( String.format( - "INSERT INTO db1.table11 (time, id1, attr1, m1) VALUES (%d, '%s', '%s', %f)", + "INSERT INTO DB1.TaBle11 (time, id1, attr1, m1) VALUES (%d, '%s', '%s', %f)", row, "id:" + row, "attr:" + row, row * 1.0)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java index 2458ff36338..9533cca1bd6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java @@ -64,7 +64,7 @@ public class SchemaValidator { public static void validate( Metadata metadata, WrappedInsertStatement insertStatement, MPPQueryContext context) { try { - insertStatement.columnsToLowerCase(); + insertStatement.toLowerCase(); insertStatement.validateTableSchema(metadata, context); insertStatement.updateAfterSchemaValidation(context); insertStatement.validateDeviceSchema(metadata, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java index f86ed3ab9f8..801ec407b90 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java @@ -219,7 +219,7 @@ public abstract class WrappedInsertStatement extends WrappedStatement return databaseName; } - public void columnsToLowerCase() { - getInnerTreeStatement().measurementsToLowerCase(); + public void toLowerCase() { + getInnerTreeStatement().toLowerCase(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 1780c5e0061..3aaabbeb3b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -476,7 +476,8 @@ public class AstBuilder extends RelationalSqlBaseVisitor<Node> { insertRowStatement.setValues(values); insertRowStatement.setNeedInferType(true); - databaseName.ifPresent(insertRowStatement::setDatabaseName); + databaseName.ifPresent( + databaseName1 -> insertRowStatement.setDatabaseName(databaseName1.toLowerCase())); return insertRowStatement; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java index c852651802c..11434865d68 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java @@ -60,6 +60,7 @@ public abstract class InsertBaseStatement extends Statement { /** * if use id table, this filed is id form of device path <br> * if not, this filed is device path<br> + * When using table model, this is the table name. */ protected PartialPath devicePath; @@ -555,7 +556,8 @@ public abstract class InsertBaseStatement extends Statement { // endregion @TableModel - public void measurementsToLowerCase() { + public void toLowerCase() { + devicePath.toLowerCase(); if (measurements == null) { return; } @@ -564,6 +566,11 @@ public abstract class InsertBaseStatement extends Statement { measurements[i] = measurements[i].toLowerCase(); } } + if (measurementSchemas != null) { + for (MeasurementSchema measurementSchema : measurementSchemas) { + measurementSchema.setMeasurementId(measurementSchema.getMeasurementId().toLowerCase()); + } + } } @TableModel @@ -576,4 +583,9 @@ public abstract class InsertBaseStatement extends Statement { } return attributeColumnNameList; } + + @TableModel + public String getTableName() { + return devicePath.getFullPath(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertMultiTabletsStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertMultiTabletsStatement.java index b92cf23ba19..67809709f7d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertMultiTabletsStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertMultiTabletsStatement.java @@ -151,8 +151,8 @@ public class InsertMultiTabletsStatement extends InsertBaseStatement { @TableModel @Override - public void measurementsToLowerCase() { - insertTabletStatementList.forEach(InsertTabletStatement::measurementsToLowerCase); + public void toLowerCase() { + insertTabletStatementList.forEach(InsertTabletStatement::toLowerCase); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java index 0ac735450f8..f2fa614fc58 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java @@ -466,7 +466,7 @@ public class InsertRowStatement extends InsertBaseStatement implements ISchemaVa public IDeviceID getTableDeviceID() { if (deviceID == null) { String[] deviceIdSegments = new String[getIdColumnIndices().size() + 1]; - deviceIdSegments[0] = this.devicePath.getFullPath(); + deviceIdSegments[0] = this.getTableName(); for (int i = 0; i < getIdColumnIndices().size(); i++) { final Integer columnIndex = getIdColumnIndices().get(i); deviceIdSegments[i + 1] = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java index b98ee464067..74b7e060131 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java @@ -174,8 +174,8 @@ public class InsertRowsOfOneDeviceStatement extends InsertBaseStatement { @TableModel @Override - public void measurementsToLowerCase() { - insertRowStatementList.forEach(InsertRowStatement::measurementsToLowerCase); + public void toLowerCase() { + insertRowStatementList.forEach(InsertRowStatement::toLowerCase); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsStatement.java index 94553269cf0..ca5cda875fb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsStatement.java @@ -180,8 +180,8 @@ public class InsertRowsStatement extends InsertBaseStatement { @TableModel @Override - public void measurementsToLowerCase() { - insertRowStatementList.forEach(InsertRowStatement::measurementsToLowerCase); + public void toLowerCase() { + insertRowStatementList.forEach(InsertRowStatement::toLowerCase); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java index 988abea1add..8cfabf92a85 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java @@ -450,7 +450,7 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem } if (deviceIDs[rowIdx] == null) { String[] deviceIdSegments = new String[getIdColumnIndices().size() + 1]; - deviceIdSegments[0] = this.devicePath.getFullPath(); + deviceIdSegments[0] = this.getTableName(); for (int i = 0; i < getIdColumnIndices().size(); i++) { final Integer columnIndex = getIdColumnIndices().get(i); Object idSeg = ((Object[]) columns[columnIndex])[rowIdx]; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java index 454a40d3ee1..e9ddcb6aabd 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java @@ -1067,4 +1067,13 @@ public class PartialPath extends Path implements Comparable<Path>, Cloneable { } return Factory.DEFAULT_FACTORY.create(segments); } + + public void toLowerCase() { + for (int i = 0; i < nodes.length; i++) { + nodes[i] = nodes[i].toLowerCase(); + } + if (fullPath != null) { + fullPath = fullPath.toLowerCase(); + } + } }
