This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch DisableObjectType
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/DisableObjectType by this push:
new a9c9582b404 remove insert object
new 2e560dd1d66 Merge branch 'DisableObjectType' of
github.com:apache/iotdb into DisableObjectType
a9c9582b404 is described below
commit a9c9582b404d7cce9e92b82e1c847e4ea0092fac
Author: HTHou <[email protected]>
AuthorDate: Tue Jan 13 12:21:37 2026 +0800
remove insert object
---
.../it/query/old/IoTDBSimpleQueryTableIT.java | 57 +--
.../iotdb/relational/it/schema/IoTDBTableIT.java | 136 ------
.../relational/it/session/IoTDBObjectDeleteIT.java | 363 ----------------
.../it/session/IoTDBObjectDeleteIT2.java | 48 ---
.../relational/it/session/IoTDBObjectInsertIT.java | 339 ---------------
.../it/session/IoTDBObjectInsertIT2.java | 170 --------
.../confignode/conf/SystemPropertiesUtils.java | 15 -
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 9 -
.../plan/relational/sql/util/AstUtil.java | 30 --
.../storageengine/dataregion/Base32ObjectPath.java | 169 --------
.../db/storageengine/dataregion/IObjectPath.java | 9 +-
.../storageengine/dataregion/PlainObjectPath.java | 126 ------
.../org/apache/iotdb/db/utils/TabletDecoder.java | 2 +-
.../object/ObjectTypeCompactionTest.java | 459 ---------------------
.../conf/iotdb-system.properties.template | 10 -
.../iotdb/commons/conf/CommonDescriptor.java | 5 -
16 files changed, 7 insertions(+), 1940 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBSimpleQueryTableIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBSimpleQueryTableIT.java
index b97dc4915a1..f2e26dd225e 100644
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBSimpleQueryTableIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBSimpleQueryTableIT.java
@@ -658,19 +658,18 @@ public class IoTDBSimpleQueryTableIT {
statement.execute("USE " + DATABASE_NAME);
statement.execute(
"CREATE TABLE table1(device STRING TAG, "
- + "s4 DATE FIELD, s5 TIMESTAMP FIELD, s6 BLOB FIELD, s7 STRING
FIELD, s8 OBJECT FIELD)");
+ + "s4 DATE FIELD, s5 TIMESTAMP FIELD, s6 BLOB FIELD, s7 STRING
FIELD)");
for (int i = 1; i <= 10; i++) {
statement.execute(
String.format(
- "insert into table1(time, device, s4, s5, s6, s7, s8) "
- + "values(%d, 'd1', '%s', %d, %s, '%s', %s)",
+ "insert into table1(time, device, s4, s5, s6, s7) "
+ + "values(%d, 'd1', '%s', %d, %s, '%s')",
i,
LocalDate.of(2024, 5, i % 31 + 1),
i,
"X'cafebabe'",
- i,
- "to_object(true, 0, X'cafebabe')"));
+ i));
}
try (ResultSet resultSet = statement.executeQuery("select * from
table1")) {
@@ -687,8 +686,6 @@ public class IoTDBSimpleQueryTableIT {
columnType.put(i, TSDataType.BLOB);
} else if (metaData.getColumnLabel(i).equals("s7")) {
columnType.put(i, TSDataType.TEXT);
- } else if (metaData.getColumnLabel(i).equals("s8")) {
- columnType.put(i, TSDataType.OBJECT);
}
}
byte[] byteArray = new byte[] {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA,
(byte) 0xBE};
@@ -698,58 +695,12 @@ public class IoTDBSimpleQueryTableIT {
long timestamp = resultSet.getLong(4);
byte[] blob = resultSet.getBytes(5);
String text = resultSet.getString(6);
- String objectSizeString = resultSet.getString(7);
assertEquals(2024 - 1900, date.getYear());
assertEquals(5 - 1, date.getMonth());
assertEquals(time % 31 + 1, date.getDate());
assertEquals(time, timestamp);
assertArrayEquals(byteArray, blob);
assertEquals(String.valueOf(time), text);
- assertEquals("(Object) 4 B", objectSizeString);
- }
- }
- try (ResultSet resultSet = statement.executeQuery("select
read_object(s8) from table1")) {
- final ResultSetMetaData metaData = resultSet.getMetaData();
- final int columnCount = metaData.getColumnCount();
- assertEquals(1, columnCount);
- byte[] byteArray = new byte[] {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA,
(byte) 0xBE};
- while (resultSet.next()) {
- byte[] blob = resultSet.getBytes(1);
- assertArrayEquals(byteArray, blob);
- }
- }
-
- } catch (SQLException e) {
- fail();
- }
- }
-
- @Test
- public void testObjectDataType() {
- try (Connection connection =
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
- Statement statement = connection.createStatement()) {
- statement.execute("CREATE DATABASE test");
- statement.execute("USE " + DATABASE_NAME);
- statement.execute("CREATE TABLE table1(device STRING TAG, s8 OBJECT
FIELD)");
- statement.execute(
- "insert into table1(time, device, s8) values(1, 'd1',
to_object(false, 0, X'cafe'))");
- statement.execute(
- "insert into table1(time, device, s8) values(1, 'd1',
to_object(true, 2, X'babe'))");
-
- try (ResultSet resultSet = statement.executeQuery("select * from
table1")) {
- while (resultSet.next()) {
- String objectSizeString = resultSet.getString(3);
- assertEquals("(Object) 4 B", objectSizeString);
- }
- }
- try (ResultSet resultSet = statement.executeQuery("select
read_object(s8) from table1")) {
- final ResultSetMetaData metaData = resultSet.getMetaData();
- final int columnCount = metaData.getColumnCount();
- assertEquals(1, columnCount);
- byte[] byteArray = new byte[] {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA,
(byte) 0xBE};
- while (resultSet.next()) {
- byte[] blob = resultSet.getBytes(1);
- assertArrayEquals(byteArray, blob);
}
}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java
index 7915d64a680..40f7c7303c6 100644
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java
@@ -783,142 +783,6 @@ public class IoTDBTableIT {
}
}
- @Test
- public void testTableObjectCheck() throws Exception {
- final Set<String> illegal = new HashSet<>(Arrays.asList("./", ".", "..",
".\\", "../hack"));
- if (SystemUtils.IS_OS_WINDOWS) {
- illegal.add("C.");
- illegal.add("a:b<|");
- illegal.add("COM1");
- }
- for (final String single : illegal) {
- testObject4SingleIllegalPath(single);
- }
- }
-
- private void testObject4SingleIllegalPath(final String illegal) throws
Exception {
- try (final Connection connection =
- EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
- final Statement statement = connection.createStatement();
- final ITableSession session =
EnvFactory.getEnv().getTableSessionConnection()) {
- statement.execute("create database if not exists db2");
- statement.execute("use db2");
-
- // Test auto-create table
- final String testObject =
- System.getProperty("user.dir")
- + File.separator
- + "target"
- + File.separator
- + "test-classes"
- + File.separator
- + "object-example.pt";
-
- final List<IMeasurementSchema> schemaList = new ArrayList<>();
- schemaList.add(new MeasurementSchema("a", TSDataType.STRING));
- schemaList.add(new MeasurementSchema("b", TSDataType.STRING));
- schemaList.add(new MeasurementSchema("c", TSDataType.INT32));
- schemaList.add(new MeasurementSchema(illegal, TSDataType.OBJECT));
- final List<ColumnCategory> columnTypes =
- Arrays.asList(
- ColumnCategory.TAG,
- ColumnCategory.ATTRIBUTE,
- ColumnCategory.FIELD,
- ColumnCategory.FIELD);
- final Tablet tablet =
- new Tablet(
- illegal,
- IMeasurementSchema.getMeasurementNameList(schemaList),
- IMeasurementSchema.getDataTypeList(schemaList),
- columnTypes,
- 1);
- tablet.addTimestamp(0, System.currentTimeMillis());
- tablet.addValue(schemaList.get(0).getMeasurementName(), 0, "d1");
- tablet.addValue(schemaList.get(1).getMeasurementName(), 0, "a1");
- tablet.addValue(schemaList.get(2).getMeasurementName(), 0, 0);
- tablet.addValue(0, 3, true, 0,
Files.readAllBytes(Paths.get(testObject)));
-
- final String expectedTableError =
- String.format(
- "701: When there are object fields, the tableName %s shall not
be '.', '..' or contain './', '.\\'."
- + (SystemUtils.IS_OS_WINDOWS ? " " +
WindowsOSUtils.OS_SEGMENT_ERROR : ""),
- illegal.toLowerCase());
- final String expectedObjectError =
- String.format(
- "701: When there are object fields, the objectName %s shall not
be '.', '..' or contain './', '.\\'."
- + (SystemUtils.IS_OS_WINDOWS ? " " +
WindowsOSUtils.OS_SEGMENT_ERROR : ""),
- illegal.toLowerCase());
-
- try {
- session.executeNonQueryStatement("use db2");
- session.insert(tablet);
- } catch (final Exception e) {
- Assert.assertEquals(expectedTableError, e.getMessage());
- }
-
- statement.execute(String.format("create table \"%s\" ()", illegal));
-
- try {
- statement.execute(String.format("alter table \"%s\" add column a
object", illegal));
- fail();
- } catch (final SQLException e) {
- Assert.assertEquals(expectedTableError, e.getMessage());
- }
-
- // Test auto-create column
- try {
- session.executeNonQueryStatement("use db2");
- session.insert(tablet);
- } catch (final Exception e) {
- Assert.assertEquals(expectedTableError, e.getMessage());
- }
-
- try {
- statement.execute(String.format("create table test (\"%s\" object)",
illegal));
- fail();
- } catch (final SQLException e) {
- Assert.assertEquals(expectedObjectError, e.getMessage());
- }
-
- statement.execute("create table test (a tag, b attribute, c int32, d
object)");
-
- // Cannot auto-extend illegal column
- tablet.setTableName("test");
- try {
- session.executeNonQueryStatement("use db2");
- session.insert(tablet);
- } catch (final Exception e) {
- Assert.assertEquals(expectedObjectError, e.getMessage());
- }
-
- // It's OK if you don't write object
- statement.execute(String.format("insert into test (a, b, c) values
('%s', 1, 1)", illegal));
- try {
- statement.execute(
- String.format(
- "insert into test (a, b, c, d) values ('%s', 1, 1,
to_object(true, 0, X'aa'))",
- illegal));
- fail();
- } catch (final SQLException e) {
- Assert.assertEquals(
- String.format(
- "507: When there are object fields, the deviceId [test, %s]
shall not be '.', '..' or contain './', '.\\'."
- + (SystemUtils.IS_OS_WINDOWS ? " " +
WindowsOSUtils.OS_SEGMENT_ERROR : ""),
- illegal),
- e.getMessage());
- }
-
- try {
- statement.execute(String.format("alter table test add column \"%s\"
object", illegal));
- fail();
- } catch (final SQLException e) {
- Assert.assertEquals(expectedObjectError, e.getMessage());
- }
-
- statement.execute("drop database db2");
- }
- }
-
@Test
public void testTreeViewTable() throws Exception {
try (final Connection connection = EnvFactory.getEnv().getConnection();
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT.java
deleted file mode 100644
index 0c697424873..00000000000
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT.java
+++ /dev/null
@@ -1,363 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.relational.it.session;
-
-import org.apache.iotdb.isession.ITableSession;
-import org.apache.iotdb.isession.SessionDataSet;
-import org.apache.iotdb.it.env.EnvFactory;
-import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
-import org.apache.iotdb.it.framework.IoTDBTestRunner;
-import org.apache.iotdb.itbase.category.TableClusterIT;
-import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
-import org.apache.iotdb.rpc.IoTDBConnectionException;
-import org.apache.iotdb.rpc.StatementExecutionException;
-
-import com.google.common.io.BaseEncoding;
-import org.apache.tsfile.enums.ColumnCategory;
-import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.utils.Binary;
-import org.apache.tsfile.write.record.Tablet;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.runner.RunWith;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.junit.Assert.assertNull;
-
-@RunWith(IoTDBTestRunner.class)
-@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
-public class IoTDBObjectDeleteIT {
-
- @BeforeClass
- public static void classSetUp() throws Exception {
- EnvFactory.getEnv().initClusterEnvironment();
- }
-
- @Before
- public void setUp() throws Exception {
- try (ITableSession session =
EnvFactory.getEnv().getTableSessionConnection()) {
- session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db1");
- }
- }
-
- @After
- public void tearDown() throws Exception {
- try (ITableSession session =
EnvFactory.getEnv().getTableSessionConnection()) {
- session.executeNonQueryStatement("DROP DATABASE IF EXISTS db1");
- }
- }
-
- @AfterClass
- public static void classTearDown() {
- EnvFactory.getEnv().cleanClusterEnvironment();
- }
-
- @Test
- public void dropObjectTableTest()
- throws IoTDBConnectionException, StatementExecutionException,
IOException {
- String testObject =
- System.getProperty("user.dir")
- + File.separator
- + "target"
- + File.separator
- + "test-classes"
- + File.separator
- + "object-example.pt";
-
- try (ITableSession session =
EnvFactory.getEnv().getTableSessionConnection()) {
- session.executeNonQueryStatement("USE \"db1\"");
- // insert table data by tablet
- List<String> columnNameList =
- Arrays.asList("region_id", "plant_id", "device_id", "temperature",
"file");
- List<TSDataType> dataTypeList =
- Arrays.asList(
- TSDataType.STRING,
- TSDataType.STRING,
- TSDataType.STRING,
- TSDataType.FLOAT,
- TSDataType.OBJECT);
- List<ColumnCategory> columnTypeList =
- new ArrayList<>(
- Arrays.asList(
- ColumnCategory.TAG,
- ColumnCategory.TAG,
- ColumnCategory.TAG,
- ColumnCategory.FIELD,
- ColumnCategory.FIELD));
- Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList,
columnTypeList, 1);
- int rowIndex = tablet.getRowSize();
- tablet.addTimestamp(rowIndex, 1);
- tablet.addValue(rowIndex, 0, "1");
- tablet.addValue(rowIndex, 1, "5");
- tablet.addValue(rowIndex, 2, "3");
- tablet.addValue(rowIndex, 3, 37.6F);
- tablet.addValue(rowIndex, 4, true, 0,
Files.readAllBytes(Paths.get(testObject)));
- session.insert(tablet);
- tablet.reset();
-
- try (SessionDataSet dataSet =
- session.executeQueryStatement(
- "select READ_OBJECT(file) from object_table where time = 1")) {
- SessionDataSet.DataIterator iterator = dataSet.iterator();
- while (iterator.next()) {
- Binary binary = iterator.getBlob(1);
- Assert.assertArrayEquals(Files.readAllBytes(Paths.get(testObject)),
binary.getValues());
- }
- session.executeNonQueryStatement("DROP TABLE IF EXISTS object_table");
- }
- }
-
- // test object file path
- boolean success = false;
- for (DataNodeWrapper dataNodeWrapper :
EnvFactory.getEnv().getDataNodeWrapperList()) {
- String objectDirStr = dataNodeWrapper.getDataNodeObjectDir();
- File objectDir = new File(objectDirStr);
- if (objectDir.exists() && objectDir.isDirectory()) {
- File[] regionDirs = objectDir.listFiles();
- if (regionDirs != null) {
- for (File regionDir : regionDirs) {
- if (regionDir.isDirectory()) {
- File objectFile =
- new File(
- regionDir,
- convertPathString("object_table")
- + File.separator
- + convertPathString("1")
- + File.separator
- + convertPathString("5")
- + File.separator
- + convertPathString("3")
- + File.separator
- + convertPathString("file")
- + File.separator
- + "1.bin");
- if (objectFile.exists() && objectFile.isFile()) {
- success = true;
- }
- }
- }
- }
- }
- }
- Assert.assertFalse(success);
- }
-
- @Test
- public void dropObjectColumnTest()
- throws IoTDBConnectionException, StatementExecutionException,
IOException {
- String testObject =
- System.getProperty("user.dir")
- + File.separator
- + "target"
- + File.separator
- + "test-classes"
- + File.separator
- + "object-example.pt";
-
- try (ITableSession session =
EnvFactory.getEnv().getTableSessionConnection()) {
- session.executeNonQueryStatement("USE \"db1\"");
- // insert table data by tablet
- List<String> columnNameList =
- Arrays.asList("region_id", "plant_id", "device_id", "temperature",
"file");
- List<TSDataType> dataTypeList =
- Arrays.asList(
- TSDataType.STRING,
- TSDataType.STRING,
- TSDataType.STRING,
- TSDataType.FLOAT,
- TSDataType.OBJECT);
- List<ColumnCategory> columnTypeList =
- new ArrayList<>(
- Arrays.asList(
- ColumnCategory.TAG,
- ColumnCategory.TAG,
- ColumnCategory.TAG,
- ColumnCategory.FIELD,
- ColumnCategory.FIELD));
- Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList,
columnTypeList, 1);
- int rowIndex = tablet.getRowSize();
- tablet.addTimestamp(rowIndex, 1);
- tablet.addValue(rowIndex, 0, "1");
- tablet.addValue(rowIndex, 1, "5");
- tablet.addValue(rowIndex, 2, "3");
- tablet.addValue(rowIndex, 3, 37.6F);
- tablet.addValue(rowIndex, 4, true, 0,
Files.readAllBytes(Paths.get(testObject)));
- session.insert(tablet);
- tablet.reset();
-
- try (SessionDataSet dataSet =
- session.executeQueryStatement(
- "select READ_OBJECT(file) from object_table where time = 1")) {
- SessionDataSet.DataIterator iterator = dataSet.iterator();
- while (iterator.next()) {
- Binary binary = iterator.getBlob(1);
- Assert.assertArrayEquals(Files.readAllBytes(Paths.get(testObject)),
binary.getValues());
- }
- session.executeNonQueryStatement("ALTER TABLE object_table drop column
file");
- }
- }
-
- // test object file path
- boolean success = false;
- for (DataNodeWrapper dataNodeWrapper :
EnvFactory.getEnv().getDataNodeWrapperList()) {
- String objectDirStr = dataNodeWrapper.getDataNodeObjectDir();
- File objectDir = new File(objectDirStr);
- if (objectDir.exists() && objectDir.isDirectory()) {
- File[] regionDirs = objectDir.listFiles();
- if (regionDirs != null) {
- for (File regionDir : regionDirs) {
- if (regionDir.isDirectory()) {
- File objectFile =
- new File(
- regionDir,
- convertPathString("object_table")
- + File.separator
- + convertPathString("1")
- + File.separator
- + convertPathString("5")
- + File.separator
- + convertPathString("3")
- + File.separator
- + convertPathString("file")
- + File.separator
- + "1.bin");
- if (objectFile.exists() && objectFile.isFile()) {
- success = true;
- }
- }
- }
- }
- }
- }
- Assert.assertFalse(success);
- }
-
- @Test
- public void deleteObjectSegmentsTest()
- throws IoTDBConnectionException, StatementExecutionException,
IOException {
- String testObject =
- System.getProperty("user.dir")
- + File.separator
- + "target"
- + File.separator
- + "test-classes"
- + File.separator
- + "object-example.pt";
- byte[] objectBytes = Files.readAllBytes(Paths.get(testObject));
- List<byte[]> objectSegments = new ArrayList<>();
- for (int i = 0; i < objectBytes.length; i += 512) {
- objectSegments.add(Arrays.copyOfRange(objectBytes, i, Math.min(i + 512,
objectBytes.length)));
- }
-
- try (ITableSession session =
EnvFactory.getEnv().getTableSessionConnection()) {
- session.executeNonQueryStatement("USE \"db1\"");
- // insert table data by tablet
- List<String> columnNameList =
- Arrays.asList("region_id", "plant_id", "device_id", "temperature",
"file");
- List<TSDataType> dataTypeList =
- Arrays.asList(
- TSDataType.STRING,
- TSDataType.STRING,
- TSDataType.STRING,
- TSDataType.FLOAT,
- TSDataType.OBJECT);
- List<ColumnCategory> columnTypeList =
- new ArrayList<>(
- Arrays.asList(
- ColumnCategory.TAG,
- ColumnCategory.TAG,
- ColumnCategory.TAG,
- ColumnCategory.FIELD,
- ColumnCategory.FIELD));
- Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList,
columnTypeList, 1);
- for (int i = 0; i < objectSegments.size() - 1; i++) {
- int rowIndex = tablet.getRowSize();
- tablet.addTimestamp(rowIndex, 1);
- tablet.addValue(rowIndex, 0, "1");
- tablet.addValue(rowIndex, 1, "5");
- tablet.addValue(rowIndex, 2, "3");
- tablet.addValue(rowIndex, 3, 37.6F);
- tablet.addValue(rowIndex, 4, false, i * 512L, objectSegments.get(i));
- session.insert(tablet);
- tablet.reset();
- }
- session.executeNonQueryStatement("DELETE FROM object_table where time =
1");
-
- try (SessionDataSet dataSet =
- session.executeQueryStatement("select file from object_table where
time = 1")) {
- SessionDataSet.DataIterator iterator = dataSet.iterator();
- while (iterator.next()) {
- assertNull(iterator.getString(1));
- }
- }
- }
-
- // test object file path
- boolean success = false;
- for (DataNodeWrapper dataNodeWrapper :
EnvFactory.getEnv().getDataNodeWrapperList()) {
- String objectDirStr = dataNodeWrapper.getDataNodeObjectDir();
- File objectDir = new File(objectDirStr);
- if (objectDir.exists() && objectDir.isDirectory()) {
- File[] regionDirs = objectDir.listFiles();
- if (regionDirs != null) {
- for (File regionDir : regionDirs) {
- if (regionDir.isDirectory()) {
- File objectTmpFile =
- new File(
- regionDir,
- convertPathString("object_table")
- + File.separator
- + convertPathString("1")
- + File.separator
- + convertPathString("5")
- + File.separator
- + convertPathString("3")
- + File.separator
- + convertPathString("file")
- + File.separator
- + "1.bin.tmp");
- if (objectTmpFile.exists() && objectTmpFile.isFile()) {
- success = true;
- }
- }
- }
- }
- }
- }
- Assert.assertFalse(success);
- }
-
- protected String convertPathString(String path) {
- return
BaseEncoding.base32().omitPadding().encode(path.getBytes(StandardCharsets.UTF_8));
- }
-}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT2.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT2.java
deleted file mode 100644
index c7c7f208bdc..00000000000
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT2.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.relational.it.session;
-
-import org.apache.iotdb.it.env.EnvFactory;
-import org.apache.iotdb.itbase.category.TableClusterIT;
-import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
-
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.experimental.categories.Category;
-
-@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
-public class IoTDBObjectDeleteIT2 extends IoTDBObjectDeleteIT {
-
- @BeforeClass
- public static void classSetUp() throws Exception {
-
EnvFactory.getEnv().getConfig().getCommonConfig().setRestrictObjectLimit(true);
- EnvFactory.getEnv().initClusterEnvironment();
- }
-
- @AfterClass
- public static void classTearDown() {
- EnvFactory.getEnv().cleanClusterEnvironment();
- }
-
- @Override
- protected String convertPathString(String path) {
- return path;
- }
-}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectInsertIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectInsertIT.java
deleted file mode 100644
index 6e0d869c86f..00000000000
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectInsertIT.java
+++ /dev/null
@@ -1,339 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.relational.it.session;
-
-import org.apache.iotdb.isession.ITableSession;
-import org.apache.iotdb.isession.SessionDataSet;
-import org.apache.iotdb.it.env.EnvFactory;
-import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
-import org.apache.iotdb.it.framework.IoTDBTestRunner;
-import org.apache.iotdb.itbase.category.TableClusterIT;
-import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
-import org.apache.iotdb.rpc.IoTDBConnectionException;
-import org.apache.iotdb.rpc.StatementExecutionException;
-import org.apache.iotdb.rpc.TSStatusCode;
-
-import com.google.common.io.BaseEncoding;
-import org.apache.tsfile.enums.ColumnCategory;
-import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.utils.Binary;
-import org.apache.tsfile.utils.BytesUtils;
-import org.apache.tsfile.write.record.Tablet;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.runner.RunWith;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.junit.Assert.assertNull;
-
-@RunWith(IoTDBTestRunner.class)
-@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
-public class IoTDBObjectInsertIT {
-
- @BeforeClass
- public static void classSetUp() throws Exception {
- EnvFactory.getEnv().initClusterEnvironment();
- }
-
- @Before
- public void setUp() throws Exception {
- try (ITableSession session =
EnvFactory.getEnv().getTableSessionConnection()) {
- session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db1");
- }
- }
-
- @After
- public void tearDown() throws Exception {
- try (ITableSession session =
EnvFactory.getEnv().getTableSessionConnection()) {
- session.executeNonQueryStatement("DROP DATABASE IF EXISTS db1");
- }
- }
-
- @AfterClass
- public static void classTearDown() {
- EnvFactory.getEnv().cleanClusterEnvironment();
- }
-
- @Test
- public void insertObjectTest()
- throws IoTDBConnectionException, StatementExecutionException,
IOException {
- String testObject =
- System.getProperty("user.dir")
- + File.separator
- + "target"
- + File.separator
- + "test-classes"
- + File.separator
- + "object-example.pt";
- File object = new File(testObject);
-
- try (ITableSession session =
EnvFactory.getEnv().getTableSessionConnection()) {
- session.executeNonQueryStatement("USE \"db1\"");
- // insert table data by tablet
- List<String> columnNameList =
- Arrays.asList("region_id", "plant_id", "device_id", "temperature",
"file");
- List<TSDataType> dataTypeList =
- Arrays.asList(
- TSDataType.STRING,
- TSDataType.STRING,
- TSDataType.STRING,
- TSDataType.FLOAT,
- TSDataType.OBJECT);
- List<ColumnCategory> columnTypeList =
- new ArrayList<>(
- Arrays.asList(
- ColumnCategory.TAG,
- ColumnCategory.TAG,
- ColumnCategory.TAG,
- ColumnCategory.FIELD,
- ColumnCategory.FIELD));
- Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList,
columnTypeList, 1);
- int rowIndex = tablet.getRowSize();
- tablet.addTimestamp(rowIndex, 1);
- tablet.addValue(rowIndex, 0, "1");
- tablet.addValue(rowIndex, 1, "5");
- tablet.addValue(rowIndex, 2, "3");
- tablet.addValue(rowIndex, 3, 37.6F);
- tablet.addValue(rowIndex, 4, true, 0,
Files.readAllBytes(Paths.get(testObject)));
- session.insert(tablet);
- tablet.reset();
-
- // insert another row without object value
- rowIndex = tablet.getRowSize();
- tablet.addTimestamp(rowIndex, 2);
- tablet.addValue(rowIndex, 0, "1");
- tablet.addValue(rowIndex, 1, "5");
- tablet.addValue(rowIndex, 2, "3");
- tablet.addValue(rowIndex, 3, 37.6F);
- session.insert(tablet);
- tablet.reset();
-
- try (SessionDataSet dataSet =
- session.executeQueryStatement("select file from object_table where
time = 1")) {
- SessionDataSet.DataIterator iterator = dataSet.iterator();
- while (iterator.next()) {
- Assert.assertEquals(
-
BytesUtils.parseObjectByteArrayToString(BytesUtils.longToBytes(object.length())),
- iterator.getString(1));
- }
- }
-
- try (SessionDataSet dataSet =
- session.executeQueryStatement(
- "select READ_OBJECT(file) from object_table where time = 1")) {
- SessionDataSet.DataIterator iterator = dataSet.iterator();
- while (iterator.next()) {
- Binary binary = iterator.getBlob(1);
- Assert.assertArrayEquals(Files.readAllBytes(Paths.get(testObject)),
binary.getValues());
- }
- }
- }
- // test object file path
- boolean success = false;
- for (DataNodeWrapper dataNodeWrapper :
EnvFactory.getEnv().getDataNodeWrapperList()) {
- String objectDirStr = dataNodeWrapper.getDataNodeObjectDir();
- File objectDir = new File(objectDirStr);
- if (objectDir.exists() && objectDir.isDirectory()) {
- File[] regionDirs = objectDir.listFiles();
- if (regionDirs != null) {
- for (File regionDir : regionDirs) {
- if (regionDir.isDirectory()) {
- File objectFile =
- new File(
- regionDir,
- convertPathString("object_table")
- + File.separator
- + convertPathString("1")
- + File.separator
- + convertPathString("5")
- + File.separator
- + convertPathString("3")
- + File.separator
- + convertPathString("file")
- + File.separator
- + "1.bin");
- if (objectFile.exists() && objectFile.isFile()) {
- success = true;
- }
- }
- }
- }
- }
- }
- Assert.assertTrue(success);
- }
-
- @Test
- public void insertObjectSegmentsTest()
- throws IoTDBConnectionException, StatementExecutionException,
IOException {
- String testObject =
- System.getProperty("user.dir")
- + File.separator
- + "target"
- + File.separator
- + "test-classes"
- + File.separator
- + "object-example.pt";
- byte[] objectBytes = Files.readAllBytes(Paths.get(testObject));
- List<byte[]> objectSegments = new ArrayList<>();
- for (int i = 0; i < objectBytes.length; i += 512) {
- objectSegments.add(Arrays.copyOfRange(objectBytes, i, Math.min(i + 512,
objectBytes.length)));
- }
-
- try (ITableSession session =
EnvFactory.getEnv().getTableSessionConnection()) {
- session.executeNonQueryStatement("USE \"db1\"");
- // insert table data by tablet
- List<String> columnNameList =
- Arrays.asList("region_id", "plant_id", "device_id", "temperature",
"file");
- List<TSDataType> dataTypeList =
- Arrays.asList(
- TSDataType.STRING,
- TSDataType.STRING,
- TSDataType.STRING,
- TSDataType.FLOAT,
- TSDataType.OBJECT);
- List<ColumnCategory> columnTypeList =
- new ArrayList<>(
- Arrays.asList(
- ColumnCategory.TAG,
- ColumnCategory.TAG,
- ColumnCategory.TAG,
- ColumnCategory.FIELD,
- ColumnCategory.FIELD));
- Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList,
columnTypeList, 1);
- for (int i = 0; i < objectSegments.size() - 1; i++) {
- int rowIndex = tablet.getRowSize();
- tablet.addTimestamp(rowIndex, 1);
- tablet.addValue(rowIndex, 0, "1");
- tablet.addValue(rowIndex, 1, "5");
- tablet.addValue(rowIndex, 2, "3");
- tablet.addValue(rowIndex, 3, 37.6F);
- tablet.addValue(rowIndex, 4, false, i * 512L, objectSegments.get(i));
- session.insert(tablet);
- tablet.reset();
- }
-
- try (SessionDataSet dataSet =
- session.executeQueryStatement("select file from object_table where
time = 1")) {
- SessionDataSet.DataIterator iterator = dataSet.iterator();
- while (iterator.next()) {
- assertNull(iterator.getString(1));
- }
- }
-
- // insert segment with wrong offset
- try {
- int rowIndex = tablet.getRowSize();
- tablet.addTimestamp(rowIndex, 1);
- tablet.addValue(rowIndex, 0, "1");
- tablet.addValue(rowIndex, 1, "5");
- tablet.addValue(rowIndex, 2, "3");
- tablet.addValue(rowIndex, 3, 37.6F);
- tablet.addValue(rowIndex, 4, false, 512L, objectSegments.get(1));
- session.insert(tablet);
- } catch (StatementExecutionException e) {
- Assert.assertEquals(TSStatusCode.OBJECT_INSERT_ERROR.getStatusCode(),
e.getStatusCode());
- Assert.assertEquals(
- String.format(
- "741: The file length %d is not equal to the offset %d",
- ((objectSegments.size() - 1) * 512), 512L),
- e.getMessage());
- } finally {
- tablet.reset();
- }
-
- // last segment
- int rowIndex = tablet.getRowSize();
- tablet.addTimestamp(rowIndex, 1);
- tablet.addValue(rowIndex, 0, "1");
- tablet.addValue(rowIndex, 1, "5");
- tablet.addValue(rowIndex, 2, "3");
- tablet.addValue(rowIndex, 3, 37.6F);
- tablet.addValue(
- rowIndex,
- 4,
- true,
- (objectSegments.size() - 1) * 512L,
- objectSegments.get(objectSegments.size() - 1));
- session.insert(tablet);
- tablet.reset();
-
- try (SessionDataSet dataSet =
- session.executeQueryStatement(
- "select READ_OBJECT(file) from object_table where time = 1")) {
- SessionDataSet.DataIterator iterator = dataSet.iterator();
- while (iterator.next()) {
- Binary binary = iterator.getBlob(1);
- Assert.assertArrayEquals(Files.readAllBytes(Paths.get(testObject)),
binary.getValues());
- }
- }
- }
-
- // test object file path
- boolean success = false;
- for (DataNodeWrapper dataNodeWrapper :
EnvFactory.getEnv().getDataNodeWrapperList()) {
- String objectDirStr = dataNodeWrapper.getDataNodeObjectDir();
- File objectDir = new File(objectDirStr);
- if (objectDir.exists() && objectDir.isDirectory()) {
- File[] regionDirs = objectDir.listFiles();
- if (regionDirs != null) {
- for (File regionDir : regionDirs) {
- if (regionDir.isDirectory()) {
- File objectFile =
- new File(
- regionDir,
- convertPathString("object_table")
- + File.separator
- + convertPathString("1")
- + File.separator
- + convertPathString("5")
- + File.separator
- + convertPathString("3")
- + File.separator
- + convertPathString("file")
- + File.separator
- + "1.bin");
- if (objectFile.exists() && objectFile.isFile()) {
- success = true;
- }
- }
- }
- }
- }
- }
- Assert.assertTrue(success);
- }
-
- protected String convertPathString(String path) {
- return
BaseEncoding.base32().omitPadding().encode(path.getBytes(StandardCharsets.UTF_8));
- }
-}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectInsertIT2.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectInsertIT2.java
deleted file mode 100644
index 9bd16938f56..00000000000
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectInsertIT2.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.relational.it.session;
-
-import org.apache.iotdb.db.it.utils.TestUtils;
-import org.apache.iotdb.isession.ITableSession;
-import org.apache.iotdb.isession.SessionDataSet;
-import org.apache.iotdb.it.env.EnvFactory;
-import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
-import org.apache.iotdb.itbase.category.TableClusterIT;
-import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
-import org.apache.iotdb.rpc.IoTDBConnectionException;
-import org.apache.iotdb.rpc.StatementExecutionException;
-
-import org.apache.tsfile.enums.ColumnCategory;
-import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.utils.Binary;
-import org.apache.tsfile.utils.BytesUtils;
-import org.apache.tsfile.write.record.Tablet;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
-public class IoTDBObjectInsertIT2 extends IoTDBObjectInsertIT {
-
- @BeforeClass
- public static void classSetUp() throws Exception {
-
EnvFactory.getEnv().getConfig().getCommonConfig().setRestrictObjectLimit(true);
- EnvFactory.getEnv().initClusterEnvironment();
- }
-
- @AfterClass
- public static void classTearDown() {
- EnvFactory.getEnv().cleanClusterEnvironment();
- }
-
- @Test
- public void changeRestrictObjectLimitTest()
- throws IoTDBConnectionException, StatementExecutionException,
IOException {
-
EnvFactory.getEnv().getConfig().getCommonConfig().setRestrictObjectLimit(false);
- TestUtils.restartCluster(EnvFactory.getEnv());
- String testObject =
- System.getProperty("user.dir")
- + File.separator
- + "target"
- + File.separator
- + "test-classes"
- + File.separator
- + "object-example.pt";
- File object = new File(testObject);
-
- try (ITableSession session =
EnvFactory.getEnv().getTableSessionConnection()) {
- session.executeNonQueryStatement("USE \"db1\"");
- // insert table data by tablet
- List<String> columnNameList =
- Arrays.asList("region_id", "plant_id", "device_id", "temperature",
"file");
- List<TSDataType> dataTypeList =
- Arrays.asList(
- TSDataType.STRING,
- TSDataType.STRING,
- TSDataType.STRING,
- TSDataType.FLOAT,
- TSDataType.OBJECT);
- List<ColumnCategory> columnTypeList =
- new ArrayList<>(
- Arrays.asList(
- ColumnCategory.TAG,
- ColumnCategory.TAG,
- ColumnCategory.TAG,
- ColumnCategory.FIELD,
- ColumnCategory.FIELD));
- Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList,
columnTypeList, 1);
- int rowIndex = tablet.getRowSize();
- tablet.addTimestamp(rowIndex, 1);
- tablet.addValue(rowIndex, 0, "1");
- tablet.addValue(rowIndex, 1, "5");
- tablet.addValue(rowIndex, 2, "3");
- tablet.addValue(rowIndex, 3, 37.6F);
- tablet.addValue(rowIndex, 4, true, 0,
Files.readAllBytes(Paths.get(testObject)));
- session.insert(tablet);
- tablet.reset();
-
- try (SessionDataSet dataSet =
- session.executeQueryStatement("select file from object_table where
time = 1")) {
- SessionDataSet.DataIterator iterator = dataSet.iterator();
- while (iterator.next()) {
- Assert.assertEquals(
-
BytesUtils.parseObjectByteArrayToString(BytesUtils.longToBytes(object.length())),
- iterator.getString(1));
- }
- }
-
- try (SessionDataSet dataSet =
- session.executeQueryStatement(
- "select READ_OBJECT(file) from object_table where time = 1")) {
- SessionDataSet.DataIterator iterator = dataSet.iterator();
- while (iterator.next()) {
- Binary binary = iterator.getBlob(1);
- Assert.assertArrayEquals(Files.readAllBytes(Paths.get(testObject)),
binary.getValues());
- }
- }
- }
- // test object file path
- boolean success = false;
- for (DataNodeWrapper dataNodeWrapper :
EnvFactory.getEnv().getDataNodeWrapperList()) {
- String objectDirStr = dataNodeWrapper.getDataNodeObjectDir();
- File objectDir = new File(objectDirStr);
- if (objectDir.exists() && objectDir.isDirectory()) {
- File[] regionDirs = objectDir.listFiles();
- if (regionDirs != null) {
- for (File regionDir : regionDirs) {
- if (regionDir.isDirectory()) {
- File objectFile =
- new File(
- regionDir,
- convertPathString("object_table")
- + File.separator
- + convertPathString("1")
- + File.separator
- + convertPathString("5")
- + File.separator
- + convertPathString("3")
- + File.separator
- + convertPathString("file")
- + File.separator
- + "1.bin");
- if (objectFile.exists() && objectFile.isFile()) {
- success = true;
- }
- }
- }
- }
- }
- }
- Assert.assertTrue(success);
- }
-
- @Override
- protected String convertPathString(String path) {
- return path;
- }
-}
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java
index d4dd53ffd43..529f15d06cd 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java
@@ -213,19 +213,6 @@ public class SystemPropertiesUtils {
COMMON_CONFIG.setEnableGrantOption(enableGrantOption);
}
}
-
- if (systemProperties.getProperty("restrict_object_limit", null) != null) {
- boolean restrictObjectLimit =
-
Boolean.parseBoolean(systemProperties.getProperty("restrict_object_limit"));
- if (restrictObjectLimit != COMMON_CONFIG.isRestrictObjectLimit()) {
- LOGGER.warn(
- format,
- "restrict_object_limit",
- COMMON_CONFIG.isRestrictObjectLimit(),
- restrictObjectLimit);
- COMMON_CONFIG.setRestrictObjectLimit(restrictObjectLimit);
- }
- }
}
/**
@@ -299,8 +286,6 @@ public class SystemPropertiesUtils {
"tag_attribute_total_size",
String.valueOf(COMMON_CONFIG.getTagAttributeTotalSize()));
systemProperties.setProperty(
"enable_grant_option",
String.valueOf(COMMON_CONFIG.getEnableGrantOption()));
- systemProperties.setProperty(
- "restrict_object_limit",
String.valueOf(COMMON_CONFIG.isRestrictObjectLimit()));
systemPropertiesHandler.overwrite(systemProperties);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 73b8764994c..6730138b2af 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -906,11 +906,6 @@ public class IoTDBDescriptor {
properties.getProperty(
"pipe_task_thread_count",
Integer.toString(conf.getPipeTaskThreadCount()).trim())));
- conf.setMaxObjectSizeInByte(
- Long.parseLong(
- properties.getProperty(
- "max_object_file_size_in_byte",
String.valueOf(conf.getMaxObjectSizeInByte()))));
-
// At the same time, set TSFileConfig
List<FSType> fsTypes = new ArrayList<>();
fsTypes.add(FSType.LOCAL);
@@ -2195,10 +2190,6 @@ public class IoTDBDescriptor {
"include_null_value_in_write_throughput_metric",
ConfigurationFileUtils.getConfigurationDefaultValue(
"include_null_value_in_write_throughput_metric"))));
- conf.setMaxObjectSizeInByte(
- Long.parseLong(
- properties.getProperty(
- "max_object_file_size_in_byte",
String.valueOf(conf.getMaxObjectSizeInByte()))));
// The buffer for cte materialization.
long cteBufferSizeInBytes =
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/AstUtil.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/AstUtil.java
index 700d5e40beb..cb17a90c04f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/AstUtil.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/AstUtil.java
@@ -21,22 +21,16 @@ package
org.apache.iotdb.db.queryengine.plan.relational.sql.util;
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.db.exception.sql.SemanticException;
-import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BinaryLiteral;
-import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression;
-import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral;
-import
org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TableExpressionType;
import com.google.common.graph.SuccessorsFunction;
import com.google.common.graph.Traverser;
-import org.apache.tsfile.utils.Binary;
-import org.apache.tsfile.utils.BytesUtils;
import java.time.ZoneId;
import java.util.List;
@@ -119,30 +113,6 @@ public final class AstUtil {
throw new SemanticException(
String.format("Cannot insert identifier %s, please use string
literal", expression));
}
- if (expression instanceof FunctionCall
- && "to_object".equals(((FunctionCall)
expression).getName().toString())) {
- List<Expression> arguments = ((FunctionCall) expression).getArguments();
- if (arguments.size() == 3
- && arguments.get(0).getExpressionType() ==
TableExpressionType.BOOLEAN_LITERAL
- && arguments.get(1).getExpressionType() ==
TableExpressionType.LONG_LITERAL
- && arguments.get(2).getExpressionType() ==
TableExpressionType.BINARY_LITERAL) {
- boolean isEOF =
- (boolean)
- ((BooleanLiteral) ((FunctionCall)
expression).getArguments().get(0)).getTsValue();
- long offset =
- (long) ((LongLiteral) ((FunctionCall)
expression).getArguments().get(1)).getTsValue();
- byte[] content =
- ((Binary)
- ((BinaryLiteral) ((FunctionCall)
expression).getArguments().get(2))
- .getTsValue())
- .getValues();
- byte[] val = new byte[content.length + 9];
- val[0] = (byte) (isEOF ? 1 : 0);
- System.arraycopy(BytesUtils.longToBytes(offset), 0, val, 1, 8);
- System.arraycopy(content, 0, val, 9, content.length);
- return new Binary(val);
- }
- }
throw new SemanticException("Unsupported expression: " + expression);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/Base32ObjectPath.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/Base32ObjectPath.java
deleted file mode 100644
index d0ea395502a..00000000000
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/Base32ObjectPath.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.storageengine.dataregion;
-
-import com.google.common.io.BaseEncoding;
-import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
-import org.apache.tsfile.utils.ReadWriteIOUtils;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-public class Base32ObjectPath implements IObjectPath {
-
- private final Path path;
- private int serializedSize = -1;
-
- private static final Deserializer DESERIALIZER =
- new Deserializer() {
- @Override
- public IObjectPath deserializeFrom(ByteBuffer byteBuffer) {
- return deserialize(byteBuffer);
- }
-
- @Override
- public IObjectPath deserializeFrom(InputStream inputStream) throws
IOException {
- return deserialize(inputStream);
- }
-
- @Override
- public IObjectPath deserializeFromObjectValue(ByteBuffer byteBuffer) {
- return deserialize(byteBuffer);
- }
- };
-
- private static final Factory FACTORY = Base32ObjectPath::new;
-
- private Base32ObjectPath(String first, String... more) {
- path = Paths.get(first, more);
- }
-
- public Base32ObjectPath(Path path) {
- this.path = path;
- }
-
- public Base32ObjectPath(int regionId, long time, IDeviceID iDeviceID, String
measurement) {
- Object[] segments = iDeviceID.getSegments();
- String[] pathSegments = new String[segments.length + 2];
- for (int i = 0; i < segments.length; i++) {
- Object segment = segments[i];
- String segmentString = segment == null ? "null" : segment.toString();
- pathSegments[i] =
- BaseEncoding.base32()
- .omitPadding()
- .encode(segmentString.getBytes(StandardCharsets.UTF_8));
- }
- pathSegments[pathSegments.length - 2] =
-
BaseEncoding.base32().omitPadding().encode(measurement.getBytes(StandardCharsets.UTF_8));
- pathSegments[pathSegments.length - 1] = time + ".bin";
- path = Paths.get(String.valueOf(regionId), pathSegments);
- }
-
- @Override
- public int serialize(ByteBuffer byteBuffer) {
- int cnt = 0;
- cnt += ReadWriteForEncodingUtils.writeUnsignedVarInt(path.getNameCount(),
byteBuffer);
- for (Path segment : path) {
- cnt += ReadWriteIOUtils.writeVar(segment.toString(), byteBuffer);
- }
- return cnt;
- }
-
- @Override
- public int serialize(OutputStream outputStream) throws IOException {
- int cnt = 0;
- cnt += ReadWriteForEncodingUtils.writeUnsignedVarInt(path.getNameCount(),
outputStream);
- for (Path segment : path) {
- cnt += ReadWriteIOUtils.writeVar(segment.toString(), outputStream);
- }
- return cnt;
- }
-
- @Override
- public int getSerializedSize() {
- if (serializedSize != -1) {
- return serializedSize;
- }
- int cnt = ReadWriteForEncodingUtils.varIntSize(path.getNameCount());
- for (Path segment : path) {
- byte[] bytes = segment.toString().getBytes(StandardCharsets.UTF_8);
- cnt += ReadWriteForEncodingUtils.varIntSize(bytes.length);
- cnt += bytes.length;
- }
- serializedSize = cnt;
- return cnt;
- }
-
- @Override
- public void serializeToObjectValue(ByteBuffer byteBuffer) {
- serialize(byteBuffer);
- }
-
- @Override
- public int getSerializeSizeToObjectValue() {
- return getSerializedSize();
- }
-
- public static Base32ObjectPath deserialize(ByteBuffer byteBuffer) {
- int cnt = ReadWriteForEncodingUtils.readUnsignedVarInt(byteBuffer);
- String first = ReadWriteIOUtils.readVarIntString(byteBuffer);
- String[] more = new String[cnt - 1];
-
- for (int i = 0; i < cnt - 1; ++i) {
- more[i] = ReadWriteIOUtils.readVarIntString(byteBuffer);
- }
- return new Base32ObjectPath(first, more);
- }
-
- public static Base32ObjectPath deserialize(InputStream stream) throws
IOException {
- int cnt = ReadWriteForEncodingUtils.readUnsignedVarInt(stream);
- String first = ReadWriteIOUtils.readVarIntString(stream);
- String[] more = new String[cnt - 1];
-
- for (int i = 0; i < cnt - 1; ++i) {
- more[i] = ReadWriteIOUtils.readVarIntString(stream);
- }
-
- return new Base32ObjectPath(first, more);
- }
-
- @Override
- public String toString() {
- return path.toString();
- }
-
- public Path getPath() {
- return path;
- }
-
- public static Factory getFACTORY() {
- return FACTORY;
- }
-
- public static Deserializer getDESERIALIZER() {
- return DESERIALIZER;
- }
-}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IObjectPath.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IObjectPath.java
index 3e860c0c3a7..2a2910a73d0 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IObjectPath.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IObjectPath.java
@@ -47,10 +47,7 @@ public interface IObjectPath {
IObjectPath create(int regionId, long time, IDeviceID iDeviceID, String
measurement);
- Factory FACTORY =
- CONFIG.isRestrictObjectLimit()
- ? PlainObjectPath.getFACTORY()
- : Base32ObjectPath.getFACTORY();
+ Factory FACTORY = null;
}
interface Deserializer {
@@ -63,8 +60,6 @@ public interface IObjectPath {
}
static Deserializer getDeserializer() {
- return CONFIG.isRestrictObjectLimit()
- ? PlainObjectPath.getDESERIALIZER()
- : Base32ObjectPath.getDESERIALIZER();
+ return null;
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PlainObjectPath.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PlainObjectPath.java
deleted file mode 100644
index 3ae3a925a3c..00000000000
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PlainObjectPath.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.storageengine.dataregion;
-
-import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.utils.ReadWriteIOUtils;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-
-public class PlainObjectPath implements IObjectPath {
-
- private final String filePath;
-
- private static final Deserializer DESERIALIZER =
- new Deserializer() {
- @Override
- public IObjectPath deserializeFrom(ByteBuffer byteBuffer) {
- return deserialize(byteBuffer);
- }
-
- @Override
- public IObjectPath deserializeFrom(InputStream inputStream) throws
IOException {
- return deserialize(inputStream);
- }
-
- @Override
- public IObjectPath deserializeFromObjectValue(ByteBuffer byteBuffer) {
- return deserializeObjectValue(byteBuffer);
- }
- };
-
- private static final Factory FACTORY = PlainObjectPath::new;
-
- public PlainObjectPath(String filePath) {
- this.filePath = filePath;
- }
-
- public PlainObjectPath(int regionId, long time, IDeviceID iDeviceID, String
measurement) {
- String objectFileName = time + ".bin";
- Object[] segments = iDeviceID.getSegments();
- StringBuilder relativePathString =
- new StringBuilder(String.valueOf(regionId)).append(File.separator);
- for (Object segment : segments) {
- relativePathString
- .append(segment == null ? "null" : segment.toString().toLowerCase())
- .append(File.separator);
- }
- relativePathString.append(measurement).append(File.separator);
- relativePathString.append(objectFileName);
- this.filePath = relativePathString.toString();
- }
-
- @Override
- public int serialize(ByteBuffer byteBuffer) {
- return ReadWriteIOUtils.write(filePath, byteBuffer);
- }
-
- @Override
- public int serialize(OutputStream outputStream) throws IOException {
- return ReadWriteIOUtils.write(filePath, outputStream);
- }
-
- @Override
- public int getSerializedSize() {
- return ReadWriteIOUtils.sizeToWrite(filePath);
- }
-
- @Override
- public void serializeToObjectValue(ByteBuffer byteBuffer) {
- byteBuffer.put(filePath.getBytes(StandardCharsets.UTF_8));
- }
-
- @Override
- public int getSerializeSizeToObjectValue() {
- return filePath.getBytes(StandardCharsets.UTF_8).length;
- }
-
- public static PlainObjectPath deserialize(ByteBuffer byteBuffer) {
- String filePath = ReadWriteIOUtils.readString(byteBuffer);
- return new PlainObjectPath(filePath);
- }
-
- public static PlainObjectPath deserialize(InputStream stream) throws
IOException {
- String filePath = ReadWriteIOUtils.readString(stream);
- return new PlainObjectPath(filePath);
- }
-
- public static PlainObjectPath deserializeObjectValue(ByteBuffer byteBuffer) {
- return new
PlainObjectPath(StandardCharsets.UTF_8.decode(byteBuffer).toString());
- }
-
- @Override
- public String toString() {
- return filePath;
- }
-
- public static Factory getFACTORY() {
- return FACTORY;
- }
-
- public static Deserializer getDESERIALIZER() {
- return DESERIALIZER;
- }
-}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TabletDecoder.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TabletDecoder.java
index bd1eba9cfa8..3229ea87256 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TabletDecoder.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TabletDecoder.java
@@ -192,7 +192,6 @@ public class TabletDecoder {
case STRING:
case BLOB:
case TEXT:
- case OBJECT:
Binary[] binaryCol = new Binary[rowSize];
if (encoding == TSEncoding.PLAIN) {
// PlainEncoder uses var int, which may cause compatibility problem
@@ -208,6 +207,7 @@ public class TabletDecoder {
break;
case UNKNOWN:
case VECTOR:
+ default:
throw new IllegalArgumentException("Unsupported data type: " +
dataType);
}
return column;
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/object/ObjectTypeCompactionTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/object/ObjectTypeCompactionTest.java
deleted file mode 100644
index 14495067da4..00000000000
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/object/ObjectTypeCompactionTest.java
+++ /dev/null
@@ -1,459 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.storageengine.dataregion.compaction.object;
-
-import org.apache.iotdb.commons.conf.CommonConfig;
-import org.apache.iotdb.commons.conf.CommonDescriptor;
-import org.apache.iotdb.commons.exception.MetadataException;
-import org.apache.iotdb.commons.schema.table.TsTable;
-import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema;
-import org.apache.iotdb.commons.schema.table.column.TagColumnSchema;
-import org.apache.iotdb.commons.utils.FileUtils;
-import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
-import org.apache.iotdb.db.exception.StorageEngineException;
-import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache;
-import org.apache.iotdb.db.storageengine.dataregion.Base32ObjectPath;
-import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
-import org.apache.iotdb.db.storageengine.dataregion.IObjectPath;
-import org.apache.iotdb.db.storageengine.dataregion.PlainObjectPath;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CrossSpaceCompactionTask;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.SettleCompactionTask;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
-import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
-import org.apache.iotdb.db.utils.ObjectTypeUtils;
-
-import com.google.common.io.BaseEncoding;
-import org.apache.tsfile.enums.ColumnCategory;
-import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.exception.write.WriteProcessException;
-import org.apache.tsfile.file.metadata.ColumnSchema;
-import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.file.metadata.StringArrayDeviceID;
-import org.apache.tsfile.file.metadata.TableSchema;
-import org.apache.tsfile.file.metadata.enums.CompressionType;
-import org.apache.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.tsfile.utils.Binary;
-import org.apache.tsfile.utils.BytesUtils;
-import org.apache.tsfile.utils.Pair;
-import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl;
-import org.apache.tsfile.write.schema.MeasurementSchema;
-import org.apache.tsfile.write.writer.TsFileIOWriter;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.util.Arrays;
-import java.util.Collections;
-
-public class ObjectTypeCompactionTest extends AbstractCompactionTest {
-
- private static final TableSchema tableSchema =
- new TableSchema(
- "t1",
- Arrays.asList(
- new ColumnSchema("device", TSDataType.STRING,
ColumnCategory.TAG),
- new ColumnSchema("s1", TSDataType.OBJECT,
ColumnCategory.FIELD)));
-
- private String threadName;
- private File objectDir;
- private File regionDir;
-
- private final CommonConfig config =
CommonDescriptor.getInstance().getConfig();
-
- @Before
- @Override
- public void setUp()
- throws IOException, WriteProcessException, MetadataException,
InterruptedException {
- config.setRestrictObjectLimit(true);
- this.threadName = Thread.currentThread().getName();
- Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1");
- DataNodeTableCache.getInstance().invalid(this.COMPACTION_TEST_SG);
- createTable("t1", 1);
- super.setUp();
- try {
- objectDir = new
File(TierManager.getInstance().getNextFolderForObjectFile());
- regionDir = new File(objectDir, "0");
- regionDir.mkdirs();
- } catch (DiskSpaceInsufficientException e) {
- throw new RuntimeException(e);
- }
- }
-
- @After
- @Override
- public void tearDown() throws IOException, StorageEngineException {
- super.tearDown();
- Thread.currentThread().setName(threadName);
- DataNodeTableCache.getInstance().invalid(this.COMPACTION_TEST_SG);
- File[] files = objectDir.listFiles();
- if (files != null) {
- for (File file : files) {
- FileUtils.deleteFileOrDirectory(file);
- }
- }
- config.setRestrictObjectLimit(false);
- }
-
- public void createTable(String tableName, long ttl) {
- TsTable tsTable = new TsTable(tableName);
- tsTable.addColumnSchema(new TagColumnSchema("device", TSDataType.STRING));
- tsTable.addColumnSchema(
- new FieldColumnSchema("s1", TSDataType.OBJECT, TSEncoding.PLAIN,
CompressionType.LZ4));
- tsTable.addProp(TsTable.TTL_PROPERTY, ttl + "");
- DataNodeTableCache.getInstance().preUpdateTable(this.COMPACTION_TEST_SG,
tsTable, null);
-
DataNodeTableCache.getInstance().commitUpdateTable(this.COMPACTION_TEST_SG,
tableName, null);
- }
-
- @Test
- public void testSeqCompactionWithTTL() throws IOException,
WriteProcessException {
- Pair<TsFileResource, File> pair1 =
- generateTsFileAndObject(true, System.currentTimeMillis() - 100000, 0);
- Pair<TsFileResource, File> pair2 =
- generateTsFileAndObject(true, System.currentTimeMillis() + 1000000,
100);
- tsFileManager.add(pair1.getLeft(), true);
- tsFileManager.add(pair2.getLeft(), true);
- InnerSpaceCompactionTask task =
- new InnerSpaceCompactionTask(
- 0,
- tsFileManager,
- tsFileManager.getTsFileList(true),
- true,
- new ReadChunkCompactionPerformer(),
- 0);
- Assert.assertTrue(task.start());
-
- Assert.assertTrue(pair1.getRight().exists());
- Assert.assertTrue(pair2.getRight().exists());
- CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir,
COMPACTION_TEST_SG);
- Assert.assertFalse(pair1.getRight().exists());
- Assert.assertTrue(pair2.getRight().exists());
- }
-
- @Test
- public void testUnseqCompactionWithTTL() throws IOException,
WriteProcessException {
- Pair<TsFileResource, File> pair1 =
- generateTsFileAndObject(false, System.currentTimeMillis() + 100000, 1);
- Pair<TsFileResource, File> pair2 =
- generateTsFileAndObject(false, System.currentTimeMillis() - 1000000,
0);
- tsFileManager.add(pair1.getLeft(), false);
- tsFileManager.add(pair2.getLeft(), false);
- InnerSpaceCompactionTask task =
- new InnerSpaceCompactionTask(
- 0,
- tsFileManager,
- tsFileManager.getTsFileList(false),
- false,
- new FastCompactionPerformer(false),
- 0);
- Assert.assertTrue(task.start());
-
- Assert.assertTrue(pair1.getRight().exists());
- Assert.assertTrue(pair2.getRight().exists());
- CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir,
COMPACTION_TEST_SG);
- Assert.assertFalse(pair2.getRight().exists());
- Assert.assertTrue(pair1.getRight().exists());
- }
-
- @Test
- public void testUnseqCompactionWithReadPointWithTTL() throws IOException,
WriteProcessException {
- Pair<TsFileResource, File> pair1 =
- generateTsFileAndObject(false, System.currentTimeMillis() + 100000, 0);
- Pair<TsFileResource, File> pair2 =
- generateTsFileAndObject(false, System.currentTimeMillis() - 1000000,
0);
- tsFileManager.add(pair1.getLeft(), false);
- tsFileManager.add(pair2.getLeft(), false);
- InnerSpaceCompactionTask task =
- new InnerSpaceCompactionTask(
- 0,
- tsFileManager,
- tsFileManager.getTsFileList(false),
- false,
- new ReadPointCompactionPerformer(),
- 0);
- Assert.assertTrue(task.start());
- Assert.assertTrue(pair1.getRight().exists());
- Assert.assertTrue(pair2.getRight().exists());
- CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir,
COMPACTION_TEST_SG);
- Assert.assertTrue(pair1.getRight().exists());
- Assert.assertFalse(pair2.getRight().exists());
- }
-
- @Test
- public void testCrossCompactionWithTTL() throws IOException,
WriteProcessException {
- Pair<TsFileResource, File> pair1 =
- generateTsFileAndObject(true, System.currentTimeMillis() + 100000, 1);
- Pair<TsFileResource, File> pair2 =
- generateTsFileAndObject(false, System.currentTimeMillis() - 1000000,
2);
- tsFileManager.add(pair1.getLeft(), true);
- tsFileManager.add(pair2.getLeft(), false);
- CrossSpaceCompactionTask task =
- new CrossSpaceCompactionTask(
- 0,
- tsFileManager,
- tsFileManager.getTsFileList(true),
- tsFileManager.getTsFileList(false),
- new FastCompactionPerformer(true),
- 1,
- 0);
- Assert.assertTrue(task.start());
-
- Assert.assertTrue(pair1.getRight().exists());
- Assert.assertTrue(pair2.getRight().exists());
- CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir,
COMPACTION_TEST_SG);
- Assert.assertFalse(pair2.getRight().exists());
- Assert.assertTrue(pair1.getRight().exists());
- }
-
- @Test
- public void testSettleCompaction() throws IOException, WriteProcessException
{
- Pair<TsFileResource, File> pair1 =
- generateTsFileAndObject(true, System.currentTimeMillis() - 100000, 3);
- Pair<TsFileResource, File> pair2 =
- generateTsFileAndObject(true, System.currentTimeMillis() + 1000000, 0);
- tsFileManager.add(pair1.getLeft(), true);
- tsFileManager.add(pair2.getLeft(), true);
- SettleCompactionTask task =
- new SettleCompactionTask(
- 0,
- tsFileManager,
- tsFileManager.getTsFileList(true),
- Collections.emptyList(),
- true,
- new FastCompactionPerformer(true),
- 0);
- Assert.assertTrue(task.start());
-
- Assert.assertTrue(pair1.getRight().exists());
- Assert.assertTrue(pair2.getRight().exists());
- CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir,
COMPACTION_TEST_SG);
- Assert.assertFalse(pair1.getRight().exists());
- Assert.assertTrue(pair2.getRight().exists());
- }
-
- @Test
- public void testBase32ObjectPathTTLCheck() throws IOException {
- config.setRestrictObjectLimit(false);
- try {
- File file1 =
- generateBase32PathObjectFile(regionDir, System.currentTimeMillis() +
100000, false);
- File file2 =
- generateBase32PathObjectFile(regionDir, System.currentTimeMillis() +
200000, true);
- File file3 =
- generateBase32PathObjectFile(regionDir, System.currentTimeMillis() -
100000, true);
- File file4 =
- generateBase32PathObjectFile(regionDir, System.currentTimeMillis() -
200000, false);
- Assert.assertTrue(file1.exists());
- Assert.assertTrue(file2.exists());
- Assert.assertTrue(file3.exists());
- Assert.assertTrue(file4.exists());
- CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir,
COMPACTION_TEST_SG);
- Assert.assertTrue(file1.exists());
- Assert.assertTrue(file2.exists());
- Assert.assertFalse(file3.exists());
- Assert.assertFalse(file4.exists());
- } finally {
- config.setRestrictObjectLimit(true);
- }
- }
-
- @Test
- public void testPlainObjectPathTTLCheck() throws IOException,
InterruptedException {
- File file1 =
- generatePlainPathObjectFile(regionDir, System.currentTimeMillis() +
100000, false, "d1");
- File file2 =
- generatePlainPathObjectFile(
- regionDir,
- System.currentTimeMillis() + 200000,
- true,
- (System.currentTimeMillis() - 100000) + ".bin");
- File file3 =
- generatePlainPathObjectFile(regionDir, System.currentTimeMillis() -
100000, true, "d1");
- File file4 =
- generatePlainPathObjectFile(regionDir, System.currentTimeMillis() -
200000, false, "d1");
- File file5 =
- generatePlainPathObjectFile(
- regionDir,
- System.currentTimeMillis() + 300000,
- false,
- (System.currentTimeMillis() - 300000) + ".bin");
- Assert.assertTrue(file1.exists());
- Assert.assertTrue(file2.exists());
- Assert.assertTrue(file3.exists());
- Assert.assertTrue(file4.exists());
- Assert.assertTrue(file5.exists());
- new DataRegion(COMPACTION_TEST_SG,
regionDir.getName()).executeTTLCheckForObjectFiles();
- Assert.assertTrue(file1.exists());
- Assert.assertTrue(file2.exists());
- Assert.assertFalse(file3.exists());
- Assert.assertFalse(file4.exists());
- Assert.assertTrue(file5.exists());
- }
-
- @Test
- public void testPlainObjectBinaryReplaceRegionId() {
- IObjectPath objectPath = new PlainObjectPath(1, 0, new
StringArrayDeviceID("t1.d1"), "s1");
- ByteBuffer buffer =
- ByteBuffer.allocate(Long.BYTES +
objectPath.getSerializeSizeToObjectValue());
- buffer.putLong(10);
- objectPath.serializeToObjectValue(buffer);
-
- Binary origin = new Binary(buffer.array());
- Binary result = ObjectTypeUtils.replaceRegionIdForObjectBinary(10, origin);
- ByteBuffer deserializeBuffer = ByteBuffer.wrap(result.getValues());
- deserializeBuffer.getLong();
- Assert.assertEquals(
- new PlainObjectPath(10, 0, new StringArrayDeviceID("t1.d1"),
"s1").toString(),
-
IObjectPath.getDeserializer().deserializeFromObjectValue(deserializeBuffer).toString());
- }
-
- @Test
- public void testBase32ObjectBinaryReplaceRegionId() {
- config.setRestrictObjectLimit(false);
- try {
- IObjectPath objectPath = new Base32ObjectPath(1, 0, new
StringArrayDeviceID("t1.d1"), "s1");
- ByteBuffer buffer =
- ByteBuffer.allocate(Long.BYTES +
objectPath.getSerializeSizeToObjectValue());
- buffer.putLong(10);
- objectPath.serializeToObjectValue(buffer);
-
- Binary origin = new Binary(buffer.array());
- Binary result = ObjectTypeUtils.replaceRegionIdForObjectBinary(10,
origin);
- ByteBuffer deserializeBuffer = ByteBuffer.wrap(result.getValues());
- deserializeBuffer.getLong();
- Assert.assertEquals(
- new Base32ObjectPath(10, 0, new StringArrayDeviceID("t1.d1"),
"s1").toString(),
-
IObjectPath.getDeserializer().deserializeFromObjectValue(deserializeBuffer).toString());
- } finally {
- config.setRestrictObjectLimit(true);
- }
- }
-
- private Pair<TsFileResource, File> generateTsFileAndObject(
- boolean seq, long timestamp, int regionIdInTsFile) throws IOException,
WriteProcessException {
- TsFileResource resource = createEmptyFileAndResource(seq);
- File dir =
- new File(
- regionDir.getPath()
- + File.separator
- + "t1"
- + File.separator
- + "d1"
- + File.separator
- + "s1");
- dir.mkdirs();
- File testFile1 = new File(dir, timestamp + ".bin");
- byte[] content = new byte[100];
- for (int i = 0; i < 100; i++) {
- content[i] = (byte) i;
- }
- Files.write(testFile1.toPath(), content);
- String relativePathInTsFile =
- regionIdInTsFile
- + File.separator
- + "t1"
- + File.separator
- + "d1"
- + File.separator
- + "s1"
- + File.separator
- + testFile1.getName();
- ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES +
relativePathInTsFile.length());
- buffer.putLong(100L);
- buffer.put(BytesUtils.stringToBytes(relativePathInTsFile));
- buffer.flip();
- IDeviceID deviceID = new StringArrayDeviceID("t1", "d1");
- try (TsFileIOWriter writer = new TsFileIOWriter(resource.getTsFile())) {
- writer.getSchema().registerTableSchema(tableSchema);
- writer.startChunkGroup(deviceID);
- AlignedChunkWriterImpl alignedChunkWriter =
- new AlignedChunkWriterImpl(
- Collections.singletonList(new MeasurementSchema("s1",
TSDataType.OBJECT)));
- alignedChunkWriter.write(timestamp);
- alignedChunkWriter.write(timestamp, new Binary(buffer.array()), false);
- alignedChunkWriter.sealCurrentPage();
- alignedChunkWriter.writeToFileWriter(writer);
- writer.endChunkGroup();
- writer.endFile();
- }
- resource.updateStartTime(deviceID, 1);
- resource.updateEndTime(deviceID, 1);
- resource.serialize();
- resource.deserialize();
- resource.setStatus(TsFileResourceStatus.NORMAL);
- return new Pair<>(resource, testFile1);
- }
-
- private File generatePlainPathObjectFile(
- File regionDir, long timestamp, boolean internalLevel, String tagValue)
throws IOException {
- File dir =
- new File(
- regionDir.getPath()
- + File.separator
- + "t1"
- + (internalLevel ? "" : (File.separator + tagValue))
- + File.separator
- + "s1");
- dir.mkdirs();
- File testFile1 = new File(dir, timestamp + ".bin");
- byte[] content = new byte[100];
- for (int i = 0; i < 100; i++) {
- content[i] = (byte) i;
- }
- Files.write(testFile1.toPath(), content);
- return testFile1;
- }
-
- private File generateBase32PathObjectFile(File regionDir, long timestamp,
boolean internalLevel)
- throws IOException {
- File dir =
- new File(
- regionDir.getPath()
- + File.separator
- + toBase32Str("t1")
- + (internalLevel ? "" : (File.separator + toBase32Str("d1")))
- + File.separator
- + toBase32Str("s1"));
- dir.mkdirs();
- File testFile1 = new File(dir, timestamp + ".bin");
- byte[] content = new byte[100];
- for (int i = 0; i < 100; i++) {
- content[i] = (byte) i;
- }
- Files.write(testFile1.toPath(), content);
- return testFile1;
- }
-
- private String toBase32Str(String str) {
- return
BaseEncoding.base32().omitPadding().encode(str.getBytes(StandardCharsets.UTF_8));
- }
-}
diff --git
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index 9d63529355a..c3b03040e8f 100644
---
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -1317,16 +1317,6 @@ enable_tsfile_validation=false
# Unit: ms
tier_ttl_in_ms=-1
-# The maximum size limit for a single object file.
-# effectiveMode: hot_reload
-# Datatype: long
-max_object_file_size_in_byte=4294967296
-
-# There are no special restrictions on table names, column names, and device
names of the OBJECT type.
-# effectiveMode: first_start
-# Datatype: boolean
-restrict_object_limit=false
-
####################
### Compaction Configurations
####################
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
index 2eaf05f3b84..526c20da093 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
@@ -281,11 +281,6 @@ public class CommonDescriptor {
properties.getProperty(
"path_log_max_size",
String.valueOf(config.getPathLogMaxSize()))));
- config.setRestrictObjectLimit(
- Boolean.parseBoolean(
- properties.getProperty(
- "restrict_object_limit",
String.valueOf(config.isRestrictObjectLimit()))));
-
loadRetryProperties(properties);
loadBinaryAllocatorProps(properties);
}
