This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch memoryEstimateRefine in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9972d9c7df15f79912eb7fdae6601c7bda1d2248 Author: JackieTien97 <[email protected]> AuthorDate: Sat Feb 28 14:57:38 2026 +0800 try fix --- AGENTS.md | 196 +++++++++++++++++++++ CLAUDE.md | 22 +++ .../relational/AbstractTableScanOperator.java | 20 ++- .../source/relational/TableScanOperatorTest.java | 91 ++++++++++ 4 files changed, 326 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000000..0cb9fde79b0 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,196 @@ +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> + +# Project Overview + +Apache IoTDB is a distributed time-series database written in Java. Version 2.0.x supports both tree-model (time-series paths) and table-model (relational) data access. It uses a ConfigNode/DataNode architecture with pluggable consensus protocols. + + +# IoTDB Java Code Style Guide + +This document defines the coding rules that **must** be followed when generating or modifying Java code in this project. These rules are enforced by CheckStyle (`checkstyle.xml`) and Spotless (`pom.xml`). + +## Imports + +1. **Star imports are forbidden**: Never use `import xxx.*` or `import static xxx.*`. Always list each required class or static member explicitly. +2. **Import ordering** (enforced by Spotless): + ``` + org.apache.iotdb.* + ← blank line + Other third-party packages (e.g. org.apache.arrow, com.google, etc.) + ← blank line + javax.* + java.* + ← blank line + static imports + ``` +3. **Remove unused imports**. + +## Formatting + +- Use **Google Java Format** (GOOGLE style). +- Indent with **2 spaces**, no tabs. +- Line width limit is **100 characters** (except `package`, `import`, and URL lines). +- Use **UNIX (LF)** line endings. + +## Naming Conventions + +| Type | Rule | Example | +|------|------|---------| +| Package | All lowercase, dot-separated | `org.apache.iotdb.flight` | +| Class | UpperCamelCase | `TsBlockToArrowConverter` | +| Method | lowerCamelCase (`^[a-z][a-z0-9][a-zA-Z0-9_]*$`) | `fillVectorSchemaRoot` | +| Member variable | lowerCamelCase (`^[a-z][a-z0-9][a-zA-Z0-9]*$`) | `allocator` | +| Parameter | lowerCamelCase | `tsBlock` | +| Constant | UPPER_SNAKE_CASE, at most 1 consecutive uppercase letter in abbreviations | `MAX_RETRY_COUNT` | + +## Code Structure + +- Only one top-level class per `.java` file. +- `switch` statements must have a `default` branch. +- Empty `try`/`catch`/`if`/`else`/`switch` blocks are not allowed (unless they contain a comment). +- Exception variables in empty `catch` blocks must be named `expected`. +- `if`/`else`/`for`/`while`/`do` must use braces `{}`. + +## License Header + +Every file must start with the Apache License 2.0 header, java file: + +```java +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +``` + +md or xml file: + +```xml +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +``` + +## Verification Commands + +```bash +# Format code +mvn spotless:apply -P with-integration-tests + +# Check style +mvn checkstyle:check -P with-integration-tests +``` + +# Integration Test Rules + +- When writing integration tests, always use `SessionConfig.DEFAULT_USER` and `SessionConfig.DEFAULT_PASSWORD` for the user and password instead of hardcoding values. + +# Build Commands + +```bash +# Full build (produces distribution in distribution/target/) +mvn clean package -pl distribution -am -DskipTests + +# Build server only +mvn clean package -DskipTests + +# Build CLI only +mvn clean package -pl iotdb-client/cli -am -DskipTests + +# Code formatting check / auto-fix (Google Java Format via Spotless) +mvn spotless:check -P with-integration-tests +mvn spotless:apply -P with-integration-tests + +# Format integration test code +mvn spotless:apply -P with-integration-tests +``` + +# Running Tests + +```bash +# Unit tests for a specific module +mvn test -pl iotdb-core/datanode + +# Single unit test class +mvn test -pl iotdb-core/datanode -Dtest=ClassName + +# Single unit test method +mvn test -pl iotdb-core/datanode -Dtest=ClassName#methodName + +# Integration tests — Simple mode (1 ConfigNode + 1 DataNode, tree model) +mvn clean verify -DskipUTs -pl integration-test -am -P with-integration-tests + +# Single IT test class — Simple mode (1 ConfigNode + 1 DataNode, tree model) +mvn clean verify -DskipUTs -Dit.test=ClassName -DfailIfNoTests=false -Dfailsafe.failIfNoSpecifiedTests=false -pl integration-test -P with-integration-tests -am -PSimpleIT + +# Integration tests — Cluster mode (1 ConfigNode + 3 DataNodes, tree model) +mvn clean verify -DskipUTs -pl integration-test -am -PClusterIT -P with-integration-tests + +# Single IT test class — Cluster mode (1 ConfigNode + 3 DataNodes, tree model) +mvn clean verify -DskipUTs -Dit.test=ClassName -DfailIfNoTests=false -Dfailsafe.failIfNoSpecifiedTests=false -pl integration-test -P with-integration-tests -am -PClusterIT + +# Integration tests — Table model simple +mvn clean verify -DskipUTs -pl integration-test -am -PTableSimpleIT -P with-integration-tests + +# Single IT test class — Table model simple +mvn clean verify -DskipUTs -Dit.test=ClassName -DfailIfNoTests=false -Dfailsafe.failIfNoSpecifiedTests=false -pl integration-test -P with-integration-tests -am -PTableSimpleIT + +# Integration tests — Table model cluster +mvn clean verify -DskipUTs -pl integration-test -am -PTableClusterIT -P with-integration-tests + +# Single IT test class — Table model cluster +mvn clean verify -DskipUTs -Dit.test=ClassName -DfailIfNoTests=false -Dfailsafe.failIfNoSpecifiedTests=false -pl integration-test -P with-integration-tests -am -PTableClusterIT + + +# Before running ITs in IDE for the first time (generates template-node): +mvn clean package -DskipTests -pl integration-test -am -P with-integration-tests +``` \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000000..fd152a916ee --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,22 @@ +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> + +See [AGENTS.md](./AGENTS.md) for all project guidelines, code style rules, build commands, and test instructions. \ No newline at end of file diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/AbstractTableScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/AbstractTableScanOperator.java index 151dd6f3f25..86dba541b9a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/AbstractTableScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/AbstractTableScanOperator.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.queryengine.execution.operator.source.relational; import org.apache.iotdb.commons.path.AlignedFullPath; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; import org.apache.iotdb.db.queryengine.execution.operator.source.AbstractSeriesScanOperator; @@ -90,6 +91,8 @@ public abstract class AbstractTableScanOperator extends AbstractSeriesScanOperat private int currentDeviceIndex; + protected final int maxPeekMemoryTimeAndFieldColumnCount; + public AbstractTableScanOperator(AbstractTableScanOperatorParameter parameter) { this.sourceId = parameter.sourceId; this.operatorContext = parameter.context; @@ -111,10 +114,19 @@ public abstract class AbstractTableScanOperator extends AbstractSeriesScanOperat this.currentDeviceIndex = 0; this.operatorContext.recordSpecifiedInfo(CURRENT_DEVICE_INDEX_STRING, Integer.toString(0)); + int timeAndFieldColumnCount = 0; + for (ColumnSchema columnSchema : parameter.columnSchemas) { + if (columnSchema.getColumnCategory() == TsTableColumnCategory.TIME + || columnSchema.getColumnCategory() == TsTableColumnCategory.FIELD) { + timeAndFieldColumnCount++; + } + } + this.maxPeekMemoryTimeAndFieldColumnCount = timeAndFieldColumnCount; + this.maxReturnSize = Math.min( maxReturnSize, - (1L + parameter.columnsIndexArray.length) + (1L + timeAndFieldColumnCount) * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte()); this.maxTsBlockLineNum = parameter.maxTsBlockLineNum; @@ -228,8 +240,10 @@ public abstract class AbstractTableScanOperator extends AbstractSeriesScanOperat @Override public long calculateMaxPeekMemory() { - return (1L + columnsIndexArray.length) - * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); + return Math.max( + maxReturnSize, + (1L + maxPeekMemoryTimeAndFieldColumnCount) + * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte()); } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableScanOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableScanOperatorTest.java new file mode 100644 index 00000000000..1aadb063f94 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableScanOperatorTest.java @@ -0,0 +1,91 @@ +package org.apache.iotdb.db.queryengine.execution.operator.source.relational; + +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.AbstractTableScanOperator.AbstractTableScanOperatorParameter; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; + +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.TypeFactory; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class TableScanOperatorTest { + + @Test + public void testCalculateMaxPeekMemory() { + OperatorContext operatorContext = Mockito.mock(OperatorContext.class); + + PlanNodeId sourceId = new PlanNodeId("test"); + + List<ColumnSchema> columnSchemas = + Arrays.asList( + new ColumnSchema( + "time", TypeFactory.getType(TSDataType.INT64), false, TsTableColumnCategory.TIME), + new ColumnSchema( + "tag1", TypeFactory.getType(TSDataType.STRING), false, TsTableColumnCategory.TAG), + new ColumnSchema( + "attr1", + TypeFactory.getType(TSDataType.STRING), + false, + TsTableColumnCategory.ATTRIBUTE), + new ColumnSchema( + "field1", + TypeFactory.getType(TSDataType.BOOLEAN), + false, + TsTableColumnCategory.FIELD), + new ColumnSchema( + "field2", + TypeFactory.getType(TSDataType.INT32), + false, + TsTableColumnCategory.FIELD)); + + int[] columnsIndexArray = new int[] {0, 1, 2, 3, 4}; + List<DeviceEntry> deviceEntries = Collections.emptyList(); + Ordering scanOrder = Ordering.ASC; + SeriesScanOptions seriesScanOptions = Mockito.mock(SeriesScanOptions.class); + List<String> measurementColumnNames = Arrays.asList("field1", "field2"); + List<IMeasurementSchema> measurementSchemas = new ArrayList<>(); + + AbstractTableScanOperatorParameter parameter = + new AbstractTableScanOperatorParameter( + new HashSet<>(), + operatorContext, + sourceId, + columnSchemas, + columnsIndexArray, + deviceEntries, + scanOrder, + seriesScanOptions, + measurementColumnNames, + measurementSchemas, + 1000); + + TableScanOperator operator = new TableScanOperator(parameter); + + long maxReturnSize = + Math.min( + TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(), + (1L + 2) * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte()); + long expectedMaxPeekMemory = + Math.max( + maxReturnSize, + (1L + 2) * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte()); + + assertEquals(expectedMaxPeekMemory, operator.calculateMaxPeekMemory()); + } +}
