This is an automated email from the ASF dual-hosted git repository.
spricoder pushed a change to branch feature/optimize_memory
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from ba88514f8ac add Metric memory_actual_size
add 51c92c3620c Compaction task memory is incorrect when
compaction_max_aligned_series_num_in_one_batch <= 0 (#14603)
add 4c3f64581ce Pipe IT: Save IT operation logs to the cluster-logs
directory (#14637)
add cfc15dec90a Update Python examples (#14640)
add 5df241c044b [Table Model] Refactor UDSF/UDAF interface
add 2669666d8d2 [To dev/1.3]Script adaptation to 4 new data types (#14296)
(#14630)
add f007e8a1407 Pipe: Support 'format'='tsfile' option in table model data
sync sinks (#14108)
add a9a0030998f Merge branch 'master' into feature/optimize_memory
add 7c4aeb9659a init first version of memory manager and memory block
add cc890215bc0 add IIoTDBMemoryBlock
add e33d234b2fe fix template
No new revisions were added by this update.
Summary of changes:
.github/workflows/pipe-it-2cluster.yml | 18 +-
example/udf/pom.xml | 6 +-
.../apache/iotdb/udf/AggregateFunctionExample.java | 35 +-
.../apache/iotdb/udf/ScalarFunctionExample.java | 57 ++-
.../db/query/udf/example/relational/AllSum.java | 49 +-
.../query/udf/example/relational/ContainNull.java | 21 +-
.../relational/{DatePlusOne.java => DatePlus.java} | 31 +-
.../query/udf/example/relational/FirstTwoSum.java | 33 +-
.../db/query/udf/example/relational/MyAvg.java | 32 +-
.../db/query/udf/example/relational/MyCount.java | 19 +-
.../pipe/it/tablemodel/IoTDBPipeDataSinkIT.java | 429 +++++++++++++++-
.../pipe/it/tablemodel/IoTDBPipeNullValueIT.java | 4 +-
.../iotdb/pipe/it/tablemodel/TableModelUtils.java | 546 +++++++++++++++++----
.../it/db/it/udf/IoTDBSQLFunctionManagementIT.java | 19 +-
.../it/udf/IoTDBUserDefinedScalarFunctionIT.java | 2 +-
.../analysis/AggregateFunctionAnalysis.java | 62 +++
.../api/customizer/analysis/FunctionAnalysis.java | 8 +-
.../ScalarFunctionAnalysis.java} | 26 +-
...ctionParameters.java => FunctionArguments.java} | 6 +-
...tion.java => UDFArgumentNotValidException.java} | 4 +-
.../udf/api/relational/AggregateFunction.java | 48 +-
.../iotdb/udf/api/relational/ScalarFunction.java | 47 +-
.../apache/iotdb/tool/data/AbstractDataTool.java | 32 +-
.../org/apache/iotdb/tool/data/ExportData.java | 43 +-
.../org/apache/iotdb/tool/data/ImportData.java | 24 +-
.../client-py/table_model_session_example.py | 12 +-
.../client-py/table_model_session_pool_example.py | 8 +-
.../batch/PipeTabletEventTsFileBatch.java | 385 +++------------
.../request/PipeTransferTabletRawReq.java | 4 +-
.../request/PipeTransferTabletRawReqV2.java | 9 +-
.../connector/protocol/opcua/OpcUaNameSpace.java | 8 +-
.../async/IoTDBDataRegionAsyncConnector.java | 10 +-
.../PipeTransferTabletBatchEventHandler.java | 2 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 10 +-
.../util/builder/PipeTableModeTsFileBuilder.java | 273 +++++++++++
.../util/builder/PipeTreeModelTsFileBuilder.java | 268 ++++++++++
.../connector/util/builder/PipeTsFileBuilder.java | 162 ++++++
.../util/{ => cacher}/LeaderCacheUtils.java | 2 +-
.../sorter/PipeTableModelTabletEventSorter.java | 271 ++++++++++
.../util/{ => sorter}/PipeTabletEventSorter.java | 107 +---
.../sorter/PipeTreeModelTabletEventSorter.java | 121 +++++
.../relational/aggregation/AccumulatorFactory.java | 18 +-
.../UserDefinedAggregateFunctionAccumulator.java | 27 +-
.../relational/ColumnTransformerBuilder.java | 14 +-
.../relational/metadata/TableMetadataImpl.java | 29 +-
...a => CompactionSourceFileDeletedException.java} | 7 +-
.../execute/task/InnerSpaceCompactionTask.java | 6 +-
.../estimator/AbstractCrossSpaceEstimator.java | 7 +-
.../estimator/AbstractInnerSpaceEstimator.java | 8 +-
.../estimator/CompactionEstimateUtils.java | 35 +-
.../FastCompactionInnerCompactionEstimator.java | 16 +-
.../FastCrossSpaceCompactionEstimator.java | 17 +-
...eQueryMemMeasurement.java => MetadataInfo.java} | 30 +-
.../ReadChunkInnerCompactionEstimator.java | 15 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 5 +-
.../batch/SubscriptionPipeTsFileEventBatch.java | 11 +-
.../pipe/connector/PipeTabletEventSorterTest.java | 210 +++++++-
.../cross/CrossSpaceCompactionSelectorTest.java | 102 +---
.../utils/CompactionTaskMemCostEstimatorTest.java | 48 ++
.../iotdb/commons/memory/IIoTDBMemoryBlock.java | 54 ++
.../iotdb/commons/memory/IoTDBMemoryBlock.java | 122 +++++
.../iotdb/commons/memory/IoTDBMemoryBlockType.java | 8 +-
.../iotdb/commons/memory/IoTDBMemoryManager.java | 193 ++++++++
.../memory/IoTDBRuntimeOutOfMemoryException.java | 86 ++++
64 files changed, 3343 insertions(+), 978 deletions(-)
rename
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/{DatePlusOne.java
=> DatePlus.java} (55%)
create mode 100644
iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/analysis/AggregateFunctionAnalysis.java
copy
integration-test/src/main/java/org/apache/iotdb/itbase/category/AIClusterIT.java
=>
iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/analysis/FunctionAnalysis.java
(79%)
copy
iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/{config/UDFConfigurations.java
=> analysis/ScalarFunctionAnalysis.java} (56%)
rename
iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/parameter/{FunctionParameters.java
=> FunctionArguments.java} (94%)
copy
iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/exception/{UDFParameterNotValidException.java
=> UDFArgumentNotValidException.java} (87%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTableModeTsFileBuilder.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTreeModelTsFileBuilder.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTsFileBuilder.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/{ =>
cacher}/LeaderCacheUtils.java (97%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/sorter/PipeTableModelTabletEventSorter.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/{ =>
sorter}/PipeTabletEventSorter.java (56%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/sorter/PipeTreeModelTabletEventSorter.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/{CompactionMemoryNotEnoughException.java
=> CompactionSourceFileDeletedException.java} (81%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/{IFileQueryMemMeasurement.java
=> MetadataInfo.java} (53%)
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java
copy
integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionRegressionConsumer.java
=>
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlockType.java
(88%)
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryManager.java
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBRuntimeOutOfMemoryException.java
