This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch rc/2.0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rc/2.0.12 by this push:
new bb7dbbd705e Pipe: Improved the total performance by disable some
useless logic
bb7dbbd705e is described below
commit bb7dbbd705e4ced7788d3acb5652cdbaa3f07f1b
Author: Caideyipi <[email protected]>
AuthorDate: Mon Nov 16 19:28:09 2026 +0800
Pipe: Improved the total performance by disable some useless logic
---
.github/workflows/daily-it.yml | 633 -----------------
.github/workflows/pipe-it.yml | 752 ---------------------
integration-test/pom.xml | 1 +
.../manual/basic/IoTDBPipeProtocolIT.java | 16 +-
.../treemodel/auto/basic/IoTDBPipeProtocolIT.java | 16 +-
.../relational/it/schema/IoTDBDatabaseIT.java | 6 +-
.../plugin/PipeConfigRegionSinkConstructor.java | 6 -
.../apache/iotdb/db/i18n/DataNodePipeMessages.java | 3 +
.../apache/iotdb/db/i18n/DataNodePipeMessages.java | 3 +
.../PipeDataRegionProcessorConstructor.java | 30 -
.../dataregion/PipeDataRegionSinkConstructor.java | 12 -
.../PipeSchemaRegionSinkConstructor.java | 6 -
.../task/subtask/sink/PipeSinkSubtaskManager.java | 13 +-
.../source/dataregion/IoTDBDataRegionSource.java | 55 +-
.../realtime/PipeRealtimeDataRegionSource.java | 12 +-
.../subtask/sink/PipeSinkSubtaskManagerTest.java | 12 +-
.../apache/iotdb/db/pipe/sink/PipeSinkTest.java | 93 ---
.../airgap/IoTDBDataRegionAirGapSinkTest.java | 5 +-
.../agent/plugin/builtin/BuiltinPipePlugin.java | 46 --
.../pipe/config/constant/PipeSinkConstant.java | 7 -
.../iotdb/commons/pipe/source/IoTDBSource.java | 12 +-
pom.xml | 1 +
22 files changed, 51 insertions(+), 1689 deletions(-)
diff --git a/.github/workflows/daily-it.yml b/.github/workflows/daily-it.yml
index 8d9b74862e4..5a71e6e5266 100644
--- a/.github/workflows/daily-it.yml
+++ b/.github/workflows/daily-it.yml
@@ -85,639 +85,6 @@ jobs:
name: table-standalone-log-java${{ matrix.java }}-${{ runner.os }}
path: integration-test/target/cluster-logs
retention-days: 3
- PipeSingle:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus
has not been supported yet.
- cluster1:
- [
- LightWeightStandaloneMode,
- ScalableSingleNodeMode,
- HighPerformanceMode,
- IoTConsensusV2BatchMode,
- IoTConsensusV2StreamMode,
- ]
- cluster2:
- [
- LightWeightStandaloneMode,
- ScalableSingleNodeMode,
- HighPerformanceMode,
- ]
- os: [ubuntu-latest]
- exclude:
- - cluster1: LightWeightStandaloneMode
- cluster2: LightWeightStandaloneMode
- - cluster1: LightWeightStandaloneMode
- cluster2: ScalableSingleNodeMode
- - cluster1: ScalableSingleNodeMode
- cluster2: LightWeightStandaloneMode
- - cluster1: ScalableSingleNodeMode
- cluster2: HighPerformanceMode
- - cluster1: HighPerformanceMode
- cluster2: LightWeightStandaloneMode
- - cluster1: HighPerformanceMode
- cluster2: HighPerformanceMode
- - cluster1: IoTConsensusV2BatchMode
- cluster2: LightWeightStandaloneMode
- - cluster1: IoTConsensusV2BatchMode
- cluster2: HighPerformanceMode
- - cluster1: IoTConsensusV2StreamMode
- cluster2: LightWeightStandaloneMode
- - cluster1: IoTConsensusV2StreamMode
- cluster2: HighPerformanceMode
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v5
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v5
- with:
- distribution: corretto
- java-version: ${{ matrix.java }}
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Cache Maven packages
- uses: actions/cache@v5
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in
client.yml
- # we can skip influxdb-protocol because it has been tested separately
in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256
-DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{
matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT1 \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log
===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log
======================"
-
- if ! mv ~/run-tests-$attempt.log
integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to
integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact";
then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt
$attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to
artifact transfer issue."
- echo "Treating this as a success because the issue is likely
transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v6
- with:
- name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{
matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- PipeDualTreeAutoBasic:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus
has not been supported yet.
- cluster:
- [
- LightWeightStandaloneMode,
- ScalableSingleNodeMode,
- HighPerformanceMode,
- IoTConsensusV2BatchMode,
- IoTConsensusV2StreamMode,
- ]
- os: [ubuntu-latest]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v5
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v5
- with:
- distribution: corretto
- java-version: ${{ matrix.java }}
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Cache Maven packages
- uses: actions/cache@v5
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in
client.yml
- # we can skip influxdb-protocol because it has been tested separately
in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256
-DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster
}} \
- -pl integration-test \
- -am -PMultiClusterIT2DualTreeAutoBasic \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log
===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log
======================"
-
- if ! mv ~/run-tests-$attempt.log
integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to
integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact";
then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt
$attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to
artifact transfer issue."
- echo "Treating this as a success because the issue is likely
transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v6
- with:
- name: cluster-log-dual-tree-auto-basic-java${{ matrix.java }}-${{
runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- PipeDualTreeAutoEnhanced:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus
has not been supported yet.
- cluster1:
- [
- LightWeightStandaloneMode,
- ScalableSingleNodeMode,
- HighPerformanceMode,
- IoTConsensusV2BatchMode,
- IoTConsensusV2StreamMode,
- ]
- cluster2:
- [
- LightWeightStandaloneMode,
- ScalableSingleNodeMode,
- HighPerformanceMode,
- ]
- os: [ubuntu-latest]
- exclude:
- - cluster1: LightWeightStandaloneMode
- cluster2: LightWeightStandaloneMode
- - cluster1: LightWeightStandaloneMode
- cluster2: ScalableSingleNodeMode
- - cluster1: ScalableSingleNodeMode
- cluster2: LightWeightStandaloneMode
- - cluster1: ScalableSingleNodeMode
- cluster2: HighPerformanceMode
- - cluster1: HighPerformanceMode
- cluster2: LightWeightStandaloneMode
- - cluster1: HighPerformanceMode
- cluster2: HighPerformanceMode
- - cluster1: IoTConsensusV2BatchMode
- cluster2: LightWeightStandaloneMode
- - cluster1: IoTConsensusV2BatchMode
- cluster2: HighPerformanceMode
- - cluster1: IoTConsensusV2StreamMode
- cluster2: LightWeightStandaloneMode
- - cluster1: IoTConsensusV2StreamMode
- cluster2: HighPerformanceMode
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v5
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v5
- with:
- distribution: corretto
- java-version: ${{ matrix.java }}
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Cache Maven packages
- uses: actions/cache@v5
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in
client.yml
- # we can skip influxdb-protocol because it has been tested separately
in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256
-DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{
matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT2DualTreeAutoEnhanced \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log
===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log
======================"
-
- if ! mv ~/run-tests-$attempt.log
integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to
integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact";
then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt
$attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to
artifact transfer issue."
- echo "Treating this as a success because the issue is likely
transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v6
- with:
- name: cluster-log-dual-tree-auto-enhanced-java${{ matrix.java }}-${{
runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- PipeDualTreeManual:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus
has not been supported yet.
- cluster1:
- [
- LightWeightStandaloneMode,
- ScalableSingleNodeMode,
- HighPerformanceMode,
- IoTConsensusV2BatchMode,
- IoTConsensusV2StreamMode,
- ]
- cluster2:
- [
- LightWeightStandaloneMode,
- ScalableSingleNodeMode,
- HighPerformanceMode,
- ]
- os: [ubuntu-latest]
- exclude:
- - cluster1: LightWeightStandaloneMode
- cluster2: LightWeightStandaloneMode
- - cluster1: LightWeightStandaloneMode
- cluster2: ScalableSingleNodeMode
- - cluster1: ScalableSingleNodeMode
- cluster2: LightWeightStandaloneMode
- - cluster1: ScalableSingleNodeMode
- cluster2: HighPerformanceMode
- - cluster1: HighPerformanceMode
- cluster2: LightWeightStandaloneMode
- - cluster1: HighPerformanceMode
- cluster2: HighPerformanceMode
- - cluster1: IoTConsensusV2BatchMode
- cluster2: LightWeightStandaloneMode
- - cluster1: IoTConsensusV2BatchMode
- cluster2: HighPerformanceMode
- - cluster1: IoTConsensusV2StreamMode
- cluster2: LightWeightStandaloneMode
- - cluster1: IoTConsensusV2StreamMode
- cluster2: HighPerformanceMode
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v5
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v5
- with:
- distribution: corretto
- java-version: ${{ matrix.java }}
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Cache Maven packages
- uses: actions/cache@v5
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in
client.yml
- # we can skip influxdb-protocol because it has been tested separately
in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256
-DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{
matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT2DualTreeManual \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log
===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log
======================"
-
- if ! mv ~/run-tests-$attempt.log
integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to
integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact";
then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt
$attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to
artifact transfer issue."
- echo "Treating this as a success because the issue is likely
transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v6
- with:
- name: cluster-log-dual-tree-manual-java${{ matrix.java }}-${{
runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- PipeDualTableManualBasic:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus
has not been supported yet.
- cluster:
- [
- LightWeightStandaloneMode,
- ScalableSingleNodeMode,
- HighPerformanceMode,
- IoTConsensusV2BatchMode,
- IoTConsensusV2StreamMode,
- ]
- os: [ubuntu-latest]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v5
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v5
- with:
- distribution: corretto
- java-version: ${{ matrix.java }}
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Cache Maven packages
- uses: actions/cache@v5
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in
client.yml
- # we can skip influxdb-protocol because it has been tested separately
in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256
-DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster
}} \
- -pl integration-test \
- -am -PMultiClusterIT2DualTableManualBasic \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log
===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log
======================"
-
- if ! mv ~/run-tests-$attempt.log
integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to
integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact";
then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt
$attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to
artifact transfer issue."
- echo "Treating this as a success because the issue is likely
transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v6
- with:
- name: cluster-log-dual-table-manual-basic-java${{ matrix.java }}-${{
runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- PipeDualTableManualEnhanced:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus
has not been supported yet.
- cluster:
- [
- LightWeightStandaloneMode,
- ScalableSingleNodeMode,
- HighPerformanceMode,
- IoTConsensusV2BatchMode,
- IoTConsensusV2StreamMode,
- ]
- os: [ubuntu-latest]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v5
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v5
- with:
- distribution: corretto
- java-version: ${{ matrix.java }}
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Cache Maven packages
- uses: actions/cache@v5
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in
client.yml
- # we can skip influxdb-protocol because it has been tested separately
in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256
-DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster
}} \
- -pl integration-test \
- -am -PMultiClusterIT2DualTableManualEnhanced \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log
===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log
======================"
-
- if ! mv ~/run-tests-$attempt.log
integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to
integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact";
then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt
$attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to
artifact transfer issue."
- echo "Treating this as a success because the issue is likely
transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v6
- with:
- name: cluster-log-dual-table-manual-enhanced-java${{ matrix.java
}}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
- path: integration-test/target/cluster-logs
- retention-days: 30
TableClusterITWithDeviceEntrySpill:
runs-on: [self-hosted, iotdb]
steps:
diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml
index 7b5d6df0b9e..e69de29bb2d 100644
--- a/.github/workflows/pipe-it.yml
+++ b/.github/workflows/pipe-it.yml
@@ -1,752 +0,0 @@
-name: Multi-Cluster IT
-
-on:
- push:
- branches:
- - master
- - "rel/*"
- - "rc/*"
- paths-ignore:
- - "docs/**"
- - "site/**"
- - "iotdb-client/client-cpp/**"
- - ".github/workflows/client-cpp-package.yml"
- - ".github/scripts/package-client-cpp-*.sh"
- - ".github/workflows/multi-language-client.yml"
- - "iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/**"
#queryengine
- pull_request:
- branches:
- - master
- - "rel/*"
- - "rc/*"
- - "force_ci/**"
- paths-ignore:
- - "docs/**"
- - "site/**"
- - "iotdb-client/client-cpp/**"
- - ".github/workflows/client-cpp-package.yml"
- - ".github/scripts/package-client-cpp-*.sh"
- - ".github/workflows/multi-language-client.yml"
- - "iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/**"
#queryengine
- # allow manually run the action:
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.http.retryHandler.class=standard
-Dmaven.wagon.http.retryHandler.count=3
- MAVEN_ARGS: --batch-mode --no-transfer-progress
- DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
-
-jobs:
- single:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus
has not been supported yet.
- cluster1: [HighPerformanceMode]
- cluster2: [HighPerformanceMode]
- os: [ubuntu-latest]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v5
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v5
- with:
- distribution: corretto
- java-version: ${{ matrix.java }}
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Cache Maven packages
- uses: actions/cache@v5
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in
client.yml
- # we can skip influxdb-protocol because it has been tested separately
in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.clusterReadyRetryCount=90 \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256
-DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{
matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT1 \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log
===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log
======================"
-
- if ! mv ~/run-tests-$attempt.log
integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to
integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact";
then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt
$attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to
artifact transfer issue."
- echo "Treating this as a success because the issue is likely
transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v6
- with:
- name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{
matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- # 12 IT classes split across 3 parallel shards to cut the historical ~42 min
- # wall clock to ~14 min. See cluster-it-1c1d.yml for the shard pattern.
- dual-tree-auto-basic:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus
has not been supported yet.
- cluster: [HighPerformanceMode]
- os: [ubuntu-latest]
- shard: [0, 1, 2]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v5
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v5
- with:
- distribution: corretto
- java-version: ${{ matrix.java }}
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Cache Maven packages
- uses: actions/cache@v5
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: Build IT shard list
- shell: bash
- # See cluster-it-1c1d.yml for the shard-list pattern. Write under
- # $RUNNER_TEMP (outside the repo) so Apache RAT doesn't flag the file.
- run: |
- set -euo pipefail
- SHARD=${{ matrix.shard }}
- TOTAL=3
- grep -rlE --include='*IT.java'
'\bMultiClusterIT2DualTreeAutoBasic\b' integration-test/src/test/java \
- | awk -F'/' '{print $NF}' | sed 's/\.java$//' \
- | sort \
- | awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \
- > "$RUNNER_TEMP/it-shard.txt"
- echo "Shard $SHARD/$TOTAL contains $(wc -l <
"$RUNNER_TEMP/it-shard.txt") test classes"
- head -5 "$RUNNER_TEMP/it-shard.txt"
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in
client.yml
- # we can skip influxdb-protocol because it has been tested separately
in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.clusterReadyRetryCount=90 \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256
-DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster
}} \
- -Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \
- -DfailIfNoTests=false \
- -Dfailsafe.failIfNoSpecifiedTests=false \
- -pl integration-test \
- -am -PMultiClusterIT2DualTreeAutoBasic \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log
===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log
======================"
-
- if ! mv ~/run-tests-$attempt.log
integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to
integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact";
then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt
$attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to
artifact transfer issue."
- echo "Treating this as a success because the issue is likely
transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v6
- with:
- name: cluster-log-dual-tree-auto-basic-shard${{ matrix.shard
}}-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{
matrix.cluster }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- # 9 IT classes split across 3 parallel shards to cut the historical ~51 min
- # wall clock to ~17 min. See cluster-it-1c1d.yml for the shard pattern.
- dual-tree-auto-enhanced:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus
has not been supported yet.
- cluster1: [HighPerformanceMode]
- cluster2: [HighPerformanceMode]
- os: [ubuntu-latest]
- shard: [0, 1, 2]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v5
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v5
- with:
- distribution: corretto
- java-version: ${{ matrix.java }}
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Cache Maven packages
- uses: actions/cache@v5
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: Build IT shard list
- shell: bash
- # See cluster-it-1c1d.yml for the shard-list pattern. Write under
- # $RUNNER_TEMP (outside the repo) so Apache RAT doesn't flag the file.
- run: |
- set -euo pipefail
- SHARD=${{ matrix.shard }}
- TOTAL=3
- grep -rlE --include='*IT.java'
'\bMultiClusterIT2DualTreeAutoEnhanced\b' integration-test/src/test/java \
- | awk -F'/' '{print $NF}' | sed 's/\.java$//' \
- | sort \
- | awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \
- > "$RUNNER_TEMP/it-shard.txt"
- echo "Shard $SHARD/$TOTAL contains $(wc -l <
"$RUNNER_TEMP/it-shard.txt") test classes"
- head -5 "$RUNNER_TEMP/it-shard.txt"
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in
client.yml
- # we can skip influxdb-protocol because it has been tested separately
in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.clusterReadyRetryCount=90 \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256
-DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{
matrix.cluster2 }} \
- -Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \
- -DfailIfNoTests=false \
- -Dfailsafe.failIfNoSpecifiedTests=false \
- -pl integration-test \
- -am -PMultiClusterIT2DualTreeAutoEnhanced \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log
===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log
======================"
-
- if ! mv ~/run-tests-$attempt.log
integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to
integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact";
then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt
$attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to
artifact transfer issue."
- echo "Treating this as a success because the issue is likely
transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v6
- with:
- name: cluster-log-dual-tree-auto-enhanced-shard${{ matrix.shard
}}-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{
matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- # 11 IT classes split across 3 parallel shards to cut the historical ~27 min
- # wall clock to ~9 min. See cluster-it-1c1d.yml for the shard pattern.
- dual-tree-manual:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus
has not been supported yet.
- cluster1: [HighPerformanceMode]
- cluster2: [HighPerformanceMode]
- os: [ubuntu-latest]
- shard: [0, 1, 2]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v5
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v5
- with:
- distribution: corretto
- java-version: ${{ matrix.java }}
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Cache Maven packages
- uses: actions/cache@v5
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: Build IT shard list
- shell: bash
- # See cluster-it-1c1d.yml for the shard-list pattern. Write under
- # $RUNNER_TEMP (outside the repo) so Apache RAT doesn't flag the file.
- run: |
- set -euo pipefail
- SHARD=${{ matrix.shard }}
- TOTAL=3
- grep -rlE --include='*IT.java' '\bMultiClusterIT2DualTreeManual\b'
integration-test/src/test/java \
- | awk -F'/' '{print $NF}' | sed 's/\.java$//' \
- | sort \
- | awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \
- > "$RUNNER_TEMP/it-shard.txt"
- echo "Shard $SHARD/$TOTAL contains $(wc -l <
"$RUNNER_TEMP/it-shard.txt") test classes"
- head -5 "$RUNNER_TEMP/it-shard.txt"
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in
client.yml
- # we can skip influxdb-protocol because it has been tested separately
in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.clusterReadyRetryCount=90 \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256
-DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{
matrix.cluster2 }} \
- -Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \
- -DfailIfNoTests=false \
- -Dfailsafe.failIfNoSpecifiedTests=false \
- -pl integration-test \
- -am -PMultiClusterIT2DualTreeManual \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log
===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log
======================"
-
- if ! mv ~/run-tests-$attempt.log
integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to
integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact";
then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt
$attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to
artifact transfer issue."
- echo "Treating this as a success because the issue is likely
transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v6
- with:
- name: cluster-log-dual-tree-manual-shard${{ matrix.shard }}-java${{
matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- dual-table-manual-basic:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus
has not been supported yet.
- cluster: [HighPerformanceMode]
- os: [ubuntu-latest]
- shard: [0, 1, 2]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v5
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v5
- with:
- distribution: corretto
- java-version: ${{ matrix.java }}
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Cache Maven packages
- uses: actions/cache@v5
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: Build IT shard list
- shell: bash
- # See cluster-it-1c1d.yml for the shard-list pattern. Write under
- # $RUNNER_TEMP (outside the repo) so Apache RAT doesn't flag the file.
- run: |
- set -euo pipefail
- SHARD=${{ matrix.shard }}
- TOTAL=3
- grep -rlE --include='*IT.java'
'\bMultiClusterIT2DualTableManualBasic\b' integration-test/src/test/java \
- | awk -F'/' '{print $NF}' | sed 's/\.java$//' \
- | sort \
- | awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \
- > "$RUNNER_TEMP/it-shard.txt"
- echo "Shard $SHARD/$TOTAL contains $(wc -l <
"$RUNNER_TEMP/it-shard.txt") test classes"
- head -5 "$RUNNER_TEMP/it-shard.txt"
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in
client.yml
- # we can skip influxdb-protocol because it has been tested separately
in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.clusterReadyRetryCount=90 \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256
-DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster
}} \
- -Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \
- -DfailIfNoTests=false \
- -Dfailsafe.failIfNoSpecifiedTests=false \
- -pl integration-test \
- -am -PMultiClusterIT2DualTableManualBasic \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log
===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log
======================"
-
- if ! mv ~/run-tests-$attempt.log
integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to
integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact";
then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt
$attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to
artifact transfer issue."
- echo "Treating this as a success because the issue is likely
transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v6
- with:
- name: cluster-log-dual-table-manual-basic-shard${{ matrix.shard
}}-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{
matrix.cluster }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- # 11 IT classes split across 3 parallel shards to cut the historical ~62 min
- # wall clock to ~22 min. See cluster-it-1c1d.yml for the shard pattern.
- dual-table-manual-enhanced:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus
has not been supported yet.
- cluster: [HighPerformanceMode]
- os: [ubuntu-latest]
- shard: [0, 1, 2]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v5
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v5
- with:
- distribution: corretto
- java-version: ${{ matrix.java }}
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Cache Maven packages
- uses: actions/cache@v5
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: Build IT shard list
- shell: bash
- # See cluster-it-1c1d.yml for the shard-list pattern. Write under
- # $RUNNER_TEMP (outside the repo) so Apache RAT doesn't flag the file.
- run: |
- set -euo pipefail
- SHARD=${{ matrix.shard }}
- TOTAL=3
- grep -rlE --include='*IT.java'
'\bMultiClusterIT2DualTableManualEnhanced\b' integration-test/src/test/java \
- | awk -F'/' '{print $NF}' | sed 's/\.java$//' \
- | sort \
- | awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \
- > "$RUNNER_TEMP/it-shard.txt"
- echo "Shard $SHARD/$TOTAL contains $(wc -l <
"$RUNNER_TEMP/it-shard.txt") test classes"
- head -5 "$RUNNER_TEMP/it-shard.txt"
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in
client.yml
- # we can skip influxdb-protocol because it has been tested separately
in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.clusterReadyRetryCount=90 \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256
-DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster
}} \
- -Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \
- -DfailIfNoTests=false \
- -Dfailsafe.failIfNoSpecifiedTests=false \
- -pl integration-test \
- -am -PMultiClusterIT2DualTableManualEnhanced \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log
===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log
======================"
-
- if ! mv ~/run-tests-$attempt.log
integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to
integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact";
then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt
$attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to
artifact transfer issue."
- echo "Treating this as a success because the issue is likely
transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v6
- with:
- name: cluster-log-dual-table-manual-enhanced-shard${{ matrix.shard
}}-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{
matrix.cluster }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- triple:
- strategy:
- fail-fast: false
- max-parallel: 1
- matrix:
- java: [ 17 ]
- cluster1: [ ScalableSingleNodeMode ]
- cluster2: [ ScalableSingleNodeMode ]
- cluster3: [ ScalableSingleNodeMode ]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v5
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v5
- with:
- distribution: corretto
- java-version: ${{ matrix.java }}
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Cache Maven packages
- uses: actions/cache@v5
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in
client.yml
- # we can skip influxdb-protocol because it has been tested separately
in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.clusterReadyRetryCount=90 \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256
-DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{
matrix.cluster2 }},${{ matrix.cluster3 }} \
- -pl integration-test \
- -am -PMultiClusterIT3 \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log
===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log
======================"
-
- if ! mv ~/run-tests-$attempt.log
integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to
integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact";
then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt
$attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to
artifact transfer issue."
- echo "Treating this as a success because the issue is likely
transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v6
- with:
- name: cluster-log-triple-java${{ matrix.java }}-${{ runner.os }}-${{
matrix.cluster1 }}-${{ matrix.cluster2 }}-${{ matrix.cluster3 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
diff --git a/integration-test/pom.xml b/integration-test/pom.xml
index f6624fbd65d..04bb09395c1 100644
--- a/integration-test/pom.xml
+++ b/integration-test/pom.xml
@@ -327,6 +327,7 @@
<excludedGroups>${integrationTest.excludedGroups}</excludedGroups>
<!-- Subscription IT is maintained and executed by
the enterprise edition. -->
<excludes>
+ <exclude>**/pipe/it/**</exclude>
<exclude>**/subscription/it/**</exclude>
</excludes>
<useSystemClassLoader>false</useSystemClassLoader>
diff --git
a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java
b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java
index 9a6fe6ddf41..ea9a61f250f 100644
---
a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java
@@ -358,11 +358,6 @@ public class IoTDBPipeProtocolIT extends
AbstractPipeTableModelDualManualIT {
doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName());
}
- @Test
- public void testAirGapConnectorUseNodeUrls() throws Exception {
-
doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName());
- }
-
private void doTestUseNodeUrls(String connectorName) throws Exception {
senderEnv
.getConfig()
@@ -407,16 +402,7 @@ public class IoTDBPipeProtocolIT extends
AbstractPipeTableModelDualManualIT {
};
for (final DataNodeWrapper wrapper : receiverEnv.getDataNodeWrapperList())
{
- if
(connectorName.equals(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName()))
{
- // Use default port for convenience
- nodeUrlsBuilder
- .append(wrapper.getIp())
- .append(":")
- .append(wrapper.getPipeAirGapReceiverPort())
- .append(",");
- } else {
- nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(",");
- }
+ nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(",");
}
try (final SyncConfigNodeIServiceClient client =
diff --git
a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java
b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java
index 5489c0691b3..ed04af9f95a 100644
---
a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java
@@ -353,11 +353,6 @@ public class IoTDBPipeProtocolIT extends
AbstractPipeDualTreeModelAutoIT {
doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName());
}
- @Test
- public void testAirGapConnectorUseNodeUrls() throws Exception {
-
doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName());
- }
-
private void doTestUseNodeUrls(String sinkName) throws Exception {
senderEnv
.getConfig()
@@ -398,16 +393,7 @@ public class IoTDBPipeProtocolIT extends
AbstractPipeDualTreeModelAutoIT {
final StringBuilder nodeUrlsBuilder = new StringBuilder();
for (final DataNodeWrapper wrapper : receiverEnv.getDataNodeWrapperList())
{
- if
(sinkName.equals(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName()))
{
- // Use default port for convenience
- nodeUrlsBuilder
- .append(wrapper.getIp())
- .append(":")
- .append(wrapper.getPipeAirGapReceiverPort())
- .append(",");
- } else {
- nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(",");
- }
+ nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(",");
}
try (final SyncConfigNodeIServiceClient client =
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java
index 3d24915c1fe..19f0fc69219 100644
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java
@@ -860,8 +860,7 @@ public class IoTDBDatabaseIT {
"test,view_table,s11,INT32,FIELD,USING,null,",
"test,view_table,s3,INT32,FIELD,USING,null,")));
- statement.execute(
- "create pipe a2b with source('double-living'='true') with sink
('sink'='write-back-sink')");
+ statement.execute("create pipe a2b with sink
('sink'='do-nothing-sink')");
TestUtils.assertResultSetEqual(
statement.executeQuery("select id from pipes where creation_time >
0"),
"id,",
@@ -1052,8 +1051,7 @@ public class IoTDBDatabaseIT {
final Statement adminStmt = adminCon.createStatement()) {
adminStmt.execute("create user test 'password123456'");
adminStmt.execute("create database db");
- adminStmt.execute(
- "create pipe a2b with source('double-living'='true') with sink
('sink'='write-back-sink')");
+ adminStmt.execute("create pipe a2b with sink
('sink'='do-nothing-sink')");
}
try (final Connection userCon =
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructor.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructor.java
index c7222e9a1bc..82c394c7fb5 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructor.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructor.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.confignode.manager.pipe.agent.plugin;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin;
import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.donothing.DoNothingSink;
import
org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeSinkConstructor;
-import
org.apache.iotdb.confignode.manager.pipe.sink.protocol.IoTDBConfigRegionAirGapSink;
import
org.apache.iotdb.confignode.manager.pipe.sink.protocol.IoTDBConfigRegionSink;
import org.apache.iotdb.pipe.api.PipeConnector;
@@ -41,9 +40,6 @@ class PipeConfigRegionSinkConstructor extends
PipeSinkConstructor {
pluginConstructors.put(
BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName(),
IoTDBConfigRegionSink::new);
- pluginConstructors.put(
- BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName(),
- IoTDBConfigRegionAirGapSink::new);
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(),
DoNothingSink::new);
@@ -55,8 +51,6 @@ class PipeConfigRegionSinkConstructor extends
PipeSinkConstructor {
BuiltinPipePlugin.IOTDB_THRIFT_SYNC_SINK.getPipePluginName(),
IoTDBConfigRegionSink::new);
pluginConstructors.put(
BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_SINK.getPipePluginName(),
IoTDBConfigRegionSink::new);
- pluginConstructors.put(
- BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(),
IoTDBConfigRegionAirGapSink::new);
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(),
DoNothingSink::new);
}
diff --git
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index 56f39342242..a18f0338535 100644
---
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -20,6 +20,9 @@
package org.apache.iotdb.db.i18n;
public final class DataNodePipeMessages {
+ public static final String
EXCEPTION_THE_PARAMETER_ARG_CANNOT_BE_SET_TO_FALSE_111C3408 =
+ "The parameter %s cannot be set to false.";
+
public static final String
LOG_SUBSCRIPTION_REPLAY_STALLED_FOR_ARG_MS_ARG_DIAGNOSTICS_ARG_INSPECT_PREFETCH_WORKER_STACKS_AND_WAL_I_O_CB012F98
=
"Subscription replay stalled for {} ms: {}; diagnostics={}. Inspect
prefetch worker stacks and WAL I/O.";
diff --git
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index f2759b87258..32a00474b7e 100644
---
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -20,6 +20,9 @@
package org.apache.iotdb.db.i18n;
public final class DataNodePipeMessages {
+ public static final String
EXCEPTION_THE_PARAMETER_ARG_CANNOT_BE_SET_TO_FALSE_111C3408 =
+ "参数 %s 不能设置为 false。";
+
public static final String
LOG_SUBSCRIPTION_REPLAY_STALLED_FOR_ARG_MS_ARG_DIAGNOSTICS_ARG_INSPECT_PREFETCH_WORKER_STACKS_AND_WAL_I_O_CB012F98
=
"订阅回放已停滞 {} ms:{};diagnostics={}。请检查预取工作线程栈和 WAL I/O。";
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionProcessorConstructor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionProcessorConstructor.java
index a6160e6b900..b83a75460f8 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionProcessorConstructor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionProcessorConstructor.java
@@ -21,18 +21,10 @@ package org.apache.iotdb.db.pipe.agent.plugin.dataregion;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin;
import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.donothing.DoNothingProcessor;
-import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.throwing.ThrowingExceptionProcessor;
import
org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeProcessorConstructor;
import
org.apache.iotdb.commons.pipe.agent.plugin.meta.DataNodePipePluginMetaKeeper;
-import org.apache.iotdb.db.pipe.processor.aggregate.AggregateProcessor;
-import
org.apache.iotdb.db.pipe.processor.aggregate.operator.processor.StandardStatisticsOperatorProcessor;
-import
org.apache.iotdb.db.pipe.processor.aggregate.window.processor.TumblingWindowingProcessor;
-import
org.apache.iotdb.db.pipe.processor.downsampling.changing.ChangingValueSamplingProcessor;
-import
org.apache.iotdb.db.pipe.processor.downsampling.sdt.SwingingDoorTrendingSamplingProcessor;
-import
org.apache.iotdb.db.pipe.processor.downsampling.tumbling.TumblingTimeSamplingProcessor;
import
org.apache.iotdb.db.pipe.processor.iotconsensusv2.IoTConsensusV2Processor;
import org.apache.iotdb.db.pipe.processor.schemachange.RenameDatabaseProcessor;
-import
org.apache.iotdb.db.pipe.processor.twostage.plugin.TwoStageCountProcessor;
class PipeDataRegionProcessorConstructor extends PipeProcessorConstructor {
@@ -44,28 +36,6 @@ class PipeDataRegionProcessorConstructor extends
PipeProcessorConstructor {
protected void initConstructors() {
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_PROCESSOR.getPipePluginName(),
DoNothingProcessor::new);
- pluginConstructors.put(
- BuiltinPipePlugin.TUMBLING_TIME_SAMPLING_PROCESSOR.getPipePluginName(),
- TumblingTimeSamplingProcessor::new);
- pluginConstructors.put(
- BuiltinPipePlugin.SDT_SAMPLING_PROCESSOR.getPipePluginName(),
- SwingingDoorTrendingSamplingProcessor::new);
- pluginConstructors.put(
-
BuiltinPipePlugin.CHANGING_VALUE_SAMPLING_PROCESSOR.getPipePluginName(),
- ChangingValueSamplingProcessor::new);
- pluginConstructors.put(
- BuiltinPipePlugin.THROWING_EXCEPTION_PROCESSOR.getPipePluginName(),
- ThrowingExceptionProcessor::new);
- pluginConstructors.put(
- BuiltinPipePlugin.AGGREGATE_PROCESSOR.getPipePluginName(),
AggregateProcessor::new);
- pluginConstructors.put(
- BuiltinPipePlugin.STANDARD_STATISTICS_PROCESSOR.getPipePluginName(),
- StandardStatisticsOperatorProcessor::new);
- pluginConstructors.put(
- BuiltinPipePlugin.TUMBLING_WINDOWING_PROCESSOR.getPipePluginName(),
- TumblingWindowingProcessor::new);
- pluginConstructors.put(
- BuiltinPipePlugin.COUNT_POINT_PROCESSOR.getPipePluginName(),
TwoStageCountProcessor::new);
pluginConstructors.put(
BuiltinPipePlugin.IOT_CONSENSUS_V2_PROCESSOR.getPipePluginName(),
IoTConsensusV2Processor::new);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java
index c8e87890afd..1838f495884 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java
@@ -23,11 +23,8 @@ import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin;
import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.donothing.DoNothingSink;
import
org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeSinkConstructor;
import
org.apache.iotdb.commons.pipe.agent.plugin.meta.DataNodePipePluginMetaKeeper;
-import org.apache.iotdb.db.pipe.sink.protocol.airgap.IoTDBDataRegionAirGapSink;
import
org.apache.iotdb.db.pipe.sink.protocol.iotconsensusv2.IoTConsensusV2AsyncSink;
import org.apache.iotdb.db.pipe.sink.protocol.legacy.IoTDBLegacyPipeSink;
-import org.apache.iotdb.db.pipe.sink.protocol.opcda.OpcDaSink;
-import org.apache.iotdb.db.pipe.sink.protocol.opcua.OpcUaSink;
import
org.apache.iotdb.db.pipe.sink.protocol.thrift.async.IoTDBDataRegionAsyncSink;
import
org.apache.iotdb.db.pipe.sink.protocol.thrift.sync.IoTDBDataRegionSyncSink;
import org.apache.iotdb.db.pipe.sink.protocol.websocket.WebSocketSink;
@@ -63,13 +60,8 @@ class PipeDataRegionSinkConstructor extends
PipeSinkConstructor {
pluginConstructors.put(
BuiltinPipePlugin.IOTDB_LEGACY_PIPE_CONNECTOR.getPipePluginName(),
IoTDBLegacyPipeSink::new);
- pluginConstructors.put(
- BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName(),
- IoTDBDataRegionAirGapSink::new);
pluginConstructors.put(
BuiltinPipePlugin.WEBSOCKET_CONNECTOR.getPipePluginName(),
WebSocketSink::new);
-
pluginConstructors.put(BuiltinPipePlugin.OPC_UA_CONNECTOR.getPipePluginName(),
OpcUaSink::new);
-
pluginConstructors.put(BuiltinPipePlugin.OPC_DA_CONNECTOR.getPipePluginName(),
OpcDaSink::new);
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(),
DoNothingSink::new);
pluginConstructors.put(
@@ -86,12 +78,8 @@ class PipeDataRegionSinkConstructor extends
PipeSinkConstructor {
IoTDBDataRegionAsyncSink::new);
pluginConstructors.put(
BuiltinPipePlugin.IOTDB_LEGACY_PIPE_SINK.getPipePluginName(),
IoTDBLegacyPipeSink::new);
- pluginConstructors.put(
- BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(),
IoTDBDataRegionAirGapSink::new);
pluginConstructors.put(
BuiltinPipePlugin.WEBSOCKET_SINK.getPipePluginName(),
WebSocketSink::new);
- pluginConstructors.put(BuiltinPipePlugin.OPC_UA_SINK.getPipePluginName(),
OpcUaSink::new);
- pluginConstructors.put(BuiltinPipePlugin.OPC_DA_SINK.getPipePluginName(),
OpcDaSink::new);
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(),
DoNothingSink::new);
pluginConstructors.put(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java
index 160ecf54c01..a7752994ee2 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.db.pipe.agent.plugin.schemaregion;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin;
import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.donothing.DoNothingSink;
import
org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeSinkConstructor;
-import
org.apache.iotdb.db.pipe.sink.protocol.airgap.IoTDBSchemaRegionAirGapSink;
import
org.apache.iotdb.db.pipe.sink.protocol.thrift.sync.IoTDBSchemaRegionSink;
import org.apache.iotdb.pipe.api.PipeConnector;
@@ -41,9 +40,6 @@ class PipeSchemaRegionSinkConstructor extends
PipeSinkConstructor {
pluginConstructors.put(
BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName(),
IoTDBSchemaRegionSink::new);
- pluginConstructors.put(
- BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName(),
- IoTDBSchemaRegionAirGapSink::new);
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(),
DoNothingSink::new);
@@ -55,8 +51,6 @@ class PipeSchemaRegionSinkConstructor extends
PipeSinkConstructor {
BuiltinPipePlugin.IOTDB_THRIFT_SYNC_SINK.getPipePluginName(),
IoTDBSchemaRegionSink::new);
pluginConstructors.put(
BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_SINK.getPipePluginName(),
IoTDBSchemaRegionSink::new);
- pluginConstructors.put(
- BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(),
IoTDBSchemaRegionAirGapSink::new);
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(),
DoNothingSink::new);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java
index 033a90dc38d..093ac211c19 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java
@@ -81,7 +81,7 @@ public class PipeSinkSubtaskManager {
connectorKey);
final boolean isDataRegionSink =
isDataRegionSink(environment.getRegionId());
- final int sinkNum = calculateSinkSubtaskNum(pipeSinkParameters,
isDataRegionSink, connectorKey);
+ final int sinkNum = calculateSinkSubtaskNum(pipeSinkParameters,
isDataRegionSink);
boolean realTimeFirst = false;
final String attributeSortedString =
generateAttributeSortedString(pipeSinkParameters,
environment.getRegionId());
@@ -351,8 +351,7 @@ public class PipeSinkSubtaskManager {
public static int calculateSinkSubtaskNum(
final PipeParameters pipeSinkParameters, final int regionId) {
- final String connectorKey = getConnectorKey(pipeSinkParameters);
- return calculateSinkSubtaskNum(pipeSinkParameters,
isDataRegionSink(regionId), connectorKey);
+ return calculateSinkSubtaskNum(pipeSinkParameters,
isDataRegionSink(regionId));
}
public static String generateAttributeSortedString(
@@ -376,9 +375,7 @@ public class PipeSinkSubtaskManager {
}
private static int calculateSinkSubtaskNum(
- final PipeParameters pipeSinkParameters,
- final boolean isDataRegionSink,
- final String connectorKey) {
+ final PipeParameters pipeSinkParameters, final boolean isDataRegionSink)
{
if (!isDataRegionSink) {
// Do not allow parallel tasks for schema region connectors to avoid the
potential disorder of
// the schema region data transfer.
@@ -391,9 +388,7 @@ public class PipeSinkSubtaskManager {
Arrays.asList(
PipeSinkConstant.CONNECTOR_IOTDB_PARALLEL_TASKS_KEY,
PipeSinkConstant.SINK_IOTDB_PARALLEL_TASKS_KEY),
- PipeSinkConstant.SINGLE_THREAD_DEFAULT_SINK.contains(connectorKey)
- ? 1
- : PipeSinkConstant.CONNECTOR_IOTDB_PARALLEL_TASKS_DEFAULT_VALUE);
+ PipeSinkConstant.CONNECTOR_IOTDB_PARALLEL_TASKS_DEFAULT_VALUE);
}
private static String generateAttributeSortedString(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java
index 5d677e10047..e215f886069 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.consensus.DataRegionId;
import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta;
+import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant;
import
org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePatternOperations;
import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern;
import org.apache.iotdb.commons.pipe.source.IoTDBSource;
@@ -51,6 +52,7 @@ import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
import org.apache.iotdb.pipe.api.exception.PipeException;
+import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException;
import org.apache.iotdb.pipe.api.exception.PipePasswordCheckException;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -146,6 +148,19 @@ public class IoTDBDataRegionSource extends IoTDBSource {
public void validate(final PipeParameterValidator validator) throws
Exception {
super.validate(validator);
+ final boolean forwardingPipeRequests =
+ validator
+ .getParameters()
+ .getBooleanOrDefault(
+ PipeSourceConstant.FORWARDING_PIPE_REQUESTS_KEYS,
+
PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE);
+ if (!forwardingPipeRequests) {
+ throw new PipeParameterNotValidException(
+ String.format(
+
DataNodePipeMessages.EXCEPTION_THE_PARAMETER_ARG_CANNOT_BE_SET_TO_FALSE_111C3408,
+ PipeSourceConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY));
+ }
+
final Pair<Boolean, Boolean> insertionDeletionListeningOptionPair =
DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(
validator.getParameters());
@@ -219,32 +234,6 @@ public class IoTDBDataRegionSource extends IoTDBSource {
Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY,
SOURCE_REALTIME_ENABLE_KEY),
EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE));
- // Validate source.realtime.mode
- if (validator
- .getParameters()
- .getBooleanOrDefault(
- Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY,
SOURCE_REALTIME_ENABLE_KEY),
- EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE)
- || validator
- .getParameters()
- .hasAnyAttributes(
- SOURCE_START_TIME_KEY,
- EXTRACTOR_START_TIME_KEY,
- SOURCE_END_TIME_KEY,
- EXTRACTOR_END_TIME_KEY)) {
- validator.validateAttributeValueRange(
- validator.getParameters().hasAttribute(EXTRACTOR_REALTIME_MODE_KEY)
- ? EXTRACTOR_REALTIME_MODE_KEY
- : SOURCE_REALTIME_MODE_KEY,
- true,
- EXTRACTOR_REALTIME_MODE_FILE_VALUE,
- EXTRACTOR_REALTIME_MODE_HYBRID_VALUE,
- EXTRACTOR_REALTIME_MODE_LOG_VALUE,
- EXTRACTOR_REALTIME_MODE_FORCED_LOG_VALUE,
- EXTRACTOR_REALTIME_MODE_STREAM_MODE_VALUE,
- EXTRACTOR_REALTIME_MODE_BATCH_MODE_VALUE);
- }
-
checkInvalidParameters(validator);
constructHistoricalSource();
@@ -405,6 +394,13 @@ public class IoTDBDataRegionSource extends IoTDBSource {
return;
}
+ if (!(pipeName != null
+ && (pipeName.startsWith(PipeStaticMeta.SUBSCRIPTION_PIPE_PREFIX)
+ || pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)))) {
+ realtimeSource = new PipeRealtimeDataRegionTsFileSource();
+ return;
+ }
+
// Use hybrid mode by default
if (!parameters.hasAnyAttributes(EXTRACTOR_MODE_STREAMING_KEY,
SOURCE_MODE_STREAMING_KEY)
&& !parameters.hasAnyAttributes(EXTRACTOR_REALTIME_MODE_KEY,
SOURCE_REALTIME_MODE_KEY)) {
@@ -464,6 +460,13 @@ public class IoTDBDataRegionSource extends IoTDBSource {
super.customize(parameters, configuration);
+ // The pipe name is available only after the runtime environment is
initialized.
+ if (pipeName.startsWith(PipeStaticMeta.SUBSCRIPTION_PIPE_PREFIX)
+ || pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) {
+ constructRealtimeSource(parameters);
+ realtimeSource.validate(new PipeParameterValidator(parameters));
+ }
+
historicalSource.customize(parameters, configuration);
realtimeSource.customize(parameters, configuration);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java
index 92ad93e09f1..f2d3e4ecb25 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java
@@ -122,7 +122,7 @@ public abstract class PipeRealtimeDataRegionSource
implements PipeExtractor {
private final AtomicReference<Pair<Long, Long>>
dataRegionTimePartitionIdBound =
new AtomicReference<>();
- protected boolean isForwardingPipeRequests;
+ protected boolean isForwardingPipeRequests = true;
private boolean shouldTransferModFile; // Whether to transfer mods
private String tsFileParser;
@@ -267,15 +267,7 @@ public abstract class PipeRealtimeDataRegionSource
implements PipeExtractor {
?
TimePartitionUtils.getTimePartitionId(realtimeDataExtractionEndTime)
:
TimePartitionUtils.getTimePartitionId(realtimeDataExtractionEndTime) - 1;
- final boolean isDoubleLiving =
PipeSourceConstant.isDoubleLiving(parameters);
- if (isDoubleLiving) {
- isForwardingPipeRequests = false;
- } else {
- isForwardingPipeRequests =
- parameters.getBooleanOrDefault(
- PipeSourceConstant.FORWARDING_PIPE_REQUESTS_KEYS,
-
PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE);
- }
+ isForwardingPipeRequests = true;
if (parameters.hasAnyAttributes(EXTRACTOR_MODS_KEY, SOURCE_MODS_KEY)) {
shouldTransferModFile =
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java
index 8c2fd176587..f24e9cd3f4d 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java
@@ -161,17 +161,7 @@ public class PipeSinkSubtaskManagerTest {
}
@Test
- public void
testCalculateSinkSubtaskNumUsesSingleThreadDefaultSinkAndSchemaRegionLimit() {
- final Map<String, String> singleThreadAttributes = new HashMap<>();
- singleThreadAttributes.put(
- PipeSinkConstant.CONNECTOR_SERIALIZE_BY_REGION_KEY,
Boolean.FALSE.toString());
- singleThreadAttributes.put(
- PipeSinkConstant.CONNECTOR_KEY,
BuiltinPipePlugin.OPC_UA_SINK.getPipePluginName());
- Assert.assertEquals(
- 1,
- PipeSinkSubtaskManager.calculateSinkSubtaskNum(
- new PipeParameters(singleThreadAttributes), -1));
-
+ public void testCalculateSinkSubtaskNumUsesSchemaRegionLimit() {
final Map<String, String> schemaRegionAttributes = new HashMap<>();
schemaRegionAttributes.put(
PipeSinkConstant.CONNECTOR_SERIALIZE_BY_REGION_KEY,
Boolean.FALSE.toString());
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeSinkTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeSinkTest.java
index b7732d85d2f..11e15450f09 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeSinkTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeSinkTest.java
@@ -27,7 +27,6 @@ import
org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskSinkRuntimeEnviro
import
org.apache.iotdb.db.pipe.event.common.statement.PipeStatementInsertionEvent;
import
org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
import org.apache.iotdb.db.pipe.sink.protocol.legacy.IoTDBLegacyPipeSink;
-import org.apache.iotdb.db.pipe.sink.protocol.opcua.OpcUaSink;
import
org.apache.iotdb.db.pipe.sink.protocol.thrift.async.IoTDBDataRegionAsyncSink;
import
org.apache.iotdb.db.pipe.sink.protocol.thrift.sync.IoTDBDataRegionSyncSink;
import
org.apache.iotdb.db.pipe.sink.protocol.websocket.WebSocketConnectorServer;
@@ -47,7 +46,6 @@ import org.junit.Test;
import org.mockito.Mockito;
import java.lang.reflect.Field;
-import java.security.SecureRandom;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -389,97 +387,6 @@ public class PipeSinkTest {
}
}
- @Test
- public void testOpcUaSink() {
- final List<IMeasurementSchema> schemaList =
- Arrays.asList(
- new MeasurementSchema("s1", TSDataType.INT64),
- new MeasurementSchema("s2", TSDataType.INT64));
-
- final Tablet tablet = new Tablet("root.db.d1.vector6", schemaList, 100);
-
- long timestamp = System.currentTimeMillis();
- for (long row = 0; row < 100; row++) {
- final int rowSize = tablet.getRowSize();
- tablet.addTimestamp(rowSize, timestamp);
- for (int i = 0; i < 2; i++) {
- tablet.addValue(
- schemaList.get(i).getMeasurementName(), rowSize, new
SecureRandom().nextLong());
- }
- timestamp++;
- }
-
- final List<IMeasurementSchema> opcSchemaList =
- Arrays.asList(
- new MeasurementSchema("value1", TSDataType.INT64),
- new MeasurementSchema("quality1", TSDataType.BOOLEAN));
- final Tablet qualityTablet = new Tablet("root.db.d1.vector6.s3",
opcSchemaList, 100);
-
- timestamp = System.currentTimeMillis();
- for (long row = 0; row < 100; row++) {
- final int rowSize = qualityTablet.getRowSize();
- qualityTablet.addTimestamp(rowSize, timestamp);
- qualityTablet.addValue(
- opcSchemaList.get(0).getMeasurementName(), rowSize, new
SecureRandom().nextLong());
- qualityTablet.addValue(opcSchemaList.get(1).getMeasurementName(),
rowSize, true);
- timestamp++;
- }
-
- try (final OpcUaSink qualityOPC = new OpcUaSink();
- final OpcUaSink normalOPC = new OpcUaSink()) {
- final PipeTaskRuntimeConfiguration configuration =
- new PipeTaskRuntimeConfiguration(new
PipeTaskSinkRuntimeEnvironment("temp", 0, 1));
- qualityOPC.customize(
- new PipeParameters(
- new HashMap<String, String>() {
- {
- put(
- PipeSinkConstant.CONNECTOR_KEY,
- BuiltinPipePlugin.OPC_UA_SINK.getPipePluginName());
- put(PipeSinkConstant.CONNECTOR_OPC_UA_WITH_QUALITY_KEY,
"true");
- put(PipeSinkConstant.CONNECTOR_OPC_UA_VALUE_NAME_KEY,
"value1");
- put(PipeSinkConstant.CONNECTOR_OPC_UA_QUALITY_NAME_KEY,
"quality1");
- }
- }),
- configuration);
- normalOPC.customize(
- new PipeParameters(
- new HashMap<String, String>() {
- {
- put(
- PipeSinkConstant.CONNECTOR_KEY,
- BuiltinPipePlugin.OPC_UA_SINK.getPipePluginName());
- }
- }),
- configuration);
- final PipeRawTabletInsertionEvent event =
- new PipeRawTabletInsertionEvent(
- false, "root.db", "db", "root.db", tablet, false, "pipe", 0L,
null, null, false);
- event.increaseReferenceCount("");
- normalOPC.transfer(event);
- // Shall not throw
- qualityOPC.transfer(event);
- event.decreaseReferenceCount("", false);
-
- qualityOPC.transfer(
- new PipeRawTabletInsertionEvent(
- false,
- "root.db",
- "db",
- "root.db",
- qualityTablet,
- false,
- "pipe",
- 0L,
- null,
- null,
- false));
-
- } catch (Exception e) {
- Assert.fail();
- }
- }
-
@Test
public void testWriteBackSinkTargetDatabaseValidation() throws Exception {
assertWriteBackSinkTargetDatabaseValid("target");
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataRegionAirGapSinkTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataRegionAirGapSinkTest.java
index 1e6f3f865c1..37e8fb5418f 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataRegionAirGapSinkTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataRegionAirGapSinkTest.java
@@ -19,7 +19,6 @@
package org.apache.iotdb.db.pipe.sink.protocol.airgap;
-import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin;
import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant;
import
org.apache.iotdb.commons.pipe.config.plugin.configuraion.PipeTaskRuntimeConfiguration;
import
org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskSinkRuntimeEnvironment;
@@ -115,9 +114,7 @@ public class IoTDBDataRegionAirGapSinkTest {
private PipeParameters buildParameters(final boolean useTsFileBatch) {
final Map<String, String> attributes = new HashMap<>();
- attributes.put(
- PipeSinkConstant.CONNECTOR_KEY,
- BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName());
+ attributes.put(PipeSinkConstant.CONNECTOR_KEY, "iotdb-air-gap-connector");
attributes.put(PipeSinkConstant.CONNECTOR_IOTDB_NODE_URLS_KEY,
"127.0.0.1:6668");
attributes.put(PipeSinkConstant.CONNECTOR_IOTDB_BATCH_DELAY_MS_KEY, "200");
attributes.put(PipeSinkConstant.CONNECTOR_IOTDB_BATCH_SIZE_KEY, "1048576");
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
index 7fd99dbc2aa..3a918175041 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
@@ -19,27 +19,16 @@
package org.apache.iotdb.commons.pipe.agent.plugin.builtin;
-import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.aggregate.AggregateProcessor;
-import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.aggregate.StandardStatisticsProcessor;
-import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.aggregate.TumblingWindowingProcessor;
import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.donothing.DoNothingProcessor;
-import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.downsampling.ChangingValueSamplingProcessor;
-import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.downsampling.SwingingDoorTrendingSamplingProcessor;
-import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.downsampling.TumblingTimeSamplingProcessor;
import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.iotconsensusv2.IoTConsensusV2Processor;
import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.schemachange.RenameDatabaseProcessor;
-import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.throwing.ThrowingExceptionProcessor;
-import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.twostage.TwoStageCountProcessor;
import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.donothing.DoNothingSink;
-import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.airgap.IoTDBAirGapSink;
import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.consensus.IoTConsensusV2AsyncSink;
import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBLegacyPipeSink;
import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftAsyncSink;
import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSink;
import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSslSink;
import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSyncSink;
-import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcda.OpcDaSink;
-import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcua.OpcUaSink;
import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.websocket.WebSocketSink;
import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.writeback.WriteBackSink;
import
org.apache.iotdb.commons.pipe.agent.plugin.builtin.source.donothing.DoNothingSource;
@@ -61,18 +50,8 @@ public enum BuiltinPipePlugin {
// processors
DO_NOTHING_PROCESSOR("do-nothing-processor", DoNothingProcessor.class),
- TUMBLING_TIME_SAMPLING_PROCESSOR(
- "tumbling-time-sampling-processor", TumblingTimeSamplingProcessor.class),
- SDT_SAMPLING_PROCESSOR("sdt-sampling-processor",
SwingingDoorTrendingSamplingProcessor.class),
- CHANGING_VALUE_SAMPLING_PROCESSOR(
- "changing-value-sampling-processor",
ChangingValueSamplingProcessor.class),
- THROWING_EXCEPTION_PROCESSOR("throwing-exception-processor",
ThrowingExceptionProcessor.class),
- AGGREGATE_PROCESSOR("aggregate-processor", AggregateProcessor.class),
- COUNT_POINT_PROCESSOR("count-point-processor", TwoStageCountProcessor.class),
// Hidden-processors, which are plugins of the processors
- STANDARD_STATISTICS_PROCESSOR("standard-statistics-processor",
StandardStatisticsProcessor.class),
- TUMBLING_WINDOWING_PROCESSOR("tumbling-windowing-processor",
TumblingWindowingProcessor.class),
IOT_CONSENSUS_V2_PROCESSOR("iot-consensus-v2-processor",
IoTConsensusV2Processor.class),
// Legacy alias for stale PipeMeta written before the PipeConsensus ->
IoTConsensusV2 rename.
PIPE_CONSENSUS_PROCESSOR("pipe-consensus-processor",
IoTConsensusV2Processor.class),
@@ -85,15 +64,12 @@ public enum BuiltinPipePlugin {
IOTDB_THRIFT_SYNC_CONNECTOR("iotdb-thrift-sync-connector",
IoTDBThriftSyncSink.class),
IOTDB_THRIFT_ASYNC_CONNECTOR("iotdb-thrift-async-connector",
IoTDBThriftAsyncSink.class),
IOTDB_LEGACY_PIPE_CONNECTOR("iotdb-legacy-pipe-connector",
IoTDBLegacyPipeSink.class),
- IOTDB_AIR_GAP_CONNECTOR("iotdb-air-gap-connector", IoTDBAirGapSink.class),
IOT_CONSENSUS_V2_ASYNC_CONNECTOR(
"iot-consensus-v2-async-connector", IoTConsensusV2AsyncSink.class),
// Legacy alias for stale PipeMeta written before the PipeConsensus ->
IoTConsensusV2 rename.
PIPE_CONSENSUS_ASYNC_CONNECTOR("pipe-consensus-async-connector",
IoTConsensusV2AsyncSink.class),
WEBSOCKET_CONNECTOR("websocket-connector", WebSocketSink.class),
- OPC_UA_CONNECTOR("opc-ua-connector", OpcUaSink.class),
- OPC_DA_CONNECTOR("opc-da-connector", OpcDaSink.class),
WRITE_BACK_CONNECTOR("write-back-connector", WriteBackSink.class),
DO_NOTHING_SINK("do-nothing-sink", DoNothingSink.class),
@@ -102,10 +78,7 @@ public enum BuiltinPipePlugin {
IOTDB_THRIFT_SYNC_SINK("iotdb-thrift-sync-sink", IoTDBThriftSyncSink.class),
IOTDB_THRIFT_ASYNC_SINK("iotdb-thrift-async-sink",
IoTDBThriftAsyncSink.class),
IOTDB_LEGACY_PIPE_SINK("iotdb-legacy-pipe-sink", IoTDBLegacyPipeSink.class),
- IOTDB_AIR_GAP_SINK("iotdb-air-gap-sink", IoTDBAirGapSink.class),
WEBSOCKET_SINK("websocket-sink", WebSocketSink.class),
- OPC_UA_SINK("opc-ua-sink", OpcUaSink.class),
- OPC_DA_SINK("opc-da-sink", OpcDaSink.class),
WRITE_BACK_SINK("write-back-sink", WriteBackSink.class),
SUBSCRIPTION_SINK("subscription-sink", DoNothingSink.class),
IOT_CONSENSUS_V2_ASYNC_SINK("iot-consensus-v2-async-sink",
IoTConsensusV2AsyncSink.class),
@@ -156,12 +129,9 @@ public enum BuiltinPipePlugin {
IOTDB_THRIFT_SYNC_CONNECTOR.getPipePluginName().toLowerCase(),
IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName().toLowerCase(),
IOTDB_LEGACY_PIPE_CONNECTOR.getPipePluginName().toLowerCase(),
- IOTDB_AIR_GAP_CONNECTOR.getPipePluginName().toLowerCase(),
IOT_CONSENSUS_V2_ASYNC_CONNECTOR.getPipePluginName().toLowerCase(),
PIPE_CONSENSUS_ASYNC_CONNECTOR.getPipePluginName().toLowerCase(),
WEBSOCKET_CONNECTOR.getPipePluginName().toLowerCase(),
- OPC_UA_CONNECTOR.getPipePluginName().toLowerCase(),
- OPC_DA_CONNECTOR.getPipePluginName().toLowerCase(),
WRITE_BACK_CONNECTOR.getPipePluginName().toLowerCase(),
DO_NOTHING_SINK.getPipePluginName().toLowerCase(),
IOTDB_THRIFT_SINK.getPipePluginName().toLowerCase(),
@@ -169,10 +139,7 @@ public enum BuiltinPipePlugin {
IOTDB_THRIFT_SYNC_SINK.getPipePluginName().toLowerCase(),
IOTDB_THRIFT_ASYNC_SINK.getPipePluginName().toLowerCase(),
IOTDB_LEGACY_PIPE_SINK.getPipePluginName().toLowerCase(),
- IOTDB_AIR_GAP_SINK.getPipePluginName().toLowerCase(),
WEBSOCKET_SINK.getPipePluginName().toLowerCase(),
- OPC_UA_SINK.getPipePluginName().toLowerCase(),
- OPC_DA_SINK.getPipePluginName().toLowerCase(),
WRITE_BACK_SINK.getPipePluginName().toLowerCase(),
SUBSCRIPTION_SINK.getPipePluginName().toLowerCase(),
IOT_CONSENSUS_V2_ASYNC_SINK.getPipePluginName().toLowerCase(),
@@ -188,14 +155,6 @@ public enum BuiltinPipePlugin {
// Sources
DO_NOTHING_SOURCE.getPipePluginName().toUpperCase(),
// Processors
-
TUMBLING_TIME_SAMPLING_PROCESSOR.getPipePluginName().toUpperCase(),
- SDT_SAMPLING_PROCESSOR.getPipePluginName().toUpperCase(),
-
CHANGING_VALUE_SAMPLING_PROCESSOR.getPipePluginName().toUpperCase(),
-
THROWING_EXCEPTION_PROCESSOR.getPipePluginName().toUpperCase(),
- AGGREGATE_PROCESSOR.getPipePluginName().toUpperCase(),
- COUNT_POINT_PROCESSOR.getPipePluginName().toUpperCase(),
-
STANDARD_STATISTICS_PROCESSOR.getPipePluginName().toUpperCase(),
-
TUMBLING_WINDOWING_PROCESSOR.getPipePluginName().toUpperCase(),
IOT_CONSENSUS_V2_PROCESSOR.getPipePluginName().toUpperCase(),
PIPE_CONSENSUS_PROCESSOR.getPipePluginName().toUpperCase(),
RENAME_DATABASE_PROCESSOR.getPipePluginName().toUpperCase(),
@@ -206,10 +165,7 @@ public enum BuiltinPipePlugin {
IOTDB_THRIFT_SYNC_CONNECTOR.getPipePluginName().toUpperCase(),
IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(),
IOTDB_LEGACY_PIPE_CONNECTOR.getPipePluginName().toUpperCase(),
- IOTDB_AIR_GAP_CONNECTOR.getPipePluginName().toUpperCase(),
WEBSOCKET_CONNECTOR.getPipePluginName().toUpperCase(),
- OPC_UA_CONNECTOR.getPipePluginName().toUpperCase(),
- OPC_DA_CONNECTOR.getPipePluginName().toUpperCase(),
WRITE_BACK_CONNECTOR.getPipePluginName().toUpperCase(),
IOT_CONSENSUS_V2_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(),
PIPE_CONSENSUS_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(),
@@ -218,8 +174,6 @@ public enum BuiltinPipePlugin {
IOTDB_THRIFT_ASYNC_SINK.getPipePluginName().toUpperCase(),
IOTDB_LEGACY_PIPE_SINK.getPipePluginName().toUpperCase(),
WEBSOCKET_SINK.getPipePluginName().toUpperCase(),
- OPC_UA_SINK.getPipePluginName().toUpperCase(),
- OPC_DA_SINK.getPipePluginName().toUpperCase(),
SUBSCRIPTION_SINK.getPipePluginName().toUpperCase(),
IOT_CONSENSUS_V2_ASYNC_SINK.getPipePluginName().toUpperCase(),
PIPE_CONSENSUS_ASYNC_SINK.getPipePluginName().toUpperCase())));
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java
index f2d9ea283c8..1afb4adb65e 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java
@@ -68,13 +68,6 @@ public class PipeSinkConstant {
public static final String SINK_IOTDB_PARALLEL_TASKS_KEY =
"sink.parallel.tasks";
public static final int CONNECTOR_IOTDB_PARALLEL_TASKS_DEFAULT_VALUE =
PipeConfig.getInstance().getPipeSubtaskExecutorMaxThreadNum();
- public static final Set<String> SINGLE_THREAD_DEFAULT_SINK =
- new HashSet<>(
- Arrays.asList(
- BuiltinPipePlugin.OPC_UA_SINK.getPipePluginName(),
- BuiltinPipePlugin.OPC_UA_CONNECTOR.getPipePluginName(),
- BuiltinPipePlugin.OPC_DA_SINK.getPipePluginName(),
- BuiltinPipePlugin.OPC_DA_CONNECTOR.getPipePluginName()));
public static final String CONNECTOR_REALTIME_FIRST_KEY =
"connector.realtime-first";
public static final String SINK_REALTIME_FIRST_KEY = "sink.realtime-first";
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBSource.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBSource.java
index 6315ae53b7f..54b93ad45c9 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBSource.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBSource.java
@@ -69,7 +69,7 @@ public abstract class IoTDBSource implements PipeExtractor {
protected int regionId;
protected PipeTaskMeta pipeTaskMeta;
- protected boolean isForwardingPipeRequests;
+ protected boolean isForwardingPipeRequests = true;
// The value is always true after the first start even the extractor is
closed
protected final AtomicBoolean hasBeenStarted = new AtomicBoolean(false);
@@ -144,15 +144,7 @@ public abstract class IoTDBSource implements PipeExtractor
{
pipeTaskMeta = ((PipeTaskSourceRuntimeEnvironment)
environment).getPipeTaskMeta();
}
- final boolean isDoubleLiving =
PipeSourceConstant.isDoubleLiving(parameters);
- if (isDoubleLiving) {
- isForwardingPipeRequests = false;
- } else {
- isForwardingPipeRequests =
- parameters.getBooleanOrDefault(
- PipeSourceConstant.FORWARDING_PIPE_REQUESTS_KEYS,
-
PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE);
- }
+ isForwardingPipeRequests = true;
userId =
parameters.getStringOrDefault(
diff --git a/pom.xml b/pom.xml
index 424b31c79c3..69503c40694 100644
--- a/pom.xml
+++ b/pom.xml
@@ -719,6 +719,7 @@
<argLine>${argLine} -Xmx1024m</argLine>
<!-- Subscription UT is maintained and executed by the
enterprise edition. -->
<excludes>
+ <exclude>**/pipe/**</exclude>
<exclude>**/subscription/**</exclude>
</excludes>
<!-- Force the working directory to be different for
each fork, so forks don't trample each other -->