This is an automated email from the ASF dual-hosted git repository.
mcvsubbu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new b4d75a3 Add absolute path support for compatibility testing(#6787)
(#6828)
b4d75a3 is described below
commit b4d75a30ccc26b30b0779433c5f009afac5b5588
Author: Sharayu <[email protected]>
AuthorDate: Mon Apr 26 20:24:28 2021 -0700
Add absolute path support for compatibility testing(#6787) (#6828)
---
compatibility-verifier/compCheck.sh | 32 ++++++++----
.../config}/FeatureTest1-schema.json | 0
.../config}/data/FeatureTest1-data-00.csv | 0
.../config}/data/FeatureTest1-data-01.csv | 0
.../config}/data/recordReaderConfig.json | 0
.../feature-test-1-realtime-stream-config.json | 0
.../config}/feature-test-1-realtime.json | 0
.../sample-test-suite/config}/feature-test-1.json | 4 +-
.../sample-test-suite/config}/kafka/topic1.cfg | 0
.../config}/queries/feature-test-1-sql.queries | 0
.../query-results/feature-test-1-rest-sql.results | 0
.../kafka-stream-produce-op.yaml | 8 +--
.../sample-test-suite}/kafka-topic-create-op.yaml | 2 +-
.../offline-table-create-op.yaml | 4 +-
.../offline-table-delete-op.yaml | 4 +-
.../sample-test-suite}/post-broker-rollback.yaml | 0
.../post-controller-rollback.yaml | 0
.../sample-test-suite}/post-server-rollback.yaml | 0
.../sample-test-suite}/post-server-upgrade.yaml | 0
.../sample-test-suite}/pre-broker-upgrade.yaml | 0
.../sample-test-suite}/pre-controller-upgrade.yaml | 0
.../sample-test-suite}/pre-server-upgrade.yaml | 0
.../sample-test-suite}/query-op.yaml | 4 +-
.../realtime-table-create-op.yaml | 4 +-
.../realtime-table-delete-op.yaml | 4 +-
.../sample-test-suite/sample.yaml | 30 +++++++-----
.../sample-test-suite}/segment-create-op.yaml | 8 +--
.../sample-test-suite}/segment-delete-op.yaml | 4 +-
pinot-integration-tests/pom.xml | 4 ++
.../java/org/apache/pinot/compat/tests/BaseOp.java | 14 ++++++
.../pinot/compat/tests/CompatibilityOpsRunner.java | 16 ++++--
.../org/apache/pinot/compat/tests/QueryOp.java | 4 +-
.../org/apache/pinot/compat/tests/SegmentOp.java | 10 ++--
.../org/apache/pinot/compat/tests/StreamOp.java | 10 ++--
.../org/apache/pinot/compat/tests/TableOp.java | 6 +--
.../src/test/resources/compat-tests/sample.yaml | 57 ----------------------
36 files changed, 109 insertions(+), 120 deletions(-)
diff --git a/compatibility-verifier/compCheck.sh
b/compatibility-verifier/compCheck.sh
index 9660ec9..ad168f5 100755
--- a/compatibility-verifier/compCheck.sh
+++ b/compatibility-verifier/compCheck.sh
@@ -43,7 +43,7 @@
# get usage of the script
function usage() {
command=$1
- echo "Usage: $command [workingDir]"
+ echo "Usage: $command workingDir testSuiteDir"
exit 1
}
@@ -146,11 +146,21 @@ function setupCompatTester() {
export CLASSPATH_PREFIX
}
+#compute absolute path for testSuiteDir if given relative
+function absPath() {
+ local testSuiteDirPath=$1
+ if [[ ! "$testSuiteDirPath" = /* ]]; then
+ #relative path
+ testSuiteDirPath=$(cd "$testSuiteDirPath"; pwd)
+ fi
+ echo "$testSuiteDirPath"
+}
+
#
# Main
#
-if [ $# -ne 1 ] ; then
+if [ $# -ne 2 ] ; then
usage compCheck
fi
@@ -158,6 +168,7 @@
COMPAT_TESTER_PATH="pinot-integration-tests/target/pinot-integration-tests-pkg/b
# create subdirectories for given commits
workingDir=$1
+testSuiteDir=$(absPath "$2")
oldTargetDir="$workingDir"/oldTargetDir
newTargetDir="$workingDir"/newTargetDir
@@ -172,30 +183,31 @@ fi
# Setup initial cluster with olderCommit and do rolling upgrade
+# Provide abspath of filepath to $COMPAT_TESTER
startServices "$oldTargetDir"
-#$COMPAT_TESTER pre-controller-upgrade.yaml 1; if [ $? -ne 0 ]; then exit 1; fi
+#$COMPAT_TESTER $testSuiteDir/pre-controller-upgrade.yaml 1; if [ $? -ne 0 ];
then exit 1; fi
stopService controller "$oldTargetDir"
startService controller "$newTargetDir"
waitForControllerReady
-#$COMPAT_TESTER pre-broker-upgrade.yaml 2; if [ $? -ne 0 ]; then exit 1; fi
+#$COMPAT_TESTER $testSuiteDir/pre-broker-upgrade.yaml 2; if [ $? -ne 0 ]; then
exit 1; fi
stopService broker "$oldTargetDir"
startService broker "$newTargetDir"
-#$COMPAT_TESTER pre-server-upgrade.yaml 3; if [ $? -ne 0 ]; then exit 1; fi
+#$COMPAT_TESTER $testSuiteDir/pre-server-upgrade.yaml 3; if [ $? -ne 0 ]; then
exit 1; fi
stopService server "$oldTargetDir"
startService server "$newTargetDir"
-#$COMPAT_TESTER post-server-upgrade.yaml 4; if [ $? -ne 0 ]; then exit 1; fi
+#$COMPAT_TESTER $testSuiteDir/post-server-upgrade.yaml 4; if [ $? -ne 0 ];
then exit 1; fi
# Upgrade completed, now do a rollback
stopService server "$newTargetDir"
startService server "$oldTargetDir"
-#$COMPAT_TESTER post-server-rollback.yaml 5; if [ $? -ne 0 ]; then exit 1; fi
+#$COMPAT_TESTER $testSuiteDir/post-server-rollback.yaml 5; if [ $? -ne 0 ];
then exit 1; fi
stopService broker "$newTargetDir"
startService broker "$oldTargetDir"
-#$COMPAT_TESTER post-broker-rollback.yaml 6; if [ $? -ne 0 ]; then exit 1; fi
+#$COMPAT_TESTER $testSuiteDir/post-broker-rollback.yaml 6; if [ $? -ne 0 ];
then exit 1; fi
stopService controller "$newTargetDir"
startService controller "$oldTargetDir"
waitForControllerReady
-#$COMPAT_TESTER post-controller-rollback.yaml 7; if [ $? -ne 0 ]; then exit 1;
fi
+#$COMPAT_TESTER $testSuiteDir/post-controller-rollback.yaml 7; if [ $? -ne 0
]; then exit 1; fi
stopServices "$oldTargetDir"
-exit 0
\ No newline at end of file
+exit 0
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/configs/FeatureTest1-schema.json
b/compatibility-verifier/sample-test-suite/config/FeatureTest1-schema.json
similarity index 100%
rename from
pinot-integration-tests/src/test/resources/compat-tests/configs/FeatureTest1-schema.json
rename to
compatibility-verifier/sample-test-suite/config/FeatureTest1-schema.json
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/data/FeatureTest1-data-00.csv
b/compatibility-verifier/sample-test-suite/config/data/FeatureTest1-data-00.csv
similarity index 100%
rename from
pinot-integration-tests/src/test/resources/compat-tests/data/FeatureTest1-data-00.csv
rename to
compatibility-verifier/sample-test-suite/config/data/FeatureTest1-data-00.csv
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/data/FeatureTest1-data-01.csv
b/compatibility-verifier/sample-test-suite/config/data/FeatureTest1-data-01.csv
similarity index 100%
rename from
pinot-integration-tests/src/test/resources/compat-tests/data/FeatureTest1-data-01.csv
rename to
compatibility-verifier/sample-test-suite/config/data/FeatureTest1-data-01.csv
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/data/recordReaderConfig.json
b/compatibility-verifier/sample-test-suite/config/data/recordReaderConfig.json
similarity index 100%
rename from
pinot-integration-tests/src/test/resources/compat-tests/data/recordReaderConfig.json
rename to
compatibility-verifier/sample-test-suite/config/data/recordReaderConfig.json
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1-realtime-stream-config.json
b/compatibility-verifier/sample-test-suite/config/feature-test-1-realtime-stream-config.json
similarity index 100%
rename from
pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1-realtime-stream-config.json
rename to
compatibility-verifier/sample-test-suite/config/feature-test-1-realtime-stream-config.json
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1-realtime.json
b/compatibility-verifier/sample-test-suite/config/feature-test-1-realtime.json
similarity index 100%
rename from
pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1-realtime.json
rename to
compatibility-verifier/sample-test-suite/config/feature-test-1-realtime.json
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1.json
b/compatibility-verifier/sample-test-suite/config/feature-test-1.json
similarity index 96%
rename from
pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1.json
rename to compatibility-verifier/sample-test-suite/config/feature-test-1.json
index a160228..77ce7ff 100644
---
a/pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1.json
+++ b/compatibility-verifier/sample-test-suite/config/feature-test-1.json
@@ -37,10 +37,10 @@
"sortedColumn": [],
"streamConfigs": {}
},
- "tableName": "FeatureTest1",
+ "tableName": "FeatureTest1_OFFLINE",
"tableType": "OFFLINE",
"tenants": {
"broker": "DefaultTenant",
"server": "DefaultTenant"
}
-}
\ No newline at end of file
+}
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/kafka/topic1.cfg
b/compatibility-verifier/sample-test-suite/config/kafka/topic1.cfg
similarity index 100%
rename from
pinot-integration-tests/src/test/resources/compat-tests/kafka/topic1.cfg
rename to compatibility-verifier/sample-test-suite/config/kafka/topic1.cfg
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/queries/feature-test-1-sql.queries
b/compatibility-verifier/sample-test-suite/config/queries/feature-test-1-sql.queries
similarity index 100%
rename from
pinot-integration-tests/src/test/resources/compat-tests/queries/feature-test-1-sql.queries
rename to
compatibility-verifier/sample-test-suite/config/queries/feature-test-1-sql.queries
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/query-results/feature-test-1-rest-sql.results
b/compatibility-verifier/sample-test-suite/config/query-results/feature-test-1-rest-sql.results
similarity index 100%
rename from
pinot-integration-tests/src/test/resources/compat-tests/query-results/feature-test-1-rest-sql.results
rename to
compatibility-verifier/sample-test-suite/config/query-results/feature-test-1-rest-sql.results
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/kafka-stream-produce-op.yaml
b/compatibility-verifier/sample-test-suite/kafka-stream-produce-op.yaml
similarity index 68%
rename from
pinot-integration-tests/src/test/resources/compat-tests/kafka-stream-produce-op.yaml
rename to compatibility-verifier/sample-test-suite/kafka-stream-produce-op.yaml
index 1475284..c02bd23 100644
---
a/pinot-integration-tests/src/test/resources/compat-tests/kafka-stream-produce-op.yaml
+++ b/compatibility-verifier/sample-test-suite/kafka-stream-produce-op.yaml
@@ -22,8 +22,8 @@ operations:
- type: streamOp
description: publish rows to PinotRealtimeFeatureTest1Event
op: PRODUCE
- streamConfigFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1-realtime-stream-config.json
+ streamConfigFileName: feature-test-1-realtime-stream-config.json
numRows: 1
- inputDataFileName:
pinot-integration-tests/src/test/resources/compat-tests/data/FeatureTest1-data-00.csv
- recordReaderConfigFileName:
pinot-integration-tests/src/test/resources/compat-tests/data/recordReaderConfig.json
- tableConfigFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1-realtime.json
+ inputDataFileName: data/FeatureTest1-data-00.csv
+ recordReaderConfigFileName: data/recordReaderConfig.json
+ tableConfigFileName: feature-test-1-realtime.json
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/kafka-topic-create-op.yaml
b/compatibility-verifier/sample-test-suite/kafka-topic-create-op.yaml
similarity index 88%
rename from
pinot-integration-tests/src/test/resources/compat-tests/kafka-topic-create-op.yaml
rename to compatibility-verifier/sample-test-suite/kafka-topic-create-op.yaml
index f2c6bea..776f694 100644
---
a/pinot-integration-tests/src/test/resources/compat-tests/kafka-topic-create-op.yaml
+++ b/compatibility-verifier/sample-test-suite/kafka-topic-create-op.yaml
@@ -22,4 +22,4 @@ operations:
- type: streamOp
description: create Kafka topic PinotRealtimeFeatureTest1Event
op: CREATE
- streamConfigFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1-realtime-stream-config.json
\ No newline at end of file
+ streamConfigFileName: feature-test-1-realtime-stream-config.json
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/offline-table-create-op.yaml
b/compatibility-verifier/sample-test-suite/offline-table-create-op.yaml
similarity index 81%
rename from
pinot-integration-tests/src/test/resources/compat-tests/offline-table-create-op.yaml
rename to compatibility-verifier/sample-test-suite/offline-table-create-op.yaml
index 391927c..7fd2046 100644
---
a/pinot-integration-tests/src/test/resources/compat-tests/offline-table-create-op.yaml
+++ b/compatibility-verifier/sample-test-suite/offline-table-create-op.yaml
@@ -22,5 +22,5 @@ operations:
- type: tableOp
description: Create table feature-test-1.json
op: CREATE
- schemaFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/FeatureTest1-schema.json
- tableConfigFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1.json
\ No newline at end of file
+ schemaFileName: FeatureTest1-schema.json
+ tableConfigFileName: feature-test-1.json
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/offline-table-delete-op.yaml
b/compatibility-verifier/sample-test-suite/offline-table-delete-op.yaml
similarity index 81%
rename from
pinot-integration-tests/src/test/resources/compat-tests/offline-table-delete-op.yaml
rename to compatibility-verifier/sample-test-suite/offline-table-delete-op.yaml
index e9dddf9..dfe6442 100644
---
a/pinot-integration-tests/src/test/resources/compat-tests/offline-table-delete-op.yaml
+++ b/compatibility-verifier/sample-test-suite/offline-table-delete-op.yaml
@@ -22,5 +22,5 @@ operations:
- type: tableOp
description: Delete table feature-test-1.json
op: DELETE
- schemaFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/FeatureTest1-schema.json
- tableConfigFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1.json
\ No newline at end of file
+ schemaFileName: FeatureTest1-schema.json
+ tableConfigFileName: feature-test-1.json
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/post-broker-rollback.yaml
b/compatibility-verifier/sample-test-suite/post-broker-rollback.yaml
similarity index 100%
copy from
pinot-integration-tests/src/test/resources/compat-tests/post-broker-rollback.yaml
copy to compatibility-verifier/sample-test-suite/post-broker-rollback.yaml
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/post-controller-rollback.yaml
b/compatibility-verifier/sample-test-suite/post-controller-rollback.yaml
similarity index 100%
rename from
pinot-integration-tests/src/test/resources/compat-tests/post-controller-rollback.yaml
rename to compatibility-verifier/sample-test-suite/post-controller-rollback.yaml
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/post-server-rollback.yaml
b/compatibility-verifier/sample-test-suite/post-server-rollback.yaml
similarity index 100%
rename from
pinot-integration-tests/src/test/resources/compat-tests/post-server-rollback.yaml
rename to compatibility-verifier/sample-test-suite/post-server-rollback.yaml
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/post-server-upgrade.yaml
b/compatibility-verifier/sample-test-suite/post-server-upgrade.yaml
similarity index 100%
rename from
pinot-integration-tests/src/test/resources/compat-tests/post-server-upgrade.yaml
rename to compatibility-verifier/sample-test-suite/post-server-upgrade.yaml
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/pre-broker-upgrade.yaml
b/compatibility-verifier/sample-test-suite/pre-broker-upgrade.yaml
similarity index 100%
rename from
pinot-integration-tests/src/test/resources/compat-tests/pre-broker-upgrade.yaml
rename to compatibility-verifier/sample-test-suite/pre-broker-upgrade.yaml
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/pre-controller-upgrade.yaml
b/compatibility-verifier/sample-test-suite/pre-controller-upgrade.yaml
similarity index 100%
rename from
pinot-integration-tests/src/test/resources/compat-tests/pre-controller-upgrade.yaml
rename to compatibility-verifier/sample-test-suite/pre-controller-upgrade.yaml
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/pre-server-upgrade.yaml
b/compatibility-verifier/sample-test-suite/pre-server-upgrade.yaml
similarity index 100%
rename from
pinot-integration-tests/src/test/resources/compat-tests/pre-server-upgrade.yaml
rename to compatibility-verifier/sample-test-suite/pre-server-upgrade.yaml
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/query-op.yaml
b/compatibility-verifier/sample-test-suite/query-op.yaml
similarity index 79%
rename from
pinot-integration-tests/src/test/resources/compat-tests/query-op.yaml
rename to compatibility-verifier/sample-test-suite/query-op.yaml
index c244f17..216b2ae 100644
--- a/pinot-integration-tests/src/test/resources/compat-tests/query-op.yaml
+++ b/compatibility-verifier/sample-test-suite/query-op.yaml
@@ -21,5 +21,5 @@ description: Yaml file for query ops
operations:
- type: queryOp
description: Run query on FeatureTest1 using SQL
- queryFileName:
pinot-integration-tests/src/test/resources/compat-tests/queries/feature-test-1-sql.queries
- expectedResultsFileName:
pinot-integration-tests/src/test/resources/compat-tests/query-results/feature-test-1-rest-sql.results
\ No newline at end of file
+ queryFileName: queries/feature-test-1-sql.queries
+ expectedResultsFileName: query-results/feature-test-1-rest-sql.results
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/realtime-table-create-op.yaml
b/compatibility-verifier/sample-test-suite/realtime-table-create-op.yaml
similarity index 80%
rename from
pinot-integration-tests/src/test/resources/compat-tests/realtime-table-create-op.yaml
rename to compatibility-verifier/sample-test-suite/realtime-table-create-op.yaml
index c71c3b3..732cccf 100644
---
a/pinot-integration-tests/src/test/resources/compat-tests/realtime-table-create-op.yaml
+++ b/compatibility-verifier/sample-test-suite/realtime-table-create-op.yaml
@@ -22,5 +22,5 @@ operations:
- type: tableOp
description: Create table feature-test-1-realtime.json
op: CREATE
- schemaFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/FeatureTest1-schema.json
- tableConfigFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1-realtime.json
\ No newline at end of file
+ schemaFileName: FeatureTest1-schema.json
+ tableConfigFileName: feature-test-1-realtime.json
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/realtime-table-delete-op.yaml
b/compatibility-verifier/sample-test-suite/realtime-table-delete-op.yaml
similarity index 80%
rename from
pinot-integration-tests/src/test/resources/compat-tests/realtime-table-delete-op.yaml
rename to compatibility-verifier/sample-test-suite/realtime-table-delete-op.yaml
index 8daf65d..32afab9 100644
---
a/pinot-integration-tests/src/test/resources/compat-tests/realtime-table-delete-op.yaml
+++ b/compatibility-verifier/sample-test-suite/realtime-table-delete-op.yaml
@@ -22,5 +22,5 @@ operations:
- type: tableOp
description: Delete table feature-test-1-realtime.json
op: DELETE
- schemaFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/FeatureTest1-schema.json
- tableConfigFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1-realtime.json
\ No newline at end of file
+ schemaFileName: FeatureTest1-schema.json
+ tableConfigFileName: feature-test-1-realtime.json
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/post-broker-rollback.yaml
b/compatibility-verifier/sample-test-suite/sample.yaml
similarity index 57%
rename from
pinot-integration-tests/src/test/resources/compat-tests/post-broker-rollback.yaml
rename to compatibility-verifier/sample-test-suite/sample.yaml
index 94d1a54..465a953 100644
---
a/pinot-integration-tests/src/test/resources/compat-tests/post-broker-rollback.yaml
+++ b/compatibility-verifier/sample-test-suite/sample.yaml
@@ -21,23 +21,31 @@
description: Operations to be run after Broker downgrade
operations:
- type: tableOp
- description: Create table t1
+ description: Create table FeatureTest1
op: CREATE
- schemaFileName: /tmp/schema
- tableConfigFileName: /tmp/tableConfig
+ schemaFileName: FeatureTest1-schema.json
+ tableConfigFileName: feature-test-1.json
- type: segmentOp
- description: Add segments X and Y to T1
+ description: Add segment FeatureTest1_Segment to table FeatureTest1
op: UPLOAD
- inputDataFileName: /tmp/data.csv
- tableConfigFileName: /tmp/tableConfig
+ inputDataFileName: data/FeatureTest1-data-00.csv
+ schemaFileName: FeatureTest1-schema.json
+ tableConfigFileName: feature-test-1.json
+ recordReaderConfigFileName: data/recordReaderConfig.json
+ segmentName: FeatureTest1_Segment
+ - type: segmentOp
+ description: Delete segment FeatureTest1_Segment from table FeatureTest1
+ op: DELETE
+ tableConfigFileName: feature-test-1.json
+ segmentName: FeatureTest1_Segment
- type: tableOp
- # description: Remove table T1
+ description: Remove table FeatureTest1
op: DELETE
- tableConfigFileName: /tmp/tableConfig
+ tableConfigFileName: feature-test-1.json
- type: queryOp
- description: Run queries on T1
- queryFileName: queries/T1.sql
- expectedResultsFileName: queries/T1-results.sql
+ description: Run queries on FeatureTest1
+ queryFileName: queries/feature-test-1-sql.queries
+ expectedResultsFileName: query-results/feature-test-1-rest-sql.results
- type: streamOp
description: Produce rows to topic1
op: PRODUCE
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/segment-create-op.yaml
b/compatibility-verifier/sample-test-suite/segment-create-op.yaml
similarity index 68%
rename from
pinot-integration-tests/src/test/resources/compat-tests/segment-create-op.yaml
rename to compatibility-verifier/sample-test-suite/segment-create-op.yaml
index f4843b9..6e3e306 100644
---
a/pinot-integration-tests/src/test/resources/compat-tests/segment-create-op.yaml
+++ b/compatibility-verifier/sample-test-suite/segment-create-op.yaml
@@ -23,7 +23,7 @@ operations:
- type: segmentOp
description: Build and upload segment using FeatureTest1-data-00.csv
op: UPLOAD
- inputDataFileName:
pinot-integration-tests/src/test/resources/compat-tests/data/FeatureTest1-data-00.csv
- schemaFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/FeatureTest1-schema.json
- tableConfigFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1.json
- recordReaderConfigFileName:
pinot-integration-tests/src/test/resources/compat-tests/data/recordReaderConfig.json
\ No newline at end of file
+ inputDataFileName: data/FeatureTest1-data-00.csv
+ schemaFileName: FeatureTest1-schema.json
+ tableConfigFileName: feature-test-1.json
+ recordReaderConfigFileName: data/recordReaderConfig.json
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/segment-delete-op.yaml
b/compatibility-verifier/sample-test-suite/segment-delete-op.yaml
similarity index 86%
rename from
pinot-integration-tests/src/test/resources/compat-tests/segment-delete-op.yaml
rename to compatibility-verifier/sample-test-suite/segment-delete-op.yaml
index f734d72..7a664b1 100644
---
a/pinot-integration-tests/src/test/resources/compat-tests/segment-delete-op.yaml
+++ b/compatibility-verifier/sample-test-suite/segment-delete-op.yaml
@@ -22,5 +22,5 @@ operations:
- type: segmentOp
description: Delete segment FeatureTest1_Segment
op: DELETE
- tableConfigFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1.json
- segmentName: FeatureTest1_Segment
\ No newline at end of file
+ tableConfigFileName: feature-test-1.json
+ segmentName: FeatureTest1_Segment
diff --git a/pinot-integration-tests/pom.xml b/pinot-integration-tests/pom.xml
index 07e635c..b6832bf 100644
--- a/pinot-integration-tests/pom.xml
+++ b/pinot-integration-tests/pom.xml
@@ -137,6 +137,10 @@
<groupId>org.apache.pinot</groupId>
<artifactId>pinot-orc</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/BaseOp.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/BaseOp.java
index fd15331..5469326 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/BaseOp.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/BaseOp.java
@@ -44,6 +44,8 @@ public abstract class BaseOp {
private final OpType _opType;
private String _description = "No description provided";
protected static final String GENERATION_NUMBER_PLACEHOLDER =
"__GENERATION_NUMBER__";
+ protected static final String CONFIG_PLACEHOLDER = "/config/";
+ private String _parentDir;
protected BaseOp(OpType opType) {
_opType = opType;
@@ -65,6 +67,18 @@ public abstract class BaseOp {
return _description;
}
+ public String getParentDir() {
+ return _parentDir;
+ }
+
+ public void setParentDir(String parentDir) {
+ _parentDir = parentDir;
+ }
+
+ public String getAbsoluteFileName(String fileName) {
+ return _parentDir + CONFIG_PLACEHOLDER + fileName;
+ }
+
public boolean run(int generationNumber) {
System.out.println("Running OpType " + _opType.toString() + ": " +
getDescription());
return runOp(generationNumber);
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/CompatibilityOpsRunner.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/CompatibilityOpsRunner.java
index 46173b0..e1e5996 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/CompatibilityOpsRunner.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/CompatibilityOpsRunner.java
@@ -21,11 +21,17 @@ package org.apache.pinot.compat.tests;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class CompatibilityOpsRunner {
- private static final String ROOT_DIR = "compat-tests";
+ private static final Logger LOGGER =
LoggerFactory.getLogger(CompatibilityOpsRunner.class);
+ private String _parentDir;
private final String _configFileName;
private int _generationNumber;
@@ -35,15 +41,17 @@ public class CompatibilityOpsRunner {
}
private boolean runOps() throws Exception {
- String filePath = ROOT_DIR + "/" + _configFileName;
- InputStream inputStream =
getClass().getClassLoader().getResourceAsStream(filePath);
+ Path path = Paths.get(_configFileName);
+ _parentDir = path.getParent().toString();
+ InputStream inputStream = Files.newInputStream(path);
ObjectMapper om = new ObjectMapper(new YAMLFactory());
CompatTestOperation operation = om.readValue(inputStream,
CompatTestOperation.class);
- System.out.println("Running compat verifications from file:" + filePath +
"(" + operation.getDescription() + ")");
+ LOGGER.info("Running compat verifications from file:{} ({})",
path.toString(), operation.getDescription());
boolean passed = true;
for (BaseOp op : operation.getOperations()) {
+ op.setParentDir(_parentDir);
if (!op.run(_generationNumber)) {
passed = false;
System.out.println("Failure");
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/QueryOp.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/QueryOp.java
index 91d2098..4ab9ca7 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/QueryOp.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/QueryOp.java
@@ -88,9 +88,9 @@ public class QueryOp extends BaseOp {
boolean testPassed = false;
try (BufferedReader queryReader = new BufferedReader(
- new InputStreamReader(new FileInputStream(_queryFileName),
StandardCharsets.UTF_8));
+ new InputStreamReader(new
FileInputStream(getAbsoluteFileName(_queryFileName)), StandardCharsets.UTF_8));
BufferedReader expectedResultReader = new BufferedReader(
- new InputStreamReader(new
FileInputStream(_expectedResultsFileName), StandardCharsets.UTF_8))) {
+ new InputStreamReader(new
FileInputStream(getAbsoluteFileName(_expectedResultsFileName)),
StandardCharsets.UTF_8))) {
int succeededQueryCount = 0;
int totalQueryCount = 0;
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/SegmentOp.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/SegmentOp.java
index 58bd3db..a4982f4 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/SegmentOp.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/SegmentOp.java
@@ -158,7 +158,7 @@ public class SegmentOp extends BaseOp {
FileUtils.forceMkdir(localOutputTempDir);
// replace the placeholder in the data file.
File localReplacedInputDataFile = new File(localTempDir, "replaced");
- Utils.replaceContent(new File(_inputDataFileName),
localReplacedInputDataFile, GENERATION_NUMBER_PLACEHOLDER,
+ Utils.replaceContent(new File(getAbsoluteFileName(_inputDataFileName)),
localReplacedInputDataFile, GENERATION_NUMBER_PLACEHOLDER,
String.valueOf(_generationNumber));
File segmentTarFile = generateSegment(localOutputTempDir,
localReplacedInputDataFile.getAbsolutePath());
@@ -182,16 +182,16 @@ public class SegmentOp extends BaseOp {
*/
private File generateSegment(File outputDir, String
localReplacedInputDataFilePath)
throws Exception {
- TableConfig tableConfig = JsonUtils.fileToObject(new
File(_tableConfigFileName), TableConfig.class);
+ TableConfig tableConfig = JsonUtils.fileToObject(new
File(getAbsoluteFileName(_tableConfigFileName)), TableConfig.class);
_tableName = tableConfig.getTableName();
// if user does not specify segmentName, use tableName_generationNumber
if (_segmentName == null || _segmentName.isEmpty()) {
_segmentName = _tableName + "_" + _generationNumber;
}
- Schema schema = JsonUtils.fileToObject(new File(_schemaFileName),
Schema.class);
+ Schema schema = JsonUtils.fileToObject(new
File(getAbsoluteFileName(_schemaFileName)), Schema.class);
RecordReaderConfig recordReaderConfig =
- RecordReaderFactory.getRecordReaderConfig(DEFAULT_FILE_FORMAT,
_recordReaderConfigFileName);
+
RecordReaderFactory.getRecordReaderConfig(DEFAULT_FILE_FORMAT,getAbsoluteFileName(_recordReaderConfigFileName));
SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig(tableConfig, schema);
segmentGeneratorConfig.setInputFilePath(localReplacedInputDataFilePath);
@@ -283,7 +283,7 @@ public class SegmentOp extends BaseOp {
*/
private boolean deleteSegment() {
try {
- TableConfig tableConfig = JsonUtils.fileToObject(new
File(_tableConfigFileName), TableConfig.class);
+ TableConfig tableConfig = JsonUtils.fileToObject(new
File(getAbsoluteFileName(_tableConfigFileName)), TableConfig.class);
_tableName = tableConfig.getTableName();
// if user does not specify segmentName, use tableName_generationNumber
if (_segmentName == null || _segmentName.isEmpty()) {
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/StreamOp.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/StreamOp.java
index 983be92..1a4361a 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/StreamOp.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/StreamOp.java
@@ -158,7 +158,7 @@ public class StreamOp extends BaseOp {
private boolean createKafkaTopic() {
try {
- Properties streamConfigMap = JsonUtils.fileToObject(new
File(_streamConfigFileName), Properties.class);
+ Properties streamConfigMap = JsonUtils.fileToObject(new
File(getAbsoluteFileName(_streamConfigFileName)), Properties.class);
String topicName = streamConfigMap.getProperty(TOPIC_NAME);
int partitions =
Integer.parseInt(streamConfigMap.getProperty(NUM_PARTITIONS));
@@ -180,12 +180,12 @@ public class StreamOp extends BaseOp {
private boolean produceData() {
try {
// get kafka topic
- Properties streamConfigMap = JsonUtils.fileToObject(new
File(_streamConfigFileName), Properties.class);
+ Properties streamConfigMap = JsonUtils.fileToObject(new
File(getAbsoluteFileName(_streamConfigFileName)), Properties.class);
String topicName = streamConfigMap.getProperty(TOPIC_NAME);
String partitionColumn = streamConfigMap.getProperty(PARTITION_COLUMN);
// get table config
- TableConfig tableConfig = JsonUtils.fileToObject(new
File(_tableConfigFileName), TableConfig.class);
+ TableConfig tableConfig = JsonUtils.fileToObject(new
File(getAbsoluteFileName(_tableConfigFileName)), TableConfig.class);
String tableName = tableConfig.getTableName();
long existingTotalDoc = 0;
@@ -205,12 +205,12 @@ public class StreamOp extends BaseOp {
localTempDir.deleteOnExit();
File localReplacedCSVFile = new File(localTempDir, "replaced");
FileUtils.forceMkdir(localTempDir);
- Utils.replaceContent(new File(_inputDataFileName), localReplacedCSVFile,
GENERATION_NUMBER_PLACEHOLDER,
+ Utils.replaceContent(new File(getAbsoluteFileName(_inputDataFileName)),
localReplacedCSVFile, GENERATION_NUMBER_PLACEHOLDER,
String.valueOf(_generationNumber));
CSVRecordReaderConfig recordReaderConfig =
- JsonUtils.fileToObject(new File(_recordReaderConfigFileName),
CSVRecordReaderConfig.class);
+ JsonUtils.fileToObject(new
File(getAbsoluteFileName(_recordReaderConfigFileName)),
CSVRecordReaderConfig.class);
Set<String> columnNames = new HashSet<>();
Collections.addAll(columnNames,
recordReaderConfig.getHeader().split(Character.toString(recordReaderConfig.getDelimiter())));
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/TableOp.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/TableOp.java
index 2508ff4..4fd20a7 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/TableOp.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/TableOp.java
@@ -108,7 +108,7 @@ public class TableOp extends BaseOp {
}};
ControllerTest
.sendPostRequest(ControllerRequestURLBuilder.baseUrl(ClusterDescriptor.CONTROLLER_URL).forSchemaCreate(),
- FileUtils.readFileToString(new File(_schemaFileName)), headers);
+ FileUtils.readFileToString(new
File(getAbsoluteFileName(_schemaFileName))), headers);
return true;
} catch (IOException e) {
LOGGER.error("Failed to create schema with file: {}", _schemaFileName,
e);
@@ -120,7 +120,7 @@ public class TableOp extends BaseOp {
try {
ControllerTest
.sendPostRequest(ControllerRequestURLBuilder.baseUrl(ClusterDescriptor.CONTROLLER_URL).forTableCreate(),
- FileUtils.readFileToString(new File(_tableConfigFileName)));
+ FileUtils.readFileToString(new
File(getAbsoluteFileName(_tableConfigFileName))));
return true;
} catch (IOException e) {
LOGGER.error("Failed to create table with file: {}",
_tableConfigFileName, e);
@@ -130,7 +130,7 @@ public class TableOp extends BaseOp {
private boolean deleteTable() {
try {
- TableConfig tableConfig = JsonUtils.fileToObject(new
File(_tableConfigFileName), TableConfig.class);
+ TableConfig tableConfig = JsonUtils.fileToObject(new
File(getAbsoluteFileName(_tableConfigFileName)), TableConfig.class);
ControllerTest.sendDeleteRequest(ControllerRequestURLBuilder.baseUrl(ClusterDescriptor.CONTROLLER_URL)
.forTableDelete(tableConfig.getTableName()));
return true;
diff --git
a/pinot-integration-tests/src/test/resources/compat-tests/sample.yaml
b/pinot-integration-tests/src/test/resources/compat-tests/sample.yaml
deleted file mode 100644
index deca988..0000000
--- a/pinot-integration-tests/src/test/resources/compat-tests/sample.yaml
+++ /dev/null
@@ -1,57 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# Operations to be done.
-description: Operations to be run after Broker downgrade
-operations:
- - type: tableOp
- description: Create table t1
- op: CREATE
- schemaFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/FeatureTest1-schema.json
- tableConfigFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1.json
- - type: segmentOp
- description: Add segments X and Y to T1
- op: UPLOAD
- inputDataFileName:
pinot-integration-tests/src/test/resources/compat-tests/data/FeatureTest1-data-00.csv
- schemaFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/FeatureTest1-schema.json
- tableConfigFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1.json
- recordReaderConfigFileName:
pinot-integration-tests/src/test/resources/compat-tests/data/recordReaderConfig.json
- segmentName: FeatureTest1_Segment
- - type: segmentOp
- description: Delete segments X and Y to T1
- op: DELETE
- tableConfigFileName:
pinot-integration-tests/src/test/resources/compat-tests/configs/feature-test-1.json
- segmentName: FeatureTest1_Segment
- - type: tableOp
- # description: Remove table T1
- op: DELETE
- tableConfigFileName: /tmp/tableConfig
- - type: queryOp
- description: Run queries on T1
- queryFileName: queries/T1.sql
- expectedResultsFileName: queries/T1-results.sql
- - type: streamOp
- description: Produce rows to topic1
- op: PRODUCE
- numRows: 66
- streamConfigFileName: kafka/topic1.cfg
- inputDataFileName: kafka/input.csv
- tableConfigFileNames:
- - T1
- - T2
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
