This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3a787b6684 [improvement](regression) syncer regression test (#22490)
3a787b6684 is described below
commit 3a787b6684e3778011a827fb1d06a417c0ac9179
Author: DeadlineFen <[email protected]>
AuthorDate: Wed Aug 2 20:09:27 2023 +0800
[improvement](regression) syncer regression test (#22490)
---
regression-test/conf/regression-conf.groovy | 1 +
.../org/apache/doris/regression/Config.groovy | 20 ++++++++++++-
.../apache/doris/regression/ConfigOptions.groovy | 10 +++++++
.../org/apache/doris/regression/suite/Suite.groovy | 19 ++++++++++++
.../doris/regression/suite/SuiteContext.groovy | 34 ++++++++++++++++++++++
.../apache/doris/regression/suite/Syncer.groovy | 6 ++--
.../doris/regression/util/SyncerUtils.groovy | 8 +++--
.../ccr_syncer_p1/test_backup_restore.groovy | 9 +++---
8 files changed, 96 insertions(+), 11 deletions(-)
diff --git a/regression-test/conf/regression-conf.groovy
b/regression-test/conf/regression-conf.groovy
index fe06831e59..437a6e6340 100644
--- a/regression-test/conf/regression-conf.groovy
+++ b/regression-test/conf/regression-conf.groovy
@@ -31,6 +31,7 @@ jdbcPassword = ""
feSourceThriftAddress = "127.0.0.1:9020"
feTargetThriftAddress = "127.0.0.1:9020"
+syncerAddress = "127.0.0.1:9190"
feSyncerUser = "root"
feSyncerPassword = ""
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
index 19dce1e9f5..71c2959d3a 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
@@ -45,6 +45,7 @@ class Config {
public String feTargetThriftAddress
public String feSyncerUser
public String feSyncerPassword
+ public String syncerAddress
public String feHttpAddress
public String feHttpUser
@@ -84,6 +85,7 @@ class Config {
public TNetworkAddress feSourceThriftNetworkAddress
public TNetworkAddress feTargetThriftNetworkAddress
+ public TNetworkAddress syncerNetworkAddress
public InetSocketAddress feHttpInetSocketAddress
public InetSocketAddress metaServiceHttpInetSocketAddress
public Integer parallel
@@ -96,7 +98,7 @@ class Config {
Config(String defaultDb, String jdbcUrl, String jdbcUser, String
jdbcPassword,
String feSourceThriftAddress, String feTargetThriftAddress, String
feSyncerUser, String feSyncerPassword,
- String feHttpAddress, String feHttpUser, String feHttpPassword,
String metaServiceHttpAddress,
+ String syncerPassword, String feHttpAddress, String feHttpUser,
String feHttpPassword, String metaServiceHttpAddress,
String suitePath, String dataPath, String realDataPath, String
cacheDataPath, Boolean enableCacheData,
String testGroups, String excludeGroups, String testSuites, String
excludeSuites,
String testDirectories, String excludeDirectories, String
pluginPath, String sslCertificatePath) {
@@ -108,6 +110,7 @@ class Config {
this.feTargetThriftAddress = feTargetThriftAddress
this.feSyncerUser = feSyncerUser
this.feSyncerPassword = feSyncerPassword
+ this.syncerAddress = syncerAddress
this.feHttpAddress = feHttpAddress
this.feHttpUser = feHttpUser
this.feHttpPassword = feHttpPassword
@@ -208,6 +211,15 @@ class Config {
throw new IllegalStateException("Can not parse fe thrift address:
${config.feTargetThriftAddress}", t)
}
+ config.syncerAddress = cmd.getOptionValue(syncerAddressOpt,
config.syncerAddress)
+ try {
+ String host = config.syncerAddress.split(":")[0]
+ int port = Integer.valueOf(config.syncerAddress.split(":")[1])
+ config.syncerNetworkAddress = new TNetworkAddress(host, port)
+ } catch (Throwable t) {
+ throw new IllegalStateException("Can not parse syncer address:
${config.syncerAddress}", t)
+ }
+
config.feHttpAddress = cmd.getOptionValue(feHttpAddressOpt,
config.feHttpAddress)
try {
Inet4Address host =
Inet4Address.getByName(config.feHttpAddress.split(":")[0]) as Inet4Address
@@ -270,6 +282,7 @@ class Config {
configToString(obj.feTargetThriftAddress),
configToString(obj.feSyncerUser),
configToString(obj.feSyncerPassword),
+ configToString(obj.syncerAddress),
configToString(obj.feHttpAddress),
configToString(obj.feHttpUser),
configToString(obj.feHttpPassword),
@@ -354,6 +367,11 @@ class Config {
log.info("Set feSyncerPassword to empty because not
specify.".toString())
}
+ if (config.syncerAddress == null) {
+ config.syncerAddress = "127.0.0.1:9190"
+ log.info("Set syncerAddress to '${config.syncerAddress}' because
not specify.".toString())
+ }
+
if (config.feHttpUser == null) {
config.feHttpUser = "root"
log.info("Set feHttpUser to '${config.feHttpUser}' because not
specify.".toString())
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
index 3f8824c38b..2f32415f76 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
@@ -36,6 +36,7 @@ class ConfigOptions {
static Option feTargetThriftAddressOpt
static Option feSyncerUserOpt
static Option feSyncerPasswordOpt
+ static Option syncerAddressOpt
static Option feHttpAddressOpt
static Option feHttpUserOpt
static Option feHttpPasswordOpt
@@ -249,6 +250,14 @@ class ConfigOptions {
.longOpt("feSyncerPassword")
.desc("the password of syncer")
.build()
+ syncerAddressOpt = Option.builder("sa")
+ .argName("address")
+ .required(false)
+ .hasArg(true)
+ .type(String.class)
+ .longOpt("syncerAddressOpt")
+ .desc("the syncer address, format is ip:port")
+ .build()
feHttpAddressOpt = Option.builder("ha")
.argName("address")
.required(false)
@@ -375,6 +384,7 @@ class ConfigOptions {
.addOption(feTargetThriftAddressOpt)
.addOption(feSyncerUserOpt)
.addOption(feSyncerPasswordOpt)
+ .addOption(syncerAddressOpt)
.addOption(feHttpAddressOpt)
.addOption(feHttpUserOpt)
.addOption(feHttpPasswordOpt)
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index bd86471780..464849cb22 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -17,9 +17,11 @@
package org.apache.doris.regression.suite
+import com.google.common.collect.Maps
import com.google.common.util.concurrent.Futures
import com.google.common.util.concurrent.ListenableFuture
import com.google.common.util.concurrent.MoreExecutors
+import com.google.gson.Gson
import groovy.json.JsonSlurper
import com.google.common.collect.ImmutableList
import org.apache.doris.regression.action.BenchmarkAction
@@ -190,6 +192,23 @@ class Suite implements GroovyInterceptable {
return context.connect(user, password, url, actionSupplier)
}
+ String get_ccr_body(String table) {
+ Gson gson = new Gson()
+
+ Map<String, String> srcSpec = context.getSrcSpec()
+ srcSpec.put("table", table)
+
+ Map<String, String> destSpec = context.getDestSpec()
+ destSpec.put("table", table)
+
+ Map<String, Object> body = Maps.newHashMap()
+ body.put("name", context.suiteName + "_" + context.dbName + "_" +
table)
+ body.put("src", srcSpec)
+ body.put("dest", destSpec)
+
+ return gson.toJson(body)
+ }
+
Syncer getSyncer() {
return context.getSyncer(this)
}
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy
index 1eec3d9c15..469fbc6ba4 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy
@@ -17,6 +17,7 @@
package org.apache.doris.regression.suite
+import com.google.common.collect.Maps
import groovy.transform.CompileStatic
import org.apache.doris.regression.Config
import org.apache.doris.regression.util.OutputUtils
@@ -126,6 +127,39 @@ class SuiteContext implements Closeable {
return threadConn
}
+ private String getJdbcNetInfo() {
+ String subJdbc =
config.jdbcUrl.substring(config.jdbcUrl.indexOf("://") + 3)
+ return subJdbc.substring(0, subJdbc.indexOf("/"))
+ }
+
+ private Map<String, String> getSpec() {
+ Map<String, String> spec = Maps.newHashMap()
+ String[] jdbc = getJdbcNetInfo().split(":")
+ spec.put("host", jdbc[0])
+ spec.put("port", jdbc[1])
+ spec.put("user", config.feSyncerUser)
+ spec.put("password", config.feSyncerPassword)
+ spec.put("cluster", "")
+
+ return spec
+ }
+
+ Map<String, String> getSrcSpec() {
+ Map<String, String> spec = getSpec()
+ spec.put("thrift_port",
config.feSourceThriftNetworkAddress.port.toString())
+ spec.put("database", dbName)
+
+ return spec
+ }
+
+ Map<String, String> getDestSpec() {
+ Map<String, String> spec = getSpec()
+ spec.put("thrift_port",
config.feTargetThriftNetworkAddress.port.toString())
+ spec.put("database", "TEST_" + dbName)
+
+ return spec
+ }
+
Connection getTargetConnection(Suite suite) {
def context = getSyncer(suite).context
if (context.targetConnection == null) {
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
index 9504f2a874..15bfec72a8 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
@@ -351,7 +351,7 @@ class Syncer {
}
Boolean checkRestoreFinish() {
- String checkSQL = "SHOW RESTORE FROM " + context.db
+ String checkSQL = "SHOW RESTORE FROM TEST_" + context.db
List<Object> row = suite.sql(checkSQL)[0]
logger.info("Now row is ${row}")
@@ -455,7 +455,7 @@ class Syncer {
return checkGetMasterToken(result)
}
- Boolean restoreSnapshot() {
+ Boolean restoreSnapshot(boolean forCCR = false) {
logger.info("Restore snapshot ${context.labelName}")
FrontendClientImpl clientImpl = context.getSourceFrontClient()
@@ -474,7 +474,7 @@ class Syncer {
context.getSnapshotResult.setJobInfo(gson.toJson(jsonMap).getBytes())
// step 2: restore
- TRestoreSnapshotResult result =
SyncerUtils.restoreSnapshot(clientImpl, context)
+ TRestoreSnapshotResult result =
SyncerUtils.restoreSnapshot(clientImpl, context, forCCR)
return checkRestoreSnapshot(result)
}
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/SyncerUtils.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/SyncerUtils.groovy
index dbcd1dba5d..3bb7be40e5 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/SyncerUtils.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/SyncerUtils.groovy
@@ -100,10 +100,14 @@ class SyncerUtils {
return clientImpl.client.getSnapshot(request)
}
- static TRestoreSnapshotResult restoreSnapshot(FrontendClientImpl
clientImpl, SyncerContext context) throws TException {
+ static TRestoreSnapshotResult restoreSnapshot(FrontendClientImpl
clientImpl, SyncerContext context, boolean forCCR) throws TException {
TRestoreSnapshotRequest request = new TRestoreSnapshotRequest()
setAuthorInformation(request, context)
- request.setDb(context.db)
+ if (forCCR) {
+ request.setDb("TEST_" + context.db)
+ } else {
+ request.setDb(context.db)
+ }
if (context.tableName != null) {
request.setTable(context.tableName)
}
diff --git a/regression-test/suites/ccr_syncer_p1/test_backup_restore.groovy
b/regression-test/suites/ccr_syncer_p1/test_backup_restore.groovy
index e5cc5e9d08..0d37a4b832 100644
--- a/regression-test/suites/ccr_syncer_p1/test_backup_restore.groovy
+++ b/regression-test/suites/ccr_syncer_p1/test_backup_restore.groovy
@@ -33,10 +33,10 @@ suite("test_backup_restore") {
UNIQUE KEY(`test`, `id`)
DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES (
- "replication_allocation" = "tag.location.default: 1"
+ "replication_allocation" = "tag.location.default: 1",
+ "binlog.enable" = "true"
)
"""
- sql """ALTER TABLE ${tableName} set ("binlog.enable" = "true")"""
logger.info("=== Test 1: Common backup and restore ===")
test_num = 1
@@ -58,11 +58,10 @@ suite("test_backup_restore") {
Thread.sleep(3000)
}
assertTrue(syncer.getSnapshot("${snapshotName}", "${tableName}"))
- sql "DROP TABLE IF EXISTS ${tableName}"
- assertTrue(syncer.restoreSnapshot())
+ assertTrue(syncer.restoreSnapshot(true))
while (syncer.checkRestoreFinish() == false) {
Thread.sleep(3000)
}
- res = sql "SELECT * FROM ${tableName}"
+ res = target_sql "SELECT * FROM ${tableName}"
assertTrue(res.size() == insert_num)
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
