This is an automated email from the ASF dual-hosted git repository.
w41ter pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 22e748432c3 [feat](snapshot) Support storage vault for create/list
snapshot (#62523)
22e748432c3 is described below
commit 22e748432c3c1678b4951434067f6957eaf7f0b7
Author: Yixuan Wang <[email protected]>
AuthorDate: Wed May 13 10:42:38 2026 +0800
[feat](snapshot) Support storage vault for create/list snapshot (#62523)
---
.../schema_cluster_snapshots_scanner.cpp | 14 +++++++
.../schema_cluster_snapshots_scanner_test.cpp | 5 +++
docker/runtime/doris-compose/cluster.py | 24 +++++++++--
docker/runtime/doris-compose/command.py | 5 ++-
docker/runtime/doris-compose/resource/common.sh | 46 +++++++++++++++-------
.../java/org/apache/doris/catalog/SchemaTable.java | 1 +
.../doris/cloud/snapshot/CloudSnapshotHandler.java | 2 +-
.../AdminCreateClusterSnapshotCommand.java | 9 ++++-
.../AdminCreateClusterSnapshotCommandTest.java | 3 ++
gensrc/proto/cloud.proto | 2 +
.../doris/regression/suite/SuiteCluster.groovy | 12 +++++-
11 files changed, 101 insertions(+), 22 deletions(-)
diff --git a/be/src/information_schema/schema_cluster_snapshots_scanner.cpp
b/be/src/information_schema/schema_cluster_snapshots_scanner.cpp
index d9bca21ce90..df1e24c8352 100644
--- a/be/src/information_schema/schema_cluster_snapshots_scanner.cpp
+++ b/be/src/information_schema/schema_cluster_snapshots_scanner.cpp
@@ -46,6 +46,7 @@ std::vector<SchemaScanner::ColumnDesc>
SchemaClusterSnapshotsScanner::_s_tbls_co
{"LABEL", TYPE_STRING, sizeof(StringRef), true},
{"MSG", TYPE_STRING, sizeof(StringRef), true},
{"COUNT", TYPE_INT, sizeof(int32_t), true},
+ {"VAULT_ID", TYPE_STRING, sizeof(StringRef), true},
};
SchemaClusterSnapshotsScanner::SchemaClusterSnapshotsScanner()
@@ -246,6 +247,19 @@ Status
SchemaClusterSnapshotsScanner::_fill_block_impl(Block* block) {
}
RETURN_IF_ERROR(fill_dest_column_for_range(block, 11, datas));
}
+ // resource_id
+ {
+ for (int i = 0; i < row_num; ++i) {
+ auto& snapshot = _snapshots[i];
+ if (snapshot.has_resource_id()) {
+ strs[i] = StringRef(snapshot.resource_id().c_str(),
snapshot.resource_id().size());
+ datas[i] = strs.data() + i;
+ } else {
+ datas[i] = nullptr;
+ }
+ }
+ RETURN_IF_ERROR(fill_dest_column_for_range(block, 12, datas));
+ }
return Status::OK();
}
diff --git
a/be/test/exec/schema_scanner/schema_cluster_snapshots_scanner_test.cpp
b/be/test/exec/schema_scanner/schema_cluster_snapshots_scanner_test.cpp
index 6415f837a7c..f1392dcb831 100644
--- a/be/test/exec/schema_scanner/schema_cluster_snapshots_scanner_test.cpp
+++ b/be/test/exec/schema_scanner/schema_cluster_snapshots_scanner_test.cpp
@@ -49,6 +49,7 @@ TEST_F(SchemaClusterSnapshotsScannerTest,
test_get_next_block_internal) {
snapshot.set_ttl_seconds(3600);
snapshot.set_snapshot_label("label");
snapshot.set_reason("reason");
+ snapshot.set_resource_id("vault_1");
snapshots.push_back(snapshot);
}
@@ -62,6 +63,10 @@ TEST_F(SchemaClusterSnapshotsScannerTest,
test_get_next_block_internal) {
auto col = data_block->safe_get_by_position(0);
auto v = (*col.column)[1].get<TYPE_STRING>();
EXPECT_EQ(v, "232ds");
+
+ auto vault_col = data_block->safe_get_by_position(12);
+ auto vault_id = (*vault_col.column)[1].get<TYPE_STRING>();
+ EXPECT_EQ(vault_id, "vault_1");
}
} // namespace doris
diff --git a/docker/runtime/doris-compose/cluster.py
b/docker/runtime/doris-compose/cluster.py
index d05556646cb..55ca5664175 100644
--- a/docker/runtime/doris-compose/cluster.py
+++ b/docker/runtime/doris-compose/cluster.py
@@ -64,6 +64,20 @@ CLUSTER_ID = "12345678"
LOG = utils.get_logger()
+def is_true(value):
+ return str(value).strip().lower() in ("1", "true", "yes", "y", "on")
+
+
+def get_env_value(envs, name):
+ for env in envs or []:
+ pos = env.find('=')
+ if pos == -1:
+ continue
+ if env[:pos] == name:
+ return env[pos + 1:]
+ return None
+
+
def get_cluster_path(cluster_name):
return os.path.join(LOCAL_DORIS_PATH, cluster_name)
@@ -397,6 +411,7 @@ class Node(object):
"STOP_GRACE": 1 if enable_coverage else 0,
"IS_CLOUD": 1 if self.cluster.is_cloud else 0,
"SQL_MODE_NODE_MGR": 1 if self.cluster.sql_mode_node_mgr else 0,
+ "ENABLE_STORAGE_VAULT": 1 if getattr(self.cluster,
"enable_storage_vault", False) else 0,
"TDE_AK": self.get_tde_ak(),
"TDE_SK": self.get_tde_sk(),
}
@@ -915,7 +930,8 @@ class Cluster(object):
local_network_ip, fe_follower, be_disks, be_cluster, reg_be,
extra_hosts, env, coverage_dir, cloud_store_config,
sql_mode_node_mgr, be_metaservice_endpoint, be_cluster_id,
tde_ak, tde_sk,
- external_ms_cluster, instance_id, cluster_snapshot=""):
+ external_ms_cluster, instance_id, cluster_snapshot="",
+ enable_storage_vault=False):
self.name = name
self.subnet = subnet
self.image = image
@@ -941,6 +957,7 @@ class Cluster(object):
self.instance_id = f"instance_{name}" if self.external_ms_cluster
else "default_instance_id"
# cluster_snapshot is not persisted to meta, only used during cluster
creation
self.cluster_snapshot = cluster_snapshot
+ self.enable_storage_vault = is_true(enable_storage_vault)
self.is_rollback = False
self.groups = {
node_type: Group(node_type)
@@ -961,7 +978,8 @@ class Cluster(object):
fe_follower, be_disks, be_cluster, reg_be, extra_hosts, env,
coverage_dir, cloud_store_config, sql_mode_node_mgr,
be_metaservice_endpoint, be_cluster_id, tde_ak, tde_sk,
- external_ms_cluster, instance_id, cluster_snapshot=""):
+ external_ms_cluster, instance_id, cluster_snapshot="",
+ enable_storage_vault=False):
if not os.path.exists(LOCAL_DORIS_PATH):
os.makedirs(LOCAL_DORIS_PATH, exist_ok=True)
os.chmod(LOCAL_DORIS_PATH, 0o777)
@@ -977,7 +995,7 @@ class Cluster(object):
coverage_dir, cloud_store_config,
sql_mode_node_mgr, be_metaservice_endpoint,
be_cluster_id, tde_ak, tde_sk,
external_ms_cluster,
- instance_id, cluster_snapshot)
+ instance_id, cluster_snapshot,
enable_storage_vault)
os.makedirs(cluster.get_path(), exist_ok=True)
os.makedirs(get_status_path(name), exist_ok=True)
cluster._save_meta()
diff --git a/docker/runtime/doris-compose/command.py
b/docker/runtime/doris-compose/command.py
index 75618eec009..14f3158a96c 100644
--- a/docker/runtime/doris-compose/command.py
+++ b/docker/runtime/doris-compose/command.py
@@ -669,6 +669,8 @@ class UpCommand(Command):
instance_id = getattr(args, 'instance_id', None)
cluster_snapshot = getattr(args, 'cluster_snapshot', '')
+ enable_storage_vault = CLUSTER.is_true(
+ CLUSTER.get_env_value(args.env, "ENABLE_STORAGE_VAULT"))
cluster = CLUSTER.Cluster.new(
args.NAME, args.IMAGE, args.cloud, args.root, args.fe_config,
@@ -677,7 +679,8 @@ class UpCommand(Command):
args.be_disks if args.be_disks is not None else ["HDD=1"],
args.be_cluster, args.reg_be, args.extra_hosts, args.env,
args.coverage_dir, cloud_store_config, args.sql_mode_node_mgr,
args.be_metaservice_endpoint, args.be_cluster_id, args.tde_ak,
args.tde_sk,
- external_ms_cluster, instance_id, cluster_snapshot)
+ external_ms_cluster, instance_id, cluster_snapshot,
+ enable_storage_vault)
LOG.info("Create new cluster {} succ, cluster path is {}".format(
args.NAME, cluster.get_path()))
diff --git a/docker/runtime/doris-compose/resource/common.sh
b/docker/runtime/doris-compose/resource/common.sh
index e05b46c2d81..cc1d43eb806 100644
--- a/docker/runtime/doris-compose/resource/common.sh
+++ b/docker/runtime/doris-compose/resource/common.sh
@@ -153,20 +153,38 @@ create_doris_instance() {
lock_cluster
- output=$(curl -s
"${META_SERVICE_ENDPOINT}/MetaService/http/create_instance?token=greedisgood9999"
\
- -d '{"instance_id":"'"${INSTANCE_ID}"'",
- "name": "'"${INSTANCE_ID}"'",
- "user_id": "'"${DORIS_CLOUD_USER}"'",
- "obj_info": {
- "ak": "'"${DORIS_CLOUD_AK}"'",
- "sk": "'"${DORIS_CLOUD_SK}"'",
- "bucket": "'"${DORIS_CLOUD_BUCKET}"'",
- "endpoint": "'"${DORIS_CLOUD_ENDPOINT}"'",
- "external_endpoint":
"'"${DORIS_CLOUD_EXTERNAL_ENDPOINT}"'",
- "prefix": "'"${DORIS_CLOUD_PREFIX}"'",
- "region": "'"${DORIS_CLOUD_REGION}"'",
- "provider": "'"${DORIS_CLOUD_PROVIDER}"'"
- }}')
+ if [[ "${ENABLE_STORAGE_VAULT}" =~
^([Tt][Rr][Uu][Ee]|[Yy][Ee][Ss]|[Yy]|[Oo][Nn]|1)$ ]]; then
+ output=$(curl -s
"${META_SERVICE_ENDPOINT}/MetaService/http/create_instance?token=greedisgood9999"
\
+ -d '{"instance_id":"'"${INSTANCE_ID}"'",
+ "name": "'"${INSTANCE_ID}"'",
+ "user_id": "'"${DORIS_CLOUD_USER}"'",
+ "vault": {
+ "obj_info": {
+ "ak": "'"${DORIS_CLOUD_AK}"'",
+ "sk": "'"${DORIS_CLOUD_SK}"'",
+ "bucket": "'"${DORIS_CLOUD_BUCKET}"'",
+ "endpoint": "'"${DORIS_CLOUD_ENDPOINT}"'",
+ "external_endpoint":
"'"${DORIS_CLOUD_EXTERNAL_ENDPOINT}"'",
+ "prefix": "'"${DORIS_CLOUD_PREFIX}"'",
+ "region": "'"${DORIS_CLOUD_REGION}"'",
+ "provider": "'"${DORIS_CLOUD_PROVIDER}"'"
+ }}}')
+ else
+ output=$(curl -s
"${META_SERVICE_ENDPOINT}/MetaService/http/create_instance?token=greedisgood9999"
\
+ -d '{"instance_id":"'"${INSTANCE_ID}"'",
+ "name": "'"${INSTANCE_ID}"'",
+ "user_id": "'"${DORIS_CLOUD_USER}"'",
+ "obj_info": {
+ "ak": "'"${DORIS_CLOUD_AK}"'",
+ "sk": "'"${DORIS_CLOUD_SK}"'",
+ "bucket": "'"${DORIS_CLOUD_BUCKET}"'",
+ "endpoint": "'"${DORIS_CLOUD_ENDPOINT}"'",
+ "external_endpoint":
"'"${DORIS_CLOUD_EXTERNAL_ENDPOINT}"'",
+ "prefix": "'"${DORIS_CLOUD_PREFIX}"'",
+ "region": "'"${DORIS_CLOUD_REGION}"'",
+ "provider": "'"${DORIS_CLOUD_PROVIDER}"'"
+ }}')
+ fi
unlock_cluster
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
index 6b8b7b4d9d4..343949eeee3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
@@ -830,6 +830,7 @@ public class SchemaTable extends Table {
.column("LABEL", ScalarType.createStringType())
.column("MSG", ScalarType.createStringType())
.column("COUNT",
ScalarType.createType(PrimitiveType.INT))
+ .column("VAULT_ID", ScalarType.createStringType())
.build()))
.put("cluster_snapshot_properties",
new SchemaTable(SystemIdGenerator.getNextId(),
"cluster_snapshot_properties", TableType.SCHEMA,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/snapshot/CloudSnapshotHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/snapshot/CloudSnapshotHandler.java
index dad391cc948..0b6dc996df5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/snapshot/CloudSnapshotHandler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/snapshot/CloudSnapshotHandler.java
@@ -62,7 +62,7 @@ public class CloudSnapshotHandler extends MasterDaemon {
// do nothing
}
- public void submitJob(long ttl, String label) throws Exception {
+ public void submitJob(long ttl, String label, String vaultName) throws
Exception {
throw new NotImplementedException("submitJob is not implemented");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCreateClusterSnapshotCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCreateClusterSnapshotCommand.java
index 425b6bb2176..c7b9cff0cd0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCreateClusterSnapshotCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCreateClusterSnapshotCommand.java
@@ -45,11 +45,13 @@ public class AdminCreateClusterSnapshotCommand extends
Command implements Forwar
public static final String PROP_TTL = "ttl";
public static final String PROP_LABEL = "label";
+ public static final String PROP_VAULT_NAME = "vault_name";
private static final Logger LOG =
LogManager.getLogger(AdminCreateClusterSnapshotCommand.class);
private Map<String, String> properties;
private long ttl;
private String label = null;
+ private String vaultName = null;
/**
* AdminCreateClusterSnapshotCommand
@@ -64,7 +66,7 @@ public class AdminCreateClusterSnapshotCommand extends
Command implements Forwar
public void run(ConnectContext ctx, StmtExecutor executor) throws
Exception {
validate(ctx);
CloudSnapshotHandler cloudSnapshotHandler = ((CloudEnv)
ctx.getEnv()).getCloudSnapshotHandler();
- cloudSnapshotHandler.submitJob(ttl, label);
+ cloudSnapshotHandler.submitJob(ttl, label, vaultName);
}
/**
@@ -106,6 +108,11 @@ public class AdminCreateClusterSnapshotCommand extends
Command implements Forwar
if (label == null || label.isEmpty()) {
throw new AnalysisException("Property 'label' cannot be
empty");
}
+ } else if (entry.getKey().equalsIgnoreCase(PROP_VAULT_NAME)) {
+ vaultName = entry.getValue();
+ if (vaultName == null || vaultName.isEmpty()) {
+ throw new AnalysisException("Property 'vault_name' cannot
be empty");
+ }
} else {
throw new AnalysisException("Unknown property: " +
entry.getKey());
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminCreateClusterSnapshotCommandTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminCreateClusterSnapshotCommandTest.java
index fccbd73e683..0beae8a01a2 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminCreateClusterSnapshotCommandTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AdminCreateClusterSnapshotCommandTest.java
@@ -97,10 +97,13 @@ public class AdminCreateClusterSnapshotCommandTest {
properties.add(Pair.of(ImmutableMap.of("ttl", "a", "label", "a"),
"Invalid value"));
properties.add(Pair.of(ImmutableMap.of("ttl", "0", "label", "a"),
"Property 'ttl' must be positive"));
properties.add(Pair.of(ImmutableMap.of("ttl", "3600", "label", ""),
"Property 'label' cannot be empty"));
+ properties.add(Pair.of(ImmutableMap.of("ttl", "3600", "label", "a",
"vault_name", ""),
+ "Property 'vault_name' cannot be empty"));
// unknown property
properties.add(Pair.of(ImmutableMap.of("ttl", "0", "a", "b"), "Unknown
property"));
// normal case
properties.add(Pair.of(ImmutableMap.of("ttl", "3600", "label", "abc"),
""));
+ properties.add(Pair.of(ImmutableMap.of("ttl", "3600", "label", "abc",
"vault_name", "vault_1"), ""));
for (Pair<Map<String, String>, String> entry : properties) {
AdminCreateClusterSnapshotCommand command0 = new
AdminCreateClusterSnapshotCommand(entry.first);
diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto
index d16f30cb53f..bff85eee942 100644
--- a/gensrc/proto/cloud.proto
+++ b/gensrc/proto/cloud.proto
@@ -2110,6 +2110,7 @@ message BeginSnapshotRequest {
optional int64 timeout_seconds = 4;
optional int64 ttl_seconds = 5;
optional string request_ip = 6;
+ optional string vault_name = 7;
}
message BeginSnapshotResponse {
@@ -2178,6 +2179,7 @@ message SnapshotInfoPB {
optional int64 snapshot_logical_data_size = 17;
optional int64 snapshot_retained_data_size = 18;
optional int64 snapshot_billable_data_size = 19;
+ optional string resource_id = 20;
}
message ListSnapshotRequest {
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
index f37ff665f26..ef5282eb218 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
@@ -117,6 +117,10 @@ class ClusterOptions {
// Example: clusterSnapshot = '{"cloud_unique_id":"1:instance_id:xxx"}'
String clusterSnapshot = null;
+ // Create cloud instance in storage-vault mode instead of legacy obj_info
mode.
+ // Docker framework will also create a default storage vault automatically
for new clusters.
+ Boolean enableStorageVault = false;
+
void enableDebugPoints() {
feConfigs.add('enable_debug_points=true')
beConfigs.add('enable_debug_points=true')
@@ -375,9 +379,13 @@ class SuiteCluster {
cmd += ['--extra-hosts']
cmd += options.extraHosts
}
- if (!options.environments.isEmpty()) {
+ def envs = new ArrayList<String>(options.environments)
+ if (options.enableStorageVault) {
+ envs.add('ENABLE_STORAGE_VAULT=1')
+ }
+ if (!envs.isEmpty()) {
cmd += ['--env']
- cmd += options.environments
+ cmd += envs
}
if (!options.cloudStoreConfigs.isEmpty()) {
cmd += ['--cloud-config']
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
