This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 5f6d0614cf6 branch-3.1: [feature](encryption) Add metadata for
transparent encryption #52771 (#52755)
5f6d0614cf6 is described below
commit 5f6d0614cf616e9ed214fcb7afbb923579c6d3c9
Author: Luwei <[email protected]>
AuthorDate: Fri Jul 11 19:11:14 2025 +0800
branch-3.1: [feature](encryption) Add metadata for transparent encryption
#52771 (#52755)
pick from #52771
---
.../main/java/org/apache/doris/catalog/Env.java | 16 ++++++++++
.../org/apache/doris/encryption/KeyManager.java | 37 ++++++++++++++++++++++
.../org/apache/doris/journal/JournalEntity.java | 6 ++++
.../java/org/apache/doris/persist/EditLog.java | 4 +++
.../org/apache/doris/persist/KeyOperationInfo.java | 37 ++++++++++++++++++++++
.../org/apache/doris/persist/OperationType.java | 2 ++
.../doris/persist/meta/MetaPersistMethod.java | 6 ++++
.../doris/persist/meta/PersistMetaModules.java | 2 +-
8 files changed, 109 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index d83013a5ab9..0a5be5fe11b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -147,6 +147,7 @@ import org.apache.doris.deploy.DeployManager;
import org.apache.doris.deploy.impl.AmbariDeployManager;
import org.apache.doris.deploy.impl.K8sDeployManager;
import org.apache.doris.deploy.impl.LocalFileDeployManager;
+import org.apache.doris.encryption.KeyManager;
import org.apache.doris.event.EventProcessor;
import org.apache.doris.event.ReplacePartitionEvent;
import org.apache.doris.ha.BDBHA;
@@ -586,6 +587,8 @@ public class Env {
private TokenManager tokenManager;
+ private KeyManager keyManager;
+
// if a config is relative to a daemon thread. record the relation here.
we will proactively change interval of it.
private final Map<String, Supplier<MasterDaemon>> configtoThreads =
ImmutableMap
.of("dynamic_partition_check_interval_seconds",
this::getDynamicPartitionScheduler);
@@ -836,6 +839,7 @@ public class Env {
this.splitSourceManager = new SplitSourceManager();
this.globalExternalTransactionInfoMgr = new
GlobalExternalTransactionInfoMgr();
this.tokenManager = new TokenManager();
+ this.keyManager = new KeyManager();
}
public static Map<String, Long> getSessionReportTimeMap() {
@@ -2502,6 +2506,12 @@ public class Env {
return checksum;
}
+ public long loadKeyManager(DataInputStream in, long checksum) throws
IOException {
+ this.keyManager = KeyManager.read(in);
+ LOG.info("finished replay KeyManager from image");
+ return checksum;
+ }
+
public long saveInsertOverwrite(CountingDataOutputStream out, long
checksum) throws IOException {
this.insertOverwriteManager.write(out);
LOG.info("finished save iot to image");
@@ -2785,6 +2795,12 @@ public class Env {
return checksum;
}
+ public long saveKeyManager(CountingDataOutputStream out, long checksum)
throws IOException {
+ this.keyManager.write(out);
+ LOG.info("finished save KeyManager to image");
+ return checksum;
+ }
+
public void createLabelCleaner() {
labelCleaner = new MasterDaemon("LoadLabelCleaner",
Config.label_clean_interval_second * 1000L) {
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/encryption/KeyManager.java
b/fe/fe-core/src/main/java/org/apache/doris/encryption/KeyManager.java
new file mode 100644
index 00000000000..1a2bb90c497
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/encryption/KeyManager.java
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.encryption;
+
+import org.apache.doris.common.io.Text;
+import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class KeyManager implements Writable {
+ @Override
+ public void write(DataOutput out) throws IOException {
+ Text.writeString(out, GsonUtils.GSON.toJson(this));
+ }
+
+ public static KeyManager read(DataInput in) throws IOException {
+ return GsonUtils.GSON.fromJson(Text.readString(in), KeyManager.class);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
index 21a3a6f849f..22e2641cea3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
@@ -95,6 +95,7 @@ import org.apache.doris.persist.DropWorkloadGroupOperationLog;
import org.apache.doris.persist.DropWorkloadSchedPolicyOperatorLog;
import org.apache.doris.persist.GlobalVarPersistInfo;
import org.apache.doris.persist.HbPackage;
+import org.apache.doris.persist.KeyOperationInfo;
import org.apache.doris.persist.LdapInfo;
import org.apache.doris.persist.ModifyCommentOperationLog;
import org.apache.doris.persist.ModifyPartitionInfo;
@@ -979,6 +980,11 @@ public class JournalEntity implements Writable {
isRead = true;
break;
}
+ case OperationType.OP_OPERATE_KEY: {
+ data = KeyOperationInfo.read(in);
+ isRead = true;
+ break;
+ }
default: {
IOException e = new IOException();
LOG.error("UNKNOWN Operation Type {}", opCode, e);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index be28a322e3a..731252eb211 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -1281,6 +1281,10 @@ public class EditLog {
}
break;
}
+ case OperationType.OP_OPERATE_KEY: {
+ //KeyOperationInfo info = (KeyOperationInfo)
journal.getData();
+ break;
+ }
default: {
IOException e = new IOException();
LOG.error("UNKNOWN Operation Type {}, log id: {}", opCode,
logId, e);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/KeyOperationInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/KeyOperationInfo.java
new file mode 100644
index 00000000000..77c9e36a44a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/KeyOperationInfo.java
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.persist;
+
+import org.apache.doris.common.io.Text;
+import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class KeyOperationInfo implements Writable {
+ @Override
+ public void write(DataOutput out) throws IOException {
+ Text.writeString(out, GsonUtils.GSON.toJson(this));
+ }
+
+ public static KeyOperationInfo read(DataInput in) throws IOException {
+ return GsonUtils.GSON.fromJson(Text.readString(in),
KeyOperationInfo.class);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
index 78da9ec14a5..455ff56589d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
@@ -420,6 +420,8 @@ public class OperationType {
public static final short OP_CREATE_INDEX_POLICY = 490;
public static final short OP_DROP_INDEX_POLICY = 491;
+ public static final short OP_OPERATE_KEY = 492;
+
// For cloud.
public static final short OP_UPDATE_CLOUD_REPLICA = 1000;
@Deprecated
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java
index ae4ae9c26b8..18a934cdde3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java
@@ -260,6 +260,12 @@ public class MetaPersistMethod {
metaPersistMethod.writeMethod =
Env.class.getDeclaredMethod("savePlsqlProcedure",
CountingDataOutputStream.class, long.class);
break;
+ case "KeyManager":
+ metaPersistMethod.readMethod =
Env.class.getDeclaredMethod("loadKeyManager", DataInputStream.class,
+ long.class);
+ metaPersistMethod.writeMethod =
Env.class.getDeclaredMethod("saveKeyManager",
+ CountingDataOutputStream.class, long.class);
+ break;
default:
break;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java
index 08b60b309ca..445a1a1a079 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java
@@ -42,7 +42,7 @@ public class PersistMetaModules {
"paloAuth", "transactionState", "colocateTableIndex",
"routineLoadJobs", "loadJobV2", "smallFiles",
"plugins", "deleteHandler", "sqlBlockRule", "policy",
"globalFunction", "workloadGroups",
"binlogs", "resourceGroups", "AnalysisMgrV2", "AsyncJobManager",
"workloadSchedPolicy",
- "insertOverwrite", "plsql", "indexPolicy");
+ "insertOverwrite", "plsql", "indexPolicy", "KeyManager");
// The modules in `CloudEnv`.
public static final ImmutableList<String> CLOUD_MODULE_NAMES =
ImmutableList.of("cloudWarmUpJob");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
