This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 9fa94bbde96 [opt](catalog) cache the Configuration object
(#45433)(#45756) (#45758)
9fa94bbde96 is described below
commit 9fa94bbde96073150e3d43b7d09f1ce20f946616
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Sun Dec 22 08:01:00 2024 +0800
[opt](catalog) cache the Configuration object (#45433)(#45756) (#45758)
---
.../apache/doris/datasource/ExternalCatalog.java | 22 +++++++++++
.../datasource/hive/HiveMetaStoreClientHelper.java | 7 +---
.../doris/datasource/ExternalCatalogTest.java | 46 ++++++++++++++++++++--
3 files changed, 65 insertions(+), 10 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index ddbf7e5e4e6..e1d37b009e1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -146,6 +146,9 @@ public abstract class ExternalCatalog
protected MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache;
protected PreExecutionAuthenticator preExecutionAuthenticator;
+ private volatile Configuration cachedConf = null;
+ private byte[] confLock = new byte[0];
+
public ExternalCatalog() {
}
@@ -157,6 +160,20 @@ public abstract class ExternalCatalog
}
public Configuration getConfiguration() {
+ // build configuration is costly, so we cache it.
+ if (cachedConf != null) {
+ return cachedConf;
+ }
+ synchronized (confLock) {
+ if (cachedConf != null) {
+ return cachedConf;
+ }
+ cachedConf = buildConf();
+ return cachedConf;
+ }
+ }
+
+ private Configuration buildConf() {
Configuration conf =
DFSFileSystem.getHdfsConf(ifNotSetFallbackToSimpleAuth());
Map<String, String> catalogProperties =
catalogProperty.getHadoopProperties();
for (Map.Entry<String, String> entry : catalogProperties.entrySet()) {
@@ -402,6 +419,10 @@ public abstract class ExternalCatalog
this.convertedProperties = null;
}
+ synchronized (this.confLock) {
+ this.cachedConf = null;
+ }
+
refreshOnlyCatalogCache(invalidCache);
}
@@ -752,6 +773,7 @@ public abstract class ExternalCatalog
}
}
this.propLock = new byte[0];
+ this.confLock = new byte[0];
this.initialized = false;
setDefaultPropsIfMissing(true);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
index f6b6af2d2d2..f407284bf44 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
@@ -42,7 +42,6 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.security.authentication.AuthenticationConfig;
import org.apache.doris.common.security.authentication.HadoopAuthenticator;
import org.apache.doris.datasource.ExternalCatalog;
-import org.apache.doris.fs.remote.dfs.DFSFileSystem;
import org.apache.doris.thrift.TExprOpcode;
import com.google.common.base.Strings;
@@ -840,11 +839,7 @@ public class HiveMetaStoreClientHelper {
}
public static Configuration getConfiguration(HMSExternalTable table) {
- Configuration conf =
DFSFileSystem.getHdfsConf(table.getCatalog().ifNotSetFallbackToSimpleAuth());
- for (Map.Entry<String, String> entry :
table.getHadoopProperties().entrySet()) {
- conf.set(entry.getKey(), entry.getValue());
- }
- return conf;
+ return table.getCatalog().getConfiguration();
}
public static Optional<String> getSerdeProperty(Table table, String key) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
index 43348ca8a0e..70e5e5f37af 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
@@ -22,9 +22,12 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.FeMetaVersion;
+import org.apache.doris.common.io.Text;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.test.TestExternalCatalog;
-import org.apache.doris.mysql.privilege.Auth;
+import org.apache.doris.meta.MetaContext;
+import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.QueryState.MysqlStateType;
import org.apache.doris.qe.StmtExecutor;
@@ -32,16 +35,20 @@ import org.apache.doris.utframe.TestWithFeService;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import org.apache.hadoop.conf.Configuration;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.nio.file.Files;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class ExternalCatalogTest extends TestWithFeService {
- private static Auth auth;
- private static Env env;
+ private Env env;
private CatalogMgr mgr;
private ConnectContext rootCtx;
@@ -51,7 +58,6 @@ public class ExternalCatalogTest extends TestWithFeService {
mgr = Env.getCurrentEnv().getCatalogMgr();
rootCtx = createDefaultCtx();
env = Env.getCurrentEnv();
- auth = env.getAuth();
// 1. create test catalog
CreateCatalogStmt testCatalog = (CreateCatalogStmt)
parseAndAnalyzeStmt(
"create catalog test1 properties(\n"
@@ -244,4 +250,36 @@ public class ExternalCatalogTest extends TestWithFeService
{
return MOCKED_META;
}
}
+
+ @Test
+ public void testSerialization() throws Exception {
+ MetaContext metaContext = new MetaContext();
+ metaContext.setMetaVersion(FeMetaVersion.VERSION_CURRENT);
+ metaContext.setThreadLocalInfo();
+
+ // 1. Write objects to file
+ File file = new File("./external_catalog_persist_test.dat");
+ file.createNewFile();
+ try {
+ DataOutputStream dos = new
DataOutputStream(Files.newOutputStream(file.toPath()));
+
+ TestExternalCatalog ctl = (TestExternalCatalog)
mgr.getCatalog("test1");
+ ctl.write(dos);
+ dos.flush();
+ dos.close();
+
+ // 2. Read objects from file
+ DataInputStream dis = new
DataInputStream(Files.newInputStream(file.toPath()));
+
+ String json = Text.readString(dis);
+ TestExternalCatalog ctl2 = GsonUtils.GSON.fromJson(json,
TestExternalCatalog.class);
+ Configuration conf = ctl2.getConfiguration();
+ Assertions.assertNotNull(conf);
+
+ // 3. delete files
+ dis.close();
+ } finally {
+ file.delete();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
