This is an automated email from the ASF dual-hosted git repository.
jinsongzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 9218023a8 [Improvement] Polish codes in AmsEnvironment and fix some
errors in GitHub CI (#3706)
9218023a8 is described below
commit 9218023a8fa21c3b0a25362cc3e07f100c1f029a
Author: ZhouJinsong <[email protected]>
AuthorDate: Mon Aug 4 11:15:41 2025 +0800
[Improvement] Polish codes in AmsEnvironment and fix some errors in GitHub
CI (#3706)
Polish codes in AmsEnvironment
---
.../org/apache/amoro/server/AmsEnvironment.java | 130 +++++----------------
.../amoro/server/RestCatalogServiceTestBase.java | 3 -
2 files changed, 28 insertions(+), 105 deletions(-)
diff --git
a/amoro-ams/src/test/java/org/apache/amoro/server/AmsEnvironment.java
b/amoro-ams/src/test/java/org/apache/amoro/server/AmsEnvironment.java
index f20e6d1d5..ff09d575e 100644
--- a/amoro-ams/src/test/java/org/apache/amoro/server/AmsEnvironment.java
+++ b/amoro-ams/src/test/java/org/apache/amoro/server/AmsEnvironment.java
@@ -33,11 +33,9 @@ import org.apache.amoro.server.catalog.DefaultCatalogManager;
import org.apache.amoro.server.catalog.ServerCatalog;
import org.apache.amoro.server.resource.InternalContainers;
import org.apache.amoro.server.resource.OptimizerManager;
-import org.apache.amoro.server.table.DefaultTableService;
import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
import org.apache.amoro.shade.guava32.com.google.common.io.MoreFiles;
import
org.apache.amoro.shade.guava32.com.google.common.io.RecursiveDeleteOption;
-import org.apache.amoro.shade.thrift.org.apache.thrift.server.TServer;
import
org.apache.amoro.shade.thrift.org.apache.thrift.transport.TTransportException;
import org.apache.amoro.table.TableIdentifier;
import org.apache.commons.io.FileUtils;
@@ -50,6 +48,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.net.BindException;
+import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.time.Duration;
@@ -57,8 +56,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Random;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicBoolean;
public class AmsEnvironment {
@@ -68,9 +65,7 @@ public class AmsEnvironment {
private static final String DEFAULT_ROOT_PATH = "/tmp/amoro_integration";
private static final String OPTIMIZE_GROUP = "default";
private final AmoroServiceContainer serviceContainer;
- private Configurations serviceConfig;
private DefaultCatalogManager catalogManager;
- private final AtomicBoolean amsExit;
private int tableServiceBindPort;
private int optimizingServiceBindPort;
private final HMSMockServer testHMS;
@@ -115,13 +110,13 @@ public class AmsEnvironment {
public AmsEnvironment(String rootPath) throws Exception {
this.rootPath = rootPath;
- LOG.info("ams environment root path: {}", rootPath);
+ LOG.info("Initializing test AMS environment with root path: {}", rootPath);
String path =
Objects.requireNonNull(this.getClass().getClassLoader().getResource("")).getPath();
- FileUtils.writeStringToFile(new File(rootPath + "/conf/config.yaml"),
getAmsConfig());
+ FileUtils.writeStringToFile(
+ new File(rootPath + "/conf/config.yaml"), getAmsConfig(),
Charset.defaultCharset());
System.setProperty(Environments.AMORO_HOME, rootPath);
System.setProperty("derby.init.sql.dir", path + "../classes/sql/derby/");
- amsExit = new AtomicBoolean(false);
serviceContainer = new AmoroServiceContainer();
TemporaryFolder hiveDir = new TemporaryFolder();
hiveDir.create();
@@ -139,29 +134,9 @@ public class AmsEnvironment {
testHMS.start();
startAms();
- DynFields.UnboundField<DefaultTableService> amsTableServiceField =
- DynFields.builder().hiddenImpl(AmoroServiceContainer.class,
"tableService").build();
DynFields.UnboundField<DefaultCatalogManager> amsCatalogManagerField =
DynFields.builder().hiddenImpl(AmoroServiceContainer.class,
"catalogManager").build();
catalogManager = amsCatalogManagerField.bind(serviceContainer).get();
- DefaultTableService tableService =
amsTableServiceField.bind(serviceContainer).get();
- DynFields.UnboundField<CompletableFuture<Boolean>>
tableServiceInitializedField =
- DynFields.builder().hiddenImpl(DefaultTableService.class,
"initialized").build();
- boolean tableServiceIsStart = false;
- long startTime = System.currentTimeMillis();
- while (!tableServiceIsStart) {
- if (System.currentTimeMillis() - startTime > 10000) {
- throw new RuntimeException("table service not start yet after 10s");
- }
- try {
- tableServiceInitializedField.bind(tableService).get().get();
- tableServiceIsStart = true;
- } catch (RuntimeException e) {
- LOG.info("table service not start yet");
- }
- Thread.sleep(1000);
- }
-
initCatalog();
started = true;
}
@@ -327,82 +302,33 @@ public class AmsEnvironment {
}
private void startAms() throws Exception {
- Thread amsRunner =
- new Thread(
- () -> {
- int retry = 10;
- try {
- while (true) {
- try {
- LOG.info("start ams");
- genThriftBindPort();
- DynFields.UnboundField<Configurations> field =
- DynFields.builder()
- .hiddenImpl(AmoroServiceContainer.class,
"serviceConfig")
- .build();
- serviceConfig = field.bind(serviceContainer).get();
- serviceConfig.set(
- AmoroManagementConf.TABLE_SERVICE_THRIFT_BIND_PORT,
tableServiceBindPort);
- serviceConfig.set(
-
AmoroManagementConf.OPTIMIZING_SERVICE_THRIFT_BIND_PORT,
- optimizingServiceBindPort);
- serviceConfig.set(
- AmoroManagementConf.REFRESH_EXTERNAL_CATALOGS_INTERVAL,
- Duration.ofMillis(1000L));
- serviceContainer.startService();
- break;
- } catch (TTransportException e) {
- if (e.getCause() instanceof BindException) {
- LOG.error("start ams failed", e);
- if (retry-- < 0) {
- throw e;
- } else {
- Thread.sleep(1000);
- }
- } else {
- throw e;
- }
- } catch (Throwable e) {
- throw e;
- }
- }
- } catch (Throwable t) {
- LOG.error("start ams failed", t);
- } finally {
- amsExit.set(true);
- }
- },
- "ams-runner");
- amsRunner.start();
-
- DynFields.UnboundField<TServer> tableManagementServerField =
- DynFields.builder()
- .hiddenImpl(AmoroServiceContainer.class, "tableManagementServer")
- .build();
- DynFields.UnboundField<TServer> optimizingServiceServerField =
- DynFields.builder()
- .hiddenImpl(AmoroServiceContainer.class, "optimizingServiceServer")
- .build();
+ int retry = 10;
while (true) {
- if (amsExit.get()) {
- LOG.error("ams exit");
- break;
- }
- TServer tableManagementServer =
tableManagementServerField.bind(serviceContainer).get();
- TServer optimizingServiceServer =
optimizingServiceServerField.bind(serviceContainer).get();
- if (tableManagementServer != null
- && tableManagementServer.isServing()
- && optimizingServiceServer != null
- && optimizingServiceServer.isServing()) {
- LOG.info("ams start");
- break;
- }
try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
- LOG.warn("interrupt ams");
- amsRunner.interrupt();
+ LOG.info("Starting test AMS...");
+ genThriftBindPort();
+ DynFields.UnboundField<Configurations> field =
+ DynFields.builder().hiddenImpl(AmoroServiceContainer.class,
"serviceConfig").build();
+ Configurations serviceConfig = field.bind(serviceContainer).get();
+ serviceConfig.set(AmoroManagementConf.TABLE_SERVICE_THRIFT_BIND_PORT,
tableServiceBindPort);
+ serviceConfig.set(
+ AmoroManagementConf.OPTIMIZING_SERVICE_THRIFT_BIND_PORT,
optimizingServiceBindPort);
+ serviceConfig.set(
+ AmoroManagementConf.REFRESH_EXTERNAL_CATALOGS_INTERVAL,
Duration.ofMillis(1000L));
+ serviceContainer.startService();
+ LOG.info("Started test AMS.");
break;
+ } catch (TTransportException e) {
+ if (e.getCause() instanceof BindException) {
+ if (retry-- < 0) {
+ throw e;
+ } else {
+ LOG.error("Started test AMS failed, retry later...", e);
+ Thread.sleep(1000);
+ }
+ } else {
+ throw e;
+ }
}
}
}
diff --git
a/amoro-ams/src/test/java/org/apache/amoro/server/RestCatalogServiceTestBase.java
b/amoro-ams/src/test/java/org/apache/amoro/server/RestCatalogServiceTestBase.java
index 7d9691367..5e324a47c 100644
---
a/amoro-ams/src/test/java/org/apache/amoro/server/RestCatalogServiceTestBase.java
+++
b/amoro-ams/src/test/java/org/apache/amoro/server/RestCatalogServiceTestBase.java
@@ -40,14 +40,11 @@ import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Map;
public abstract class RestCatalogServiceTestBase {
- private static final Logger LOG =
LoggerFactory.getLogger(RestCatalogServiceTestBase.class);
static AmsEnvironment ams = AmsEnvironment.getIntegrationInstances();
static String restCatalogUri = RestCatalogService.ICEBERG_REST_API_PREFIX;
