This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new fdae388 [HUDI-1203] add port configuration for
EmbeddedTimelineService (#2142)
fdae388 is described below
commit fdae388626b8d97acc01191aa0e7075c36a41132
Author: lw0090 <[email protected]>
AuthorDate: Tue Oct 6 02:36:54 2020 +0800
[HUDI-1203] add port configuration for EmbeddedTimelineService (#2142)
---
.../apache/hudi/client/AbstractHoodieClient.java | 2 +-
.../client/embedded/EmbeddedTimelineService.java | 6 ++--
.../org/apache/hudi/config/HoodieWriteConfig.java | 11 +++++++
.../hudi/timeline/service/TimelineService.java | 37 +++++++++++++++++++---
4 files changed, 48 insertions(+), 8 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java
index 9628e41..7a59ebc 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java
@@ -103,7 +103,7 @@ public abstract class AbstractHoodieClient implements
Serializable, AutoCloseabl
LOG.info("Starting Timeline service !!");
Option<String> hostAddr =
context.getProperty(EngineProperty.EMBEDDED_SERVER_HOST);
timelineServer = Option.of(new EmbeddedTimelineService(context,
hostAddr.orElse(null),
- config.getClientSpecifiedViewStorageConfig()));
+ config.getEmbeddedTimelineServerPort(),
config.getClientSpecifiedViewStorageConfig()));
try {
timelineServer.get().startServer();
// Allow executor to find this newly instantiated timeline service
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
index ca0c713..738119c 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
@@ -39,17 +39,19 @@ public class EmbeddedTimelineService {
private static final Logger LOG =
LogManager.getLogger(EmbeddedTimelineService.class);
private int serverPort;
+ private int preferredPort;
private String hostAddr;
private final SerializableConfiguration hadoopConf;
private final FileSystemViewStorageConfig config;
private transient FileSystemViewManager viewManager;
private transient TimelineService server;
- public EmbeddedTimelineService(HoodieEngineContext context, String
embeddedTimelineServiceHostAddr, FileSystemViewStorageConfig config) {
+ public EmbeddedTimelineService(HoodieEngineContext context, String
embeddedTimelineServiceHostAddr, int embeddedTimelineServerPort,
FileSystemViewStorageConfig config) {
setHostAddr(embeddedTimelineServiceHostAddr);
this.config = config;
this.hadoopConf = context.getHadoopConf();
this.viewManager = createViewManager();
+ this.preferredPort = embeddedTimelineServerPort;
}
private FileSystemViewManager createViewManager() {
@@ -66,7 +68,7 @@ public class EmbeddedTimelineService {
}
public void startServer() throws IOException {
- server = new TimelineService(0, viewManager, hadoopConf.newCopy());
+ server = new TimelineService(preferredPort, viewManager,
hadoopConf.newCopy());
serverPort = server.startService();
LOG.info("Started embedded timeline server at " + hostAddr + ":" +
serverPort);
}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index 60ca3c2..bb65600 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -99,6 +99,8 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
public static final String EMBEDDED_TIMELINE_SERVER_ENABLED =
"hoodie.embed.timeline.server";
public static final String DEFAULT_EMBEDDED_TIMELINE_SERVER_ENABLED = "true";
+ public static final String EMBEDDED_TIMELINE_SERVER_PORT =
"hoodie.embed.timeline.server.port";
+ public static final String DEFAULT_EMBEDDED_TIMELINE_SERVER_PORT = "0";
public static final String FAIL_ON_TIMELINE_ARCHIVING_ENABLED_PROP =
"hoodie.fail.on.timeline.archiving";
public static final String DEFAULT_FAIL_ON_TIMELINE_ARCHIVING_ENABLED =
"true";
@@ -255,6 +257,10 @@ public class HoodieWriteConfig extends DefaultHoodieConfig
{
return
Boolean.parseBoolean(props.getProperty(EMBEDDED_TIMELINE_SERVER_ENABLED));
}
+ public int getEmbeddedTimelineServerPort() {
+ return Integer.parseInt(props.getProperty(EMBEDDED_TIMELINE_SERVER_PORT,
DEFAULT_EMBEDDED_TIMELINE_SERVER_PORT));
+ }
+
public boolean isFailOnTimelineArchivingEnabled() {
return
Boolean.parseBoolean(props.getProperty(FAIL_ON_TIMELINE_ARCHIVING_ENABLED_PROP));
}
@@ -953,6 +959,11 @@ public class HoodieWriteConfig extends DefaultHoodieConfig
{
return this;
}
+ public Builder withEmbeddedTimelineServerPort(int port) {
+ props.setProperty(EMBEDDED_TIMELINE_SERVER_PORT, String.valueOf(port));
+ return this;
+ }
+
public Builder withBulkInsertSortMode(String mode) {
props.setProperty(BULKINSERT_SORT_MODE, mode);
return this;
diff --git
a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java
b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java
index f38909f..969f824 100644
---
a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java
+++
b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java
@@ -41,6 +41,7 @@ import java.io.Serializable;
public class TimelineService {
private static final Logger LOG =
LogManager.getLogger(TimelineService.class);
+ private static final int START_SERVICE_MAX_RETRIES = 16;
private int serverPort;
private Configuration conf;
@@ -98,16 +99,42 @@ public class TimelineService {
public Boolean help = false;
}
+ private int startServiceOnPort(int port) throws IOException {
+ if (!(port == 0 || (1024 <= port && port < 65536))) {
+ throw new IllegalArgumentException(String.format("startPort should be
between 1024 and 65535 (inclusive), "
+ + "or 0 for a random free port. but now is %s.", port));
+ }
+ for (int attempt = 0; attempt < START_SERVICE_MAX_RETRIES; attempt++) {
+ // Returns port to try when trying to bind a service. Handles wrapping
and skipping privileged ports.
+ int tryPort = port == 0 ? port : (port + attempt - 1024) % (65536 -
1024) + 1024;
+ try {
+ app.start(tryPort);
+ return app.port();
+ } catch (Exception e) {
+ if (e.getMessage() != null && e.getMessage().contains("Failed to bind
to")) {
+ if (tryPort == 0) {
+ LOG.warn("Timeline server could not bind on a random free port.");
+ } else {
+ LOG.warn(String.format("Timeline server could not bind on port %d.
"
+ + "Attempting port %d + 1.",tryPort, tryPort));
+ }
+ } else {
+ LOG.warn(String.format("Timeline server start failed on port %d.
Attempting port %d + 1.",tryPort, tryPort), e);
+ }
+ }
+ }
+ throw new IOException(String.format("Timeline server start failed on port
%d, after retry %d times", port, START_SERVICE_MAX_RETRIES));
+ }
+
public int startService() throws IOException {
app = Javalin.create();
FileSystemViewHandler router = new FileSystemViewHandler(app, conf,
fsViewsManager);
app.get("/", ctx -> ctx.result("Hello World"));
router.register();
- app.start(serverPort);
- // If port = 0, a dynamic port is assigned. Store it.
- serverPort = app.port();
- LOG.info("Starting Timeline server on port :" + serverPort);
- return serverPort;
+ int realServerPort = startServiceOnPort(serverPort);
+ LOG.info("Starting Timeline server on port :" + realServerPort);
+ this.serverPort = realServerPort;
+ return realServerPort;
}
public void run() throws IOException {
