This is an automated email from the ASF dual-hosted git repository.
suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 8110e88 [GOBBLIN-1279] Fix flaky unit tests involving REST.li calls
to FlowConfig endpoint in Gobblin-as-a-Service[]
8110e88 is described below
commit 8110e88a1c09c4ec7795560748321891013aadcc
Author: suvasude <[email protected]>
AuthorDate: Wed Oct 7 10:19:37 2020 -0700
[GOBBLIN-1279] Fix flaky unit tests involving REST.li calls to FlowConfig
endpoint in Gobblin-as-a-Service[]
Closes #3119 from sv2000/flowConfigFlakiness
---
.../main/java/org/apache/gobblin/service/FlowConfigClient.java | 7 ++++++-
.../java/org/apache/gobblin/service/FlowConfigV2Client.java | 4 ++++
.../test/java/org/apache/gobblin/service/FlowConfigTest.java | 7 +++++--
.../test/java/org/apache/gobblin/service/FlowConfigV2Test.java | 7 +++++--
.../org/apache/gobblin/service/GobblinServiceManagerTest.java | 8 ++++++--
.../gobblin/service/modules/core/GobblinServiceHATest.java | 10 ++++++++--
6 files changed, 34 insertions(+), 9 deletions(-)
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/main/java/org/apache/gobblin/service/FlowConfigClient.java
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/main/java/org/apache/gobblin/service/FlowConfigClient.java
index 7e78e29..58ae8d1 100644
---
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/main/java/org/apache/gobblin/service/FlowConfigClient.java
+++
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/main/java/org/apache/gobblin/service/FlowConfigClient.java
@@ -20,6 +20,7 @@ package org.apache.gobblin.service;
import java.io.Closeable;
import java.io.IOException;
import java.util.Collections;
+import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -59,10 +60,14 @@ public class FlowConfigClient implements Closeable {
* @param serverUri address and port of the REST server
*/
public FlowConfigClient(String serverUri) {
+ this(serverUri, Collections.<String, String>emptyMap());
+ }
+
+ public FlowConfigClient(String serverUri, Map<String, String> properties) {
LOG.debug("FlowConfigClient with serverUri " + serverUri);
_httpClientFactory = Optional.of(new HttpClientFactory());
- Client r2Client = new
TransportClientAdapter(_httpClientFactory.get().getClient(Collections.<String,
String>emptyMap()));
+ Client r2Client = new
TransportClientAdapter(_httpClientFactory.get().getClient(properties));
_restClient = Optional.of(new RestClient(r2Client, serverUri));
_flowconfigsRequestBuilders = createRequestBuilders();
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/main/java/org/apache/gobblin/service/FlowConfigV2Client.java
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/main/java/org/apache/gobblin/service/FlowConfigV2Client.java
index 56130f6..98b9e90 100644
---
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/main/java/org/apache/gobblin/service/FlowConfigV2Client.java
+++
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/main/java/org/apache/gobblin/service/FlowConfigV2Client.java
@@ -70,6 +70,10 @@ public class FlowConfigV2Client implements Closeable {
* @param serverUri address and port of the REST server
*/
public FlowConfigV2Client(String serverUri) {
+ this(serverUri, Collections.emptyMap());
+ }
+
+ public FlowConfigV2Client(String serverUri, Map<String, String> properties) {
LOG.debug("FlowConfigClient with serverUri " + serverUri);
_httpClientFactory = Optional.of(new HttpClientFactory());
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/test/java/org/apache/gobblin/service/FlowConfigTest.java
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/test/java/org/apache/gobblin/service/FlowConfigTest.java
index 38bc2d5..e89f65f 100644
---
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/test/java/org/apache/gobblin/service/FlowConfigTest.java
+++
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/test/java/org/apache/gobblin/service/FlowConfigTest.java
@@ -38,6 +38,7 @@ import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.name.Names;
import com.linkedin.data.template.StringMap;
+import com.linkedin.r2.transport.http.client.HttpClientFactory;
import com.linkedin.restli.client.RestLiResponseException;
import com.linkedin.restli.common.HttpStatus;
import com.linkedin.restli.server.resources.BaseResource;
@@ -50,7 +51,7 @@ import org.apache.gobblin.runtime.spec_catalog.FlowCatalog;
import org.apache.gobblin.runtime.spec_store.FSSpecStore;
-@Test(groups = { "gobblin.service" })
+@Test(groups = { "gobblin.service" }, singleThreaded = true)
public class FlowConfigTest {
private FlowConfigClient _client;
private EmbeddedRestliServer _server;
@@ -102,8 +103,10 @@ public class FlowConfigTest {
_server.startAsync();
_server.awaitRunning();
+ Map<String, String> transportClientProperties = Maps.newHashMap();
+ transportClientProperties.put(HttpClientFactory.HTTP_REQUEST_TIMEOUT,
"10000");
_client =
- new FlowConfigClient(String.format("http://localhost:%s/";,
_server.getPort()));
+ new FlowConfigClient(String.format("http://localhost:%s/";,
_server.getPort()), transportClientProperties);
}
private void cleanUpDir(String dir) throws Exception {
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/test/java/org/apache/gobblin/service/FlowConfigV2Test.java
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/test/java/org/apache/gobblin/service/FlowConfigV2Test.java
index eea986e..50fb947 100644
---
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/test/java/org/apache/gobblin/service/FlowConfigV2Test.java
+++
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/test/java/org/apache/gobblin/service/FlowConfigV2Test.java
@@ -38,6 +38,7 @@ import com.google.inject.Module;
import com.google.inject.name.Names;
import com.linkedin.data.DataMap;
import com.linkedin.data.template.StringMap;
+import com.linkedin.r2.transport.http.client.HttpClientFactory;
import com.linkedin.restli.client.RestLiResponseException;
import com.linkedin.restli.common.PatchRequest;
import com.linkedin.restli.internal.server.util.DataMapUtils;
@@ -55,7 +56,7 @@ import org.apache.gobblin.runtime.spec_catalog.FlowCatalog;
import org.apache.gobblin.runtime.spec_store.FSSpecStore;
-@Test(groups = { "gobblin.service" })
+@Test(groups = { "gobblin.service" }, singleThreaded = true)
public class FlowConfigV2Test {
private FlowConfigV2Client _client;
private EmbeddedRestliServer _server;
@@ -106,8 +107,10 @@ public class FlowConfigV2Test {
_server.startAsync();
_server.awaitRunning();
+ Map<String, String> transportClientProperties = Maps.newHashMap();
+ transportClientProperties.put(HttpClientFactory.HTTP_REQUEST_TIMEOUT,
"10000");
_client =
- new FlowConfigV2Client(String.format("http://localhost:%s/";,
_server.getPort()));
+ new FlowConfigV2Client(String.format("http://localhost:%s/";,
_server.getPort()), transportClientProperties);
}
protected void cleanUpDir(String dir) throws Exception {
diff --git
a/gobblin-service/src/test/java/org/apache/gobblin/service/GobblinServiceManagerTest.java
b/gobblin-service/src/test/java/org/apache/gobblin/service/GobblinServiceManagerTest.java
index f1eafbb..d3cb898 100644
---
a/gobblin-service/src/test/java/org/apache/gobblin/service/GobblinServiceManagerTest.java
+++
b/gobblin-service/src/test/java/org/apache/gobblin/service/GobblinServiceManagerTest.java
@@ -47,6 +47,7 @@ import com.google.common.io.Files;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.linkedin.data.template.StringMap;
+import com.linkedin.r2.transport.http.client.HttpClientFactory;
import com.linkedin.restli.client.RestLiResponseException;
import com.typesafe.config.Config;
@@ -106,6 +107,7 @@ public class GobblinServiceManagerTest {
private TestingServer testingServer;
Properties serviceCoreProperties = new Properties();
Map<String, String> flowProperties = Maps.newHashMap();
+ Map<String, String> transportClientProperties = Maps.newHashMap();
public GobblinServiceManagerTest() throws Exception {
}
@@ -153,6 +155,8 @@ public class GobblinServiceManagerTest {
serviceCoreProperties.put(ServiceConfigKeys.GOBBLIN_SERVICE_FLOWCOMPILER_CLASS_KEY,
MockedSpecCompiler.class.getCanonicalName());
+ transportClientProperties.put(HttpClientFactory.HTTP_REQUEST_TIMEOUT,
"10000");
+
// Create a bare repository
RepositoryCache.FileKey fileKey = RepositoryCache.FileKey.exact(new
File(GIT_REMOTE_REPO_DIR), FS.DETECTED);
fileKey.open(false).create(true);
@@ -168,7 +172,7 @@ public class GobblinServiceManagerTest {
this.gobblinServiceManager.start();
this.flowConfigClient = new
FlowConfigV2Client(String.format("http://127.0.0.1:%s/";,
-
this.gobblinServiceManager.getRestLiServer().getListeningURI().getPort()));
+
this.gobblinServiceManager.getRestLiServer().getListeningURI().getPort()),
transportClientProperties);
}
private void cleanUpDir(String dir) throws Exception {
@@ -499,7 +503,7 @@ null, null, null, null);
this.gobblinServiceManager = new MockGobblinServiceManager("CoreService",
"1",
ConfigUtils.propertiesToConfig(serviceCoreProperties), Optional.of(new
Path(SERVICE_WORK_DIR)));
this.flowConfigClient = new
FlowConfigV2Client(String.format("http://127.0.0.1:%s/";,
- this.gobblinServiceManager.getRestLiServer().getPort()));
+ this.gobblinServiceManager.getRestLiServer().getPort()),
transportClientProperties);
this.gobblinServiceManager.start();
}
diff --git
a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceHATest.java
b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceHATest.java
index 2132778..d8bc99e 100644
---
a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceHATest.java
+++
b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/core/GobblinServiceHATest.java
@@ -36,6 +36,10 @@ import org.testng.annotations.Test;
import com.google.common.base.Optional;
import com.google.common.collect.Maps;
import com.linkedin.data.template.StringMap;
+import com.linkedin.r2.transport.common.Client;
+import com.linkedin.r2.transport.common.bridge.client.TransportClientAdapter;
+import com.linkedin.r2.transport.http.client.HttpClientFactory;
+import com.linkedin.restli.client.RestClient;
import com.linkedin.restli.client.RestLiResponseException;
import org.apache.gobblin.configuration.ConfigurationKeys;
@@ -168,12 +172,14 @@ public class GobblinServiceHATest {
this.node2GobblinServiceManager.start();
// Initialize Node 1 Client
+ Map<String, String> transportClientProperties = Maps.newHashMap();
+ transportClientProperties.put(HttpClientFactory.HTTP_REQUEST_TIMEOUT,
"10000");
this.node1FlowConfigClient = new
FlowConfigClient(String.format("http://localhost:%s/";,
- this.node1GobblinServiceManager.restliServer.getPort()));
+ this.node1GobblinServiceManager.restliServer.getPort()),
transportClientProperties);
// Initialize Node 2 Client
this.node2FlowConfigClient = new
FlowConfigClient(String.format("http://localhost:%s/";,
- this.node2GobblinServiceManager.restliServer.getPort()));
+ this.node2GobblinServiceManager.restliServer.getPort()),
transportClientProperties);
}
private void cleanUpDir(String dir) throws Exception {
