This is an automated email from the ASF dual-hosted git repository.
wanghailin pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new fa5b7d3d83 [Improve][HttpConnector]Increase custom configuration
timeout. (#6223)
fa5b7d3d83 is described below
commit fa5b7d3d83812f11f819d032fc56f4f8e812318e
Author: lightzhao <[email protected]>
AuthorDate: Wed Jan 17 12:06:34 2024 +0800
[Improve][HttpConnector]Increase custom configuration timeout. (#6223)
---
docs/en/connector-v2/sink/Http.md | 2 ++
docs/en/connector-v2/source/Http.md | 2 ++
.../seatunnel/http/client/HttpClientProvider.java | 25 +++++++++++-----------
.../seatunnel/http/config/HttpConfig.java | 14 ++++++++++++
.../seatunnel/http/config/HttpParameter.java | 8 +++++++
5 files changed, 38 insertions(+), 13 deletions(-)
diff --git a/docs/en/connector-v2/sink/Http.md
b/docs/en/connector-v2/sink/Http.md
index 55a69e9333..1eb89af0d0 100644
--- a/docs/en/connector-v2/sink/Http.md
+++ b/docs/en/connector-v2/sink/Http.md
@@ -39,6 +39,8 @@ They can be downloaded via install-plugin.sh or from the
Maven central repositor
| retry | Int | No | - | The max retry
times if request http return to `IOException`
|
| retry_backoff_multiplier_ms | Int | No | 100 | The
retry-backoff times(millis) multiplier if request http failed
|
| retry_backoff_max_ms | Int | No | 10000 | The maximum
retry-backoff times(millis) if request http failed
|
+| connect_timeout_ms | Int | No | 12000 | Connection
timeout setting, default 12s.
|
+| socket_timeout_ms | Int | No | 60000 | Socket timeout
setting, default 60s.
|
| common-options | | No | - | Sink plugin
common parameters, please refer to [Sink Common Options](common-options.md) for
details |
## Example
diff --git a/docs/en/connector-v2/source/Http.md
b/docs/en/connector-v2/source/Http.md
index 199bef7dee..e4d021c05c 100644
--- a/docs/en/connector-v2/source/Http.md
+++ b/docs/en/connector-v2/source/Http.md
@@ -63,6 +63,8 @@ They can be downloaded via install-plugin.sh or from the
Maven central repositor
| retry_backoff_multiplier_ms | Int | No | 100 | The
retry-backoff times(millis) multiplier if request http failed.
|
| retry_backoff_max_ms | Int | No | 10000 | The maximum
retry-backoff times(millis) if request http failed
|
| enable_multi_lines | Boolean | No | false |
|
+| connect_timeout_ms | Int | No | 12000 | Connection
timeout setting, default 12s.
|
+| socket_timeout_ms | Int | No | 60000 | Socket timeout
setting, default 60s.
|
| common-options | | No | - | Source plugin
common parameters, please refer to [Source Common Options](common-options.md)
for details |
## How to Create a Http Data Synchronization Jobs
diff --git
a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/client/HttpClientProvider.java
b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/client/HttpClientProvider.java
index 2c6fe67b79..b666058ce1 100644
---
a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/client/HttpClientProvider.java
+++
b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/client/HttpClientProvider.java
@@ -66,20 +66,19 @@ import java.util.concurrent.TimeUnit;
public class HttpClientProvider implements AutoCloseable {
private static final String ENCODING = "UTF-8";
private static final String APPLICATION_JSON = "application/json";
- private static final int CONNECT_TIMEOUT = 6000 * 2;
- private static final int SOCKET_TIMEOUT = 6000 * 10;
private static final int INITIAL_CAPACITY = 16;
- private static final RequestConfig REQUEST_CONFIG =
- RequestConfig.custom()
- .setConnectTimeout(CONNECT_TIMEOUT)
- .setSocketTimeout(SOCKET_TIMEOUT)
- .build();
+ private RequestConfig requestConfig;
private final CloseableHttpClient httpClient;
private final Retryer<CloseableHttpResponse> retryer;
public HttpClientProvider(HttpParameter httpParameter) {
this.httpClient = HttpClients.createDefault();
this.retryer = buildRetryer(httpParameter);
+ this.requestConfig =
+ RequestConfig.custom()
+ .setConnectTimeout(httpParameter.getConnectTimeoutMs())
+ .setSocketTimeout(httpParameter.getSocketTimeoutMs())
+ .build();
}
private Retryer<CloseableHttpResponse> buildRetryer(HttpParameter
httpParameter) {
@@ -176,7 +175,7 @@ public class HttpClientProvider implements AutoCloseable {
// create a new http get
HttpGet httpGet = new HttpGet(uriBuilder.build());
// set default request config
- httpGet.setConfig(REQUEST_CONFIG);
+ httpGet.setConfig(requestConfig);
// set request header
addHeaders(httpGet, headers);
// return http response
@@ -220,7 +219,7 @@ public class HttpClientProvider implements AutoCloseable {
// create a new http get
HttpPost httpPost = new HttpPost(url);
// set default request config
- httpPost.setConfig(REQUEST_CONFIG);
+ httpPost.setConfig(requestConfig);
// set request header
addHeaders(httpPost, headers);
// set request params
@@ -255,7 +254,7 @@ public class HttpClientProvider implements AutoCloseable {
// create a new http post
HttpPost httpPost = new HttpPost(url);
// set default request config
- httpPost.setConfig(REQUEST_CONFIG);
+ httpPost.setConfig(requestConfig);
// set request header
addHeaders(httpPost, headers);
// add body in request
@@ -280,7 +279,7 @@ public class HttpClientProvider implements AutoCloseable {
// create a new http get
HttpPost httpPost = new HttpPost(url);
// set default request config
- httpPost.setConfig(REQUEST_CONFIG);
+ httpPost.setConfig(requestConfig);
// set request header
addHeaders(httpPost, headers);
// set request params
@@ -314,7 +313,7 @@ public class HttpClientProvider implements AutoCloseable {
// create a new http put
HttpPut httpPut = new HttpPut(url);
// set default request config
- httpPut.setConfig(REQUEST_CONFIG);
+ httpPut.setConfig(requestConfig);
// set request params
addParameters(httpPut, params);
// return http response
@@ -332,7 +331,7 @@ public class HttpClientProvider implements AutoCloseable {
// create a new http delete
HttpDelete httpDelete = new HttpDelete(url);
// set default request config
- httpDelete.setConfig(REQUEST_CONFIG);
+ httpDelete.setConfig(requestConfig);
// return http response
return getResponse(httpDelete);
}
diff --git
a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/config/HttpConfig.java
b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/config/HttpConfig.java
index 5db4c40e97..043f907e44 100644
---
a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/config/HttpConfig.java
+++
b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/config/HttpConfig.java
@@ -29,6 +29,8 @@ public class HttpConfig {
public static final int DEFAULT_RETRY_BACKOFF_MULTIPLIER_MS = 100;
public static final int DEFAULT_RETRY_BACKOFF_MAX_MS = 10000;
public static final boolean DEFAULT_ENABLE_MULTI_LINES = false;
+ public static final int DEFAULT_CONNECT_TIMEOUT_MS = 6000 * 2;
+ public static final int DEFAULT_SOCKET_TIMEOUT_MS = 6000 * 10;
public static final Option<String> URL =
Options.key("url").stringType().noDefaultValue().withDescription("Http request
url");
public static final Option<Long> TOTAL_PAGE_SIZE =
@@ -112,6 +114,18 @@ public class HttpConfig {
.withDescription(
"SeaTunnel enableMultiLines.This parameter can
support http splitting response text by line.");
+ public static final Option<Integer> CONNECT_TIMEOUT_MS =
+ Options.key("connect_timeout_ms")
+ .intType()
+ .defaultValue(DEFAULT_CONNECT_TIMEOUT_MS)
+ .withDescription("Connection timeout setting, default
12s.");
+
+ public static final Option<Integer> SOCKET_TIMEOUT_MS =
+ Options.key("socket_timeout_ms")
+ .intType()
+ .defaultValue(DEFAULT_SOCKET_TIMEOUT_MS)
+ .withDescription("Socket timeout setting, default 60s.");
+
public enum ResponseFormat {
JSON("json"),
TEXT("text");
diff --git
a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/config/HttpParameter.java
b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/config/HttpParameter.java
index 74b5c6e4e2..e4dffb539a 100644
---
a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/config/HttpParameter.java
+++
b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/config/HttpParameter.java
@@ -38,6 +38,8 @@ public class HttpParameter implements Serializable {
protected int retryBackoffMultiplierMillis =
HttpConfig.DEFAULT_RETRY_BACKOFF_MULTIPLIER_MS;
protected int retryBackoffMaxMillis =
HttpConfig.DEFAULT_RETRY_BACKOFF_MAX_MS;
protected boolean enableMultilines;
+ protected int connectTimeoutMs = HttpConfig.DEFAULT_CONNECT_TIMEOUT_MS;
+ protected int socketTimeoutMs = HttpConfig.DEFAULT_SOCKET_TIMEOUT_MS;
public void buildWithConfig(Config pluginConfig) {
// set url
@@ -85,6 +87,12 @@ public class HttpParameter implements Serializable {
} else {
this.setEnableMultilines(HttpConfig.ENABLE_MULTI_LINES.defaultValue());
}
+ if (pluginConfig.hasPath(HttpConfig.CONNECT_TIMEOUT_MS.key())) {
+
this.setConnectTimeoutMs(pluginConfig.getInt(HttpConfig.CONNECT_TIMEOUT_MS.key()));
+ }
+ if (pluginConfig.hasPath(HttpConfig.SOCKET_TIMEOUT_MS.key())) {
+
this.setSocketTimeoutMs(pluginConfig.getInt(HttpConfig.SOCKET_TIMEOUT_MS.key()));
+ }
}
public void setRetryParameters(Config pluginConfig) {
