This is an automated email from the ASF dual-hosted git repository.
snagel pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new 309bc1863 NUTCH-3066 Protocol plugin unit tests fail randomly
309bc1863 is described below
commit 309bc18632d61c592de0fc030b28af9cbaccdd29
Author: Sebastian Nagel <[email protected]>
AuthorDate: Fri Sep 6 13:04:19 2024 +0200
NUTCH-3066 Protocol plugin unit tests fail randomly
- use random free ephemeral port in AbstractHttpProtocolPluginTest
- store used port in field "serverPort", inherited by plugin tests
- use serverPort to sleep longer until server is up and set its port
number
---
.../apache/nutch/protocol/http/TestResponse.java | 2 +-
.../apache/nutch/protocol/okhttp/TestResponse.java | 2 +-
.../protocol/AbstractHttpProtocolPluginTest.java | 37 ++++++++++++----------
3 files changed, 23 insertions(+), 18 deletions(-)
diff --git
a/src/plugin/protocol-http/src/test/org/apache/nutch/protocol/http/TestResponse.java
b/src/plugin/protocol-http/src/test/org/apache/nutch/protocol/http/TestResponse.java
index 9d65b6df8..53b605568 100644
---
a/src/plugin/protocol-http/src/test/org/apache/nutch/protocol/http/TestResponse.java
+++
b/src/plugin/protocol-http/src/test/org/apache/nutch/protocol/http/TestResponse.java
@@ -70,7 +70,7 @@ public class TestResponse extends
AbstractHttpProtocolPluginTest {
protected HttpResponse getResponse(int statusCode, String headerName) {
try {
- URL url = new URL(protocol, localHost, defaultPort, "/" + headerName);
+ URL url = new URL(protocol, localHost, serverPort, "/" + headerName);
LOG.info("Emulating fetch of {}", url);
return new HttpResponse((Http) http, url, new CrawlDatum(statusCode,
1000));
} catch (ProtocolException | IOException e) {
diff --git
a/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/TestResponse.java
b/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/TestResponse.java
index 695a6c539..5cff6ddbc 100644
---
a/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/TestResponse.java
+++
b/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/TestResponse.java
@@ -70,7 +70,7 @@ public class TestResponse extends
AbstractHttpProtocolPluginTest {
protected OkHttpResponse getResponse(int statusCode, String headerName) {
try {
- URL url = new URL(protocol, localHost, defaultPort, "/" + headerName);
+ URL url = new URL(protocol, localHost, serverPort, "/" + headerName);
LOG.info("Emulating fetch of {}", url);
return new OkHttpResponse((OkHttp) http, url, new CrawlDatum(statusCode,
1000));
} catch (ProtocolException | IOException e) {
diff --git
a/src/test/org/apache/nutch/protocol/AbstractHttpProtocolPluginTest.java
b/src/test/org/apache/nutch/protocol/AbstractHttpProtocolPluginTest.java
index 322b34e99..be26d9147 100644
--- a/src/test/org/apache/nutch/protocol/AbstractHttpProtocolPluginTest.java
+++ b/src/test/org/apache/nutch/protocol/AbstractHttpProtocolPluginTest.java
@@ -22,7 +22,6 @@ import static org.junit.Assert.assertEquals;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.lang.invoke.MethodHandles;
-import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.SocketException;
@@ -70,7 +69,7 @@ public abstract class AbstractHttpProtocolPluginTest {
protected String localHost = "127.0.0.1";
/** Port used to send/receive test requests */
- protected int defaultPort = 47505;
+ protected int serverPort = 0;
protected static final String responseHeader = "HTTP/1.1 200 OK\r\n";
protected static final String simpleContent = "Content-Type:
text/html\r\n\r\nThis is a text.";
@@ -121,8 +120,10 @@ public abstract class AbstractHttpProtocolPluginTest {
/**
* Starts the test server at a specified port and constant response.
*
- * @param portno
- * Port number.
+ * @param port
+ * Port number. If 0, a random free port is used. The port number
+ * actually used by the test server is always stored in
+ * {@link #serverPort}.
* @param responder
* function to return a response (byte[] containing HTTP response
* header and payload content) for a given request header
represented
@@ -134,15 +135,16 @@ public abstract class AbstractHttpProtocolPluginTest {
protected void runServer(int port,
BiFunction<String, String[], byte[]> responder,
Predicate<List<String>> requestChecker) throws Exception {
- server = new ServerSocket();
- server.bind(new InetSocketAddress(localHost, port));
+ server = new ServerSocket(port);
+ serverPort = server.getLocalPort();
Pattern requestPattern = Pattern.compile("(?i)^GET\\s+(\\S+)");
while (true) {
- LOG.info("Listening on port {}", port);
if (server.isClosed()) {
- server = new ServerSocket();
- server.bind(new InetSocketAddress(localHost, port));
+ LOG.info("Server closed, restarting ...");
+ server = new ServerSocket(0); // request a free port
+ serverPort = server.getLocalPort();
}
+ LOG.info("Listening on port {}", serverPort);
Socket socket = server.accept();
LOG.info("Connection received");
try (BufferedReader in = new BufferedReader(new InputStreamReader(
@@ -195,13 +197,16 @@ public abstract class AbstractHttpProtocolPluginTest {
try {
runServer(port, responder, requestChecker);
} catch (SocketException e) {
- LOG.info("Socket on port {} closed: {}", port, e.getMessage());
+ LOG.info("Socket on port {} closed: {}",
+ (port == 0 ? serverPort : port), e.getMessage());
} catch (Exception e) {
LOG.warn("Test server died:", e);
}
});
serverThread.start();
- Thread.sleep(50);
+ do {
+ Thread.sleep(50);
+ } while (serverPort == 0); // runServer(...) sets serverPort
}
protected void launchServer(Function<String, byte[]> responder)
@@ -211,7 +216,7 @@ public abstract class AbstractHttpProtocolPluginTest {
protected void launchServer(Function<String, byte[]> responder,
Predicate<List<String>> requestChecker) throws InterruptedException {
- launchServer(defaultPort, responder, requestChecker);
+ launchServer(0, responder, requestChecker);
}
protected void launchServer(int port, Function<String, byte[]> responder,
@@ -224,7 +229,7 @@ public abstract class AbstractHttpProtocolPluginTest {
protected void launchServer(Map<String, byte[]> responses)
throws InterruptedException {
- launchServer(defaultPort, (String requestPath) -> {
+ launchServer(0, (String requestPath) -> {
return responses.get(requestPath);
}, null);
}
@@ -247,12 +252,12 @@ public abstract class AbstractHttpProtocolPluginTest {
protected ProtocolOutput fetchPage(String page, int expectedCode)
throws Exception {
- return fetchPage(defaultPort, page, expectedCode, null);
+ return fetchPage(serverPort, page, expectedCode, null);
}
protected ProtocolOutput fetchPage(String page, int expectedCode,
String expectedContentType) throws Exception {
- return fetchPage(defaultPort, page, expectedCode, null);
+ return fetchPage(serverPort, page, expectedCode, null);
}
/**
@@ -261,7 +266,7 @@ public abstract class AbstractHttpProtocolPluginTest {
* code.
*
* @param port
- * port server is running on
+ * port server is listening on
* @param page
* Page to be fetched
* @param expectedCode
