This is an automated email from the ASF dual-hosted git repository. ppkarwasz pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/logging-flume-legacy.git
commit 8b5c18822c9594911440d3ddf4e30fe3aed6ebcd Author: Andras Beni <[email protected]> AuthorDate: Thu Oct 26 15:31:04 2017 -0700 FLUME-199. Remove hardcoded ports from unit tests Many unit tests use hardcoded port numbers which leads to flakiness and causes problems when running builds in parallel. This patch fixes this issue by searching for available ports instead of the hardcoded ones. This closes #124 Reviewers: Miklos Csanady, Ferenc Szabo (Andras Beni via Denes Arvay) --- .../TestLoadBalancingLog4jAppender.java | 73 +++++++++++----------- .../clients/log4jappender/TestLog4jAppender.java | 11 +++- .../log4jappender/TestLog4jAppenderWithAvro.java | 14 ++++- ...lume-loadbalancing-backoff-log4jtest.properties | 2 +- .../flume-loadbalancing-rnd-log4jtest.properties | 2 +- .../flume-loadbalancinglog4jtest.properties | 2 +- .../flume-log4jtest-avro-generic.properties | 2 +- .../flume-log4jtest-avro-reflect.properties | 2 +- .../src/test/resources/flume-log4jtest.properties | 2 +- 9 files changed, 67 insertions(+), 43 deletions(-) diff --git a/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLoadBalancingLog4jAppender.java b/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLoadBalancingLog4jAppender.java index ec5d6df..0546c23 100644 --- a/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLoadBalancingLog4jAppender.java +++ b/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLoadBalancingLog4jAppender.java @@ -18,9 +18,10 @@ */ package org.apache.flume.clients.log4jappender; -import java.io.File; -import java.io.FileReader; import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.ServerSocket; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -31,6 +32,7 @@ import java.util.concurrent.atomic.AtomicInteger; import junit.framework.Assert; +import org.apache.commons.lang.StringUtils; import org.apache.flume.Channel; import org.apache.flume.ChannelSelector; import org.apache.flume.Context; @@ -64,6 +66,25 @@ public class TestLoadBalancingLog4jAppender { private Logger fixture; private boolean slowDown = false; + private static List<Integer> getFreePorts(int numberOfPorts) throws IOException { + List<Integer> ports = new ArrayList<>(numberOfPorts); + for (int index = 0; index < numberOfPorts; ++index) { + try (ServerSocket socket = new ServerSocket(0)) { + ports.add(socket.getLocalPort()); + } + } + return ports; + } + + private static String toHostList(List<Integer> ports) { + List<String> addresses = new ArrayList<String>(ports.size()); + for (Integer port : ports) { + addresses.add("localhost:" + port); + } + String hostList = StringUtils.join(addresses, " "); + return hostList; + } + @Before public void initiate() throws InterruptedException { ch = new MemoryChannel(); @@ -92,10 +113,8 @@ public class TestLoadBalancingLog4jAppender { public void testLog4jAppenderRoundRobin() throws IOException { int numberOfMsgs = 1000; int expectedPerSource = 500; - File TESTFILE = new File(TestLoadBalancingLog4jAppender.class - .getClassLoader() - .getResource("flume-loadbalancinglog4jtest.properties").getFile()); - startSources(TESTFILE, false, new int[] { 25430, 25431 }); + String propertiesFile = "flume-loadbalancinglog4jtest.properties"; + startSources(propertiesFile, false, getFreePorts(2)); sendAndAssertMessages(numberOfMsgs); @@ -107,12 +126,8 @@ public class TestLoadBalancingLog4jAppender { @Test public void testLog4jAppenderRandom() throws IOException { int numberOfMsgs = 1000; - File TESTFILE = new File(TestLoadBalancingLog4jAppender.class - .getClassLoader() - .getResource("flume-loadbalancing-rnd-log4jtest.properties").getFile()); - startSources(TESTFILE, false, new int[] { 25430, 25431, 25432, 25433, - 25434, - 25435, 25436, 25437, 25438, 25439 }); + String propertiesFile = "flume-loadbalancing-rnd-log4jtest.properties"; + startSources(propertiesFile, false, getFreePorts(10)); sendAndAssertMessages(numberOfMsgs); @@ -129,11 +144,8 @@ public class TestLoadBalancingLog4jAppender { @Test public void testRandomBackoff() throws Exception { - File TESTFILE = new File(TestLoadBalancingLog4jAppender.class - .getClassLoader() - .getResource("flume-loadbalancing-backoff-log4jtest.properties") - .getFile()); - startSources(TESTFILE, false, new int[] { 25430, 25431, 25432 }); + String propertiesFile = "flume-loadbalancing-backoff-log4jtest.properties"; + startSources(propertiesFile, false, getFreePorts(3)); sources.get(0).setFail(); sources.get(2).setFail(); @@ -163,11 +175,8 @@ public class TestLoadBalancingLog4jAppender { @Test public void testRandomBackoffUnsafeMode() throws Exception { - File TESTFILE = new File(TestLoadBalancingLog4jAppender.class - .getClassLoader() - .getResource("flume-loadbalancing-backoff-log4jtest.properties") - .getFile()); - startSources(TESTFILE, true, new int[]{25430, 25431, 25432}); + String propertiesFile = "flume-loadbalancing-backoff-log4jtest.properties"; + startSources(propertiesFile, true, getFreePorts(3)); sources.get(0).setFail(); sources.get(1).setFail(); @@ -178,15 +187,12 @@ public class TestLoadBalancingLog4jAppender { @Test (expected = EventDeliveryException.class) public void testTimeout() throws Throwable { - File TESTFILE = new File(TestLoadBalancingLog4jAppender.class - .getClassLoader() - .getResource("flume-loadbalancinglog4jtest.properties") - .getFile()); + String propertiesFile = "flume-loadbalancinglog4jtest.properties"; ch = new TestLog4jAppender.SlowMemoryChannel(2000); configureChannel(); slowDown = true; - startSources(TESTFILE, false, new int[]{25430, 25431, 25432}); + startSources(propertiesFile, false, getFreePorts(3)); int level = 20000; String msg = "This is log message number" + String.valueOf(level); try { @@ -194,16 +200,12 @@ public class TestLoadBalancingLog4jAppender { } catch (FlumeException ex) { throw ex.getCause(); } - } @Test(expected = EventDeliveryException.class) public void testRandomBackoffNotUnsafeMode() throws Throwable { - File TESTFILE = new File(TestLoadBalancingLog4jAppender.class - .getClassLoader() - .getResource("flume-loadbalancing-backoff-log4jtest.properties") - .getFile()); - startSources(TESTFILE, false, new int[]{25430, 25431, 25432}); + String propertiesFile = "flume-loadbalancing-backoff-log4jtest.properties"; + startSources(propertiesFile, false, getFreePorts(3)); sources.get(0).setFail(); sources.get(1).setFail(); @@ -272,7 +274,7 @@ public class TestLoadBalancingLog4jAppender { } - private void startSources(File log4jProps, boolean unsafeMode, int... ports) + private void startSources(String log4jProps, boolean unsafeMode, List<Integer> ports) throws IOException { for (int port : ports) { CountingAvroSource source = new CountingAvroSource(port); @@ -294,9 +296,10 @@ public class TestLoadBalancingLog4jAppender { // log4j setup is completed before the @Before calls also. // This will cause the test to fail even before it starts! - FileReader reader = new FileReader(log4jProps); + Reader reader = new InputStreamReader(getClass().getResourceAsStream("/" + log4jProps)); Properties props = new Properties(); props.load(reader); + props.setProperty("log4j.appender.out2.Hosts", toHostList(ports)); props.setProperty("log4j.appender.out2.UnsafeMode", String.valueOf(unsafeMode)); if (slowDown) { diff --git a/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppender.java b/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppender.java index 25698c5..3f8ba37 100644 --- a/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppender.java +++ b/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppender.java @@ -21,6 +21,8 @@ package org.apache.flume.clients.log4jappender; import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.net.ServerSocket; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; @@ -58,9 +60,15 @@ public class TestLog4jAppender { private Channel ch; private Properties props; + private static int getFreePort() throws Exception { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } + @Before public void initiate() throws Exception { - int port = 25430; + int port = getFreePort(); source = Mockito.spy(new AvroSource()); ch = new MemoryChannel(); Configurables.configure(ch, new Context()); @@ -76,6 +84,7 @@ public class TestLog4jAppender { FileReader reader = new FileReader(TESTFILE); props = new Properties(); props.load(reader); + props.put("log4j.appender.out2.Port", String.valueOf(port)); reader.close(); } diff --git a/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java b/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java index 7c2a964..9ccebc8 100644 --- a/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java +++ b/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java @@ -23,6 +23,7 @@ import com.google.common.io.Resources; import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.net.ServerSocket; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; @@ -59,6 +60,13 @@ public class TestLog4jAppenderWithAvro { private AvroSource source; private Channel ch; private Properties props; + private int port; + + private static int getFreePort() throws Exception { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } @Before public void setUp() throws Exception { @@ -66,7 +74,7 @@ public class TestLog4jAppenderWithAvro { Files.copy(Resources.newInputStreamSupplier(schemaUrl), new File("/tmp/myrecord.avsc")); - int port = 25430; + port = getFreePort(); source = new AvroSource(); ch = new MemoryChannel(); Configurables.configure(ch, new Context()); @@ -99,6 +107,7 @@ public class TestLog4jAppenderWithAvro { @Test public void testAvroGeneric() throws IOException { loadProperties("flume-log4jtest-avro-generic.properties"); + props.put("log4j.appender.out2.Port", String.valueOf(port)); PropertyConfigurator.configure(props); Logger logger = LogManager.getLogger(TestLog4jAppenderWithAvro.class); String msg = "This is log message number " + String.valueOf(0); @@ -136,6 +145,7 @@ public class TestLog4jAppenderWithAvro { @Test public void testAvroReflect() throws IOException { loadProperties("flume-log4jtest-avro-reflect.properties"); + props.put("log4j.appender.out2.Port", String.valueOf(port)); PropertyConfigurator.configure(props); Logger logger = LogManager.getLogger(TestLog4jAppenderWithAvro.class); String msg = "This is log message number " + String.valueOf(0); @@ -173,6 +183,7 @@ public class TestLog4jAppenderWithAvro { @Test public void testDifferentEventTypesInBatchWithAvroReflect() throws IOException { loadProperties("flume-log4jtest-avro-reflect.properties"); + props.put("log4j.appender.out2.Port", String.valueOf(port)); PropertyConfigurator.configure(props); Logger logger = LogManager.getLogger(getClass()); List<Object> events = Arrays.asList("string", new AppEvent("appEvent")); @@ -203,6 +214,7 @@ public class TestLog4jAppenderWithAvro { @Test public void testDifferentEventTypesInBatchWithAvroGeneric() throws IOException { loadProperties("flume-log4jtest-avro-generic.properties"); + props.put("log4j.appender.out2.Port", String.valueOf(port)); PropertyConfigurator.configure(props); Logger logger = LogManager.getLogger(getClass()); String msg = "Avro log message"; diff --git a/flume-ng-log4jappender/src/test/resources/flume-loadbalancing-backoff-log4jtest.properties b/flume-ng-log4jappender/src/test/resources/flume-loadbalancing-backoff-log4jtest.properties index 6e8235e..6f11e17 100644 --- a/flume-ng-log4jappender/src/test/resources/flume-loadbalancing-backoff-log4jtest.properties +++ b/flume-ng-log4jappender/src/test/resources/flume-loadbalancing-backoff-log4jtest.properties @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. log4j.appender.out2 = org.apache.flume.clients.log4jappender.LoadBalancingLog4jAppender -log4j.appender.out2.Hosts = localhost:25430 localhost:25431 localhost:25432 +#log4j.appender.out2.Hosts = Set from java source log4j.appender.out2.Selector = ROUND_ROBIN log4j.appender.out2.MaxBackoff = 30000 log4j.logger.org.apache.flume.clients.log4jappender = DEBUG,out2 \ No newline at end of file diff --git a/flume-ng-log4jappender/src/test/resources/flume-loadbalancing-rnd-log4jtest.properties b/flume-ng-log4jappender/src/test/resources/flume-loadbalancing-rnd-log4jtest.properties index fd43d19..753feb6 100644 --- a/flume-ng-log4jappender/src/test/resources/flume-loadbalancing-rnd-log4jtest.properties +++ b/flume-ng-log4jappender/src/test/resources/flume-loadbalancing-rnd-log4jtest.properties @@ -15,6 +15,6 @@ # specific language governing permissions and limitations # under the License. log4j.appender.out2 = org.apache.flume.clients.log4jappender.LoadBalancingLog4jAppender -log4j.appender.out2.Hosts = localhost:25430 localhost:25431 localhost:25432 localhost:25433 localhost:25434 localhost:25435 localhost:25436 localhost:25437 localhost:25438 localhost:25439 +#log4j.appender.out2.Hosts = Set from java source log4j.appender.out2.Selector = RANDOM log4j.logger.org.apache.flume.clients.log4jappender = DEBUG,out2 \ No newline at end of file diff --git a/flume-ng-log4jappender/src/test/resources/flume-loadbalancinglog4jtest.properties b/flume-ng-log4jappender/src/test/resources/flume-loadbalancinglog4jtest.properties index 618e504..cfa4e11 100644 --- a/flume-ng-log4jappender/src/test/resources/flume-loadbalancinglog4jtest.properties +++ b/flume-ng-log4jappender/src/test/resources/flume-loadbalancinglog4jtest.properties @@ -15,5 +15,5 @@ # specific language governing permissions and limitations # under the License. log4j.appender.out2 = org.apache.flume.clients.log4jappender.LoadBalancingLog4jAppender -log4j.appender.out2.Hosts = localhost:25430 localhost:25431 +#log4j.appender.out2.Hosts = Set from java source log4j.logger.org.apache.flume.clients.log4jappender = DEBUG,out2 \ No newline at end of file diff --git a/flume-ng-log4jappender/src/test/resources/flume-log4jtest-avro-generic.properties b/flume-ng-log4jappender/src/test/resources/flume-log4jtest-avro-generic.properties index ffdab8b..92d7d16 100644 --- a/flume-ng-log4jappender/src/test/resources/flume-log4jtest-avro-generic.properties +++ b/flume-ng-log4jappender/src/test/resources/flume-log4jtest-avro-generic.properties @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. log4j.appender.out2 = org.apache.flume.clients.log4jappender.Log4jAppender -log4j.appender.out2.Port = 25430 +#log4j.appender.out2.Port = Set from java source log4j.appender.out2.Hostname = localhost log4j.appender.out2.AvroSchemaUrl = file:///tmp/myrecord.avsc log4j.logger.org.apache.flume.clients.log4jappender = DEBUG,out2 \ No newline at end of file diff --git a/flume-ng-log4jappender/src/test/resources/flume-log4jtest-avro-reflect.properties b/flume-ng-log4jappender/src/test/resources/flume-log4jtest-avro-reflect.properties index b50ffcc..06a0c68 100644 --- a/flume-ng-log4jappender/src/test/resources/flume-log4jtest-avro-reflect.properties +++ b/flume-ng-log4jappender/src/test/resources/flume-log4jtest-avro-reflect.properties @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. log4j.appender.out2 = org.apache.flume.clients.log4jappender.Log4jAppender -log4j.appender.out2.Port = 25430 +#log4j.appender.out2.Port = Set from java source log4j.appender.out2.Hostname = localhost log4j.appender.out2.AvroReflectionEnabled = true log4j.logger.org.apache.flume.clients.log4jappender = DEBUG,out2 \ No newline at end of file diff --git a/flume-ng-log4jappender/src/test/resources/flume-log4jtest.properties b/flume-ng-log4jappender/src/test/resources/flume-log4jtest.properties index 6575c0e..97dcab5 100644 --- a/flume-ng-log4jappender/src/test/resources/flume-log4jtest.properties +++ b/flume-ng-log4jappender/src/test/resources/flume-log4jtest.properties @@ -15,6 +15,6 @@ # specific language governing permissions and limitations # under the License. log4j.appender.out2 = org.apache.flume.clients.log4jappender.Log4jAppender -log4j.appender.out2.Port = 25430 +#log4j.appender.out2.Port = Set from java source log4j.appender.out2.Hostname = localhost log4j.logger.org.apache.flume.clients.log4jappender = DEBUG,out2 \ No newline at end of file
