Ottomata has submitted this change and it was merged. Change subject: Remove Java part of kraken-etl ......................................................................
Remove Java part of kraken-etl We now use a separate branch of the camus repository to build the jar for etl. Hence, no need to keep the now unused Java part of kraken-etl around. Change-Id: I9277a8d66871984e0773b31e7a6e8be1444c2140 --- D kraken-etl/pom.xml D kraken-etl/src/main/java/org/wikimedia/analytics/kraken/etl/camus/kafka/coders/JsonStringMessageDecoder.java D kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/camus/kafka/coders/JsonStringMessageDecoderTest.java D kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/testutil/LoggingMockingTestCase.java D kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/testutil/MockingTestCase.java D kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/testutil/log/CollectionAppender.java D kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/testutil/log/LogUtil.java M pom.xml 8 files changed, 0 insertions(+), 862 deletions(-) Approvals: Ottomata: Verified; Looks good to me, approved diff --git a/kraken-etl/pom.xml b/kraken-etl/pom.xml deleted file mode 100644 index 4fcc6d6..0000000 --- a/kraken-etl/pom.xml +++ /dev/null @@ -1,76 +0,0 @@ -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - - <parent> - <groupId>org.wikimedia.analytics.kraken</groupId> - <artifactId>kraken</artifactId> - <version>0.0.2-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - - <artifactId>kraken-etl</artifactId> - <name>Kraken ETL</name> - <packaging>jar</packaging> - - <dependencies> - <dependency> - <groupId>com.google.code.gson</groupId> - <artifactId>gson</artifactId> - <version>2.2.4</version> - <scope>compile</scope> - </dependency> - - <dependency> - <groupId>com.linkedin.camus</groupId> - <artifactId>camus-etl-kafka</artifactId> - <version>0.1.0-SNAPSHOT-wmf-1</version> - <scope>compile</scope> - </dependency> - - <dependency> - <groupId>org.easymock</groupId> - <artifactId>easymock</artifactId> - <version>3.0</version> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.powermock</groupId> - <artifactId>powermock-module-junit4</artifactId> - <version>1.5</version> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.powermock</groupId> - <artifactId>powermock-api-easymock</artifactId> - <version>1.5</version> - <scope>test</scope> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <version>2.0</version> - <configuration> - <shadedArtifactAttached>false</shadedArtifactAttached> - </configuration> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <createDependencyReducedPom>false</createDependencyReducedPom> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> diff --git a/kraken-etl/src/main/java/org/wikimedia/analytics/kraken/etl/camus/kafka/coders/JsonStringMessageDecoder.java b/kraken-etl/src/main/java/org/wikimedia/analytics/kraken/etl/camus/kafka/coders/JsonStringMessageDecoder.java deleted file mode 100644 index fa524af..0000000 --- a/kraken-etl/src/main/java/org/wikimedia/analytics/kraken/etl/camus/kafka/coders/JsonStringMessageDecoder.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (C) 2013 Wikimedia Foundation -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License -// as published by the Free Software Foundation; either version 2 -// of the License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -package org.wikimedia.analytics.kraken.etl.camus.kafka.coders; - -import java.util.Properties; -import java.text.SimpleDateFormat; - -import com.google.gson.JsonParser; -import com.google.gson.JsonObject; - -import com.linkedin.camus.coders.CamusWrapper; -import com.linkedin.camus.coders.MessageDecoder; - -import org.apache.log4j.Logger; - - -/** - * MessageDecoder class that will convert the payload into a JSON object, - * look for a field named 'timestamp', and then set the CamusWrapper's - * timestamp property to the record's timestamp. If the JSON does not have - * a timestamp, then System.currentTimeMillis() will be used. - * This MessageDecoder returns a CamusWrapper that works with Strings. - */ -public class JsonStringMessageDecoder extends MessageDecoder<byte[], String> { - private static org.apache.log4j.Logger log = Logger.getLogger(JsonStringMessageDecoder.class); - - public static final String CAMUS_MESSAGE_TIMESTAMP_FORMAT = "camus.message.timestamp.format"; - public static final String DEFAULT_TIMESTAMP_FORMAT = "[dd/MMM/yyyy:HH:mm:ss Z]"; - - public static final String CAMUS_MESSAGE_TIMESTAMP_FIELD = "camus.message.timestamp.field"; - public static final String DEFAULT_TIMESTAMP_FIELD = "timestamp"; - - private String timestampFormat; - private String timestampField; - - @Override - public void init(Properties props, String topicName) { - this.props = props; - this.topicName = topicName; - - timestampFormat = props.getProperty(CAMUS_MESSAGE_TIMESTAMP_FORMAT, DEFAULT_TIMESTAMP_FORMAT); - timestampField = props.getProperty(CAMUS_MESSAGE_TIMESTAMP_FIELD, DEFAULT_TIMESTAMP_FIELD); - } - - @Override - public CamusWrapper<String> decode(byte[] payload) { - long timestamp = 0; - String payloadString; - JsonObject jsonObject; - - payloadString = new String(payload); - - // Parse the payload into a JsonObject. - try { - jsonObject = new JsonParser().parse(payloadString).getAsJsonObject(); - } catch (RuntimeException e) { - log.error("Caught exception while parsing JSON string '" + payloadString + "'."); - throw new RuntimeException(e); - } - - // Attempt to read and parse the timestamp element into a long. - if (jsonObject.has(timestampField)) { - String timestampString = jsonObject.get(timestampField).getAsString(); - try { - timestamp = new SimpleDateFormat(timestampFormat).parse(timestampString).getTime(); - } catch (Exception e) { - log.error("Could not parse timestamp '" + timestampString + "' while decoding JSON message."); - } - } - - // If timestamp wasn't set in the above block, - // then set it to current time. - if (timestamp == 0) { - log.warn("Couldn't find or parse timestamp field '" + timestampField + "' in JSON message, defaulting to current time."); - timestamp = System.currentTimeMillis(); - } - - return new CamusWrapper<String>(payloadString, timestamp); - } -} diff --git a/kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/camus/kafka/coders/JsonStringMessageDecoderTest.java b/kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/camus/kafka/coders/JsonStringMessageDecoderTest.java deleted file mode 100644 index 4412c1a..0000000 --- a/kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/camus/kafka/coders/JsonStringMessageDecoderTest.java +++ /dev/null @@ -1,266 +0,0 @@ -// Copyright (C) 2013 Wikimedia Foundation -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License -// as published by the Free Software Foundation; either version 2 -// of the License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -package org.wikimedia.analytics.kraken.etl.camus.kafka.coders; - -import java.text.DateFormatSymbols; -import java.util.Properties; - -import org.wikimedia.analytics.kraken.etl.testutil.LoggingMockingTestCase; - -import com.linkedin.camus.coders.CamusWrapper; - -public class JsonStringMessageDecoderTest extends LoggingMockingTestCase { - public void testPropertiesEmpty() { - JsonStringMessageDecoder decoder = new JsonStringMessageDecoder(); - - decoder.init(new Properties(), "someTopic"); - - // The default timestamp format uses localized month names :-/ so we - // have to localize month names to be able to test them. - String localizedJune = DateFormatSymbols.getInstance().getMonths()[5]; - - StringBuilder jsonStringBuilder = new StringBuilder(); - jsonStringBuilder.append("{"); - jsonStringBuilder.append("\"timestamp\": \"[14/"); - jsonStringBuilder.append(localizedJune); - jsonStringBuilder.append("/2012:17:58:52 +0200]\""); - jsonStringBuilder.append("}"); - - String jsonString = jsonStringBuilder.toString(); - - byte[] payload = jsonString.getBytes(); - - CamusWrapper<String> res = decoder.decode(payload); - - assertEquals("Parsed time stamp is not 2012-06-14 15:58:52", - 1339689532L * 1000, res.getTimestamp()); - - assertEquals("Parsed record does not match JSON", jsonString, res.getRecord()); - } - - public void testPropertiesTimestampField() { - JsonStringMessageDecoder decoder = new JsonStringMessageDecoder(); - - Properties properties = new Properties(); - properties.setProperty("camus.message.timestamp.field", "real_timestamp"); - decoder.init(properties, "someTopic"); - - // The default timestamp format uses localized month names :-/ so we - // have to localize month names to be able to test them. - String localizedJune = DateFormatSymbols.getInstance().getMonths()[5]; - - StringBuilder jsonStringBuilder = new StringBuilder(); - jsonStringBuilder.append("{"); - jsonStringBuilder.append("\"timestamp\": \"[13/"); - jsonStringBuilder.append(localizedJune); - jsonStringBuilder.append("/2012:17:58:52 +0200]\""); - jsonStringBuilder.append(","); - jsonStringBuilder.append("\"real_timestamp\": \"[14/"); - jsonStringBuilder.append(localizedJune); - jsonStringBuilder.append("/2012:17:58:52 +0200]\""); - jsonStringBuilder.append(","); - jsonStringBuilder.append("\"timestamp\": \"[15/"); - jsonStringBuilder.append(localizedJune); - jsonStringBuilder.append("/2012:17:58:52 +0200]\""); - jsonStringBuilder.append("}"); - - String jsonString = jsonStringBuilder.toString(); - - byte[] payload = jsonString.getBytes(); - - CamusWrapper<String> res = decoder.decode(payload); - - assertEquals("Parsed time stamp is not 2012-06-14 15:58:52", - 1339689532L * 1000, res.getTimestamp()); - - assertEquals("Parsed record does not match JSON", jsonString, res.getRecord()); - } - - public void testPropertiesTimestampFormatIsoPlain() { - JsonStringMessageDecoder decoder = new JsonStringMessageDecoder(); - - Properties properties = new Properties(); - properties.setProperty("camus.message.timestamp.format", "yyyy-MM-dd'T'HH:mm:ssZ"); - decoder.init(properties, "someTopic"); - - String jsonString = "{\"timestamp\": \"2012-06-14T17:58:52 +0200]\"}"; - - byte[] payload = jsonString.getBytes(); - - CamusWrapper<String> res = decoder.decode(payload); - - assertEquals("Parsed time stamp is not 2012-06-14 15:58:52", - 1339689532L * 1000, res.getTimestamp()); - - assertEquals("Parsed record does not match JSON", jsonString, res.getRecord()); - } - - public void testPropertiesTimestampFormatIsoSubsecond() { - JsonStringMessageDecoder decoder = new JsonStringMessageDecoder(); - - Properties properties = new Properties(); - properties.setProperty("camus.message.timestamp.format", "yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - decoder.init(properties, "someTopic"); - - String jsonString = "{\"timestamp\": \"2012-06-14T17:58:52.332 +0200]\"}"; - - byte[] payload = jsonString.getBytes(); - - CamusWrapper<String> res = decoder.decode(payload); - - assertEquals("Parsed time stamp is not 2012-06-14 15:58:52.332", - 1339689532L * 1000 + 332, res.getTimestamp()); - - assertEquals("Parsed record does not match JSON", jsonString, res.getRecord()); - } - - public void testPropertiesManyValues() { - JsonStringMessageDecoder decoder = new JsonStringMessageDecoder(); - - Properties properties = new Properties(); - properties.setProperty("someKey", "someValue"); - properties.setProperty("camus.message.timestamp.field", "real_ts"); - properties.setProperty("someOtherKey", "someOtherValue"); - properties.setProperty("camus.message.timestamp.format", "yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - properties.setProperty("yetAnotherKey", "yetAnotherValue"); - decoder.init(properties, "someTopic"); - - String jsonString = "{\"real_ts\": \"2012-06-14T17:58:52.332 +0200]\"}"; - - byte[] payload = jsonString.getBytes(); - - CamusWrapper<String> res = decoder.decode(payload); - - assertEquals("Parsed time stamp is not 2012-06-14 15:58:52.332", - 1339689532L * 1000 + 332, res.getTimestamp()); - - assertEquals("Parsed record does not match JSON", jsonString, res.getRecord()); - } - - public void testJsonWithPayload() { - JsonStringMessageDecoder decoder = new JsonStringMessageDecoder(); - - Properties properties = new Properties(); - properties.setProperty("camus.message.timestamp.format", "yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - decoder.init(properties, "someTopic"); - - StringBuilder jsonStringBuilder = new StringBuilder(); - jsonStringBuilder.append("{"); - jsonStringBuilder.append("\"payload1\": true,"); - jsonStringBuilder.append("\"payload2\": 42,"); - jsonStringBuilder.append("\"timestamp\": \"2012-06-14T17:58:52.332 +0200]\","); - jsonStringBuilder.append("\"payload3\": \"foo\","); - jsonStringBuilder.append("\"payload4\": \"bar\","); - jsonStringBuilder.append("\"payload5\": [ { \"foo\": \"bar\", \"baz\": 4711 }, 169 ]"); - jsonStringBuilder.append("}"); - - String jsonString = jsonStringBuilder.toString(); - - byte[] payload = jsonString.getBytes(); - - CamusWrapper<String> res = decoder.decode(payload); - - assertEquals("Parsed time stamp is not 2012-06-14 15:58:52.332", - 1339689532L * 1000 + 332, res.getTimestamp()); - - assertEquals("Parsed record does not match JSON", jsonString, res.getRecord()); - } - - public void testMalformedJson() { - JsonStringMessageDecoder decoder = new JsonStringMessageDecoder(); - - Properties properties = new Properties(); - properties.setProperty("camus.message.timestamp.format", "yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - decoder.init(properties, "someTopic"); - - byte[] payload = "{".getBytes(); - - try { - decoder.decode(payload); - } catch (RuntimeException e) { - assertLogMessageContains("pars"); - } - } - - public void testJsonWithoutTimestamp() { - JsonStringMessageDecoder decoder = new JsonStringMessageDecoder(); - - Properties properties = new Properties(); - properties.setProperty("camus.message.timestamp.format", "yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - decoder.init(properties, "someTopic"); - - String jsonString = "{}"; - byte[] payload = jsonString.getBytes(); - - long tsPreDecode = System.currentTimeMillis(); - CamusWrapper<String> res = decoder.decode(payload); - long tsPostDecode = System.currentTimeMillis(); - - assertLogMessageContains("current time"); - assertTrue("Parsed time stamp is before start of decoding", - tsPreDecode <= res.getTimestamp()); - assertTrue("Parsed time stamp is after end of decoding", - res.getTimestamp() <= tsPostDecode); - assertEquals("Parsed record does not match JSON", jsonString, res.getRecord()); - } - - public void testJsonWithMalformedTimestamp() { - JsonStringMessageDecoder decoder = new JsonStringMessageDecoder(); - - Properties properties = new Properties(); - properties.setProperty("camus.message.timestamp.format", "yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - decoder.init(properties, "someTopic"); - - String jsonString = "{\"timestamp\": \"foo\"}"; - byte[] payload = jsonString.getBytes(); - - long tsPreDecode = System.currentTimeMillis(); - CamusWrapper<String> res = decoder.decode(payload); - long tsPostDecode = System.currentTimeMillis(); - - assertLogMessageContains("foo"); - assertLogMessageContains("current time"); - assertTrue("Parsed time stamp is before start of decoding", - tsPreDecode <= res.getTimestamp()); - assertTrue("Parsed time stamp is after end of decoding", - res.getTimestamp() <= tsPostDecode); - assertEquals("Parsed record does not match JSON", jsonString, res.getRecord()); - } - - public void testJsonWithTimestampOfWrongType() { - JsonStringMessageDecoder decoder = new JsonStringMessageDecoder(); - - Properties properties = new Properties(); - properties.setProperty("camus.message.timestamp.format", "yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - decoder.init(properties, "someTopic"); - - String jsonString = "{\"timestamp\": 4711}"; - byte[] payload = jsonString.getBytes(); - - long tsPreDecode = System.currentTimeMillis(); - CamusWrapper<String> res = decoder.decode(payload); - long tsPostDecode = System.currentTimeMillis(); - - assertLogMessageContains("4711"); - assertLogMessageContains("current time"); - assertTrue("Parsed time stamp is before start of decoding", - tsPreDecode <= res.getTimestamp()); - assertTrue("Parsed time stamp is after end of decoding", - res.getTimestamp() <= tsPostDecode); - assertEquals("Parsed record does not match JSON", jsonString, res.getRecord()); - } -} diff --git a/kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/testutil/LoggingMockingTestCase.java b/kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/testutil/LoggingMockingTestCase.java deleted file mode 100644 index fc199a0..0000000 --- a/kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/testutil/LoggingMockingTestCase.java +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (C) 2013 Wikimedia Foundation -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License -// as published by the Free Software Foundation; either version 2 -// of the License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -package org.wikimedia.analytics.kraken.etl.testutil; - -import com.google.common.collect.Lists; - -import org.wikimedia.analytics.kraken.etl.testutil.log.LogUtil; -import org.apache.log4j.spi.LoggingEvent; -import org.junit.After; - -import java.util.Iterator; - -import org.apache.log4j.Level; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; - -public abstract class LoggingMockingTestCase extends MockingTestCase { - private String loggerName; - private LogUtil.LoggerSettings loggerSettings; - private java.util.Collection<LoggingEvent> loggedEvents; - - protected final void assertLogMessageContains(String needle, Level level) { - LoggingEvent hit = null; - Iterator<LoggingEvent> iter = loggedEvents.iterator(); - while (hit == null && iter.hasNext()) { - LoggingEvent event = iter.next(); - if (event.getRenderedMessage().contains(needle)) { - if (level == null || level.equals(event.getLevel())) { - hit = event; - } - } - } - assertNotNull("Could not find log message containing '" + needle + "'", - hit); - assertTrue("Could not remove log message containing '" + needle + "'", - loggedEvents.remove(hit)); - } - - protected final void assertLogMessageContains(String needle) { - assertLogMessageContains(needle, null); - } - - protected final void assertLogThrowableMessageContains(String needle) { - LoggingEvent hit = null; - Iterator<LoggingEvent> iter = loggedEvents.iterator(); - while (hit == null && iter.hasNext()) { - LoggingEvent event = iter.next(); - - if (event.getThrowableInformation().getThrowable().toString() - .contains(needle)) { - hit = event; - } - } - assertNotNull("Could not find log message with a Throwable containing '" - + needle + "'", hit); - assertTrue("Could not remove log message with a Throwable containing '" - + needle + "'", loggedEvents.remove(hit)); - } - - // As the PowerMock runner does not pass through runTest, we inject log - // verification through @After - @After - public final void assertNoUnassertedLogEvents() { - if (loggedEvents.size() > 0) { - LoggingEvent event = loggedEvents.iterator().next(); - String msg = "Found untreated logged events. First one is:\n"; - msg += event.getRenderedMessage(); - if (event.getThrowableInformation() != null) { - msg += "\n" + event.getThrowableInformation().getThrowable(); - } - fail(msg); - } - } - - @Override - public void setUp() throws Exception { - super.setUp(); - loggedEvents = Lists.newArrayList(); - - // The logger we're interested is class name without the trailing "Test". - // While this is not the most general approach it is sufficient for now, - // and we can improve later to allow tests to specify which loggers are - // to check. - loggerName = this.getClass().getCanonicalName(); - loggerName = loggerName.substring(0, loggerName.length()-4); - loggerSettings = LogUtil.logToCollection(loggerName, loggedEvents); - } - - @Override - protected void runTest() throws Throwable { - super.runTest(); - // Plain JUnit runner does not pick up @After, so we add it here - // explicitly. Note, that we cannot put this into tearDown, as failure - // to verify mocks would bail out and might leave open resources from - // subclasses open. - assertNoUnassertedLogEvents(); - } - - @Override - public void tearDown() throws Exception { - if (loggerName != null && loggerSettings != null) { - Logger logger = LogManager.getLogger(loggerName); - loggerSettings.pushOntoLogger(logger); - } - super.tearDown(); - } -} diff --git a/kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/testutil/MockingTestCase.java b/kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/testutil/MockingTestCase.java deleted file mode 100644 index f2cb3c7..0000000 --- a/kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/testutil/MockingTestCase.java +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright (C) 2013 Wikimedia Foundation -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License -// as published by the Free Software Foundation; either version 2 -// of the License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -package org.wikimedia.analytics.kraken.etl.testutil; - -import junit.framework.TestCase; - -import org.easymock.EasyMock; -import org.easymock.IMocksControl; -import org.junit.After; -import org.junit.runner.RunWith; -import org.powermock.api.easymock.PowerMock; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.util.ArrayList; -import java.util.Collection; - -/** - * Test case with some support for automatically verifying mocks. - */ -public abstract class MockingTestCase extends TestCase { - private Collection<Object> mocks; - private Collection<IMocksControl> mockControls; - private boolean mocksReplayed; - private boolean usePowerMock; - - /** - * Create and register a mock control. - * - * @return The mock control instance. - */ - protected final IMocksControl createMockControl() { - IMocksControl mockControl = EasyMock.createControl(); - assertTrue("Adding mock control failed", mockControls.add(mockControl)); - return mockControl; - } - - /** - * Create and register a mock. - * - * Creates a mock and registers it in the list of created mocks, so it gets - * treated automatically upon {@code replay} and {@code verify}; - * @param toMock The class to create a mock for. - * @return The mock instance. - */ - protected final <T> T createMock(Class<T> toMock) { - return createMock(toMock, null); - } - - /** - * Create a mock for a mock control and register a mock. - * - * Creates a mock and registers it in the list of created mocks, so it gets - * treated automatically upon {@code replay} and {@code verify}; - * @param toMock The class to create a mock for. - * @param control The mock control to create the mock on. If null, do not use - * a specific control. - * @return The mock instance. - */ - protected final <T> T createMock(Class<T> toMock, IMocksControl control) { - assertFalse("Mocks have already been set to replay", mocksReplayed); - final T mock; - if (control == null) { - if (usePowerMock) { - mock = PowerMock.createMock(toMock); - } else { - mock = EasyMock.createMock(toMock); - } - assertTrue("Adding " + toMock.getName() + " mock failed", - mocks.add(mock)); - } else { - mock = control.createMock(toMock); - } - return mock; - } - - /** - * Set all registered mocks to replay - */ - protected final void replayMocks() { - assertFalse("Mocks have already been set to replay", mocksReplayed); - if (usePowerMock) { - PowerMock.replayAll(); - } else { - EasyMock.replay(mocks.toArray()); - } - for (IMocksControl mockControl : mockControls) { - mockControl.replay(); - } - mocksReplayed = true; - } - - /** - * Verify all registered mocks - * - * This method is called automatically at the end of a test. Nevertheless, - * it is safe to also call it beforehand, if this better meets the - * verification part of a test. - */ - // As the PowerMock runner does not pass through runTest, we inject mock - // verification through @After - @After - public final void verifyMocks() { - if (!mocks.isEmpty() || !mockControls.isEmpty()) { - assertTrue("Created mocks have not been set to replay. Call replayMocks " - + "within the test", mocksReplayed); - if (usePowerMock) { - PowerMock.verifyAll(); - } else { - EasyMock.verify(mocks.toArray()); - } - for (IMocksControl mockControl : mockControls) { - mockControl.verify(); - } - } - } - - @Override - public void setUp() throws Exception { - super.setUp(); - - usePowerMock = false; - RunWith runWith = this.getClass().getAnnotation(RunWith.class); - if (runWith != null) { - usePowerMock = PowerMockRunner.class.isAssignableFrom(runWith.value()); - } - - mocks = new ArrayList<Object>(); - mockControls = new ArrayList<IMocksControl>(); - mocksReplayed = false; - } - - @Override - protected void runTest() throws Throwable { - super.runTest(); - // Plain JUnit runner does not pick up @After, so we add it here - // explicitly. Note, that we cannot put this into tearDown, as failure - // to verify mocks would bail out and might leave open resources from - // subclasses open. - verifyMocks(); - } -} diff --git a/kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/testutil/log/CollectionAppender.java b/kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/testutil/log/CollectionAppender.java deleted file mode 100644 index 948aa28..0000000 --- a/kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/testutil/log/CollectionAppender.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (C) 2013 Wikimedia Foundation -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License -// as published by the Free Software Foundation; either version 2 -// of the License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -package org.wikimedia.analytics.kraken.etl.testutil.log; - -import com.google.common.collect.Lists; - -import org.apache.log4j.AppenderSkeleton; -import org.apache.log4j.spi.LoggingEvent; - -import java.util.Collection; -import java.util.LinkedList; - -/** - * Log4j appender that logs into a list - */ -public class CollectionAppender extends AppenderSkeleton { - private Collection<LoggingEvent> events; - - public CollectionAppender() { - events = new LinkedList<LoggingEvent>(); - } - - public CollectionAppender(Collection<LoggingEvent> events) { - this.events = events; - } - - @Override - public boolean requiresLayout() { - return false; - } - - @Override - protected void append(LoggingEvent event) { - if (! events.add(event)) { - throw new RuntimeException("Could not append event " + event); - } - } - - @Override - public void close() { - } - - public Collection<LoggingEvent> getLoggedEvents() { - return Lists.newLinkedList(events); - } -} diff --git a/kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/testutil/log/LogUtil.java b/kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/testutil/log/LogUtil.java deleted file mode 100644 index 12bb3fd..0000000 --- a/kraken-etl/src/test/java/org/wikimedia/analytics/kraken/etl/testutil/log/LogUtil.java +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (C) 2013 Wikimedia Foundation -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License -// as published by the Free Software Foundation; either version 2 -// of the License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -package org.wikimedia.analytics.kraken.etl.testutil.log; - -import org.apache.log4j.Appender; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; -import org.apache.log4j.spi.LoggingEvent; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Enumeration; -import java.util.List; - -public class LogUtil { - /** - * Change logger's setting so it only logs to a collection. - * - * @param logName Name of the logger to modify. - * @param collection The collection to log into. - * @return The logger's original settings. - */ - public static LoggerSettings logToCollection(String logName, - Collection<LoggingEvent> collection) { - Logger logger = LogManager.getLogger(logName); - LoggerSettings loggerSettings = new LoggerSettings(logger); - logger.removeAllAppenders(); - logger.setAdditivity(false); - CollectionAppender listAppender = new CollectionAppender(collection); - logger.addAppender(listAppender); - return loggerSettings; - } - - /** - * Capsule for a logger's settings that get mangled by rerouting logging to a collection - */ - public static class LoggerSettings { - private final boolean additive; - private final List<Appender> appenders; - - /** - * Read off logger settings from an instance. - * - * @param logger The logger to read the settings off from. - */ - private LoggerSettings(Logger logger) { - this.additive = logger.getAdditivity(); - - Enumeration<?> appenders = logger.getAllAppenders(); - this.appenders = new ArrayList<Appender>(); - while (appenders.hasMoreElements()) { - Object appender = appenders.nextElement(); - if (appender instanceof Appender) { - this.appenders.add((Appender)appender); - } else { - throw new RuntimeException("getAllAppenders of " + logger - + " contained an object that is not an Appender"); - } - } - } - - /** - * Pushes this settings back onto a logger. - * - * @param logger the logger on which to push the settings. - */ - public void pushOntoLogger(Logger logger) { - logger.setAdditivity(additive); - logger.removeAllAppenders(); - for (Appender appender : appenders) { - logger.addAppender(appender); - } - } - } -} diff --git a/pom.xml b/pom.xml index 4f037dc..4b017f3 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,6 @@ <module>kraken-generic</module> <module>kraken-dclass</module> <module>kraken-pig</module> - <module>kraken-etl</module> <module>kraken-toolbelt</module> </modules> -- To view, visit https://gerrit.wikimedia.org/r/119724 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I9277a8d66871984e0773b31e7a6e8be1444c2140 Gerrit-PatchSet: 2 Gerrit-Project: analytics/kraken Gerrit-Branch: master Gerrit-Owner: QChris <[email protected]> Gerrit-Reviewer: Ottomata <[email protected]> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
