http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/test/java/org/apache/kudu/client/TestPartitionPruner.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestPartitionPruner.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestPartitionPruner.java index 3a5610f..c05083b 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestPartitionPruner.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestPartitionPruner.java @@ -16,18 +16,18 @@ // under the License. package org.apache.kudu.client; +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.List; + import com.google.common.collect.ImmutableList; +import org.junit.Test; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.Type; import org.apache.kudu.client.KuduPredicate.ComparisonOp; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.*; public class TestPartitionPruner extends BaseKuduTest {
http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/test/java/org/apache/kudu/client/TestRemoteTablet.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestRemoteTablet.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRemoteTablet.java index ea77c27..22cfec0 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestRemoteTablet.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRemoteTablet.java @@ -16,15 +16,19 @@ // under the License. package org.apache.kudu.client; -import com.google.protobuf.ByteString; -import org.apache.kudu.consensus.Metadata; -import org.apache.kudu.master.Master; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.*; +import com.google.protobuf.ByteString; +import org.junit.Test; + +import org.apache.kudu.consensus.Metadata; +import org.apache.kudu.master.Master; public class TestRemoteTablet { http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowErrors.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowErrors.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowErrors.java index f4929b5..a488b76 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowErrors.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowErrors.java @@ -16,14 +16,13 @@ // under the License. package org.apache.kudu.client; -import org.junit.BeforeClass; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.util.List; -import static org.junit.Assert.*; - -import com.google.common.collect.ImmutableList; +import org.junit.BeforeClass; +import org.junit.Test; public class TestRowErrors extends BaseKuduTest { http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowResult.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowResult.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowResult.java index 0db5b44..223ddc4 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowResult.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowResult.java @@ -16,16 +16,17 @@ // under the License. package org.apache.kudu.client; -import org.junit.BeforeClass; -import org.junit.Test; -import org.apache.kudu.Type; - -import java.nio.ByteBuffer; - import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.nio.ByteBuffer; + +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.kudu.Type; + public class TestRowResult extends BaseKuduTest { // Generate a unique table name http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/test/java/org/apache/kudu/client/TestRpcTraces.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestRpcTraces.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRpcTraces.java index 6e8842c..fb0887f 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestRpcTraces.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRpcTraces.java @@ -16,11 +16,13 @@ // under the License. package org.apache.kudu.client; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; import java.util.List; -import static org.junit.Assert.*; +import org.junit.Test; public class TestRpcTraces { http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/test/java/org/apache/kudu/client/TestScanPredicate.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestScanPredicate.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestScanPredicate.java index 8d6d06e..3af4dfe 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestScanPredicate.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestScanPredicate.java @@ -17,20 +17,21 @@ package org.apache.kudu.client; +import java.util.ArrayList; +import java.util.List; +import java.util.NavigableSet; +import java.util.TreeSet; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSortedSet; import org.junit.Assert; import org.junit.Test; + import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.Type; import org.apache.kudu.client.KuduPredicate.ComparisonOp; -import java.util.ArrayList; -import java.util.List; -import java.util.NavigableSet; -import java.util.TreeSet; - public class TestScanPredicate extends BaseKuduTest { private Schema createTableSchema(Type type) { http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java index 14454c0..1165ea4 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java @@ -16,19 +16,20 @@ // under the License. package org.apache.kudu.client; +import static org.apache.kudu.Type.STRING; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.util.ArrayList; + import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.stumbleupon.async.Deferred; -import org.apache.kudu.ColumnSchema; -import org.apache.kudu.Schema; import org.junit.BeforeClass; import org.junit.Test; -import java.util.ArrayList; - -import static org.junit.Assert.assertNull; -import static org.apache.kudu.Type.STRING; -import static org.junit.Assert.assertEquals; +import org.apache.kudu.ColumnSchema; +import org.apache.kudu.Schema; public class TestScannerMultiTablet extends BaseKuduTest { // Generate a unique table name http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatistics.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatistics.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatistics.java index 365f95e..fadbfde 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatistics.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatistics.java @@ -16,11 +16,12 @@ // under the License. package org.apache.kudu.client; +import static org.junit.Assert.assertEquals; + import org.junit.BeforeClass; import org.junit.Test; -import org.apache.kudu.client.Statistics.Statistic; -import static org.junit.Assert.assertEquals; +import org.apache.kudu.client.Statistics.Statistic; public class TestStatistics extends BaseKuduTest { http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatus.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatus.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatus.java index dcd061a..2da9055 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatus.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatus.java @@ -16,10 +16,11 @@ // under the License. package org.apache.kudu.client; -import org.junit.Test; -import org.apache.kudu.client.Status; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; -import static org.junit.Assert.*; +import org.junit.Test; public class TestStatus { http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/test/java/org/apache/kudu/client/TestTestUtils.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestTestUtils.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestTestUtils.java index 856a5e5..ae164fe 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestTestUtils.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestTestUtils.java @@ -16,10 +16,8 @@ // under the License. package org.apache.kudu.client; -import org.junit.After; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.io.BufferedReader; import java.io.IOException; @@ -27,8 +25,10 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.util.concurrent.atomic.AtomicLong; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import org.junit.After; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Tests for non-trivial helper methods in TestUtils. http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/test/java/org/apache/kudu/client/TestTimeouts.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestTimeouts.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestTimeouts.java index 12db70a..2a22ecd 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestTimeouts.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestTimeouts.java @@ -19,7 +19,6 @@ package org.apache.kudu.client; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import com.stumbleupon.async.TimeoutException; import org.junit.Test; public class TestTimeouts extends BaseKuduTest { http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java index 6cdd23d..16a5497 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java @@ -16,18 +16,6 @@ // under the License. package org.apache.kudu.client; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; -import com.google.protobuf.ByteString; -import com.sun.security.auth.module.UnixSystem; -import org.apache.kudu.Common; -import org.apache.kudu.consensus.Metadata; -import org.apache.kudu.master.Master; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import sun.management.VMManagement; - import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -48,6 +36,19 @@ import java.nio.file.StandardCopyOption; import java.util.List; import java.util.Set; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.protobuf.ByteString; +import com.sun.security.auth.module.UnixSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sun.management.VMManagement; + +import org.apache.kudu.Common; +import org.apache.kudu.consensus.Metadata; +import org.apache.kudu.master.Master; + /** * A grouping of methods that help unit testing. */ http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/test/java/org/apache/kudu/util/TestAsyncUtil.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/util/TestAsyncUtil.java b/java/kudu-client/src/test/java/org/apache/kudu/util/TestAsyncUtil.java index 2e82586..630dac5 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/util/TestAsyncUtil.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/util/TestAsyncUtil.java @@ -16,14 +16,14 @@ // under the License. package org.apache.kudu.util; +import static org.junit.Assert.assertEquals; + import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import static org.junit.Assert.assertEquals; - /** * Test for {@link AsyncUtil}. */ http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/test/java/org/apache/kudu/util/TestByteVec.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/util/TestByteVec.java b/java/kudu-client/src/test/java/org/apache/kudu/util/TestByteVec.java index 8761512..f77258a 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/util/TestByteVec.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/util/TestByteVec.java @@ -16,14 +16,16 @@ // under the License. package org.apache.kudu.util; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Random; -import static org.junit.Assert.*; +import org.junit.Test; public class TestByteVec { private static final Random RAND = new Random(); http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/test/java/org/apache/kudu/util/TestMurmurHash.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/util/TestMurmurHash.java b/java/kudu-client/src/test/java/org/apache/kudu/util/TestMurmurHash.java index f095f6c..cf3ac22 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/util/TestMurmurHash.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/util/TestMurmurHash.java @@ -16,12 +16,12 @@ // under the License. package org.apache.kudu.util; +import static org.junit.Assert.assertEquals; + import com.google.common.primitives.UnsignedLongs; import com.sangupta.murmur.Murmur2; import org.junit.Test; -import static org.junit.Assert.assertEquals; - /** * Test Murmur2 Hash64 returns the expected values for inputs. * http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/test/java/org/apache/kudu/util/TestNetUtil.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/util/TestNetUtil.java b/java/kudu-client/src/test/java/org/apache/kudu/util/TestNetUtil.java index aa9f4b1..9f1f3b0 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/util/TestNetUtil.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/util/TestNetUtil.java @@ -16,14 +16,17 @@ // under the License. package org.apache.kudu.util; -import com.google.common.net.HostAndPort; -import org.junit.Test; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.net.InetAddress; import java.util.Arrays; import java.util.List; -import static org.junit.Assert.*; +import com.google.common.net.HostAndPort; +import org.junit.Test; /** * Test for {@link NetUtil}. http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducer.java ---------------------------------------------------------------------- diff --git a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducer.java b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducer.java index a3d6021..a139eb1 100644 --- a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducer.java +++ b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducer.java @@ -19,11 +19,6 @@ package org.apache.kudu.flume.sink; -import com.google.common.base.Preconditions; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.util.concurrent.UncheckedExecutionException; import java.io.IOException; import java.io.InputStream; import java.net.URI; @@ -33,6 +28,12 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.concurrent.ExecutionException; + +import com.google.common.base.Preconditions; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.util.concurrent.UncheckedExecutionException; import org.apache.avro.Schema; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericRecord; @@ -45,14 +46,16 @@ import org.apache.flume.FlumeException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; + import org.apache.kudu.ColumnSchema; import org.apache.kudu.client.KuduTable; import org.apache.kudu.client.Operation; import org.apache.kudu.client.PartialRow; /** - * <p>An Avro serializer that generates one operation per event by deserializing the event + * An Avro serializer that generates one operation per event by deserializing the event * body as an Avro record and mapping its fields to columns in a Kudu table. + * * <p><strong>Avro Kudu Operations Producer configuration parameters</strong> * <table cellpadding=3 cellspacing=0 border=1> * <tr><th>Property Name</th> @@ -87,7 +90,7 @@ public class AvroKuduOperationsProducer implements KuduOperationsProducer { private String operation; private GenericRecord reuse; private KuduTable table; - private String defaultSchemaURL; + private String defaultSchemaUrl; /** * The binary decoder to reuse for event parsing. @@ -157,7 +160,7 @@ public class AvroKuduOperationsProducer implements KuduOperationsProducer { String schemaPath = context.getString(SCHEMA_PROP); if (schemaPath != null) { - defaultSchemaURL = schemaPath; + defaultSchemaUrl = schemaPath; } } @@ -250,26 +253,26 @@ public class AvroKuduOperationsProducer implements KuduOperationsProducer { private Schema getSchema(Event event) throws FlumeException { Map<String, String> headers = event.getHeaders(); - String schemaURL = headers.get(SCHEMA_URL_HEADER); + String schemaUrl = headers.get(SCHEMA_URL_HEADER); String schemaLiteral = headers.get(SCHEMA_LITERAL_HEADER); try { - if (schemaURL != null) { - return schemasFromURL.get(schemaURL); + if (schemaUrl != null) { + return schemasFromURL.get(schemaUrl); } else if (schemaLiteral != null) { return schemasFromLiteral.get(schemaLiteral); - } else if (defaultSchemaURL != null) { - return schemasFromURL.get(defaultSchemaURL); + } else if (defaultSchemaUrl != null) { + return schemasFromURL.get(defaultSchemaUrl); } else { throw new FlumeException( String.format("No schema for event. " + - "Specify configuration property '%s' or event header '%s'", + "Specify configuration property '%s' or event header '%s'", SCHEMA_PROP, SCHEMA_URL_HEADER)); } - } catch (ExecutionException ex) { - throw new FlumeException("Cannot get schema", ex); - } catch (UncheckedExecutionException ex) { - throw new FlumeException("Cannot parse schema", ex); + } catch (ExecutionException e) { + throw new FlumeException("Cannot get schema", e); + } catch (UncheckedExecutionException e) { + throw new FlumeException("Cannot parse schema", e); } } http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduOperationsProducer.java ---------------------------------------------------------------------- diff --git a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduOperationsProducer.java b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduOperationsProducer.java index 8816e95..b38f387 100644 --- a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduOperationsProducer.java +++ b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduOperationsProducer.java @@ -16,17 +16,19 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.kudu.flume.sink; +import java.util.List; + import org.apache.flume.Event; import org.apache.flume.conf.Configurable; + import org.apache.kudu.annotations.InterfaceAudience; import org.apache.kudu.annotations.InterfaceStability; import org.apache.kudu.client.KuduTable; import org.apache.kudu.client.Operation; -import java.util.List; - /** * Interface for an operations producer that produces Kudu Operations from * Flume events. @@ -42,8 +44,7 @@ public interface KuduOperationsProducer extends Configurable, AutoCloseable { void initialize(KuduTable table); /** - * Returns the operations that should be written to Kudu as a result of this - * event. + * Returns the operations that should be written to Kudu as a result of this event. * @return List of {@link org.apache.kudu.client.Operation} that * should be written to Kudu */ http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java ---------------------------------------------------------------------- diff --git a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java index b80462c..b186709 100644 --- a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java +++ b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.kudu.flume.sink; import static org.apache.kudu.flume.sink.KuduSinkConfigurationConstants.BATCH_SIZE; @@ -25,6 +26,8 @@ import static org.apache.kudu.flume.sink.KuduSinkConfigurationConstants.PRODUCER import static org.apache.kudu.flume.sink.KuduSinkConfigurationConstants.TABLE_NAME; import static org.apache.kudu.flume.sink.KuduSinkConfigurationConstants.TIMEOUT_MILLIS; +import java.util.List; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; @@ -37,6 +40,9 @@ import org.apache.flume.Transaction; import org.apache.flume.conf.Configurable; import org.apache.flume.instrumentation.SinkCounter; import org.apache.flume.sink.AbstractSink; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.kudu.annotations.InterfaceAudience; import org.apache.kudu.annotations.InterfaceStability; import org.apache.kudu.client.AsyncKuduClient; @@ -46,27 +52,33 @@ import org.apache.kudu.client.KuduTable; import org.apache.kudu.client.Operation; import org.apache.kudu.client.OperationResponse; import org.apache.kudu.client.SessionConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; /** - * <p>A Flume sink that reads events from a channel and writes them to a Kudu table. + * A Flume sink that reads events from a channel and writes them to a Kudu table. * * <p><strong>Flume Kudu Sink configuration parameters</strong> * * <table cellpadding=3 cellspacing=0 border=1> * <tr><th>Property Name</th><th>Default</th><th>Required?</th><th>Description</th></tr> - * <tr><td>channel</td><td></td><td>Yes</td><td>The name of the Flume channel to read from.</td></tr> - * <tr><td>type</td><td></td><td>Yes</td><td>Component name. Must be {@code org.apache.kudu.flume.sink.KuduSink}</td></tr> - * <tr><td>masterAddresses</td><td></td><td>Yes</td><td>Comma-separated list of "host:port" Kudu master addresses. The port is optional.</td></tr> - * <tr><td>tableName</td><td></td><td>Yes</td><td>The name of the Kudu table to write to.</td></tr> - * <tr><td>batchSize</td><td>100</td><td>No</td><td>The maximum number of events the sink will attempt to take from the channel per transaction.</td></tr> - * <tr><td>ignoreDuplicateRows</td><td>true</td><td>No</td><td>Whether to ignore duplicate primary key errors caused by inserts.</td></tr> - * <tr><td>timeoutMillis</td><td>10000</td><td>No</td><td>Timeout period for Kudu write operations, in milliseconds.</td></tr> - * <tr><td>producer</td><td>{@link SimpleKuduOperationsProducer}</td><td>No</td><td>The fully qualified class name of the {@link KuduOperationsProducer} the sink should use.</td></tr> - * <tr><td>producer.*</td><td></td><td>(Varies by operations producer)</td><td>Configuration properties to pass to the operations producer implementation.</td></tr> + * <tr><td>channel</td><td></td><td>Yes</td><td>The name of the Flume channel to read.</td></tr> + * <tr><td>type</td><td></td><td>Yes</td> + * <td>Component name. Must be {@code org.apache.kudu.flume.sink.KuduSink}</td></tr> + * <tr><td>masterAddresses</td><td></td><td>Yes</td> + * <td>Comma-separated list of "host:port" Kudu master addresses. + * The port is optional.</td></tr> + * <tr><td>tableName</td><td></td><td>Yes</td> + * <td>The name of the Kudu table to write to.</td></tr> + * <tr><td>batchSize</td><td>100</td><td>No</td> + * <td>The maximum number of events the sink takes from the channel per transaction.</td></tr> + * <tr><td>ignoreDuplicateRows</td><td>true</td> + * <td>No</td><td>Whether to ignore duplicate primary key errors caused by inserts.</td></tr> + * <tr><td>timeoutMillis</td><td>10000</td><td>No</td> + * <td>Timeout period for Kudu write operations, in milliseconds.</td></tr> + * <tr><td>producer</td><td>{@link SimpleKuduOperationsProducer}</td><td>No</td> + * <td>The fully-qualified class name of the {@link KuduOperationsProducer} + * the sink should use.</td></tr> + * <tr><td>producer.*</td><td></td><td>(Varies by operations producer)</td> + * <td>Configuration properties to pass to the operations producer implementation.</td></tr> * </table> * * <p><strong>Installation</strong> @@ -126,11 +138,11 @@ public class KuduSink extends AbstractSink implements Configurable { try { table = client.openTable(tableName); - } catch (Exception e) { + } catch (Exception ex) { sinkCounter.incrementConnectionFailedCount(); String msg = String.format("Could not open Kudu table '%s'", tableName); - logger.error(msg, e); - throw new FlumeException(msg, e); + logger.error(msg, ex); + throw new FlumeException(msg, ex); } operationsProducer.initialize(table); http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/RegexpKuduOperationsProducer.java ---------------------------------------------------------------------- diff --git a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/RegexpKuduOperationsProducer.java b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/RegexpKuduOperationsProducer.java index b905ef1..3ed147c 100644 --- a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/RegexpKuduOperationsProducer.java +++ b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/RegexpKuduOperationsProducer.java @@ -19,26 +19,31 @@ package org.apache.kudu.flume.sink; +import java.nio.charset.Charset; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import java.nio.charset.Charset; import org.apache.flume.Context; import org.apache.flume.Event; import org.apache.flume.FlumeException; -import org.apache.kudu.ColumnSchema; -import org.apache.kudu.Schema; -import org.apache.kudu.Type; -import org.apache.kudu.client.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.regex.PatternSyntaxException; +import org.apache.kudu.ColumnSchema; +import org.apache.kudu.Schema; +import org.apache.kudu.Type; +import org.apache.kudu.client.Insert; +import org.apache.kudu.client.KuduTable; +import org.apache.kudu.client.Operation; +import org.apache.kudu.client.PartialRow; +import org.apache.kudu.client.Upsert; /** - * <p>A regular expression serializer that generates one {@link Insert} or + * A regular expression serializer that generates one {@link Insert} or * {@link Upsert} per {@link Event} by parsing the payload into values using a * regular expression. Values are coerced to the proper column types. * @@ -47,7 +52,7 @@ import java.util.regex.PatternSyntaxException; * key INT32 * name STRING * - * and producer.pattern is '(?<key>\\d+),(?<name>\w+)', then the + * and producer.pattern is '(?<key>\\d+),(?<name>\w+)', then the * RegexpKuduOperationsProducer will parse the string * * |12345,Mike||54321,Todd| @@ -57,7 +62,7 @@ import java.util.regex.PatternSyntaxException; * Note: this class relies on JDK7 named capturing groups, which are documented * in {@link Pattern}. * - * <p><strong>Regular Expression Kudu Operations Producer configuration parameters</strong> + * <p><strong>Regular Expression Kudu Operations Producer configuration parameters</strong></p> * * <table cellpadding=3 cellspacing=0 border=1> * <tr> @@ -195,22 +200,22 @@ public class RegexpKuduOperationsProducer implements KuduOperationsProducer { default: throw new FlumeException( String.format("Unrecognized operation type '%s' in getOperations: " + - "this should never happen!", operation)); + "this should never happen!", operation)); } PartialRow row = op.getRow(); for (ColumnSchema col : schema.getColumns()) { try { - CoerceAndSet(m.group(col.getName()), col.getName(), col.getType(), row); + coerceAndSet(m.group(col.getName()), col.getName(), col.getType(), row); } catch (NumberFormatException e) { String msg = String.format( "Raw value '%s' couldn't be parsed to type %s for column '%s'", raw, col.getType(), col.getName()); - LogOrThrow(skipBadColumnValue, msg, e); + logOrThrow(skipBadColumnValue, msg, e); } catch (IllegalArgumentException e) { String msg = String.format( "Column '%s' has no matching group in '%s'", col.getName(), raw); - LogOrThrow(skipMissingColumn, msg, e); + logOrThrow(skipMissingColumn, msg, e); } catch (Exception e) { throw new FlumeException("Failed to create Kudu operation", e); } @@ -233,7 +238,7 @@ public class RegexpKuduOperationsProducer implements KuduOperationsProducer { * @param row the row to set the value in * @throws NumberFormatException if `rawVal` cannot be cast as `type`. */ - private void CoerceAndSet(String rawVal, String colName, Type type, PartialRow row) + private void coerceAndSet(String rawVal, String colName, Type type, PartialRow row) throws NumberFormatException { switch (type) { case INT8: @@ -272,7 +277,7 @@ public class RegexpKuduOperationsProducer implements KuduOperationsProducer { } } - private void LogOrThrow(boolean log, String msg, Exception e) + private void logOrThrow(boolean log, String msg, Exception e) throws FlumeException { if (log) { logger.warn(msg, e); http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKeyedKuduOperationsProducer.java ---------------------------------------------------------------------- diff --git a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKeyedKuduOperationsProducer.java b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKeyedKuduOperationsProducer.java index 8360df8..897738a 100644 --- a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKeyedKuduOperationsProducer.java +++ b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKeyedKuduOperationsProducer.java @@ -19,20 +19,21 @@ package org.apache.kudu.flume.sink; +import java.util.Collections; +import java.util.List; + import org.apache.flume.Context; import org.apache.flume.Event; import org.apache.flume.FlumeException; + import org.apache.kudu.client.Insert; -import org.apache.kudu.client.Upsert; import org.apache.kudu.client.KuduTable; import org.apache.kudu.client.Operation; import org.apache.kudu.client.PartialRow; - -import java.util.Collections; -import java.util.List; +import org.apache.kudu.client.Upsert; /** - * <p>A simple serializer that generates one {@link Insert} or {@link Upsert} + * A simple serializer that generates one {@link Insert} or {@link Upsert} * per {@link Event} by writing the event body into a BINARY column. The pair * (key column name, key column value) should be a header in the {@link Event}; * the column name is configurable but the column type must be STRING. Multiple @@ -121,7 +122,7 @@ public class SimpleKeyedKuduOperationsProducer implements KuduOperationsProducer row.addBinary(payloadColumn, event.getBody()); return Collections.singletonList(op); - } catch (Exception e){ + } catch (Exception e) { throw new FlumeException("Failed to create Kudu Operation object", e); } } http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKuduOperationsProducer.java ---------------------------------------------------------------------- diff --git a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKuduOperationsProducer.java b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKuduOperationsProducer.java index f5f5838..e2f4452 100644 --- a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKuduOperationsProducer.java +++ b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKuduOperationsProducer.java @@ -19,19 +19,20 @@ package org.apache.kudu.flume.sink; +import java.util.Collections; +import java.util.List; + import org.apache.flume.Context; import org.apache.flume.Event; import org.apache.flume.FlumeException; + import org.apache.kudu.client.Insert; import org.apache.kudu.client.KuduTable; import org.apache.kudu.client.Operation; import org.apache.kudu.client.PartialRow; -import java.util.Collections; -import java.util.List; - /** - * <p>A simple serializer that generates one {@link Insert} per {@link Event} + * A simple serializer that generates one {@link Insert} per {@link Event} * by writing the event body into a BINARY column. The headers are discarded. * * <p><strong>Simple Kudu Event Producer configuration parameters</strong> @@ -58,7 +59,7 @@ public class SimpleKuduOperationsProducer implements KuduOperationsProducer { private KuduTable table; private String payloadColumn; - public SimpleKuduOperationsProducer(){ + public SimpleKuduOperationsProducer() { } @Override @@ -79,7 +80,7 @@ public class SimpleKuduOperationsProducer implements KuduOperationsProducer { row.addBinary(payloadColumn, event.getBody()); return Collections.singletonList((Operation) insert); - } catch (Exception e){ + } catch (Exception e) { throw new FlumeException("Failed to create Kudu Insert object", e); } } http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java ---------------------------------------------------------------------- diff --git a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java index 2465428..c9310b8 100644 --- a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java +++ b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java @@ -29,11 +29,6 @@ import static org.apache.kudu.flume.sink.KuduSinkConfigurationConstants.TABLE_NA import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import com.google.common.base.Charsets; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.common.io.Files; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -41,6 +36,12 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; + +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.io.Files; import org.apache.avro.io.DatumWriter; import org.apache.avro.io.Encoder; import org.apache.avro.io.EncoderFactory; @@ -54,14 +55,15 @@ import org.apache.flume.Transaction; import org.apache.flume.channel.MemoryChannel; import org.apache.flume.conf.Configurables; import org.apache.flume.event.EventBuilder; +import org.junit.BeforeClass; +import org.junit.Test; + import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.Type; import org.apache.kudu.client.BaseKuduTest; import org.apache.kudu.client.CreateTableOptions; import org.apache.kudu.client.KuduTable; -import org.junit.BeforeClass; -import org.junit.Test; public class AvroKuduOperationsProducerTest extends BaseKuduTest { private static final String schemaPath = "src/test/avro/testAvroKuduOperationsProducer.avsc"; http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KeyedKuduOperationsProducerTest.java ---------------------------------------------------------------------- diff --git a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KeyedKuduOperationsProducerTest.java b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KeyedKuduOperationsProducerTest.java index 5fc2547..6b62528 100644 --- a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KeyedKuduOperationsProducerTest.java +++ b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KeyedKuduOperationsProducerTest.java @@ -28,12 +28,13 @@ import static org.apache.kudu.flume.sink.SimpleKeyedKuduOperationsProducer.PAYLO import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import com.google.common.base.Charsets; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import java.util.ArrayList; import java.util.HashMap; import java.util.List; + +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.apache.flume.Channel; import org.apache.flume.Context; import org.apache.flume.Event; @@ -42,15 +43,16 @@ import org.apache.flume.Transaction; import org.apache.flume.channel.MemoryChannel; import org.apache.flume.conf.Configurables; import org.apache.flume.event.EventBuilder; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.Type; import org.apache.kudu.client.BaseKuduTest; import org.apache.kudu.client.CreateTableOptions; import org.apache.kudu.client.KuduTable; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class KeyedKuduOperationsProducerTest extends BaseKuduTest { private static final Logger LOG = LoggerFactory.getLogger(KeyedKuduOperationsProducerTest.class); http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java ---------------------------------------------------------------------- diff --git a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java index 7c426cb..a00b63f 100644 --- a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java +++ b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java @@ -18,10 +18,17 @@ */ package org.apache.kudu.flume.sink; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + import com.google.common.base.Charsets; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; - import org.apache.flume.Channel; import org.apache.flume.Context; import org.apache.flume.Event; @@ -34,22 +41,15 @@ import org.apache.flume.conf.Configurables; import org.apache.flume.event.EventBuilder; import org.junit.Assert; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.Type; import org.apache.kudu.client.BaseKuduTest; import org.apache.kudu.client.CreateTableOptions; import org.apache.kudu.client.KuduTable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; public class KuduSinkTest extends BaseKuduTest { private static final Logger LOG = LoggerFactory.getLogger(KuduSinkTest.class); http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/RegexpKuduOperationsProducerTest.java ---------------------------------------------------------------------- diff --git a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/RegexpKuduOperationsProducerTest.java b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/RegexpKuduOperationsProducerTest.java index d7734b6..d672a31 100644 --- a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/RegexpKuduOperationsProducerTest.java +++ b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/RegexpKuduOperationsProducerTest.java @@ -27,11 +27,12 @@ import static org.apache.kudu.flume.sink.RegexpKuduOperationsProducer.PATTERN_PR import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; + +import com.google.common.collect.ImmutableList; import org.apache.flume.Channel; import org.apache.flume.Context; import org.apache.flume.Event; @@ -40,13 +41,14 @@ import org.apache.flume.Transaction; import org.apache.flume.channel.MemoryChannel; import org.apache.flume.conf.Configurables; import org.apache.flume.event.EventBuilder; +import org.junit.Test; + import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.Type; import org.apache.kudu.client.BaseKuduTest; import org.apache.kudu.client.CreateTableOptions; import org.apache.kudu.client.KuduTable; -import org.junit.Test; public class RegexpKuduOperationsProducerTest extends BaseKuduTest { private static final String TEST_REGEXP = http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/CommandLineParser.java ---------------------------------------------------------------------- diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/CommandLineParser.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/CommandLineParser.java index 385752e..830cb7b 100644 --- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/CommandLineParser.java +++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/CommandLineParser.java @@ -14,12 +14,14 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. + package org.apache.kudu.mapreduce; +import org.apache.hadoop.conf.Configuration; + import org.apache.kudu.annotations.InterfaceAudience; import org.apache.kudu.annotations.InterfaceStability; import org.apache.kudu.client.AsyncKuduClient; -import org.apache.hadoop.conf.Configuration; import org.apache.kudu.client.KuduClient; /** @@ -131,11 +133,11 @@ public class CommandLineParser { public static String getHelpSnippet() { return "\nAdditionally, the following options are available:" + " -D" + OPERATION_TIMEOUT_MS_KEY + "=TIME - timeout for read and write " + - "operations, defaults to " + OPERATION_TIMEOUT_MS_DEFAULT + " \n"+ + "operations, defaults to " + OPERATION_TIMEOUT_MS_DEFAULT + " \n" + " -D" + ADMIN_OPERATION_TIMEOUT_MS_KEY + "=TIME - timeout for admin operations " + - ", defaults to " + OPERATION_TIMEOUT_MS_DEFAULT + " \n"+ + ", defaults to " + OPERATION_TIMEOUT_MS_DEFAULT + " \n" + " -D" + SOCKET_READ_TIMEOUT_MS_KEY + "=TIME - timeout for socket reads " + - ", defaults to " + SOCKET_READ_TIMEOUT_MS_DEFAULT + " \n"+ + ", defaults to " + SOCKET_READ_TIMEOUT_MS_DEFAULT + " \n" + " -D" + MASTER_ADDRESSES_KEY + "=ADDRESSES - addresses to reach the Masters, " + "defaults to " + MASTER_ADDRESSES_DEFAULT + " which is usually wrong.\n" + " -D " + NUM_REPLICAS_KEY + "=NUM - number of replicas to use when configuring a new " + http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java ---------------------------------------------------------------------- diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java index 3dbfa04..e1d6a0c 100644 --- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java +++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java @@ -14,9 +14,8 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package org.apache.kudu.mapreduce; -import com.google.common.base.Preconditions; +package org.apache.kudu.mapreduce; import java.io.BufferedOutputStream; import java.io.File; @@ -34,6 +33,8 @@ import java.util.jar.Manifest; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import com.google.common.base.Preconditions; + /** * Finds the Jar for a class. If the class is in a directory in the * classpath, it creates a Jar on the fly with the contents of the directory @@ -65,7 +66,7 @@ public class JarFinder { } public static void jarDir(File dir, String relativePath, ZipOutputStream zos) - throws IOException { + throws IOException { Preconditions.checkNotNull(relativePath, "relativePath"); Preconditions.checkNotNull(zos, "zos"); @@ -100,8 +101,7 @@ public class JarFinder { String filePath = f.getPath(); File file = new File(filePath); zipDir(file, relativePath + f.getName() + "/", zos, false); - } - else { + } else { String path = relativePath + f.getName(); if (!path.equals(JarFile.MANIFEST_NAME)) { ZipEntry anEntry = new ZipEntry(path); @@ -138,9 +138,9 @@ public class JarFinder { Preconditions.checkNotNull(klass, "klass"); ClassLoader loader = klass.getClassLoader(); if (loader != null) { - String class_file = klass.getName().replaceAll("\\.", "/") + ".class"; + String classFile = klass.getName().replaceAll("\\.", "/") + ".class"; try { - for (Enumeration itr = loader.getResources(class_file); + for (Enumeration itr = loader.getResources(classFile); itr.hasMoreElements(); ) { URL url = (URL) itr.nextElement(); String path = url.getPath(); @@ -151,17 +151,16 @@ public class JarFinder { if ("jar".equals(url.getProtocol())) { path = URLDecoder.decode(path, "UTF-8"); return path.replaceAll("!.*$", ""); - } - else if ("file".equals(url.getProtocol())) { + } else if ("file".equals(url.getProtocol())) { String klassName = klass.getName(); klassName = klassName.replace(".", "/") + ".class"; path = path.substring(0, path.length() - klassName.length()); - File baseDir = new File(path); File testDir = new File(System.getProperty("test.build.dir", "target/test-dir")); testDir = testDir.getAbsoluteFile(); if (!testDir.exists()) { testDir.mkdirs(); } + File baseDir = new File(path); File tempJar = File.createTempFile("hadoop-", "", testDir); tempJar = new File(tempJar.getAbsolutePath() + ".jar"); tempJar.deleteOnExit(); @@ -169,8 +168,7 @@ public class JarFinder { return tempJar.getAbsolutePath(); } } - } - catch (IOException e) { + } catch (IOException e) { throw new RuntimeException(e); } } http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java ---------------------------------------------------------------------- diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java index 6a15cc1..91f4825 100644 --- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java +++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java @@ -12,12 +12,8 @@ * See the License for the specific language governing permissions and * limitations under the License. See accompanying LICENSE file. */ -package org.apache.kudu.mapreduce; -import com.google.common.base.Objects; -import com.google.common.base.Splitter; -import com.google.common.collect.Lists; -import com.google.common.primitives.UnsignedBytes; +package org.apache.kudu.mapreduce; import java.io.ByteArrayInputStream; import java.io.DataInput; @@ -33,6 +29,10 @@ import java.util.List; import java.util.Map; import javax.naming.NamingException; +import com.google.common.base.Objects; +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import com.google.common.primitives.UnsignedBytes; import org.apache.commons.net.util.Base64; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; @@ -44,6 +44,9 @@ import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.net.DNS; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.kudu.Common; import org.apache.kudu.Schema; import org.apache.kudu.annotations.InterfaceAudience; @@ -52,14 +55,12 @@ import org.apache.kudu.client.AsyncKuduClient; import org.apache.kudu.client.Bytes; import org.apache.kudu.client.KuduClient; import org.apache.kudu.client.KuduPredicate; +import org.apache.kudu.client.KuduScanToken; import org.apache.kudu.client.KuduScanner; import org.apache.kudu.client.KuduTable; import org.apache.kudu.client.LocatedTablet; import org.apache.kudu.client.RowResult; import org.apache.kudu.client.RowResultIterator; -import org.apache.kudu.client.KuduScanToken; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * <p> @@ -207,12 +208,12 @@ public class KuduTableInputFormat extends InputFormat<NullWritable, RowResult> String masterAddresses = conf.get(MASTER_ADDRESSES_KEY); this.operationTimeoutMs = conf.getLong(OPERATION_TIMEOUT_MS_KEY, AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS); - this.nameServer = conf.get(NAME_SERVER_KEY); - this.cacheBlocks = conf.getBoolean(SCAN_CACHE_BLOCKS, false); - this.client = new KuduClient.KuduClientBuilder(masterAddresses) .defaultOperationTimeoutMs(operationTimeoutMs) .build(); + this.nameServer = conf.get(NAME_SERVER_KEY); + this.cacheBlocks = conf.getBoolean(SCAN_CACHE_BLOCKS, false); + try { this.table = client.openTable(tableName); } catch (Exception ex) { @@ -261,8 +262,9 @@ public class KuduTableInputFormat extends InputFormat<NullWritable, RowResult> * */ private static String domainNamePointerToHostName(String dnPtr) { - if (dnPtr == null) + if (dnPtr == null) { return null; + } return dnPtr.endsWith(".") ? dnPtr.substring(0, dnPtr.length() - 1) : dnPtr; } @@ -344,8 +346,12 @@ public class KuduTableInputFormat extends InputFormat<NullWritable, RowResult> @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } TableSplit that = (TableSplit) o; @@ -370,7 +376,8 @@ public class KuduTableInputFormat extends InputFormat<NullWritable, RowResult> private TableSplit split; @Override - public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { + public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) + throws IOException, InterruptedException { if (!(inputSplit instanceof TableSplit)) { throw new IllegalArgumentException("TableSplit is the only accepted input split"); } http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java ---------------------------------------------------------------------- diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java index f0e5b62..6488163 100644 --- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java +++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java @@ -12,6 +12,7 @@ * See the License for the specific language governing permissions and * limitations under the License. See accompanying LICENSE file. */ + package org.apache.kudu.mapreduce; import java.io.ByteArrayOutputStream; @@ -38,6 +39,7 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.TaskInputOutputContext; import org.apache.hadoop.util.StringUtils; + import org.apache.kudu.annotations.InterfaceAudience; import org.apache.kudu.annotations.InterfaceStability; import org.apache.kudu.client.AsyncKuduClient; @@ -65,7 +67,7 @@ public class KuduTableMapReduceUtil { /** * Base class for MR I/O formats, contains the common configurations. */ - private static abstract class AbstractMapReduceConfigurator<S> { + private abstract static class AbstractMapReduceConfigurator<S> { protected final Job job; protected final String table; @@ -106,7 +108,7 @@ public class KuduTableMapReduceUtil { * <p> * Use either child classes when configuring the table output format. */ - private static abstract class AbstractTableOutputFormatConfigurator + private abstract static class AbstractTableOutputFormatConfigurator <S extends AbstractTableOutputFormatConfigurator<? super S>> extends AbstractMapReduceConfigurator<S> { @@ -144,7 +146,7 @@ public class KuduTableMapReduceUtil { * <p> * Use either child classes when configuring the table input format. */ - private static abstract class AbstractTableInputFormatConfigurator + private abstract static class AbstractTableInputFormatConfigurator <S extends AbstractTableInputFormatConfigurator<? super S>> extends AbstractMapReduceConfigurator<S> { @@ -403,7 +405,9 @@ public class KuduTableMapReduceUtil { // Add jars containing the specified classes for (Class<?> clazz : classes) { - if (clazz == null) continue; + if (clazz == null) { + continue; + } Path path = findOrCreateJar(clazz, localFs, packagedClasses); if (path == null) { @@ -412,13 +416,14 @@ public class KuduTableMapReduceUtil { continue; } if (!localFs.exists(path)) { - LOG.warn("Could not validate jar file " + path + " for class " - + clazz); + LOG.warn("Could not validate jar file " + path + " for class " + clazz); continue; } jars.add(path.toString()); } - if (jars.isEmpty()) return; + if (jars.isEmpty()) { + return; + } conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[jars.size()]))); } @@ -451,20 +456,20 @@ public class KuduTableMapReduceUtil { * created, it is created in the system temporary directory. Otherwise, * returns an existing jar that contains a class of the same name. Maintains * a mapping from jar contents to the tmp jar created. - * @param my_class the class to find. + * @param myClass the class to find. * @param fs the FileSystem with which to qualify the returned path. * @param packagedClasses a map of class name to path. * @return a jar file that contains the class. * @throws IOException */ @SuppressWarnings("deprecation") - private static Path findOrCreateJar(Class<?> my_class, FileSystem fs, + private static Path findOrCreateJar(Class<?> myClass, FileSystem fs, Map<String, String> packagedClasses) throws IOException { // attempt to locate an existing jar for the class. - String jar = findContainingJar(my_class, packagedClasses); + String jar = findContainingJar(myClass, packagedClasses); if (null == jar || jar.isEmpty()) { - jar = JarFinder.getJar(my_class); + jar = JarFinder.getJar(myClass); updateMap(jar, packagedClasses); } @@ -472,7 +477,7 @@ public class KuduTableMapReduceUtil { return null; } - LOG.debug(String.format("For class %s, using jar %s", my_class.getName(), jar)); + LOG.debug(String.format("For class %s, using jar %s", myClass.getName(), jar)); return new Path(jar).makeQualified(fs); } @@ -481,17 +486,17 @@ public class KuduTableMapReduceUtil { * a jar file, even if that is not the first thing on the class path that * has a class with the same name. Looks first on the classpath and then in * the <code>packagedClasses</code> map. - * @param my_class the class to find. + * @param myClass the class to find. * @return a jar file that contains the class, or null. * @throws IOException */ - private static String findContainingJar(Class<?> my_class, Map<String, String> packagedClasses) + private static String findContainingJar(Class<?> myClass, Map<String, String> packagedClasses) throws IOException { - ClassLoader loader = my_class.getClassLoader(); - String class_file = my_class.getName().replaceAll("\\.", "/") + ".class"; + ClassLoader loader = myClass.getClassLoader(); + String classFile = myClass.getName().replaceAll("\\.", "/") + ".class"; // first search the classpath - for (Enumeration<URL> itr = loader.getResources(class_file); itr.hasMoreElements();) { + for (Enumeration<URL> itr = loader.getResources(classFile); itr.hasMoreElements();) { URL url = itr.nextElement(); if ("jar".equals(url.getProtocol())) { String toReturn = url.getPath(); @@ -512,7 +517,7 @@ public class KuduTableMapReduceUtil { // now look in any jars we've packaged using JarFinder. Returns null when // no jar is found. - return packagedClasses.get(class_file); + return packagedClasses.get(classFile); } /** @@ -521,7 +526,8 @@ public class KuduTableMapReduceUtil { * @param jar The jar who's content to list. * @param packagedClasses map[class -> jar] */ - private static void updateMap(String jar, Map<String, String> packagedClasses) throws IOException { + private static void updateMap(String jar, Map<String, String> packagedClasses) + throws IOException { if (null == jar || jar.isEmpty()) { return; } @@ -535,7 +541,9 @@ public class KuduTableMapReduceUtil { } } } finally { - if (null != zip) zip.close(); + if (null != zip) { + zip.close(); + } } } } http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputCommitter.java ---------------------------------------------------------------------- diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputCommitter.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputCommitter.java index c84c3d5..3ee3774 100644 --- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputCommitter.java +++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputCommitter.java @@ -14,16 +14,18 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. + package org.apache.kudu.mapreduce; +import java.io.IOException; + import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.TaskAttemptContext; + import org.apache.kudu.annotations.InterfaceAudience; import org.apache.kudu.annotations.InterfaceStability; -import java.io.IOException; - /** * Small committer class that does not do anything. */ http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputFormat.java ---------------------------------------------------------------------- diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputFormat.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputFormat.java index 69b076d..bb64a1e 100644 --- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputFormat.java +++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputFormat.java @@ -14,11 +14,14 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. + package org.apache.kudu.mapreduce; -import org.apache.kudu.annotations.InterfaceAudience; -import org.apache.kudu.annotations.InterfaceStability; -import org.apache.kudu.client.*; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; @@ -30,10 +33,16 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; +import org.apache.kudu.annotations.InterfaceAudience; +import org.apache.kudu.annotations.InterfaceStability; +import org.apache.kudu.client.AsyncKuduClient; +import org.apache.kudu.client.AsyncKuduSession; +import org.apache.kudu.client.KuduClient; +import org.apache.kudu.client.KuduSession; +import org.apache.kudu.client.KuduTable; +import org.apache.kudu.client.Operation; +import org.apache.kudu.client.OperationResponse; +import org.apache.kudu.client.RowError; /** * <p> @@ -106,7 +115,6 @@ public class KuduTableOutputFormat extends OutputFormat<NullWritable,Operation> String tableName = this.conf.get(OUTPUT_TABLE_KEY); this.operationTimeoutMs = this.conf.getLong(OPERATION_TIMEOUT_MS_KEY, AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS); - int bufferSpace = this.conf.getInt(BUFFER_ROW_COUNT_KEY, 1000); this.client = new KuduClient.KuduClientBuilder(masterAddress) .defaultOperationTimeoutMs(operationTimeoutMs) @@ -120,10 +128,10 @@ public class KuduTableOutputFormat extends OutputFormat<NullWritable,Operation> } this.session = client.newSession(); this.session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_BACKGROUND); - this.session.setMutationBufferSpace(bufferSpace); + this.session.setMutationBufferSpace(this.conf.getInt(BUFFER_ROW_COUNT_KEY, 1000)); this.session.setIgnoreAllDuplicateRows(true); String multitonKey = String.valueOf(Thread.currentThread().getId()); - assert(MULTITON.get(multitonKey) == null); + assert MULTITON.get(multitonKey) == null; MULTITON.put(multitonKey, this); entries.set(MULTITON_KEY, multitonKey); } @@ -150,7 +158,8 @@ public class KuduTableOutputFormat extends OutputFormat<NullWritable,Operation> } @Override - public RecordWriter<NullWritable, Operation> getRecordWriter(TaskAttemptContext taskAttemptContext) + public RecordWriter<NullWritable, Operation> + getRecordWriter(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { return new TableRecordWriter(this.session); } http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/TableReducer.java ---------------------------------------------------------------------- diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/TableReducer.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/TableReducer.java index e319fe4..9e28046 100644 --- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/TableReducer.java +++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/TableReducer.java @@ -14,15 +14,17 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. + package org.apache.kudu.mapreduce; +import org.apache.hadoop.mapreduce.Reducer; + import org.apache.kudu.annotations.InterfaceAudience; import org.apache.kudu.annotations.InterfaceStability; import org.apache.kudu.client.Operation; -import org.apache.hadoop.mapreduce.Reducer; @InterfaceAudience.Public @InterfaceStability.Evolving -public abstract class TableReducer<KEYIN, VALUEIN, KEYOUT> - extends Reducer<KEYIN, VALUEIN, KEYOUT, Operation> { +public abstract class TableReducer<KeyinT, ValueinT, KeyoutT> + extends Reducer<KeyinT, ValueinT, KeyoutT, Operation> { } http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java ---------------------------------------------------------------------- diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java index 0db2c9a..c8ced88 100644 --- a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java +++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java @@ -18,6 +18,9 @@ */ package org.apache.kudu.mapreduce; +import java.io.File; +import java.io.IOException; + import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -25,9 +28,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import java.io.File; -import java.io.IOException; - /** * This class is analog to HBaseTestingUtility except that we only need it for the MR tests. */ http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java ---------------------------------------------------------------------- diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java index 2f4b05e..f4b1d4f 100644 --- a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java +++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java @@ -16,6 +16,13 @@ // under the License. package org.apache.kudu.mapreduce; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + import com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; @@ -25,17 +32,12 @@ import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import org.apache.kudu.client.BaseKuduTest; -import org.apache.kudu.client.KuduPredicate; -import org.apache.kudu.client.RowResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.*; +import org.apache.kudu.client.BaseKuduTest; +import org.apache.kudu.client.KuduPredicate; +import org.apache.kudu.client.RowResult; public class ITInputFormatJob extends BaseKuduTest { private static final Logger LOG = LoggerFactory.getLogger(ITInputFormatJob.class); http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java ---------------------------------------------------------------------- diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java index 818f33f..963ab9a 100644 --- a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java +++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java @@ -16,20 +16,30 @@ // under the License. package org.apache.kudu.mapreduce; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.List; + import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import org.apache.kudu.Schema; -import org.apache.kudu.client.*; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.RecordReader; import org.junit.Test; -import java.io.IOException; -import java.util.List; - -import static org.junit.Assert.*; +import org.apache.kudu.Schema; +import org.apache.kudu.client.AsyncKuduSession; +import org.apache.kudu.client.BaseKuduTest; +import org.apache.kudu.client.Insert; +import org.apache.kudu.client.KuduPredicate; +import org.apache.kudu.client.KuduTable; +import org.apache.kudu.client.PartialRow; +import org.apache.kudu.client.RowResult; public class ITKuduTableInputFormat extends BaseKuduTest { http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java ---------------------------------------------------------------------- diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java index 57193b3..0223cd8 100644 --- a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java +++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java @@ -16,14 +16,21 @@ // under the License. package org.apache.kudu.mapreduce; -import org.apache.kudu.client.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.RecordWriter; import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.*; +import org.apache.kudu.client.AsyncKuduScanner; +import org.apache.kudu.client.BaseKuduTest; +import org.apache.kudu.client.Insert; +import org.apache.kudu.client.KuduTable; +import org.apache.kudu.client.Operation; +import org.apache.kudu.client.PartialRow; public class ITKuduTableOutputFormat extends BaseKuduTest { http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITOutputFormatJob.java ---------------------------------------------------------------------- diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITOutputFormatJob.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITOutputFormatJob.java index d3f1dbf..06874c2 100644 --- a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITOutputFormatJob.java +++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITOutputFormatJob.java @@ -16,7 +16,13 @@ // under the License. package org.apache.kudu.mapreduce; -import org.apache.kudu.client.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.NullWritable; @@ -29,11 +35,12 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; - -import static org.junit.Assert.*; +import org.apache.kudu.client.AsyncKuduScanner; +import org.apache.kudu.client.BaseKuduTest; +import org.apache.kudu.client.Insert; +import org.apache.kudu.client.KuduTable; +import org.apache.kudu.client.Operation; +import org.apache.kudu.client.PartialRow; public class ITOutputFormatJob extends BaseKuduTest { http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java ---------------------------------------------------------------------- diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java index acf155f..710d5a1 100644 --- a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java +++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java @@ -16,11 +16,6 @@ // under the License. package org.apache.kudu.mapreduce; -import org.apache.commons.logging.LogFactory; -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; @@ -35,6 +30,10 @@ import java.util.jar.JarInputStream; import java.util.jar.JarOutputStream; import java.util.jar.Manifest; +import org.apache.commons.logging.LogFactory; +import org.junit.Assert; +import org.junit.Test; + /** * This file was forked from hbase/branches/master@4ce6f48. */ http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala ---------------------------------------------------------------------- diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala index 3bf152b..87999ba 100644 --- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala +++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala @@ -19,16 +19,17 @@ package org.apache.kudu.spark.kudu import java.sql.Timestamp +import scala.collection.JavaConverters._ + import org.apache.spark.rdd.RDD import org.apache.spark.sql.sources._ import org.apache.spark.sql.types._ -import org.apache.spark.sql.{DataFrame, Row, SaveMode, SQLContext} +import org.apache.spark.sql.{DataFrame, Row, SQLContext, SaveMode} + import org.apache.kudu.Type import org.apache.kudu.annotations.InterfaceStability -import org.apache.kudu.client._ import org.apache.kudu.client.KuduPredicate.ComparisonOp - -import scala.collection.JavaConverters._ +import org.apache.kudu.client._ /** * Data source for integration with Spark's [[DataFrame]] API. http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala ---------------------------------------------------------------------- diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala index 1293901..5a9c267 100644 --- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala +++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala @@ -18,18 +18,19 @@ package org.apache.kudu.spark.kudu import java.util + +import scala.collection.mutable + import org.apache.hadoop.util.ShutdownHookManager import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD +import org.apache.spark.sql.types.{DataType, DataTypes, StructType} import org.apache.spark.sql.{DataFrame, Row} -import org.apache.spark.sql.types.{StructType, DataType, DataTypes} -import org.apache.kudu.{ColumnSchema, Schema, Type} + import org.apache.kudu.annotations.InterfaceStability import org.apache.kudu.client.SessionConfiguration.FlushMode import org.apache.kudu.client._ - - -import scala.collection.mutable +import org.apache.kudu.{ColumnSchema, Schema, Type} /** * KuduContext is a serializable container for Kudu client connections. http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala ---------------------------------------------------------------------- diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala index f9686be..5ba7c35 100644 --- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala +++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala @@ -16,14 +16,15 @@ */ package org.apache.kudu.spark.kudu +import scala.collection.JavaConverters._ + import org.apache.spark.rdd.RDD import org.apache.spark.sql.Row import org.apache.spark.{Partition, SparkContext, TaskContext} + import org.apache.kudu.client._ import org.apache.kudu.{Type, client} -import scala.collection.JavaConverters._ - /** * A Resilient Distributed Dataset backed by a Kudu table. */ http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala ---------------------------------------------------------------------- diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala index 007bb8d..4cb15db 100644 --- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala +++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala @@ -20,17 +20,18 @@ import java.sql.Timestamp import java.text.SimpleDateFormat import java.util.TimeZone +import scala.collection.JavaConverters._ +import scala.collection.immutable.IndexedSeq +import scala.util.control.NonFatal + import org.apache.spark.sql.SQLContext import org.apache.spark.sql.functions._ import org.junit.Assert._ import org.junit.runner.RunWith -import org.apache.kudu.client.CreateTableOptions import org.scalatest.junit.JUnitRunner import org.scalatest.{BeforeAndAfter, FunSuite} -import scala.collection.JavaConverters._ -import scala.collection.immutable.IndexedSeq -import scala.util.control.NonFatal; +import org.apache.kudu.client.CreateTableOptions; @RunWith(classOf[JUnitRunner]) class DefaultSourceTest extends FunSuite with TestContext with BeforeAndAfter { http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala ---------------------------------------------------------------------- diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala index be0ed57..e35f276 100644 --- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala +++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala @@ -19,10 +19,10 @@ package org.apache.kudu.spark.kudu import java.sql.Timestamp import org.apache.spark.sql.SQLContext +import org.apache.spark.sql.functions.decode import org.junit.runner.RunWith -import org.scalatest.{FunSuite, Matchers} import org.scalatest.junit.JUnitRunner -import org.apache.spark.sql.functions.decode +import org.scalatest.{FunSuite, Matchers} @RunWith(classOf[JUnitRunner]) class KuduContextTest extends FunSuite with TestContext with Matchers { http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala ---------------------------------------------------------------------- diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala index ebd09bf..f46d290 100644 --- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala +++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala @@ -18,17 +18,18 @@ package org.apache.kudu.spark.kudu import java.util.Date +import scala.collection.JavaConverters._ +import scala.collection.immutable.IndexedSeq + import com.google.common.collect.ImmutableList import org.apache.spark.{SparkConf, SparkContext} +import org.scalatest.{BeforeAndAfterAll, Suite} + import org.apache.kudu.ColumnSchema.ColumnSchemaBuilder import org.apache.kudu.client.KuduClient.KuduClientBuilder import org.apache.kudu.client.MiniKuduCluster.MiniKuduClusterBuilder import org.apache.kudu.client.{CreateTableOptions, KuduClient, KuduTable, MiniKuduCluster} import org.apache.kudu.{Schema, Type} -import org.scalatest.{BeforeAndAfterAll, Suite} - -import scala.collection.JavaConverters._ -import scala.collection.immutable.IndexedSeq trait TestContext extends BeforeAndAfterAll { self: Suite =>
