[java] Add checkstyle coverage to Java build This patch introduces the Maven checkstyle plugin to Kudu. The style rules are mostly Google style guide, with the notable exception being a different include order. The include order mirrors that of the C++ client and is described by the IntelliJ rule
import static java.* import static javax.* <blank line> import static all other imports <blank line> import static org.apache.kudu.* <blank line> import java.* import javax.* <blank line> import all other imports <blank line> import org.apache.kudu.* Additional deviations from Google style are noted in kudu_style.xml. This patch also addresses all style warnings. Checkstyle warnings will not fail the build at this time. Change-Id: I1578371509f7fcaaffa63bec0795bb3cf56c03c6 Reviewed-on: http://gerrit.cloudera.org:8080/5016 Tested-by: Kudu Jenkins Reviewed-by: Dan Burkert <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/22067edb Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/22067edb Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/22067edb Branch: refs/heads/master Commit: 22067edb431e9ebf7317bf7821b90bb095f78f2c Parents: e836ac1 Author: Will Berkeley <[email protected]> Authored: Tue Nov 8 13:38:36 2016 -0500 Committer: Dan Burkert <[email protected]> Committed: Wed Nov 16 19:41:09 2016 +0000 ---------------------------------------------------------------------- .../kudu/annotations/InterfaceAudience.java | 15 +- .../kudu/annotations/InterfaceStability.java | 9 +- .../ExcludePrivateAnnotationsJDiffDoclet.java | 11 +- ...ExcludePrivateAnnotationsStandardDoclet.java | 10 +- .../IncludePublicAnnotationsStandardDoclet.java | 10 +- .../annotations/tools/RootDocProcessor.java | 42 ++-- .../annotations/tools/StabilityOptions.java | 21 +- .../kudu/annotations/tools/package-info.java | 1 + .../apache/kudu/mapreduce/tools/CsvParser.java | 8 +- .../apache/kudu/mapreduce/tools/ImportCsv.java | 14 +- .../kudu/mapreduce/tools/ImportCsvMapper.java | 20 +- .../tools/IntegrationTestBigLinkedList.java | 129 ++++++---- .../apache/kudu/mapreduce/tools/RowCounter.java | 21 +- .../kudu/mapreduce/tools/ITImportCsv.java | 32 +-- .../tools/ITIntegrationTestBigLinkedList.java | 1 + .../kudu/mapreduce/tools/ITRowCounter.java | 11 +- .../protobuf/ZeroCopyLiteralByteString.java | 5 +- .../main/java/org/apache/kudu/ColumnSchema.java | 25 +- .../src/main/java/org/apache/kudu/Schema.java | 76 +++--- .../src/main/java/org/apache/kudu/Type.java | 41 +-- .../kudu/client/AbstractKuduScannerBuilder.java | 13 +- .../apache/kudu/client/AlterTableOptions.java | 9 +- .../apache/kudu/client/AlterTableRequest.java | 21 +- .../apache/kudu/client/AlterTableResponse.java | 1 + .../org/apache/kudu/client/AsyncKuduClient.java | 136 ++++++---- .../apache/kudu/client/AsyncKuduScanner.java | 82 +++--- .../apache/kudu/client/AsyncKuduSession.java | 64 ++--- .../main/java/org/apache/kudu/client/Batch.java | 18 +- .../org/apache/kudu/client/BatchResponse.java | 5 +- .../main/java/org/apache/kudu/client/Bytes.java | 161 ++++++------ .../org/apache/kudu/client/CallResponse.java | 29 ++- .../kudu/client/ColumnRangePredicate.java | 49 ++-- .../org/apache/kudu/client/ConnectionCache.java | 34 +-- .../apache/kudu/client/CreateTableOptions.java | 5 +- .../apache/kudu/client/CreateTableRequest.java | 11 +- .../apache/kudu/client/CreateTableResponse.java | 2 +- .../org/apache/kudu/client/DeadlineTracker.java | 5 +- .../java/org/apache/kudu/client/Delete.java | 3 +- .../apache/kudu/client/DeleteTableRequest.java | 10 +- .../apache/kudu/client/DeleteTableResponse.java | 1 + .../org/apache/kudu/client/ErrorCollector.java | 8 +- .../kudu/client/ExternalConsistencyMode.java | 2 + .../client/GetMasterRegistrationReceived.java | 20 +- .../client/GetMasterRegistrationRequest.java | 14 +- .../client/GetMasterRegistrationResponse.java | 1 + .../kudu/client/GetTableLocationsRequest.java | 16 +- .../kudu/client/GetTableSchemaRequest.java | 12 +- .../kudu/client/GetTableSchemaResponse.java | 1 + .../kudu/client/HasFailedRpcException.java | 1 + .../java/org/apache/kudu/client/IPCUtil.java | 18 +- .../java/org/apache/kudu/client/Insert.java | 1 + .../kudu/client/IsAlterTableDoneRequest.java | 12 +- .../kudu/client/IsAlterTableDoneResponse.java | 1 + .../kudu/client/IsCreateTableDoneRequest.java | 8 +- .../java/org/apache/kudu/client/KeyEncoder.java | 34 ++- .../java/org/apache/kudu/client/KuduClient.java | 17 +- .../org/apache/kudu/client/KuduException.java | 6 +- .../org/apache/kudu/client/KuduPredicate.java | 141 +++++++---- .../java/org/apache/kudu/client/KuduRpc.java | 34 +-- .../org/apache/kudu/client/KuduRpcResponse.java | 1 + .../org/apache/kudu/client/KuduScanToken.java | 20 +- .../org/apache/kudu/client/KuduScanner.java | 3 +- .../org/apache/kudu/client/KuduSession.java | 10 +- .../java/org/apache/kudu/client/KuduTable.java | 69 ++--- .../apache/kudu/client/ListTablesRequest.java | 14 +- .../apache/kudu/client/ListTablesResponse.java | 5 +- .../kudu/client/ListTabletServersRequest.java | 18 +- .../kudu/client/ListTabletServersResponse.java | 5 +- .../apache/kudu/client/ListTabletsRequest.java | 14 +- .../apache/kudu/client/ListTabletsResponse.java | 5 +- .../org/apache/kudu/client/LocatedTablet.java | 5 +- .../kudu/client/NoLeaderFoundException.java | 4 +- .../kudu/client/NonCoveredRangeException.java | 2 +- .../kudu/client/NonRecoverableException.java | 1 + .../java/org/apache/kudu/client/Operation.java | 39 +-- .../apache/kudu/client/OperationResponse.java | 7 +- .../java/org/apache/kudu/client/PartialRow.java | 91 +++++-- .../java/org/apache/kudu/client/Partition.java | 20 +- .../org/apache/kudu/client/PartitionPruner.java | 60 +++-- .../org/apache/kudu/client/PartitionSchema.java | 9 +- .../org/apache/kudu/client/PingRequest.java | 7 +- .../org/apache/kudu/client/PingResponse.java | 1 + .../kudu/client/PleaseThrottleException.java | 2 + .../org/apache/kudu/client/ProtobufHelper.java | 27 +- .../apache/kudu/client/RangePartitionBound.java | 1 + .../kudu/client/RecoverableException.java | 1 + .../org/apache/kudu/client/RemoteTablet.java | 26 +- .../apache/kudu/client/ReplicaSelection.java | 1 + .../org/apache/kudu/client/RequestTracker.java | 5 +- .../java/org/apache/kudu/client/RowError.java | 1 + .../kudu/client/RowErrorsAndOverflowStatus.java | 1 + .../java/org/apache/kudu/client/RowResult.java | 82 +++--- .../apache/kudu/client/RowResultIterator.java | 2 + .../org/apache/kudu/client/RpcTraceFrame.java | 6 +- .../org/apache/kudu/client/SecureRpcHelper.java | 37 +-- .../java/org/apache/kudu/client/ServerInfo.java | 1 + .../kudu/client/SessionConfiguration.java | 1 + .../java/org/apache/kudu/client/Statistics.java | 23 +- .../java/org/apache/kudu/client/Status.java | 23 +- .../apache/kudu/client/TableLocationsCache.java | 23 +- .../org/apache/kudu/client/TabletClient.java | 141 +++++------ .../java/org/apache/kudu/client/Update.java | 1 + .../java/org/apache/kudu/client/Upsert.java | 1 + .../java/org/apache/kudu/util/AsyncUtil.java | 7 +- .../main/java/org/apache/kudu/util/ByteVec.java | 47 +++- .../org/apache/kudu/util/HybridTimeUtil.java | 7 +- .../main/java/org/apache/kudu/util/NetUtil.java | 15 +- .../main/java/org/apache/kudu/util/Pair.java | 18 +- .../main/java/org/apache/kudu/util/Slice.java | 167 +++++-------- .../main/java/org/apache/kudu/util/Slices.java | 78 +++--- .../org/apache/kudu/client/BaseKuduTest.java | 23 +- .../java/org/apache/kudu/client/ITClient.java | 12 +- .../kudu/client/ITScannerMultiTablet.java | 11 +- .../java/org/apache/kudu/client/MiniKdc.java | 26 +- .../org/apache/kudu/client/MiniKuduCluster.java | 26 +- .../org/apache/kudu/client/TestAlterTable.java | 22 +- .../apache/kudu/client/TestAsyncKuduClient.java | 20 +- .../kudu/client/TestAsyncKuduSession.java | 21 +- .../java/org/apache/kudu/client/TestBitSet.java | 6 +- .../java/org/apache/kudu/client/TestBytes.java | 4 +- .../kudu/client/TestColumnRangePredicate.java | 12 +- .../apache/kudu/client/TestConnectionCache.java | 10 +- .../apache/kudu/client/TestDeadlineTracker.java | 8 +- .../kudu/client/TestFlexiblePartitioning.java | 22 +- .../TestGetMasterRegistrationReceived.java | 15 +- .../org/apache/kudu/client/TestHybridTime.java | 27 +- .../org/apache/kudu/client/TestKeyEncoding.java | 7 +- .../org/apache/kudu/client/TestKuduClient.java | 20 +- .../apache/kudu/client/TestKuduPredicate.java | 20 +- .../org/apache/kudu/client/TestKuduSession.java | 12 +- .../org/apache/kudu/client/TestKuduTable.java | 24 +- .../apache/kudu/client/TestLeaderFailover.java | 5 +- .../apache/kudu/client/TestMasterFailover.java | 5 +- .../org/apache/kudu/client/TestMiniKdc.java | 4 +- .../apache/kudu/client/TestMiniKuduCluster.java | 12 +- .../org/apache/kudu/client/TestOperation.java | 8 +- .../apache/kudu/client/TestPartitionPruner.java | 12 +- .../apache/kudu/client/TestRemoteTablet.java | 14 +- .../org/apache/kudu/client/TestRowErrors.java | 9 +- .../org/apache/kudu/client/TestRowResult.java | 13 +- .../org/apache/kudu/client/TestRpcTraces.java | 6 +- .../apache/kudu/client/TestScanPredicate.java | 11 +- .../kudu/client/TestScannerMultiTablet.java | 15 +- .../org/apache/kudu/client/TestStatistics.java | 5 +- .../java/org/apache/kudu/client/TestStatus.java | 7 +- .../org/apache/kudu/client/TestTestUtils.java | 12 +- .../org/apache/kudu/client/TestTimeouts.java | 1 - .../java/org/apache/kudu/client/TestUtils.java | 25 +- .../org/apache/kudu/util/TestAsyncUtil.java | 4 +- .../java/org/apache/kudu/util/TestByteVec.java | 6 +- .../org/apache/kudu/util/TestMurmurHash.java | 4 +- .../java/org/apache/kudu/util/TestNetUtil.java | 9 +- .../flume/sink/AvroKuduOperationsProducer.java | 39 +-- .../kudu/flume/sink/KuduOperationsProducer.java | 9 +- .../org/apache/kudu/flume/sink/KuduSink.java | 46 ++-- .../sink/RegexpKuduOperationsProducer.java | 41 +-- .../sink/SimpleKeyedKuduOperationsProducer.java | 13 +- .../sink/SimpleKuduOperationsProducer.java | 13 +- .../sink/AvroKuduOperationsProducerTest.java | 16 +- .../sink/KeyedKuduOperationsProducerTest.java | 14 +- .../apache/kudu/flume/sink/KuduSinkTest.java | 22 +- .../sink/RegexpKuduOperationsProducerTest.java | 6 +- .../kudu/mapreduce/CommandLineParser.java | 10 +- .../org/apache/kudu/mapreduce/JarFinder.java | 22 +- .../kudu/mapreduce/KuduTableInputFormat.java | 37 +-- .../kudu/mapreduce/KuduTableMapReduceUtil.java | 48 ++-- .../mapreduce/KuduTableOutputCommitter.java | 6 +- .../kudu/mapreduce/KuduTableOutputFormat.java | 31 ++- .../org/apache/kudu/mapreduce/TableReducer.java | 8 +- .../kudu/mapreduce/HadoopTestingUtility.java | 6 +- .../apache/kudu/mapreduce/ITInputFormatJob.java | 18 +- .../kudu/mapreduce/ITKuduTableInputFormat.java | 22 +- .../kudu/mapreduce/ITKuduTableOutputFormat.java | 11 +- .../kudu/mapreduce/ITOutputFormatJob.java | 19 +- .../apache/kudu/mapreduce/TestJarFinder.java | 9 +- .../apache/kudu/spark/kudu/DefaultSource.scala | 9 +- .../apache/kudu/spark/kudu/KuduContext.scala | 11 +- .../org/apache/kudu/spark/kudu/KuduRDD.scala | 5 +- .../kudu/spark/kudu/DefaultSourceTest.scala | 9 +- .../kudu/spark/kudu/KuduContextTest.scala | 4 +- .../apache/kudu/spark/kudu/TestContext.scala | 9 +- java/kudu_style.xml | 249 +++++++++++++++++++ java/pom.xml | 31 +++ 183 files changed, 2416 insertions(+), 1575 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceAudience.java ---------------------------------------------------------------------- diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceAudience.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceAudience.java index d254c56..6afbd2a 100644 --- a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceAudience.java +++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceAudience.java @@ -16,6 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.kudu.annotations; import java.lang.annotation.Documented; @@ -29,17 +30,17 @@ import java.lang.annotation.RetentionPolicy; * All public classes must have InterfaceAudience annotation. <br> * <ul> * <li>Public classes that are not marked with this annotation must be - * considered by default as {@link Private}.</li> - * + * considered by default as {@link Private}.</li> + * * <li>External applications must only use classes that are marked * {@link Public}. Avoid using non public classes as these classes * could be removed or change in incompatible ways.</li> - * + * * <li>Hadoop projects must only use classes that are marked * {@link LimitedPrivate} or {@link Public}</li> - * + * * <li> Methods may have a different annotation that it is more restrictive - * compared to the audience classification of the class. Example: A class + * compared to the audience classification of the class. Example: A class * might be {@link Public}, but a method may be {@link LimitedPrivate} * </li></ul> */ @@ -52,7 +53,7 @@ public class InterfaceAudience { @Documented @Retention(RetentionPolicy.RUNTIME) public @interface Public {}; - + /** * Intended only for the project(s) specified in the annotation. * For example, "Common", "HDFS", "MapReduce", "ZooKeeper", "HBase". @@ -62,7 +63,7 @@ public class InterfaceAudience { public @interface LimitedPrivate { String[] value(); }; - + /** * Intended for use only within Kudu itself. */ http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceStability.java ---------------------------------------------------------------------- diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceStability.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceStability.java index f589ada..cba19d9 100644 --- a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceStability.java +++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceStability.java @@ -16,6 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.kudu.annotations; import java.lang.annotation.Documented; @@ -30,7 +31,7 @@ import org.apache.kudu.annotations.InterfaceAudience.Public; * Annotation to inform users of how much to rely on a particular package, * class or method not changing over time. Currently the stability can be * {@link Stable}, {@link Evolving} or {@link Unstable}. <br> - * + * * <ul><li>All classes that are annotated with {@link Public} or * {@link LimitedPrivate} must have InterfaceStability annotation. </li> * <li>Classes that are {@link Private} are to be considered unstable unless @@ -42,20 +43,20 @@ import org.apache.kudu.annotations.InterfaceAudience.Public; @InterfaceStability.Evolving public class InterfaceStability { /** - * Can evolve while retaining compatibility for minor release boundaries.; + * Can evolve while retaining compatibility for minor release boundaries.; * can break compatibility only at major release (ie. at m.0). */ @Documented @Retention(RetentionPolicy.RUNTIME) public @interface Stable {}; - + /** * Evolving, but can break compatibility at minor release (i.e. m.x) */ @Documented @Retention(RetentionPolicy.RUNTIME) public @interface Evolving {}; - + /** * No guarantee is provided as to reliability or stability across any * level of release granularity. http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java ---------------------------------------------------------------------- diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java index dd04806..0d5c6e2 100644 --- a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java +++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java @@ -16,13 +16,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.kudu.annotations.tools; import com.sun.javadoc.DocErrorReporter; import com.sun.javadoc.LanguageVersion; import com.sun.javadoc.RootDoc; - import jdiff.JDiff; + import org.apache.kudu.annotations.InterfaceAudience; /** @@ -33,17 +34,17 @@ import org.apache.kudu.annotations.InterfaceAudience; * It delegates to the JDiff Doclet, and takes the same options. */ public class ExcludePrivateAnnotationsJDiffDoclet { - + public static LanguageVersion languageVersion() { return LanguageVersion.JAVA_1_5; } - + public static boolean start(RootDoc root) { System.out.println( ExcludePrivateAnnotationsJDiffDoclet.class.getSimpleName()); return JDiff.start(RootDocProcessor.process(root)); } - + public static int optionLength(String option) { Integer length = StabilityOptions.optionLength(option); if (length != null) { @@ -51,7 +52,7 @@ public class ExcludePrivateAnnotationsJDiffDoclet { } return JDiff.optionLength(option); } - + public static boolean validOptions(String[][] options, DocErrorReporter reporter) { StabilityOptions.validOptions(options, reporter); http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java ---------------------------------------------------------------------- diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java index 4b1165c..ecfd7e9 100644 --- a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java +++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java @@ -16,12 +16,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.kudu.annotations.tools; import com.sun.javadoc.DocErrorReporter; import com.sun.javadoc.LanguageVersion; import com.sun.javadoc.RootDoc; import com.sun.tools.doclets.standard.Standard; + import org.apache.kudu.annotations.InterfaceAudience; /** @@ -32,17 +34,17 @@ import org.apache.kudu.annotations.InterfaceAudience; * It delegates to the Standard Doclet, and takes the same options. */ public class ExcludePrivateAnnotationsStandardDoclet { - + public static LanguageVersion languageVersion() { return LanguageVersion.JAVA_1_5; } - + public static boolean start(RootDoc root) { System.out.println( ExcludePrivateAnnotationsStandardDoclet.class.getSimpleName()); return Standard.start(RootDocProcessor.process(root)); } - + public static int optionLength(String option) { Integer length = StabilityOptions.optionLength(option); if (length != null) { @@ -50,7 +52,7 @@ public class ExcludePrivateAnnotationsStandardDoclet { } return Standard.optionLength(option); } - + public static boolean validOptions(String[][] options, DocErrorReporter reporter) { StabilityOptions.validOptions(options, reporter); http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/IncludePublicAnnotationsStandardDoclet.java ---------------------------------------------------------------------- diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/IncludePublicAnnotationsStandardDoclet.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/IncludePublicAnnotationsStandardDoclet.java index ea10e6a..7c004fb 100644 --- a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/IncludePublicAnnotationsStandardDoclet.java +++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/IncludePublicAnnotationsStandardDoclet.java @@ -16,12 +16,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.kudu.annotations.tools; import com.sun.javadoc.DocErrorReporter; import com.sun.javadoc.LanguageVersion; import com.sun.javadoc.RootDoc; import com.sun.tools.doclets.standard.Standard; + import org.apache.kudu.annotations.InterfaceAudience; /** @@ -36,18 +38,18 @@ import org.apache.kudu.annotations.InterfaceAudience; * It delegates to the Standard Doclet, and takes the same options. */ public class IncludePublicAnnotationsStandardDoclet { - + public static LanguageVersion languageVersion() { return LanguageVersion.JAVA_1_5; } - + public static boolean start(RootDoc root) { System.out.println( IncludePublicAnnotationsStandardDoclet.class.getSimpleName()); RootDocProcessor.treatUnannotatedClassesAsPrivate = true; return Standard.start(RootDocProcessor.process(root)); } - + public static int optionLength(String option) { Integer length = StabilityOptions.optionLength(option); if (length != null) { @@ -55,7 +57,7 @@ public class IncludePublicAnnotationsStandardDoclet { } return Standard.optionLength(option); } - + public static boolean validOptions(String[][] options, DocErrorReporter reporter) { StabilityOptions.validOptions(options, reporter); http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/RootDocProcessor.java ---------------------------------------------------------------------- diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/RootDocProcessor.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/RootDocProcessor.java index 9fa5f4b..2a2c12a 100644 --- a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/RootDocProcessor.java +++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/RootDocProcessor.java @@ -16,8 +16,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.kudu.annotations.tools; +import java.lang.reflect.Array; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.WeakHashMap; + import com.sun.javadoc.AnnotationDesc; import com.sun.javadoc.AnnotationTypeDoc; import com.sun.javadoc.ClassDoc; @@ -29,16 +40,6 @@ import com.sun.javadoc.PackageDoc; import com.sun.javadoc.ProgramElementDoc; import com.sun.javadoc.RootDoc; -import java.lang.reflect.Array; -import java.lang.reflect.InvocationHandler; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.Proxy; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.WeakHashMap; - import org.apache.kudu.annotations.InterfaceAudience; import org.apache.kudu.annotations.InterfaceStability; @@ -78,8 +79,7 @@ class RootDocProcessor { return obj; } - private static Map<Object, Object> proxies = - new WeakHashMap<Object, Object>(); + private static Map<Object, Object> proxies = new WeakHashMap<Object, Object>(); private static Object getProxy(Object obj) { Object proxy = proxies.get(obj); @@ -160,9 +160,8 @@ class RootDocProcessor { } if (args != null) { - if (methodName.equals("compareTo") || methodName.equals("equals") - || methodName.equals("overrides") - || methodName.equals("subclassOf")) { + if (methodName.equals("compareTo") || methodName.equals("equals") || + methodName.equals("overrides") || methodName.equals("subclassOf")) { args[0] = unwrap(args[0]); } } @@ -184,9 +183,9 @@ class RootDocProcessor { for (AnnotationDesc annotation : annotations) { String qualifiedTypeName = annotation.annotationType().qualifiedTypeName(); if (qualifiedTypeName.equals( - InterfaceAudience.Private.class.getCanonicalName()) - || qualifiedTypeName.equals( - InterfaceAudience.LimitedPrivate.class.getCanonicalName())) { + InterfaceAudience.Private.class.getCanonicalName()) || + qualifiedTypeName.equals( + InterfaceAudience.LimitedPrivate.class.getCanonicalName())) { return true; } if (stability.equals(StabilityOptions.EVOLVING_OPTION)) { @@ -197,8 +196,8 @@ class RootDocProcessor { } if (stability.equals(StabilityOptions.STABLE_OPTION)) { if (qualifiedTypeName.equals( - InterfaceStability.Unstable.class.getCanonicalName()) - || qualifiedTypeName.equals( + InterfaceStability.Unstable.class.getCanonicalName()) || + qualifiedTypeName.equals( InterfaceStability.Evolving.class.getCanonicalName())) { return true; } @@ -234,8 +233,9 @@ class RootDocProcessor { } private Object unwrap(Object proxy) { - if (proxy instanceof Proxy) + if (proxy instanceof Proxy) { return ((ExcludeHandler) Proxy.getInvocationHandler(proxy)).target; + } return proxy; } http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/StabilityOptions.java ---------------------------------------------------------------------- diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/StabilityOptions.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/StabilityOptions.java index c435bcf..c74f033 100644 --- a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/StabilityOptions.java +++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/StabilityOptions.java @@ -16,14 +16,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kudu.annotations.tools; -import com.sun.javadoc.DocErrorReporter; +package org.apache.kudu.annotations.tools; import java.util.ArrayList; import java.util.List; import java.util.Locale; +import com.sun.javadoc.DocErrorReporter; + class StabilityOptions { public static final String STABLE_OPTION = "-stable"; public static final String EVOLVING_OPTION = "-evolving"; @@ -31,9 +32,11 @@ class StabilityOptions { public static Integer optionLength(String option) { String opt = option.toLowerCase(Locale.ENGLISH); - if (opt.equals(UNSTABLE_OPTION)) return 1; - if (opt.equals(EVOLVING_OPTION)) return 1; - if (opt.equals(STABLE_OPTION)) return 1; + if (opt.equals(UNSTABLE_OPTION) || + opt.equals(EVOLVING_OPTION) || + opt.equals(STABLE_OPTION)) { + return 1; + } return null; } @@ -50,13 +53,13 @@ class StabilityOptions { } } } - + public static String[][] filterOptions(String[][] options) { List<String[]> optionsList = new ArrayList<String[]>(); for (int i = 0; i < options.length; i++) { - if (!options[i][0].equalsIgnoreCase(UNSTABLE_OPTION) - && !options[i][0].equalsIgnoreCase(EVOLVING_OPTION) - && !options[i][0].equalsIgnoreCase(STABLE_OPTION)) { + if (!options[i][0].equalsIgnoreCase(UNSTABLE_OPTION) && + !options[i][0].equalsIgnoreCase(EVOLVING_OPTION) && + !options[i][0].equalsIgnoreCase(STABLE_OPTION)) { optionsList.add(options[i]); } } http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/package-info.java ---------------------------------------------------------------------- diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/package-info.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/package-info.java index 8659a5a..7675154 100644 --- a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/package-info.java +++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/package-info.java @@ -17,6 +17,7 @@ * limitations under the License. */ @InterfaceAudience.Private + package org.apache.kudu.annotations.tools; import org.apache.kudu.annotations.InterfaceAudience; http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/CsvParser.java ---------------------------------------------------------------------- diff --git a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/CsvParser.java b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/CsvParser.java index 7e42bd4..94f7707 100644 --- a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/CsvParser.java +++ b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/CsvParser.java @@ -12,18 +12,20 @@ * See the License for the specific language governing permissions and * limitations under the License. See accompanying LICENSE file. */ + package org.apache.kudu.mapreduce.tools; +import java.util.ArrayList; +import java.util.List; + import com.google.common.base.Preconditions; import com.google.common.base.Splitter; import com.google.common.collect.Lists; + import org.apache.kudu.annotations.InterfaceAudience; import org.apache.kudu.annotations.InterfaceStability; import org.apache.kudu.client.Bytes; -import java.util.ArrayList; -import java.util.List; - /** * Column-separated values parser that gives access to the different columns inside each line of * data. http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsv.java ---------------------------------------------------------------------- diff --git a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsv.java b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsv.java index 8c1c64c..55d1254 100644 --- a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsv.java +++ b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsv.java @@ -12,12 +12,11 @@ * See the License for the specific language governing permissions and * limitations under the License. See accompanying LICENSE file. */ + package org.apache.kudu.mapreduce.tools; -import org.apache.kudu.annotations.InterfaceAudience; -import org.apache.kudu.annotations.InterfaceStability; -import org.apache.kudu.mapreduce.CommandLineParser; -import org.apache.kudu.mapreduce.KuduTableMapReduceUtil; +import java.io.IOException; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; @@ -27,7 +26,10 @@ import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import java.io.IOException; +import org.apache.kudu.annotations.InterfaceAudience; +import org.apache.kudu.annotations.InterfaceStability; +import org.apache.kudu.mapreduce.CommandLineParser; +import org.apache.kudu.mapreduce.KuduTableMapReduceUtil; /** * Map-only job that reads CSV files and inserts them into a single Kudu table. @@ -36,7 +38,7 @@ import java.io.IOException; @InterfaceStability.Unstable public class ImportCsv extends Configured implements Tool { - public static enum Counters { BAD_LINES }; + public static enum Counters { BAD_LINES } static final String NAME = "importcsv"; static final String DEFAULT_SEPARATOR = "\t"; http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsvMapper.java ---------------------------------------------------------------------- diff --git a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsvMapper.java b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsvMapper.java index 6d6e11b..365fddb 100644 --- a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsvMapper.java +++ b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsvMapper.java @@ -12,14 +12,11 @@ * See the License for the specific language governing permissions and * limitations under the License. See accompanying LICENSE file. */ + package org.apache.kudu.mapreduce.tools; -import org.apache.kudu.ColumnSchema; -import org.apache.kudu.Schema; -import org.apache.kudu.annotations.InterfaceAudience; -import org.apache.kudu.annotations.InterfaceStability; -import org.apache.kudu.client.*; -import org.apache.kudu.mapreduce.KuduTableMapReduceUtil; +import java.io.IOException; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.NullWritable; @@ -27,7 +24,16 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Counter; import org.apache.hadoop.mapreduce.Mapper; -import java.io.IOException; +import org.apache.kudu.ColumnSchema; +import org.apache.kudu.Schema; +import org.apache.kudu.annotations.InterfaceAudience; +import org.apache.kudu.annotations.InterfaceStability; +import org.apache.kudu.client.Bytes; +import org.apache.kudu.client.Insert; +import org.apache.kudu.client.KuduTable; +import org.apache.kudu.client.Operation; +import org.apache.kudu.client.PartialRow; +import org.apache.kudu.mapreduce.KuduTableMapReduceUtil; /** * Mapper that ingests CSV lines and turns them into Kudu Inserts. http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java ---------------------------------------------------------------------- diff --git a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java index 5a0c0ea..8efac49 100644 --- a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java +++ b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java @@ -12,19 +12,22 @@ * See the License for the specific language governing permissions and * limitations under the License. See accompanying LICENSE file. */ + package org.apache.kudu.mapreduce.tools; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; -import org.apache.kudu.ColumnSchema; -import org.apache.kudu.Schema; -import org.apache.kudu.Type; -import org.apache.kudu.annotations.InterfaceAudience; -import org.apache.kudu.annotations.InterfaceStability; -import org.apache.kudu.client.*; -import org.apache.kudu.mapreduce.CommandLineParser; -import org.apache.kudu.mapreduce.KuduTableMapReduceUtil; -import org.apache.kudu.util.Pair; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; @@ -60,18 +63,27 @@ import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.math.BigInteger; -import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Random; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; +import org.apache.kudu.ColumnSchema; +import org.apache.kudu.Schema; +import org.apache.kudu.Type; +import org.apache.kudu.annotations.InterfaceAudience; +import org.apache.kudu.annotations.InterfaceStability; +import org.apache.kudu.client.AbstractKuduScannerBuilder; +import org.apache.kudu.client.Bytes; +import org.apache.kudu.client.CreateTableOptions; +import org.apache.kudu.client.KuduClient; +import org.apache.kudu.client.KuduScanner; +import org.apache.kudu.client.KuduSession; +import org.apache.kudu.client.KuduTable; +import org.apache.kudu.client.Operation; +import org.apache.kudu.client.PartialRow; +import org.apache.kudu.client.RowResult; +import org.apache.kudu.client.RowResultIterator; +import org.apache.kudu.client.SessionConfiguration; +import org.apache.kudu.client.Update; +import org.apache.kudu.mapreduce.CommandLineParser; +import org.apache.kudu.mapreduce.KuduTableMapReduceUtil; +import org.apache.kudu.util.Pair; /** * <p> @@ -308,13 +320,16 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { * Copied under the public domain from SquidLib. */ private static class Xoroshiro128PlusRandom { - private long state0, state1; + private long state0; + private long state1; + public Xoroshiro128PlusRandom() { this((long) (Math.random() * Long.MAX_VALUE)); } + public Xoroshiro128PlusRandom(long seed) { - long state = seed + 0x9E3779B97F4A7C15L, - z = state; + long state = seed + 0x9E3779B97F4A7C15L; + long z = state; z = (z ^ (z >>> 30)) * 0xBF58476D1CE4E5B9L; z = (z ^ (z >>> 27)) * 0x94D049BB133111EBL; state0 = z ^ (z >>> 31); @@ -324,6 +339,7 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { z = (z ^ (z >>> 27)) * 0x94D049BB133111EBL; state1 = z ^ (z >>> 31); } + public long nextLong() { final long s0 = state0; long s1 = state1; @@ -335,8 +351,10 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { return result; } + public void nextBytes(final byte[] bytes) { - int i = bytes.length, n = 0; + int i = bytes.length; + int n = 0; while (i != 0) { n = Math.min(i, 8); for (long bits = nextLong(); n-- != 0; bits >>>= 8) { @@ -354,18 +372,22 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { private static final Log LOG = LogFactory.getLog(Generator.class); static class GeneratorInputFormat extends InputFormat<BytesWritable,NullWritable> { + static class GeneratorInputSplit extends InputSplit implements Writable { @Override public long getLength() throws IOException, InterruptedException { return 1; } + @Override public String[] getLocations() throws IOException, InterruptedException { return new String[0]; } + @Override public void readFields(DataInput arg0) throws IOException { } + @Override public void write(DataOutput arg0) throws IOException { } @@ -567,8 +589,9 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { private static <T> void circularLeftShift(T[] first) { T ez = first[0]; - for (int i = 0; i < first.length - 1; i++) + for (int i = 0; i < first.length - 1; i++) { first[i] = first[i + 1]; + } first[first.length - 1] = ez; } @@ -635,6 +658,15 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { return run(numMappers, numNodes, numTablets, tmpOutput, width, wrapMultiplier); } + public int run(int numMappers, long numNodes, int numTablets, Path tmpOutput, + Integer width, Integer wrapMultiplier) throws Exception { + int ret = runRandomInputGenerator(numMappers, numNodes, tmpOutput, width, wrapMultiplier); + if (ret > 0) { + return ret; + } + return runGenerator(numMappers, numNodes, numTablets, tmpOutput, width, wrapMultiplier); + } + protected void createTables(int numTablets) throws Exception { createSchema(getTableName(getConf()), getTableSchema(), numTablets); @@ -659,8 +691,8 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { if (numTablets > 1) { BigInteger min = BigInteger.valueOf(Long.MIN_VALUE); BigInteger max = BigInteger.valueOf(Long.MAX_VALUE); - BigInteger step = max.multiply(BigInteger.valueOf(2)).divide(BigInteger.valueOf - (numTablets)); + BigInteger step = max.multiply(BigInteger.valueOf(2)) + .divide(BigInteger.valueOf(numTablets)); LOG.info(min.longValue()); LOG.info(max.longValue()); LOG.info(step.longValue()); @@ -683,8 +715,8 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { public int runRandomInputGenerator(int numMappers, long numNodes, Path tmpOutput, Integer width, Integer wrapMultiplier) throws Exception { - LOG.info("Running RandomInputGenerator with numMappers=" + numMappers - + ", numNodes=" + numNodes); + LOG.info("Running RandomInputGenerator with numMappers=" + numMappers + + ", numNodes=" + numNodes); Job job = new Job(getConf()); job.setJobName("Random Input Generator"); @@ -709,7 +741,7 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { public int runGenerator(int numMappers, long numNodes, int numTablets, Path tmpOutput, Integer width, Integer wrapMultiplier) throws Exception { - LOG.info("Running Generator with numMappers=" + numMappers +", numNodes=" + numNodes); + LOG.info("Running Generator with numMappers=" + numMappers + ", numNodes=" + numNodes); createTables(numTablets); Job job = new Job(getConf()); @@ -719,7 +751,8 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { job.setJarByClass(getClass()); FileInputFormat.setInputPaths(job, tmpOutput); - job.setInputFormatClass(OneFilePerMapperSFIF.class); + job.setInputFormatClass(org.apache.kudu.mapreduce.tools.IntegrationTestBigLinkedList + .Generator.OneFilePerMapperSFIF.class); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(NullWritable.class); @@ -742,15 +775,6 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { return success ? 0 : 1; } - - public int run(int numMappers, long numNodes, int numTablets, Path tmpOutput, - Integer width, Integer wrapMultiplier) throws Exception { - int ret = runRandomInputGenerator(numMappers, numNodes, tmpOutput, width, wrapMultiplier); - if (ret > 0) { - return ret; - } - return runGenerator(numMappers, numNodes, numTablets, tmpOutput, width, wrapMultiplier); - } } /** @@ -861,7 +885,8 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { public int run(String[] args) throws Exception { if (args.length != 2) { - System.out.println("Usage : " + Verify.class.getSimpleName() + " <output dir> <num reducers>"); + System.out.println("Usage : " + Verify.class.getSimpleName() + + " <output dir> <num reducers>"); return 0; } @@ -876,7 +901,7 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { } public int run(Path outputDir, int numReducers) throws Exception { - LOG.info("Running Verify with outputDir=" + outputDir +", numReducers=" + numReducers); + LOG.info("Running Verify with outputDir=" + outputDir + ", numReducers=" + numReducers); job = new Job(getConf()); @@ -927,8 +952,9 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { if (unreferenced.getValue() > 0) { boolean couldBeMultiRef = (multiref.getValue() == unreferenced.getValue()); - LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue() - + (couldBeMultiRef ? "; could be due to duplicate random numbers" : "")); + LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + + unreferenced.getValue() + + (couldBeMultiRef ? "; could be due to duplicate random numbers" : "")); success = false; } @@ -975,8 +1001,9 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { FileSystem fs; - protected void runGenerator(int numMappers, long numNodes, int numTablets, - String outputDir, Integer width, Integer wrapMultiplier) throws Exception { + protected void runGenerator(int numMappers, long numNodes, + int numTablets, String outputDir, + Integer width, Integer wrapMultiplier) throws Exception { Path outputPath = new Path(outputDir); UUID uuid = UUID.randomUUID(); //create a random UUID. Path generatorOutput = new Path(outputPath, uuid.toString()); @@ -1184,7 +1211,7 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { * Schema we use when getting rows from the linked list, we only need the reference and * its update count. */ - private final List<String> SCAN_COLUMN_NAMES = ImmutableList.of( + private static final List<String> SCAN_COLUMN_NAMES = ImmutableList.of( COLUMN_PREV_ONE, COLUMN_PREV_TWO, COLUMN_UPDATE_COUNT, COLUMN_CLIENT); private long numUpdatesPerMapper; @@ -1308,7 +1335,7 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { */ private RowResult nextNode(long prevKeyOne, long prevKeyTwo) throws IOException { KuduScanner.KuduScannerBuilder builder = client.newScannerBuilder(table) - .setProjectedColumnNames(SCAN_COLUMN_NAMES); + .setProjectedColumnNames(SCAN_COLUMN_NAMES); configureScannerForRandomRead(builder, table, prevKeyOne, prevKeyTwo); @@ -1676,9 +1703,7 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { tool = new Updater(); } else if (toRun.equals("Walker")) { tool = new Walker(); - } /*else if (toRun.equals("Delete")) { - tool = new Delete(); - }*/ else { + } else { usage(); throw new RuntimeException("Unknown arg"); } http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/RowCounter.java ---------------------------------------------------------------------- diff --git a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/RowCounter.java b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/RowCounter.java index 112a8cc..e256acb 100644 --- a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/RowCounter.java +++ b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/RowCounter.java @@ -14,13 +14,11 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. + package org.apache.kudu.mapreduce.tools; -import org.apache.kudu.annotations.InterfaceAudience; -import org.apache.kudu.annotations.InterfaceStability; -import org.apache.kudu.mapreduce.CommandLineParser; -import org.apache.kudu.mapreduce.KuduTableMapReduceUtil; -import org.apache.kudu.client.RowResult; +import java.io.IOException; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.io.NullWritable; @@ -30,7 +28,11 @@ import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import java.io.IOException; +import org.apache.kudu.annotations.InterfaceAudience; +import org.apache.kudu.annotations.InterfaceStability; +import org.apache.kudu.client.RowResult; +import org.apache.kudu.mapreduce.CommandLineParser; +import org.apache.kudu.mapreduce.KuduTableMapReduceUtil; /** * Map-only job that counts all the rows in the provided table. @@ -69,18 +71,15 @@ public class RowCounter extends Configured implements Tool { @SuppressWarnings("deprecation") public static Job createSubmittableJob(Configuration conf, String[] args) throws IOException, ClassNotFoundException { - - String columnProjection = conf.get(COLUMN_PROJECTION_KEY); - - Class<RowCounterMapper> mapperClass = RowCounterMapper.class; String tableName = args[0]; - String jobName = NAME + "_" + tableName; Job job = new Job(conf, jobName); + Class<RowCounterMapper> mapperClass = RowCounterMapper.class; job.setJarByClass(mapperClass); job.setMapperClass(mapperClass); job.setNumReduceTasks(0); job.setOutputFormatClass(NullOutputFormat.class); + String columnProjection = conf.get(COLUMN_PROJECTION_KEY); new KuduTableMapReduceUtil.TableInputFormatConfiguratorWithCommandLineParser( job, tableName, http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java ---------------------------------------------------------------------- diff --git a/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java index 3e76382..4af8c59 100644 --- a/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java +++ b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java @@ -16,29 +16,29 @@ // under the License. package org.apache.kudu.mapreduce.tools; -import org.apache.kudu.ColumnSchema; -import org.apache.kudu.Schema; -import org.apache.kudu.Type; -import org.apache.kudu.mapreduce.CommandLineParser; -import org.apache.kudu.mapreduce.HadoopTestingUtility; -import org.apache.kudu.client.BaseKuduTest; -import org.apache.kudu.client.CreateTableOptions; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.util.GenericOptionsParser; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import com.google.common.collect.ImmutableList; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.util.GenericOptionsParser; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.kudu.ColumnSchema; +import org.apache.kudu.Schema; +import org.apache.kudu.Type; +import org.apache.kudu.client.BaseKuduTest; +import org.apache.kudu.client.CreateTableOptions; +import org.apache.kudu.mapreduce.CommandLineParser; +import org.apache.kudu.mapreduce.HadoopTestingUtility; public class ITImportCsv extends BaseKuduTest { http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITIntegrationTestBigLinkedList.java ---------------------------------------------------------------------- diff --git a/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITIntegrationTestBigLinkedList.java b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITIntegrationTestBigLinkedList.java index 32e5d15..56fb813 100644 --- a/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITIntegrationTestBigLinkedList.java +++ b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITIntegrationTestBigLinkedList.java @@ -22,6 +22,7 @@ import org.apache.hadoop.util.ToolRunner; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Test; + import org.apache.kudu.client.BaseKuduTest; import org.apache.kudu.mapreduce.CommandLineParser; import org.apache.kudu.mapreduce.HadoopTestingUtility; http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITRowCounter.java ---------------------------------------------------------------------- diff --git a/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITRowCounter.java b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITRowCounter.java index 6d590a6..29ed60d 100644 --- a/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITRowCounter.java +++ b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITRowCounter.java @@ -16,9 +16,9 @@ // under the License. package org.apache.kudu.mapreduce.tools; -import org.apache.kudu.mapreduce.CommandLineParser; -import org.apache.kudu.mapreduce.HadoopTestingUtility; -import org.apache.kudu.client.BaseKuduTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.util.GenericOptionsParser; @@ -26,8 +26,9 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import org.apache.kudu.client.BaseKuduTest; +import org.apache.kudu.mapreduce.CommandLineParser; +import org.apache.kudu.mapreduce.HadoopTestingUtility; public class ITRowCounter extends BaseKuduTest { http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/main/java/com/google/protobuf/ZeroCopyLiteralByteString.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/main/java/com/google/protobuf/ZeroCopyLiteralByteString.java b/java/kudu-client/src/main/java/com/google/protobuf/ZeroCopyLiteralByteString.java index 5bd933d..f2ea261 100644 --- a/java/kudu-client/src/main/java/com/google/protobuf/ZeroCopyLiteralByteString.java +++ b/java/kudu-client/src/main/java/com/google/protobuf/ZeroCopyLiteralByteString.java @@ -24,6 +24,7 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ + package com.google.protobuf; // This is a lie. /** @@ -60,8 +61,8 @@ public final class ZeroCopyLiteralByteString extends LiteralByteString { if (buf instanceof LiteralByteString) { return ((LiteralByteString) buf).bytes; } - throw new UnsupportedOperationException("Need a LiteralByteString, got a " - + buf.getClass().getName()); + throw new UnsupportedOperationException("Need a LiteralByteString, got a " + + buf.getClass().getName()); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java b/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java index ac71ea2..e4d93e4 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java @@ -14,6 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. + package org.apache.kudu; import org.apache.kudu.Common.CompressionType; @@ -63,7 +64,7 @@ public class ColumnSchema { public EncodingType getInternalPbType() { return internalPbType; } - }; + } /** * Specifies the compression algorithm of data for a column on disk. @@ -86,7 +87,7 @@ public class ColumnSchema { public CompressionType getInternalPbType() { return internalPbType; } - }; + } private ColumnSchema(String name, Type type, boolean key, boolean nullable, Object defaultValue, int desiredBlockSize, Encoding encoding, @@ -168,14 +169,24 @@ public class ColumnSchema { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } ColumnSchema that = (ColumnSchema) o; - if (key != that.key) return false; - if (!name.equals(that.name)) return false; - if (!type.equals(that.type)) return false; + if (key != that.key) { + return false; + } + if (!name.equals(that.name)) { + return false; + } + if (!type.equals(that.type)) { + return false; + } return true; } http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/main/java/org/apache/kudu/Schema.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/main/java/org/apache/kudu/Schema.java b/java/kudu-client/src/main/java/org/apache/kudu/Schema.java index 74a0587..d9eae40 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/Schema.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/Schema.java @@ -14,19 +14,21 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. + package org.apache.kudu; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import com.google.common.collect.ImmutableList; + import org.apache.kudu.annotations.InterfaceAudience; import org.apache.kudu.annotations.InterfaceStability; import org.apache.kudu.client.Bytes; import org.apache.kudu.client.PartialRow; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - /** * Represents table's schema which is essentially a list of columns. * This class offers a few utility methods for querying it. @@ -158,6 +160,24 @@ public class Schema { } /** + * Gives the size in bytes for a single row given the specified schema + * @param columns the row's columns + * @return row size in bytes + */ + private static int getRowSize(List<ColumnSchema> columns) { + int totalSize = 0; + boolean hasNullables = false; + for (ColumnSchema column : columns) { + totalSize += column.getType().getSize(); + hasNullables |= column.isNullable(); + } + if (hasNullables) { + totalSize += Bytes.getBitSetSize(columns.size()); + } + return totalSize; + } + + /** * Get the index at which this column can be found in the backing byte array * @param idx column's index * @return column's offset @@ -181,29 +201,33 @@ public class Schema { } /** - * Get the column at the specified index in the original list - * @param idx column's index - * @return the column - */ - public ColumnSchema getColumnByIndex(int idx) { - return this.columnsByIndex.get(idx); - } - - /** * Get the column index of the column with the provided ID. * This method is not part of the stable API. * @param columnId the column id of the column * @return the column index of the column. */ public int getColumnIndex(int columnId) { - if (!hasColumnIds()) throw new IllegalStateException("Schema does not have Column IDs"); + if (!hasColumnIds()) { + throw new IllegalStateException("Schema does not have Column IDs"); + } Integer index = this.columnsById.get(columnId); - if (index == null) throw new IllegalArgumentException( - String.format("Unknown column id: %s", columnId)); + if (index == null) { + throw new IllegalArgumentException( + String.format("Unknown column id: %s", columnId)); + } return index; } /** + * Get the column at the specified index in the original list + * @param idx column's index + * @return the column + */ + public ColumnSchema getColumnByIndex(int idx) { + return this.columnsByIndex.get(idx); + } + + /** * Get the column associated with the specified name * @param columnName column's name * @return the column @@ -264,24 +288,6 @@ public class Schema { } /** - * Gives the size in bytes for a single row given the specified schema - * @param columns the row's columns - * @return row size in bytes - */ - private static int getRowSize(List<ColumnSchema> columns) { - int totalSize = 0; - boolean hasNullables = false; - for (ColumnSchema column : columns) { - totalSize += column.getType().getSize(); - hasNullables |= column.isNullable(); - } - if (hasNullables) { - totalSize += Bytes.getBitSetSize(columns.size()); - } - return totalSize; - } - - /** * Creates a new partial row for the schema. * @return a new partial row */ http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/main/java/org/apache/kudu/Type.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/main/java/org/apache/kudu/Type.java b/java/kudu-client/src/main/java/org/apache/kudu/Type.java index 400555d..133baaf 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/Type.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/Type.java @@ -14,16 +14,18 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. + package org.apache.kudu; +import static org.apache.kudu.Common.DataType; + import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import com.google.common.primitives.Shorts; + import org.apache.kudu.annotations.InterfaceAudience; import org.apache.kudu.annotations.InterfaceStability; -import static org.apache.kudu.Common.DataType; - /** * Describes all the types available to build table schemas. */ @@ -31,16 +33,16 @@ import static org.apache.kudu.Common.DataType; @InterfaceStability.Evolving public enum Type { - INT8 (DataType.INT8, "int8"), - INT16 (DataType.INT16, "int16"), - INT32 (DataType.INT32, "int32"), - INT64 (DataType.INT64, "int64"), - BINARY (DataType.BINARY, "binary"), - STRING (DataType.STRING, "string"), - BOOL (DataType.BOOL, "bool"), - FLOAT (DataType.FLOAT, "float"), - DOUBLE (DataType.DOUBLE, "double"), - UNIXTIME_MICROS (DataType.UNIXTIME_MICROS, "unixtime_micros"); + INT8(DataType.INT8, "int8"), + INT16(DataType.INT16, "int16"), + INT32(DataType.INT32, "int32"), + INT64(DataType.INT64, "int64"), + BINARY(DataType.BINARY, "binary"), + STRING(DataType.STRING, "string"), + BOOL(DataType.BOOL, "bool"), + FLOAT(DataType.FLOAT, "float"), + DOUBLE(DataType.DOUBLE, "double"), + UNIXTIME_MICROS(DataType.UNIXTIME_MICROS, "unixtime_micros"); private final DataType dataType; private final String name; @@ -94,15 +96,20 @@ public enum Type { static int getTypeSize(DataType type) { switch (type) { case STRING: - case BINARY: return 8 + 8; // offset then string length + case BINARY: + return 8 + 8; // offset then string length case BOOL: - case INT8: return 1; - case INT16: return Shorts.BYTES; + case INT8: + return 1; + case INT16: + return Shorts.BYTES; case INT32: - case FLOAT: return Ints.BYTES; + case FLOAT: + return Ints.BYTES; case INT64: case DOUBLE: - case UNIXTIME_MICROS: return Longs.BYTES; + case UNIXTIME_MICROS: + return Longs.BYTES; default: throw new IllegalArgumentException("The provided data type doesn't map" + " to know any known one."); } http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/main/java/org/apache/kudu/client/AbstractKuduScannerBuilder.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AbstractKuduScannerBuilder.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AbstractKuduScannerBuilder.java index f58ae41..f1e0f10 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/client/AbstractKuduScannerBuilder.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AbstractKuduScannerBuilder.java @@ -14,6 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. + package org.apache.kudu.client; import java.util.HashMap; @@ -21,6 +22,7 @@ import java.util.List; import java.util.Map; import com.google.common.collect.ImmutableList; + import org.apache.kudu.Common; import org.apache.kudu.annotations.InterfaceAudience; import org.apache.kudu.annotations.InterfaceStability; @@ -42,7 +44,7 @@ public abstract class AbstractKuduScannerBuilder AsyncKuduScanner.ReadMode readMode = AsyncKuduScanner.ReadMode.READ_LATEST; Common.OrderMode orderMode = Common.OrderMode.UNORDERED; - int batchSizeBytes = 1024*1024; + int batchSizeBytes = 1024 * 1024; long limit = Long.MAX_VALUE; boolean prefetching = false; boolean cacheBlocks = true; @@ -274,7 +276,8 @@ public abstract class AbstractKuduScannerBuilder } /** - * Like lowerBoundPrimaryKey() but the encoded primary key is an opaque byte array obtained elsewhere. + * Like lowerBoundPrimaryKey() but the encoded primary key is an opaque byte + * array obtained elsewhere. * @param startPrimaryKey bytes containing an encoded start key * @return this instance * @deprecated use {@link #lowerBound(PartialRow)} @@ -299,7 +302,8 @@ public abstract class AbstractKuduScannerBuilder } /** - * Like exclusiveUpperBound() but the encoded primary key is an opaque byte array obtained elsewhere. + * Like exclusiveUpperBound() but the encoded primary key is an opaque byte + * array obtained elsewhere. * @param endPrimaryKey bytes containing an encoded end key * @return this instance * @deprecated use {@link #exclusiveUpperBound(PartialRow)} @@ -344,7 +348,8 @@ public abstract class AbstractKuduScannerBuilder * @return this instance */ S exclusiveUpperBoundPartitionKeyRaw(byte[] partitionKey) { - if (upperBoundPartitionKey.length == 0 || Bytes.memcmp(partitionKey, upperBoundPartitionKey) < 0) { + if (upperBoundPartitionKey.length == 0 || + Bytes.memcmp(partitionKey, upperBoundPartitionKey) < 0) { this.upperBoundPartitionKey = partitionKey; } return (S) this; http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java index 0642bf8..1eab671 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java @@ -14,8 +14,11 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. + package org.apache.kudu.client; +import static org.apache.kudu.master.Master.AlterTableRequestPB; + import com.google.common.base.Preconditions; import org.apache.kudu.ColumnSchema; @@ -23,8 +26,6 @@ import org.apache.kudu.Type; import org.apache.kudu.annotations.InterfaceAudience; import org.apache.kudu.annotations.InterfaceStability; -import static org.apache.kudu.master.Master.AlterTableRequestPB; - /** * This builder must be used to alter a table. At least one change must be specified. */ @@ -168,8 +169,8 @@ public class AlterTableOptions { AlterTableRequestPB.AddRangePartition.Builder builder = AlterTableRequestPB.AddRangePartition.newBuilder(); builder.setRangeBounds( - new Operation.OperationsEncoder().encodeLowerAndUpperBounds(lowerBound, upperBound, - lowerBoundType, upperBoundType)); + new Operation.OperationsEncoder() + .encodeLowerAndUpperBounds(lowerBound, upperBound, lowerBoundType, upperBoundType)); step.setAddRangePartition(builder); if (!pb.hasSchema()) { pb.setSchema(ProtobufHelper.schemaToPb(lowerBound.getSchema())); http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java index 8e6cb7f..e1921bb 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java @@ -14,18 +14,23 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. + package org.apache.kudu.client; +import static org.apache.kudu.master.Master.AlterTableRequestPB; +import static org.apache.kudu.master.Master.AlterTableResponsePB; +import static org.apache.kudu.master.Master.MasterFeatures; +import static org.apache.kudu.master.Master.TableIdentifierPB; + +import java.util.Collection; +import java.util.List; + import com.google.common.collect.ImmutableList; import com.google.protobuf.Message; -import org.apache.kudu.annotations.InterfaceAudience; -import org.apache.kudu.util.Pair; import org.jboss.netty.buffer.ChannelBuffer; -import static org.apache.kudu.master.Master.*; - -import java.util.Collection; -import java.util.List; +import org.apache.kudu.annotations.InterfaceAudience; +import org.apache.kudu.util.Pair; /** * RPC used to alter a table. When it returns it doesn't mean that the table is altered, @@ -57,7 +62,9 @@ class AlterTableRequest extends KuduRpc<AlterTableResponse> { } @Override - String serviceName() { return MASTER_SERVICE_NAME; } + String serviceName() { + return MASTER_SERVICE_NAME; + } @Override String method() { http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableResponse.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableResponse.java index af582e1..4d3b131 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableResponse.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableResponse.java @@ -14,6 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. + package org.apache.kudu.client; import org.apache.kudu.annotations.InterfaceAudience; http://git-wip-us.apache.org/repos/asf/kudu/blob/22067edb/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java index 9274087..9c28458 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java @@ -26,6 +26,25 @@ package org.apache.kudu.client; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.kudu.client.ExternalConsistencyMode.CLIENT_PROPAGATED; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; +import javax.annotation.concurrent.GuardedBy; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -34,14 +53,6 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.protobuf.Message; import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; -import org.apache.kudu.Schema; -import org.apache.kudu.annotations.InterfaceAudience; -import org.apache.kudu.annotations.InterfaceStability; -import org.apache.kudu.master.Master; -import org.apache.kudu.master.Master.GetTableLocationsResponsePB; -import org.apache.kudu.util.AsyncUtil; -import org.apache.kudu.util.NetUtil; -import org.apache.kudu.util.Pair; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.channel.socket.ClientSocketChannelFactory; import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory; @@ -52,24 +63,14 @@ import org.jboss.netty.util.TimerTask; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.concurrent.GuardedBy; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Random; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; -import java.util.concurrent.Semaphore; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.apache.kudu.client.ExternalConsistencyMode.CLIENT_PROPAGATED; +import org.apache.kudu.Schema; +import org.apache.kudu.annotations.InterfaceAudience; +import org.apache.kudu.annotations.InterfaceStability; +import org.apache.kudu.master.Master; +import org.apache.kudu.master.Master.GetTableLocationsResponsePB; +import org.apache.kudu.util.AsyncUtil; +import org.apache.kudu.util.NetUtil; +import org.apache.kudu.util.Pair; /** * A fully asynchronous and thread-safe client for Kudu. @@ -207,7 +208,7 @@ public class AsyncKuduClient implements AutoCloseable { @VisibleForTesting public synchronized void updateLastPropagatedTimestamp(long lastPropagatedTimestamp) { if (this.lastPropagatedTimestamp == -1 || - this.lastPropagatedTimestamp < lastPropagatedTimestamp) { + this.lastPropagatedTimestamp < lastPropagatedTimestamp) { this.lastPropagatedTimestamp = lastPropagatedTimestamp; } } @@ -252,11 +253,12 @@ public class AsyncKuduClient implements AutoCloseable { create.setTimeoutMillis(defaultAdminOperationTimeoutMs); return sendRpcToTablet(create).addCallbackDeferring( new Callback<Deferred<KuduTable>, CreateTableResponse>() { - @Override - public Deferred<KuduTable> call(CreateTableResponse createTableResponse) throws Exception { - return openTable(name); - } - }); + @Override + public Deferred<KuduTable> call(CreateTableResponse createTableResponse) + throws Exception { + return openTable(name); + } + }); } /** @@ -301,6 +303,7 @@ public class AsyncKuduClient implements AutoCloseable { } return resp; } + @Override public String toString() { return "ClearTableLocationsCacheCB"; @@ -313,6 +316,7 @@ public class AsyncKuduClient implements AutoCloseable { tableLocations.clear(); return e; } + @Override public String toString() { return "ClearTableLocationsCacheEB"; @@ -408,10 +412,14 @@ public class AsyncKuduClient implements AutoCloseable { // timeouts and use its Deferred. final KuduRpc<KuduTable> fakeRpc = new KuduRpc<KuduTable>(null) { @Override - ChannelBuffer serialize(Message header) { return null; } + ChannelBuffer serialize(Message header) { + return null; + } @Override - String serviceName() { return null; } + String serviceName() { + return null; + } @Override String method() { @@ -420,7 +428,9 @@ public class AsyncKuduClient implements AutoCloseable { @Override Pair<KuduTable, Object> deserialize(CallResponse callResponse, String tsUUID) - throws KuduException { return null; } + throws KuduException { + return null; + } }; fakeRpc.setTimeoutMillis(defaultAdminOperationTimeoutMs); @@ -589,7 +599,6 @@ public class AsyncKuduClient implements AutoCloseable { KuduRpc<AsyncKuduScanner.Response> nextRequest = scanner.getNextRowsRequest(); String uuid = tablet.getReplicaSelectedUUID(nextRequest.getReplicaSelection()); TabletClient client = connectionCache.getClient(uuid); - Deferred<AsyncKuduScanner.Response> d = nextRequest.getDeferred(); // Important to increment the attempts before the next if statement since // getSleepTimeForRpc() relies on it if the client is null or dead. nextRequest.attempt++; @@ -597,12 +606,12 @@ public class AsyncKuduClient implements AutoCloseable { // A null client means we either don't know about this tablet anymore (unlikely) or we // couldn't find a leader (which could be triggered by a read timeout). // We'll first delay the RPC in case things take some time to settle down, then retry. - Status statusRemoteError = Status.RemoteError("Not connected to server " + uuid - + " will retry after a delay"); + Status statusRemoteError = Status.RemoteError("Not connected to server " + uuid + + " will retry after a delay"); return delayedSendRpcToTablet(nextRequest, new RecoverableException(statusRemoteError)); } client.sendRpc(nextRequest); - return d; + return nextRequest.getDeferred(); } /** @@ -668,7 +677,7 @@ public class AsyncKuduClient implements AutoCloseable { // the server the message includes the last propagated timestamp. long lastPropagatedTs = getLastPropagatedTimestamp(); if (request.getExternalConsistencyMode() == CLIENT_PROPAGATED && - lastPropagatedTs != NO_TIMESTAMP) { + lastPropagatedTs != NO_TIMESTAMP) { request.setPropagatedTimestamp(lastPropagatedTs); } @@ -727,13 +736,16 @@ public class AsyncKuduClient implements AutoCloseable { */ final class RetryRpcCB<R, D> implements Callback<Deferred<R>, D> { private final KuduRpc<R> request; + RetryRpcCB(KuduRpc<R> request) { this.request = request; } + public Deferred<R> call(final D arg) { LOG.debug("Retrying sending RPC {} after lookup", request); return sendRpcToTablet(request); // Retry the RPC. } + public String toString() { return "retry RPC"; } @@ -840,6 +852,7 @@ public class AsyncKuduClient implements AutoCloseable { d.addCallbacks(retryCB, errback); } } + long sleepTime = getSleepTimeForRpc(rpc); if (rpc.deadlineTracker.wouldSleepingTimeout(sleepTime)) { return tooManyAttemptsOrTimeout(rpc, null); @@ -854,6 +867,7 @@ public class AsyncKuduClient implements AutoCloseable { releaseMasterLookupPermit(); return arg; } + public String toString() { return "release master lookup permit"; } @@ -863,10 +877,14 @@ public class AsyncKuduClient implements AutoCloseable { private final class IsCreateTableDoneCB implements Callback<Master.IsCreateTableDoneResponsePB, Master.IsCreateTableDoneResponsePB> { final String tableName; + IsCreateTableDoneCB(String tableName) { this.tableName = tableName; + } - public Master.IsCreateTableDoneResponsePB call(final Master.IsCreateTableDoneResponsePB response) { + + public Master.IsCreateTableDoneResponsePB + call(final Master.IsCreateTableDoneResponsePB response) { if (response.getDone()) { LOG.debug("Table {} was created", tableName); tablesNotServed.remove(tableName); @@ -875,6 +893,7 @@ public class AsyncKuduClient implements AutoCloseable { } return response; } + public String toString() { return "ask the master if " + tableName + " was created"; } @@ -884,7 +903,6 @@ public class AsyncKuduClient implements AutoCloseable { return tablesNotServed.contains(tableId); } - long getSleepTimeForRpc(KuduRpc<?> rpc) { byte attemptCount = rpc.attempt; assert (attemptCount > 0); @@ -894,8 +912,8 @@ public class AsyncKuduClient implements AutoCloseable { attemptCount = 1; } // Randomized exponential backoff, truncated at 4096ms. - long sleepTime = (long)(Math.pow(2.0, Math.min(attemptCount, 12)) - * sleepRandomizer.nextDouble()); + long sleepTime = (long)(Math.pow(2.0, Math.min(attemptCount, 12)) * + sleepRandomizer.nextDouble()); if (LOG.isDebugEnabled()) { LOG.debug("Going to sleep for " + sleepTime + " at retry " + rpc.attempt); } @@ -981,8 +999,8 @@ public class AsyncKuduClient implements AutoCloseable { // looked up the tablet we're interested in. Every once in a while // this will save us a Master lookup. TableLocationsCache.Entry entry = getTableLocationEntry(tableId, partitionKey); - if (entry != null && !entry.isNonCoveredRange() - && entry.getTablet().getLeaderUUID() != null) { + if (entry != null && !entry.isNonCoveredRange() && + entry.getTablet().getLeaderUUID() != null) { return Deferred.fromResult(null); // Looks like no lookup needed. } } @@ -1111,6 +1129,7 @@ public class AsyncKuduClient implements AutoCloseable { public Deferred<List<LocatedTablet>> call(GetTableLocationsResponsePB resp) { return loopLocateTable(table, lookupKey, endPartitionKey, ret, deadlineTracker); } + @Override public String toString() { return "LoopLocateTableCB"; @@ -1188,6 +1207,7 @@ public class AsyncKuduClient implements AutoCloseable { sendRpcToTablet(rpc); } } + assert (ex != null); Status reasonForRetry = ex.getStatus(); rpc.addTrace( @@ -1221,10 +1241,12 @@ public class AsyncKuduClient implements AutoCloseable { Master.GetTableLocationsResponsePB> { final KuduTable table; private final byte[] partitionKey; + MasterLookupCB(KuduTable table, byte[] partitionKey) { this.table = table; this.partitionKey = partitionKey; } + public Object call(final GetTableLocationsResponsePB response) { if (response.hasError()) { if (response.getError().getCode() == Master.MasterErrorPB.Code.TABLET_NOT_RUNNING) { @@ -1247,6 +1269,7 @@ public class AsyncKuduClient implements AutoCloseable { } return null; } + public String toString() { return "get tablet locations from the master for table " + table.getName(); } @@ -1496,6 +1519,7 @@ public class AsyncKuduClient implements AutoCloseable { ShutdownThread() { super("AsyncKuduClient@" + AsyncKuduClient.super.hashCode() + " shutdown"); } + public void run() { // This terminates the Executor. channelFactory.releaseExternalResources(); @@ -1510,6 +1534,7 @@ public class AsyncKuduClient implements AutoCloseable { new ShutdownThread().start(); return arg; } + public String toString() { return "release resources callback"; } @@ -1521,6 +1546,7 @@ public class AsyncKuduClient implements AutoCloseable { public Deferred<ArrayList<Void>> call(ArrayList<List<OperationResponse>> ignoredResponses) { return connectionCache.disconnectEverything().addCallback(new ReleaseResourcesCB()); } + public String toString() { return "disconnect callback"; } @@ -1562,15 +1588,15 @@ public class AsyncKuduClient implements AutoCloseable { return MASTER_TABLE_NAME_PLACEHOLDER == tableId; } - void newTimeout(final TimerTask task, final long timeout_ms) { + void newTimeout(final TimerTask task, final long timeoutMs) { try { - timer.newTimeout(task, timeout_ms, MILLISECONDS); + timer.newTimeout(task, timeoutMs, MILLISECONDS); } catch (IllegalStateException e) { // This can happen if the timer fires just before shutdown() // is called from another thread, and due to how threads get // scheduled we tried to call newTimeout() after timer.stop(). - LOG.warn("Failed to schedule timer." - + " Ignore this if we're shutting down.", e); + LOG.warn("Failed to schedule timer." + + " Ignore this if we're shutting down.", e); } } @@ -1580,7 +1606,7 @@ public class AsyncKuduClient implements AutoCloseable { */ @InterfaceAudience.Public @InterfaceStability.Evolving - public final static class AsyncKuduClientBuilder { + public static final class AsyncKuduClientBuilder { private static final int DEFAULT_MASTER_PORT = 7051; private static final int DEFAULT_BOSS_COUNT = 1; private static final int DEFAULT_WORKER_COUNT = 2 * Runtime.getRuntime().availableProcessors(); @@ -1722,8 +1748,12 @@ public class AsyncKuduClient implements AutoCloseable { .setNameFormat("kudu-nio-%d") .setDaemon(true) .build()); - if (boss == null) boss = defaultExec; - if (worker == null) worker = defaultExec; + if (boss == null) { + boss = defaultExec; + } + if (worker == null) { + worker = defaultExec; + } } // Share the timer with the socket channel factory so that it does not // create an internal timer with a non-daemon thread.
