[
https://issues.apache.org/jira/browse/BEAM-4323?focusedWorklogId=107595&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-107595
]
ASF GitHub Bot logged work on BEAM-4323:
----------------------------------------
Author: ASF GitHub Bot
Created on: 31/May/18 07:50
Start Date: 31/May/18 07:50
Worklog Time Spent: 10m
Work Description: iemejia closed pull request #5501: [BEAM-4323] Enforce
ErrorProne analysis in sketching extensions
URL: https://github.com/apache/beam/pull/5501
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/sdks/java/extensions/sketching/build.gradle
b/sdks/java/extensions/sketching/build.gradle
index 1ddc2269ab5..6392b0842ac 100644
--- a/sdks/java/extensions/sketching/build.gradle
+++ b/sdks/java/extensions/sketching/build.gradle
@@ -17,7 +17,7 @@
*/
apply from: project(":").file("build_rules.gradle")
-applyJavaNature()
+applyJavaNature(failOnWarning: true)
description = "Apache Beam :: SDKs :: Java :: Extensions :: Sketching"
@@ -26,10 +26,12 @@ def tdigest_version = "3.2"
dependencies {
compile library.java.guava
+ compileOnly library.java.findbugs_annotations
shadow project(path: ":beam-sdks-java-core", configuration: "shadow")
shadow "com.clearspring.analytics:stream:$streamlib_version"
shadow "com.tdunning:t-digest:$tdigest_version"
shadow library.java.slf4j_api
+ testCompileOnly library.java.findbugs_annotations
shadowTest library.java.avro
shadowTest library.java.commons_lang3
shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest")
diff --git
a/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinct.java
b/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinct.java
index e8d85f7c057..2d0a56ee4d4 100644
---
a/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinct.java
+++
b/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinct.java
@@ -534,9 +534,8 @@ protected long getEncodedElementByteSize(HyperLogLogPlus
value) throws IOExcepti
* Utility class that provides {@link DoFn}s to retrieve the cardinality
from a {@link
* HyperLogLogPlus} structure in a global or perKey context.
*/
- public static class RetrieveCardinality {
-
- public static <K> DoFn<KV<K, HyperLogLogPlus>, KV<K, Long>> perKey() {
+ private static class RetrieveCardinality {
+ private static <K> DoFn<KV<K, HyperLogLogPlus>, KV<K, Long>> perKey() {
return new DoFn<KV<K, HyperLogLogPlus>, KV<K, Long>>() {
@ProcessElement
public void processElement(ProcessContext c) {
@@ -546,7 +545,7 @@ public void processElement(ProcessContext c) {
};
}
- public static DoFn<HyperLogLogPlus, Long> globally() {
+ private static DoFn<HyperLogLogPlus, Long> globally() {
return new DoFn<HyperLogLogPlus, Long>() {
@ProcessElement
public void apply(ProcessContext c) {
diff --git
a/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/SketchFrequencies.java
b/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/SketchFrequencies.java
index 5d06ec00ac5..0451dcebd23 100644
---
a/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/SketchFrequencies.java
+++
b/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/SketchFrequencies.java
@@ -148,7 +148,7 @@
* {@literal PCollection<KV<MyObject, Long>>} pairs = pc.apply(ParDo.of(
* {@literal new DoFn<Long, KV<MyObject, Long>>()} {
* {@literal @ProcessElement}
- * public void procesElement(ProcessContext c) {
+ * public void processElement(ProcessContext c) {
* Long elem = c.element();
* CountMinSketch sketch = c.sideInput(sketchView);
* c.output(sketch.estimateCount(elem, coder));
@@ -260,10 +260,11 @@
@Override
public PCollection<Sketch<InputT>> expand(PCollection<InputT> input) {
- return input.apply("Compute Count-Min Sketch",
- Combine.<InputT, Sketch<InputT>>globally(CountMinSketchFn
- .<InputT>create(input.getCoder())
- .withAccuracy(relativeError(), confidence())));
+ return input.apply(
+ "Compute Count-Min Sketch",
+ Combine.globally(
+ CountMinSketchFn.create(input.getCoder())
+ .withAccuracy(relativeError(), confidence())));
}
}
@@ -324,10 +325,11 @@
@Override
public PCollection<KV<K, Sketch<V>>> expand(PCollection<KV<K, V>> input) {
KvCoder<K, V> inputCoder = (KvCoder<K, V>) input.getCoder();
- return input.apply("Compute Count-Min Sketch perKey",
- Combine.<K, V, Sketch<V>>perKey(CountMinSketchFn
- .<V>create(inputCoder.getValueCoder())
- .withAccuracy(relativeError(), confidence())));
+ return input.apply(
+ "Compute Count-Min Sketch perKey",
+ Combine.perKey(
+ CountMinSketchFn.create(inputCoder.getValueCoder())
+ .withAccuracy(relativeError(), confidence())));
}
}
@@ -391,7 +393,7 @@ private CountMinSketchFn(final Coder<InputT> coder, double
eps, double confidenc
}
@Override public Sketch<InputT> createAccumulator() {
- return Sketch.<InputT>create(epsilon, confidence);
+ return Sketch.create(epsilon, confidence);
}
@Override public Sketch<InputT> addInput(Sketch<InputT> accumulator,
InputT element) {
@@ -413,7 +415,7 @@ private CountMinSketchFn(final Coder<InputT> coder, double
eps, double confidenc
throw new IllegalStateException("The accumulators cannot be merged:" +
e.getMessage());
}
- return Sketch.<InputT>create(mergedSketches);
+ return Sketch.create(mergedSketches);
}
/** Output the whole structure so it can be queried, reused or stored
easily. */
@@ -520,7 +522,7 @@ public void encode(Sketch<T> value, OutputStream outStream)
throws IOException {
public Sketch<T> decode(InputStream inStream) throws IOException {
byte[] sketchBytes = BYTE_ARRAY_CODER.decode(inStream);
CountMinSketch sketch = CountMinSketch.deserialize(sketchBytes);
- return Sketch.<T>create(sketch);
+ return Sketch.create(sketch);
}
@Override
diff --git
a/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/TDigestQuantiles.java
b/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/TDigestQuantiles.java
index 28e95ac4a84..a06d6d588f8 100644
---
a/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/TDigestQuantiles.java
+++
b/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/TDigestQuantiles.java
@@ -109,7 +109,7 @@
* {@literal PCollection<KV<Double, Double>>} quantiles = pc.apply(ParDo.of(
* {@literal new DoFn<MergingDigest, KV<Double, Double>>()} {
* {@literal @ProcessElement}
- * public void procesElement(ProcessContext c) {
+ * public void processElement(ProcessContext c) {
* double[] quantiles = {0.01, 0.25, 0.5, 0.75, 0.99}
* for (double q : quantiles) {
* c.output(KV.of(q, c.element().quantile(q));
diff --git
a/sdks/java/extensions/sketching/src/test/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinctTest.java
b/sdks/java/extensions/sketching/src/test/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinctTest.java
index 27655f8fcda..2d9933955d3 100644
---
a/sdks/java/extensions/sketching/src/test/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinctTest.java
+++
b/sdks/java/extensions/sketching/src/test/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinctTest.java
@@ -174,7 +174,7 @@ public void testDisplayData() {
assertThat(DisplayData.from(fnWithPrecision), hasDisplayItem("sp", 0));
}
- class VerifyAccuracy implements SerializableFunction<Iterable<Long>, Void> {
+ private static class VerifyAccuracy implements
SerializableFunction<Iterable<Long>, Void> {
private final int expectedCard;
diff --git
a/sdks/java/extensions/sketching/src/test/java/org/apache/beam/sdk/extensions/sketching/SketchFrequenciesTest.java
b/sdks/java/extensions/sketching/src/test/java/org/apache/beam/sdk/extensions/sketching/SketchFrequenciesTest.java
index bb3389b9c75..e7cb0d67761 100644
---
a/sdks/java/extensions/sketching/src/test/java/org/apache/beam/sdk/extensions/sketching/SketchFrequenciesTest.java
+++
b/sdks/java/extensions/sketching/src/test/java/org/apache/beam/sdk/extensions/sketching/SketchFrequenciesTest.java
@@ -53,7 +53,7 @@
@Rule public final transient TestPipeline tp = TestPipeline.create();
- private List<Long> smallStream = Arrays.asList(
+ private final List<Long> smallStream = Arrays.asList(
1L,
2L, 2L,
3L, 3L, 3L,
@@ -64,16 +64,16 @@
8L, 8L, 8L, 8L, 8L, 8L, 8L, 8L,
9L, 9L, 9L, 9L, 9L, 9L, 9L, 9L, 9L);
- private Long[] distinctElems = {1L, 2L, 3L, 4L, 5L, 6L, 8L, 9L};
- private Long[] frequencies = distinctElems.clone();
+ private final Long[] distinctElems = {1L, 2L, 3L, 4L, 5L, 6L, 8L, 9L};
+ private final Long[] frequencies = distinctElems.clone();
@Test
public void perKeyDefault() {
PCollection<Long> stream = tp.apply(Create.of(smallStream));
PCollection<Sketch<Long>> sketch = stream
- .apply(WithKeys.<Integer, Long>of(1))
- .apply(SketchFrequencies.<Integer, Long>perKey())
- .apply(Values.<Sketch<Long>>create());
+ .apply(WithKeys.of(1))
+ .apply(SketchFrequencies.perKey())
+ .apply(Values.create());
Coder<Long> coder = stream.getCoder();
@@ -114,7 +114,7 @@ public void merge() {
// n sketches each containing [0, 1, 2]
for (int i = 0; i < nOccurrences; i++) {
- Sketch<Integer> sketch = Sketch.<Integer>create(eps, conf);
+ Sketch<Integer> sketch = Sketch.create(eps, conf);
for (int j = 0; j < size; j++) {
sketch.add(j, coder);
}
@@ -134,7 +134,7 @@ public void customObject() {
long occurrences = 2L; // occurrence of each user in the stream
double eps = 0.01;
double conf = 0.8;
- Sketch<GenericRecord> sketch = Sketch.<GenericRecord>create(eps, conf);
+ Sketch<GenericRecord> sketch = Sketch.create(eps, conf);
Schema schema =
SchemaBuilder.record("User")
.fields()
@@ -154,14 +154,13 @@ public void customObject() {
@Test
public void testCoder() throws Exception {
- Sketch<Integer> cMSketch = Sketch.<Integer>create(0.01, 0.8);
+ Sketch<Integer> cMSketch = Sketch.create(0.01, 0.8);
Coder<Integer> coder = VarIntCoder.of();
for (int i = 0; i < 3; i++) {
cMSketch.add(i, coder);
}
- CoderProperties.<Sketch<Integer>>coderDecodeEncodeEqual(
- new SketchFrequencies.CountMinSketchCoder<>(), cMSketch);
+ CoderProperties.coderDecodeEncodeEqual(new
SketchFrequencies.CountMinSketchCoder<>(), cMSketch);
}
@Test
@@ -181,10 +180,9 @@ public void testDisplayData() {
}
static class VerifyStreamFrequencies<T> implements
SerializableFunction<Sketch<T>, Void> {
-
- Coder<T> coder;
- Long[] expectedHits;
- T[] elements;
+ final Coder<T> coder;
+ final Long[] expectedHits;
+ final T[] elements;
VerifyStreamFrequencies(Coder<T> coder, T[] elements, Long[] expectedHits)
{
this.coder = coder;
diff --git
a/sdks/java/extensions/sketching/src/test/java/org/apache/beam/sdk/extensions/sketching/TDigestQuantilesTest.java
b/sdks/java/extensions/sketching/src/test/java/org/apache/beam/sdk/extensions/sketching/TDigestQuantilesTest.java
index c01bd987ca3..d547deedcde 100644
---
a/sdks/java/extensions/sketching/src/test/java/org/apache/beam/sdk/extensions/sketching/TDigestQuantilesTest.java
+++
b/sdks/java/extensions/sketching/src/test/java/org/apache/beam/sdk/extensions/sketching/TDigestQuantilesTest.java
@@ -82,9 +82,9 @@ public void globally() {
@Test
public void perKey() {
PCollection<KV<Double, Double>> col = tp.apply(Create.of(stream))
- .apply(WithKeys.<Integer, Double>of(1))
+ .apply(WithKeys.of(1))
.apply(TDigestQuantiles.<Integer>perKey().withCompression(compression))
- .apply(Values.<MergingDigest>create())
+ .apply(Values.create())
.apply(ParDo.of(new RetrieveQuantiles(quantiles)));
PAssert.that("Verify Accuracy", col).satisfies(new VerifyAccuracy());
@@ -150,7 +150,7 @@ public void testDisplayData() {
static class RetrieveQuantiles extends DoFn<MergingDigest, KV<Double,
Double>> {
private final double[] quantiles;
- public RetrieveQuantiles(double[] quantiles) {
+ RetrieveQuantiles(double[] quantiles) {
this.quantiles = quantiles;
}
@@ -162,9 +162,9 @@ public RetrieveQuantiles(double[] quantiles) {
}
static class VerifyAccuracy implements
SerializableFunction<Iterable<KV<Double, Double>>, Void> {
+ final double expectedError = 3D / compression;
- double expectedError = 3D / compression;
-
+ @Override
public Void apply(Iterable<KV<Double, Double>> input) {
for (KV<Double, Double> pair : input) {
double expectedValue = pair.getKey() * (size + 1);
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 107595)
Time Spent: 40m (was: 0.5h)
> Enforce ErrorProne analysis in sketching extensions project
> -----------------------------------------------------------
>
> Key: BEAM-4323
> URL: https://issues.apache.org/jira/browse/BEAM-4323
> Project: Beam
> Issue Type: Improvement
> Components: sdk-java-sketching
> Reporter: Scott Wegner
> Assignee: Ismaël Mejía
> Priority: Minor
> Labels: errorprone, starter
> Time Spent: 40m
> Remaining Estimate: 0h
>
> Java ErrorProne static analysis was [recently
> enabled|https://github.com/apache/beam/pull/5161] in the Gradle build
> process, but only as warnings. ErrorProne errors are generally useful and
> easy to fix. Some work was done to [make sdks-java-core
> ErrorProne-clean|https://github.com/apache/beam/pull/5319] and add
> enforcement. This task is clean ErrorProne warnings and add enforcement in
> {{beam-sdks-java-extensions-sketching}}. Additional context discussed on the
> [dev
> list|https://lists.apache.org/thread.html/95aae2785c3cd728c2d3378cbdff2a7ba19caffcd4faa2049d2e2f46@%3Cdev.beam.apache.org%3E].
> Fixing this issue will involve:
> # Follow instructions in the [Contribution
> Guide|https://beam.apache.org/contribute/] to set up a {{beam}} development
> environment.
> # Run the following command to compile and run ErrorProne analysis on the
> project: {{./gradlew :beam-sdks-java-extensions-sketching:assemble}}
> # Fix each ErrorProne warning from the {{sdks/java/extensions/sketching}}
> project.
> # In {{sdks/java/extensions/sketching/build.gradle}}, add {{failOnWarning:
> true}} to the call the {{applyJavaNature()}}
> ([example|https://github.com/apache/beam/pull/5319/files#diff-9390c20635aed5f42f83b97506a87333R20]).
> This starter issue is sponsored by [~swegner]. Feel free to [reach
> out|https://beam.apache.org/community/contact-us/] with questions or code
> review:
> * JIRA: [~swegner]
> * GitHub: [@swegner|https://github.com/swegner]
> * Slack: [@Scott Wegner|https://s.apache.org/beam-slack-channel]
> * Email: swegner at google dot com
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
