from:"bchambers"

incubator-beam git commit: Change counter name in TestDataflowRunner

2016-12-19 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master d13f11f8c -> a2f44923c


Change counter name in TestDataflowRunner


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a2f44923
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a2f44923
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a2f44923

Branch: refs/heads/master
Commit: a2f44923c8a8db533ad0bb6f545a96cf5007cfa5
Parents: d13f11f
Author: Joshua Litt <joshual...@google.com>
Authored: Sat Dec 17 11:12:12 2016 -0800
Committer: bchambers <bchamb...@google.com>
Committed: Mon Dec 19 12:28:19 2016 -0800

--
 .../dataflow/testing/TestDataflowRunner.java| 29 
 .../testing/TestDataflowRunnerTest.java | 16 ++-
 2 files changed, 39 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a2f44923/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunner.java
--
diff --git 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunner.java
 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunner.java
index 4b0fcf2..0564448 100644
--- 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunner.java
+++ 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunner.java
@@ -61,7 +61,12 @@ import org.slf4j.LoggerFactory;
  */
 public class TestDataflowRunner extends PipelineRunner {
   private static final String TENTATIVE_COUNTER = "tentative";
-  private static final String WATERMARK_METRIC_SUFFIX = 
"windmill-data-watermark";
+  // See https://issues.apache.org/jira/browse/BEAM-1170
+  // we need to either fix the API or pipe the DRAINED signal through
+  @VisibleForTesting
+  static final String LEGACY_WATERMARK_METRIC_SUFFIX = 
"windmill-data-watermark";
+  @VisibleForTesting
+  static final String WATERMARK_METRIC_SUFFIX = "DataWatermark";
   private static final long MAX_WATERMARK_VALUE = -2L;
   private static final Logger LOG = 
LoggerFactory.getLogger(TestDataflowRunner.class);
 
@@ -248,6 +253,23 @@ public class TestDataflowRunner extends 
PipelineRunner {
   }
 
   /**
+   * Checks wether a metric is a streaming watermark.
+   *
+   * @return true if the metric is a watermark.
+   */
+  boolean isWatermark(MetricUpdate metric) {
+if (metric.getName() == null || metric.getName().getName() == null) {
+  return false; // no name -> shouldn't happen, not the watermark
+}
+if (metric.getScalar() == null) {
+  return false; // no scalar value -> not the watermark
+}
+String name = metric.getName().getName();
+return name.endsWith(LEGACY_WATERMARK_METRIC_SUFFIX)
+|| name.endsWith(WATERMARK_METRIC_SUFFIX);
+  }
+
+  /**
* Check watermarks of the streaming job. At least one watermark metric must 
exist.
*
* @return true if all watermarks are at max, false otherwise.
@@ -256,10 +278,7 @@ public class TestDataflowRunner extends 
PipelineRunner {
   boolean atMaxWatermark(DataflowPipelineJob job, JobMetrics metrics) {
 boolean hasMaxWatermark = false;
 for (MetricUpdate metric : metrics.getMetrics()) {
-  if (metric.getName() == null
-  || metric.getName().getName() == null
-  || !metric.getName().getName().endsWith(WATERMARK_METRIC_SUFFIX)
-  || metric.getScalar() == null) {
+  if (!isWatermark(metric)) {
 continue;
   }
   BigDecimal watermark = (BigDecimal) metric.getScalar();

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a2f44923/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunnerTest.java
--
diff --git 
a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunnerTest.java
 
b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunnerTest.java
index 366c6a1..da5630b 100644
--- 
a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunnerTest.java
+++ 
b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunnerTest.java
@@ -17,6 +17,8 @@
  */
 package org.apache.beam.runners.dataflow.testing;
 
+import static 
org.apache.beam.runners.dataflow.testing.Test

[2/2] incubator-beam git commit: Closes #1638

2016-12-19 Thread bchambers

Closes #1638


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a972b233
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a972b233
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a972b233

Branch: refs/heads/master
Commit: a972b23302146e686e0f0e141093f899e5e70555
Parents: 66318d8 998cabc
Author: bchambers <bchamb...@google.com>
Authored: Mon Dec 19 11:30:20 2016 -0800
Committer: bchambers <bchamb...@google.com>
Committed: Mon Dec 19 11:30:20 2016 -0800

--
 runners/apex/pom.xml|  3 +-
 .../beam/runners/direct/DirectRunnerTest.java   | 39 
 runners/flink/runner/pom.xml|  6 +-
 runners/google-cloud-dataflow-java/pom.xml  |  3 +-
 runners/spark/pom.xml   |  3 +-
 .../org/apache/beam/sdk/metrics/MetricName.java |  3 +-
 .../org/apache/beam/sdk/metrics/Metrics.java|  5 +-
 .../apache/beam/sdk/testing/UsesMetrics.java| 24 
 .../apache/beam/sdk/metrics/MetricMatchers.java |  4 +-
 .../apache/beam/sdk/metrics/MetricsTest.java| 63 +++-
 10 files changed, 103 insertions(+), 50 deletions(-)
--

[1/2] incubator-beam git commit: Add RunnableOnService test for Metrics

2016-12-19 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 66318d829 -> a972b2330


Add RunnableOnService test for Metrics

Add UsesMetrics interface and exclude from runners that don't yet
support Metrics

Add Serializability as needed for Metrics to be created during pipeline
construction

Remove test from DirectRunnerTest


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/998cabc8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/998cabc8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/998cabc8

Branch: refs/heads/master
Commit: 998cabc8bbbf8d08d7bfad71e9376707388f5c5c
Parents: 66318d8
Author: bchambers <bchamb...@google.com>
Authored: Thu Dec 15 17:04:59 2016 -0800
Committer: bchambers <bchamb...@google.com>
Committed: Mon Dec 19 11:29:39 2016 -0800

--
 runners/apex/pom.xml|  3 +-
 .../beam/runners/direct/DirectRunnerTest.java   | 39 
 runners/flink/runner/pom.xml|  6 +-
 runners/google-cloud-dataflow-java/pom.xml  |  3 +-
 runners/spark/pom.xml   |  3 +-
 .../org/apache/beam/sdk/metrics/MetricName.java |  3 +-
 .../org/apache/beam/sdk/metrics/Metrics.java|  5 +-
 .../apache/beam/sdk/testing/UsesMetrics.java| 24 
 .../apache/beam/sdk/metrics/MetricMatchers.java |  4 +-
 .../apache/beam/sdk/metrics/MetricsTest.java| 63 +++-
 10 files changed, 103 insertions(+), 50 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/998cabc8/runners/apex/pom.xml
--
diff --git a/runners/apex/pom.xml b/runners/apex/pom.xml
index f71637c..d03964d 100644
--- a/runners/apex/pom.xml
+++ b/runners/apex/pom.xml
@@ -186,7 +186,8 @@
   
 org.apache.beam.sdk.testing.UsesStatefulParDo,
 org.apache.beam.sdk.testing.UsesTimersInParDo,
-org.apache.beam.sdk.testing.UsesSplittableParDo
+org.apache.beam.sdk.testing.UsesSplittableParDo,
+org.apache.beam.sdk.testing.UsesMetrics
   
   none
   true

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/998cabc8/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
--
diff --git 
a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
 
b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
index eb0f344..eafb788 100644
--- 
a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
+++ 
b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
@@ -18,8 +18,6 @@
 package org.apache.beam.runners.direct;
 
 import static com.google.common.base.Preconditions.checkState;
-import static org.apache.beam.sdk.metrics.MetricMatchers.metricResult;
-import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.isA;
@@ -37,7 +35,6 @@ import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.beam.runners.direct.DirectRunner.DirectPipelineResult;
 import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
@@ -48,13 +45,6 @@ import org.apache.beam.sdk.io.BoundedSource;
 import org.apache.beam.sdk.io.CountingInput;
 import org.apache.beam.sdk.io.CountingSource;
 import org.apache.beam.sdk.io.Read;
-import org.apache.beam.sdk.metrics.Counter;
-import org.apache.beam.sdk.metrics.Distribution;
-import org.apache.beam.sdk.metrics.DistributionResult;
-import org.apache.beam.sdk.metrics.MetricNameFilter;
-import org.apache.beam.sdk.metrics.MetricQueryResults;
-import org.apache.beam.sdk.metrics.Metrics;
-import org.apache.beam.sdk.metrics.MetricsFilter;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.runners.PipelineRunner;
@@ -467,35 +457,6 @@ public class DirectRunnerTest implements Serializable {
 }
   }
 
-  @Test
-  public void testMetrics() throws Exception {
-Pipeline pipeline = getPipeline();
-pipeline
-.apply(Create.of(5, 8, 13))
-.apply("MyStep", ParDo.of(new DoFn<Integer, Void>() {
-  @ProcessElement
-  public void processElement(ProcessContext c) {
-Counter count = Metrics.counter

[2/2] incubator-beam git commit: Closes #1649

2016-12-16 Thread bchambers

Closes #1649


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/abdbee61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/abdbee61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/abdbee61

Branch: refs/heads/master
Commit: abdbee61c7fa2383343f86f3e88f134163177439
Parents: 1ee191f 92dbab8
Author: bchambers <bchamb...@google.com>
Authored: Fri Dec 16 16:26:43 2016 -0800
Committer: bchambers <bchamb...@google.com>
Committed: Fri Dec 16 16:26:43 2016 -0800

--
 .../main/java/org/apache/beam/sdk/metrics/DistributionCell.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--

[1/2] incubator-beam git commit: checkstyle: missed newline in DistributionCell

2016-12-16 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 1ee191fb8 -> abdbee61c


checkstyle: missed newline in DistributionCell


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/92dbab8e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/92dbab8e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/92dbab8e

Branch: refs/heads/master
Commit: 92dbab8ecf2761893759c494d33aa8583e919a6f
Parents: 1ee191f
Author: Eugene Kirpichov 
Authored: Fri Dec 16 15:56:00 2016 -0800
Committer: Eugene Kirpichov 
Committed: Fri Dec 16 15:56:00 2016 -0800

--
 .../main/java/org/apache/beam/sdk/metrics/DistributionCell.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/92dbab8e/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionCell.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionCell.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionCell.java
index e095158..7f684a8 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionCell.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionCell.java
@@ -66,4 +66,5 @@ public class DistributionCell implements 
MetricCell

[2/2] incubator-beam git commit: Make {Metric, Counter, Distribution}Cell public

2016-12-16 Thread bchambers

Make {Metric,Counter,Distribution}Cell public

These classes may be useful to hold on to within a runner where the step
context is known to be unchanging and there may be a desire to avoid the
extra indirection of using {Counter,Distribution} directly.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/cfd1633a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/cfd1633a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/cfd1633a

Branch: refs/heads/master
Commit: cfd1633a441f98d100bd995bd811b6b8fa2cd62d
Parents: cc28f0c
Author: bchambers <bchamb...@google.com>
Authored: Wed Dec 14 15:46:50 2016 -0800
Committer: bchambers <bchamb...@google.com>
Committed: Fri Dec 16 14:26:35 2016 -0800

--
 .../org/apache/beam/sdk/metrics/CounterCell.java | 10 +-
 .../apache/beam/sdk/metrics/DistributionCell.java| 15 +--
 .../java/org/apache/beam/sdk/metrics/MetricCell.java |  2 +-
 3 files changed, 23 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/cfd1633a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/CounterCell.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/CounterCell.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/CounterCell.java
index bb65833..93700e6 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/CounterCell.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/CounterCell.java
@@ -26,13 +26,21 @@ import org.apache.beam.sdk.annotations.Experimental.Kind;
  *
  * This class generally shouldn't be used directly. The only exception is 
within a runner where
  * a counter is being reported for a specific step (rather than the counter in 
the current context).
+ * In that case retrieving the underlying cell and reporting directly to it 
avoids a step of
+ * indirection.
  */
 @Experimental(Kind.METRICS)
-class CounterCell implements MetricCell<Counter, Long>, Counter {
+public class CounterCell implements MetricCell<Counter, Long>, Counter {
 
   private final DirtyState dirty = new DirtyState();
   private final AtomicLong value = new AtomicLong();
 
+  /**
+   * Package-visibility because all {@link CounterCell CounterCells} should be 
created by
+   * {@link MetricsContainer#getCounter(MetricName)}.
+   */
+  CounterCell() {}
+
   /** Increment the counter by the given amount. */
   private void add(long n) {
 value.addAndGet(n);

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/cfd1633a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionCell.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionCell.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionCell.java
index f0074a9..e095158 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionCell.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionCell.java
@@ -23,14 +23,25 @@ import org.apache.beam.sdk.annotations.Experimental.Kind;
 
 /**
  * Tracks the current value (and delta) for a Distribution metric.
+ *
+ * This class generally shouldn't be used directly. The only exception is 
within a runner where
+ * a distribution is being reported for a specific step (rather than the 
distribution in the current
+ * context). In that case retrieving the underlying cell and reporting 
directly to it avoids a step
+ * of indirection.
  */
 @Experimental(Kind.METRICS)
-class DistributionCell implements MetricCell<Distribution, DistributionData>, 
Distribution {
+public class DistributionCell implements MetricCell<Distribution, 
DistributionData>, Distribution {
 
   private final DirtyState dirty = new DirtyState();
   private final AtomicReference value =
   new AtomicReference(DistributionData.EMPTY);
 
+  /**
+   * Package-visibility because all {@link DistributionCell DistributionCells} 
should be created by
+   * {@link MetricsContainer#getDistribution(MetricName)}.
+   */
+  DistributionCell() {}
+
   /** Increment the counter by the given amount. */
   @Override
   public void update(long n) {
@@ -55,4 +66,4 @@ class DistributionCell implements MetricCell<Distribution, 
DistributionData>, Di
   public Distribution getInterface() {
 return this;
   }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/cfd1633a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricCell.java
--
diff --git 
a/sdks/java/core

[1/2] incubator-beam git commit: Closes #1622

2016-12-16 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master cc28f0cb4 -> 1ee191fb8


Closes #1622


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/1ee191fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/1ee191fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/1ee191fb

Branch: refs/heads/master
Commit: 1ee191fb80ed1644b275a87fc02c11eae47904c2
Parents: cc28f0c cfd1633
Author: bchambers <bchamb...@google.com>
Authored: Fri Dec 16 14:26:35 2016 -0800
Committer: bchambers <bchamb...@google.com>
Committed: Fri Dec 16 14:26:35 2016 -0800

--
 .../org/apache/beam/sdk/metrics/CounterCell.java | 10 +-
 .../apache/beam/sdk/metrics/DistributionCell.java| 15 +--
 .../java/org/apache/beam/sdk/metrics/MetricCell.java |  2 +-
 3 files changed, 23 insertions(+), 4 deletions(-)
--

[2/2] incubator-beam git commit: Closes #1566

2016-12-12 Thread bchambers

Closes #1566


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/98543e96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/98543e96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/98543e96

Branch: refs/heads/master
Commit: 98543e96fb1dc84d7ef566dc12407208b8d48144
Parents: bfd21d7 cfb71e7
Author: bchambers <bchamb...@google.com>
Authored: Mon Dec 12 12:48:09 2016 -0800
Committer: bchambers <bchamb...@google.com>
Committed: Mon Dec 12 12:48:09 2016 -0800

--
 .../beam/runners/direct/DirectRunnerTest.java   |  7 ++-
 .../apache/beam/sdk/metrics/MetricMatchers.java | 45 +---
 2 files changed, 45 insertions(+), 7 deletions(-)
--

[1/2] incubator-beam git commit: Display data keys in Python should be snake_case

2016-12-06 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/python-sdk f19f767b0 -> d5e8c79a3


Display data keys in Python should be snake_case

This fixes existing mis-cased display data keys.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/0dcc9bc0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/0dcc9bc0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/0dcc9bc0

Branch: refs/heads/python-sdk
Commit: 0dcc9bc03772d8bd5ab1e0d138c445e9418ebf66
Parents: f19f767
Author: Pablo <pabl...@google.com>
Authored: Mon Nov 28 13:05:47 2016 -0800
Committer: bchambers <bchamb...@google.com>
Committed: Tue Dec 6 17:17:16 2016 -0800

--
 sdks/python/apache_beam/io/avroio.py|  8 +++-
 sdks/python/apache_beam/io/avroio_test.py   |  8 
 sdks/python/apache_beam/io/filebasedsource.py   |  3 ++-
 .../apache_beam/io/filebasedsource_test.py  |  4 ++--
 sdks/python/apache_beam/io/fileio.py|  8 
 sdks/python/apache_beam/io/fileio_test.py   | 12 ++--
 sdks/python/apache_beam/io/pubsub.py|  2 +-
 sdks/python/apache_beam/io/pubsub_test.py   |  2 +-
 sdks/python/apache_beam/io/textio.py| 20 ++--
 sdks/python/apache_beam/io/textio_test.py   |  8 +---
 .../apache_beam/transforms/combiners_test.py|  8 
 sdks/python/apache_beam/transforms/core.py  |  8 
 12 files changed, 46 insertions(+), 45 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dcc9bc0/sdks/python/apache_beam/io/avroio.py
--
diff --git a/sdks/python/apache_beam/io/avroio.py 
b/sdks/python/apache_beam/io/avroio.py
index 6cba12d..35d0e94 100644
--- a/sdks/python/apache_beam/io/avroio.py
+++ b/sdks/python/apache_beam/io/avroio.py
@@ -75,8 +75,7 @@ class ReadFromAvro(PTransform):
   **kwargs: Additional keyword arguments to be passed to the base class.
 """
 super(ReadFromAvro, self).__init__()
-self._args = (file_pattern, min_bundle_size)
-self._source = _AvroSource(*self._args, validate=validate)
+self._source = _AvroSource(file_pattern, min_bundle_size, 
validate=validate)
 
   def apply(self, pvalue):
 return pvalue.pipeline | Read(self._source)
@@ -292,9 +291,8 @@ class WriteToAvro(beam.transforms.PTransform):
 Returns:
   A WriteToAvro transform usable for writing.
 """
-self._args = (file_path_prefix, schema, codec, file_name_suffix, 
num_shards,
-  shard_name_template, mime_type)
-self._sink = _AvroSink(*self._args)
+self._sink = _AvroSink(file_path_prefix, schema, codec, file_name_suffix,
+   num_shards, shard_name_template, mime_type)
 
   def apply(self, pcoll):
 return pcoll | beam.io.iobase.Write(self._sink)

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dcc9bc0/sdks/python/apache_beam/io/avroio_test.py
--
diff --git a/sdks/python/apache_beam/io/avroio_test.py 
b/sdks/python/apache_beam/io/avroio_test.py
index e8fb12b..e5db196 100644
--- a/sdks/python/apache_beam/io/avroio_test.py
+++ b/sdks/python/apache_beam/io/avroio_test.py
@@ -157,7 +157,7 @@ class TestAvro(unittest.TestCase):
 # No extra avro parameters for AvroSource.
 expected_items = [
 DisplayDataItemMatcher('compression', 'auto'),
-DisplayDataItemMatcher('filePattern', file_name)]
+DisplayDataItemMatcher('file_pattern', file_name)]
 hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
 
   def test_read_display_data(self):
@@ -168,7 +168,7 @@ class TestAvro(unittest.TestCase):
 # No extra avro parameters for AvroSource.
 expected_items = [
 DisplayDataItemMatcher('compression', 'auto'),
-DisplayDataItemMatcher('filePattern', file_name)]
+DisplayDataItemMatcher('file_pattern', file_name)]
 hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
 
   def test_sink_display_data(self):
@@ -186,7 +186,7 @@ class TestAvro(unittest.TestCase):
 'schema',
 str(self.SCHEMA)),
 DisplayDataItemMatcher(
-'filePattern',
+'file_pattern',
 'some_avro_sink-%(shard_num)05d-of-%(num_shards)05d.end'),
 DisplayDataItemMatcher(
 'shards',
@@ -209,7 +209,7 @@ class TestAvro(unittest.TestCase):
 'schema',
 str(self.SCHEMA)),
 DisplayDataItemMatcher(
-'filePattern',
+'file_pattern',
 'some_avro_sink-%(shard_num)05d-of-%(num_shards)05d'),
 DisplayDataItemMatcher(

[2/2] incubator-beam git commit: Closes #1443

2016-12-06 Thread bchambers

Closes #1443


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/d5e8c79a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/d5e8c79a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/d5e8c79a

Branch: refs/heads/python-sdk
Commit: d5e8c79a3f9f4ffabaff06629a8763e74254688c
Parents: f19f767 0dcc9bc
Author: bchambers <bchamb...@google.com>
Authored: Tue Dec 6 17:17:50 2016 -0800
Committer: bchambers <bchamb...@google.com>
Committed: Tue Dec 6 17:17:50 2016 -0800

--
 sdks/python/apache_beam/io/avroio.py|  8 +++-
 sdks/python/apache_beam/io/avroio_test.py   |  8 
 sdks/python/apache_beam/io/filebasedsource.py   |  3 ++-
 .../apache_beam/io/filebasedsource_test.py  |  4 ++--
 sdks/python/apache_beam/io/fileio.py|  8 
 sdks/python/apache_beam/io/fileio_test.py   | 12 ++--
 sdks/python/apache_beam/io/pubsub.py|  2 +-
 sdks/python/apache_beam/io/pubsub_test.py   |  2 +-
 sdks/python/apache_beam/io/textio.py| 20 ++--
 sdks/python/apache_beam/io/textio_test.py   |  8 +---
 .../apache_beam/transforms/combiners_test.py|  8 
 sdks/python/apache_beam/transforms/core.py  |  8 
 12 files changed, 46 insertions(+), 45 deletions(-)
--

[2/4] incubator-beam git commit: [BEAM-1047] Add DataflowClient wrapper on top of JSON library.

2016-12-06 Thread bchambers

[BEAM-1047] Add DataflowClient wrapper on top of JSON library.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/e8c9686a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/e8c9686a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/e8c9686a

Branch: refs/heads/master
Commit: e8c9686a2e898d38afd692328eb171c542084747
Parents: ded5883
Author: Pei He <pe...@google.com>
Authored: Wed Nov 23 15:59:56 2016 -0800
Committer: bchambers <bchamb...@google.com>
Committed: Tue Dec 6 17:08:12 2016 -0800

--
 .../beam/runners/dataflow/DataflowClient.java   | 130 +++
 1 file changed, 130 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e8c9686a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowClient.java
--
diff --git 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowClient.java
 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowClient.java
new file mode 100644
index 000..f2081db
--- /dev/null
+++ 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowClient.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.dataflow;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.api.services.dataflow.Dataflow;
+import com.google.api.services.dataflow.Dataflow.Projects.Jobs;
+import com.google.api.services.dataflow.model.Job;
+import com.google.api.services.dataflow.model.JobMetrics;
+import com.google.api.services.dataflow.model.LeaseWorkItemRequest;
+import com.google.api.services.dataflow.model.LeaseWorkItemResponse;
+import com.google.api.services.dataflow.model.ListJobMessagesResponse;
+import com.google.api.services.dataflow.model.ListJobsResponse;
+import com.google.api.services.dataflow.model.ReportWorkItemStatusRequest;
+import com.google.api.services.dataflow.model.ReportWorkItemStatusResponse;
+import java.io.IOException;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
+
+/**
+ * Client library for {@link Dataflow}.
+ */
+public class DataflowClient {
+
+  public static DataflowClient create(DataflowPipelineOptions options) {
+return new DataflowClient(options.getDataflowClient(), options);
+  }
+
+  private final Dataflow dataflow;
+  private final DataflowPipelineOptions options;
+
+  private DataflowClient(Dataflow dataflow, DataflowPipelineOptions options) {
+this.dataflow = checkNotNull(dataflow, "dataflow");
+this.options = checkNotNull(options, "options");
+  }
+
+  /**
+   * Creates the Dataflow {@link Job}.
+   */
+  public Job createJob(@Nonnull Job job) throws IOException {
+Jobs.Create jobsCreate = 
dataflow.projects().jobs().create(options.getProject(), job);
+return jobsCreate.execute();
+  }
+
+  /**
+   * Lists Dataflow {@link Job Jobs} in the project associated with
+   * the {@link DataflowPipelineOptions}.
+   */
+  public ListJobsResponse listJobs(@Nullable String pageToken) throws 
IOException {
+Jobs.List jobsList = dataflow.projects().jobs()
+.list(options.getProject())
+.setPageToken(pageToken);
+return jobsList.execute();
+  }
+
+  /**
+   * Updates the Dataflow {@link Job} with the given {@code jobId}.
+   */
+  public Job updateJob(@Nonnull String jobId, @Nonnull Job content) throws 
IOException {
+Jobs.Update jobsUpdate = dataflow.projects().jobs()
+.update(options.getProject(), jobId, content);
+return jobsUpdate.execute();
+  }
+
+  /**
+   * Gets the Dataflow {@link Job} with the given {@code jobId}.
+   */
+  public Job getJob(@Nonnull String jobId) throws IOException {
+Jobs.Get jobsGet = dataflow.project

[3/4] incubator-beam git commit: [Code Health] Remove redundant projectId from DataflowPipelineJob.

2016-12-06 Thread bchambers

[Code Health] Remove redundant projectId from DataflowPipelineJob.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ded58832
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ded58832
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ded58832

Branch: refs/heads/master
Commit: ded58832ceaef487f4590d9396f09744288c955d
Parents: afedd68
Author: Pei He <pe...@google.com>
Authored: Wed Nov 23 16:14:27 2016 -0800
Committer: bchambers <bchamb...@google.com>
Committed: Tue Dec 6 17:08:12 2016 -0800

--
 .../runners/dataflow/DataflowPipelineJob.java   | 22 +++--
 .../beam/runners/dataflow/DataflowRunner.java   |  4 +-
 .../dataflow/util/DataflowTemplateJob.java  |  2 +-
 .../dataflow/DataflowPipelineJobTest.java   | 48 ++--
 .../testing/TestDataflowRunnerTest.java | 36 +--
 5 files changed, 45 insertions(+), 67 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ded58832/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
--
diff --git 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
index a2b632f..58e85e0 100644
--- 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
+++ 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
@@ -62,11 +62,6 @@ public class DataflowPipelineJob implements PipelineResult {
   private String jobId;
 
   /**
-   * Google cloud project to associate this pipeline with.
-   */
-  private String projectId;
-
-  /**
* Client for the Dataflow service. This can be used to query the service
* for information about the job.
*/
@@ -119,17 +114,14 @@ public class DataflowPipelineJob implements 
PipelineResult {
   /**
* Constructs the job.
*
-   * @param projectId the project id
* @param jobId the job id
* @param dataflowOptions used to configure the client for the Dataflow 
Service
* @param aggregatorTransforms a mapping from aggregators to PTransforms
*/
   public DataflowPipelineJob(
-  String projectId,
   String jobId,
   DataflowPipelineOptions dataflowOptions,
   DataflowAggregatorTransforms aggregatorTransforms) {
-this.projectId = projectId;
 this.jobId = jobId;
 this.dataflowOptions = dataflowOptions;
 this.aggregatorTransforms = aggregatorTransforms;
@@ -146,7 +138,7 @@ public class DataflowPipelineJob implements PipelineResult {
* Get the project this job exists in.
*/
   public String getProjectId() {
-return projectId;
+return dataflowOptions.getProject();
   }
 
   /**
@@ -249,7 +241,7 @@ public class DataflowPipelineJob implements PipelineResult {
   MonitoringUtil.JobMessagesHandler messageHandler,
   Sleeper sleeper,
   NanoClock nanoClock) throws IOException, InterruptedException {
-MonitoringUtil monitor = new MonitoringUtil(projectId, 
dataflowOptions.getDataflowClient());
+MonitoringUtil monitor = new MonitoringUtil(getProjectId(), 
dataflowOptions.getDataflowClient());
 
 long lastTimestamp = 0;
 BackOff backoff;
@@ -338,12 +330,12 @@ public class DataflowPipelineJob implements 
PipelineResult {
   @Override
   public State cancel() throws IOException {
 Job content = new Job();
-content.setProjectId(projectId);
+content.setProjectId(getProjectId());
 content.setId(jobId);
 content.setRequestedState("JOB_STATE_CANCELLED");
 try {
   dataflowOptions.getDataflowClient().projects().jobs()
-  .update(projectId, jobId, content)
+  .update(getProjectId(), jobId, content)
   .execute();
   return State.CANCELLED;
 } catch (IOException e) {
@@ -412,13 +404,13 @@ public class DataflowPipelineJob implements 
PipelineResult {
 Job job = dataflowOptions.getDataflowClient()
 .projects()
 .jobs()
-.get(projectId, jobId)
+.get(getProjectId(), jobId)
 .execute();
 State currentState = MonitoringUtil.toState(job.getCurrentState());
 if (currentState.isTerminal()) {
   terminalState = currentState;
   replacedByJob = new DataflowPipelineJob(
-  getProjectId(), job.getReplacedByJobId(), dataflowOptions, 
aggregatorTransforms);
+  job.getReplacedByJobId(), dataflowOptions, aggregatorTransforms);
 }
 return job;
   } catch (IOException e

[4/4] incubator-beam git commit: Closes #1434

2016-12-06 Thread bchambers

Closes #1434


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/b2b570f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/b2b570f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/b2b570f2

Branch: refs/heads/master
Commit: b2b570f27808b1671bf6cd0fc60f874da671d4ca
Parents: afedd68 ce03f30
Author: bchambers <bchamb...@google.com>
Authored: Tue Dec 6 17:08:13 2016 -0800
Committer: bchambers <bchamb...@google.com>
Committed: Tue Dec 6 17:08:13 2016 -0800

--
 .../beam/runners/dataflow/DataflowClient.java   | 140 +++
 .../runners/dataflow/DataflowPipelineJob.java   |  31 ++--
 .../beam/runners/dataflow/DataflowRunner.java   |  20 +--
 .../dataflow/testing/TestDataflowRunner.java|   6 +-
 .../dataflow/util/DataflowTemplateJob.java  |   2 +-
 .../runners/dataflow/util/MonitoringUtil.java   |  22 +--
 .../dataflow/DataflowPipelineJobTest.java   |  49 ---
 .../testing/TestDataflowRunnerTest.java |  36 ++---
 .../transforms/DataflowGroupByKeyTest.java  |  14 +-
 .../dataflow/transforms/DataflowViewTest.java   |  16 ++-
 .../dataflow/util/MonitoringUtilTest.java   |  21 +--
 11 files changed, 235 insertions(+), 122 deletions(-)
--

[1/4] incubator-beam git commit: [BEAM-1047] Update dataflow runner code to use DataflowClient wrapper.

2016-12-06 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master afedd68e8 -> b2b570f27


[BEAM-1047] Update dataflow runner code to use DataflowClient wrapper.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ce03f30c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ce03f30c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ce03f30c

Branch: refs/heads/master
Commit: ce03f30c1ee0b84ad2e7f10a6272ffb25548244a
Parents: e8c9686
Author: Pei He <pe...@google.com>
Authored: Mon Nov 28 11:47:42 2016 -0800
Committer: bchambers <bchamb...@google.com>
Committed: Tue Dec 6 17:08:12 2016 -0800

--
 .../beam/runners/dataflow/DataflowClient.java   | 36 +---
 .../runners/dataflow/DataflowPipelineJob.java   | 23 ++---
 .../beam/runners/dataflow/DataflowRunner.java   | 16 +++--
 .../dataflow/testing/TestDataflowRunner.java|  6 ++--
 .../runners/dataflow/util/MonitoringUtil.java   | 22 +++-
 .../dataflow/DataflowPipelineJobTest.java   |  1 +
 .../transforms/DataflowGroupByKeyTest.java  | 14 +++-
 .../dataflow/transforms/DataflowViewTest.java   | 16 +++--
 .../dataflow/util/MonitoringUtilTest.java   | 21 +++-
 9 files changed, 80 insertions(+), 75 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ce03f30c/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowClient.java
--
diff --git 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowClient.java
 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowClient.java
index f2081db..3536d72 100644
--- 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowClient.java
+++ 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowClient.java
@@ -35,27 +35,28 @@ import javax.annotation.Nullable;
 import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
 
 /**
- * Client library for {@link Dataflow}.
+ * Wrapper around the generated {@link Dataflow} client to provide common 
functionality.
  */
 public class DataflowClient {
 
   public static DataflowClient create(DataflowPipelineOptions options) {
-return new DataflowClient(options.getDataflowClient(), options);
+return new DataflowClient(options.getDataflowClient(), 
options.getProject());
   }
 
   private final Dataflow dataflow;
-  private final DataflowPipelineOptions options;
+  private final String projectId;
 
-  private DataflowClient(Dataflow dataflow, DataflowPipelineOptions options) {
+  private DataflowClient(Dataflow dataflow, String projectId) {
 this.dataflow = checkNotNull(dataflow, "dataflow");
-this.options = checkNotNull(options, "options");
+this.projectId = checkNotNull(projectId, "options");
   }
 
   /**
* Creates the Dataflow {@link Job}.
*/
   public Job createJob(@Nonnull Job job) throws IOException {
-Jobs.Create jobsCreate = 
dataflow.projects().jobs().create(options.getProject(), job);
+checkNotNull(job, "job");
+Jobs.Create jobsCreate = dataflow.projects().jobs().create(projectId, job);
 return jobsCreate.execute();
   }
 
@@ -65,7 +66,7 @@ public class DataflowClient {
*/
   public ListJobsResponse listJobs(@Nullable String pageToken) throws 
IOException {
 Jobs.List jobsList = dataflow.projects().jobs()
-.list(options.getProject())
+.list(projectId)
 .setPageToken(pageToken);
 return jobsList.execute();
   }
@@ -74,8 +75,10 @@ public class DataflowClient {
* Updates the Dataflow {@link Job} with the given {@code jobId}.
*/
   public Job updateJob(@Nonnull String jobId, @Nonnull Job content) throws 
IOException {
+checkNotNull(jobId, "jobId");
+checkNotNull(content, "content");
 Jobs.Update jobsUpdate = dataflow.projects().jobs()
-.update(options.getProject(), jobId, content);
+.update(projectId, jobId, content);
 return jobsUpdate.execute();
   }
 
@@ -83,8 +86,9 @@ public class DataflowClient {
* Gets the Dataflow {@link Job} with the given {@code jobId}.
*/
   public Job getJob(@Nonnull String jobId) throws IOException {
+checkNotNull(jobId, "jobId");
 Jobs.Get jobsGet = dataflow.projects().jobs()
-.get(options.getProject(), jobId);
+.get(projectId, jobId);
 return jobsGet.execute();
   }
 
@@ -92,8 +96,9 @@ public class DataflowClient {
* Gets the {@link JobMetrics} with the given {@code jobId}.
*/
   public JobMetrics getJ

[1/2] incubator-beam git commit: Closes #1395

2016-11-21 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/python-sdk 50afc87cd -> b4fc20e44


Closes #1395


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/b4fc20e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/b4fc20e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/b4fc20e4

Branch: refs/heads/python-sdk
Commit: b4fc20e4432d2b956a0333d8e0c6b9aeb472a804
Parents: 50afc87 c127f7a
Author: bchambers <bchamb...@google.com>
Authored: Mon Nov 21 16:21:26 2016 -0800
Committer: bchambers <bchamb...@google.com>
Committed: Mon Nov 21 16:21:26 2016 -0800

--
 sdks/python/apache_beam/internal/apiclient.py   |  2 +-
 .../apache_beam/internal/apiclient_test.py  |  1 +
 sdks/python/apache_beam/transforms/display.py   | 27 
 .../apache_beam/transforms/display_test.py  | 17 
 4 files changed, 46 insertions(+), 1 deletion(-)
--

[2/2] incubator-beam git commit: Fixing error with PipelineOptions DisplayData of lists

2016-11-21 Thread bchambers

Fixing error with PipelineOptions DisplayData of lists


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/c127f7a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/c127f7a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/c127f7a9

Branch: refs/heads/python-sdk
Commit: c127f7a9e6f9885bc595e1462b27b6961c0f465d
Parents: 50afc87
Author: Pablo <pabl...@google.com>
Authored: Mon Nov 21 11:15:37 2016 -0800
Committer: bchambers <bchamb...@google.com>
Committed: Mon Nov 21 16:21:26 2016 -0800

--
 sdks/python/apache_beam/internal/apiclient.py   |  2 +-
 .../apache_beam/internal/apiclient_test.py  |  1 +
 sdks/python/apache_beam/transforms/display.py   | 27 
 .../apache_beam/transforms/display_test.py  | 17 
 4 files changed, 46 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c127f7a9/sdks/python/apache_beam/internal/apiclient.py
--
diff --git a/sdks/python/apache_beam/internal/apiclient.py 
b/sdks/python/apache_beam/internal/apiclient.py
index 8992ec3..5612631 100644
--- a/sdks/python/apache_beam/internal/apiclient.py
+++ b/sdks/python/apache_beam/internal/apiclient.py
@@ -242,7 +242,7 @@ class Environment(object):
   dataflow.Environment.SdkPipelineOptionsValue.AdditionalProperty(
   key='options', value=to_json_value(options_dict)))
 
-  dd = DisplayData.create_from(options)
+  dd = DisplayData.create_from_options(options)
   items = [item.get_dict() for item in dd.items]
   self.proto.sdkPipelineOptions.additionalProperties.append(
   dataflow.Environment.SdkPipelineOptionsValue.AdditionalProperty(

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c127f7a9/sdks/python/apache_beam/internal/apiclient_test.py
--
diff --git a/sdks/python/apache_beam/internal/apiclient_test.py 
b/sdks/python/apache_beam/internal/apiclient_test.py
index 8fddae7..66cc8db 100644
--- a/sdks/python/apache_beam/internal/apiclient_test.py
+++ b/sdks/python/apache_beam/internal/apiclient_test.py
@@ -31,5 +31,6 @@ class UtilTest(unittest.TestCase):
 pipeline_options,
 DataflowPipelineRunner.BATCH_ENVIRONMENT_MAJOR_VERSION)
 
+
 if __name__ == '__main__':
   unittest.main()

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c127f7a9/sdks/python/apache_beam/transforms/display.py
--
diff --git a/sdks/python/apache_beam/transforms/display.py 
b/sdks/python/apache_beam/transforms/display.py
index 365abaf..c38fd9b 100644
--- a/sdks/python/apache_beam/transforms/display.py
+++ b/sdks/python/apache_beam/transforms/display.py
@@ -108,6 +108,33 @@ class DisplayData(object):
   key=key))
 
   @classmethod
+  def create_from_options(cls, pipeline_options):
+""" Creates DisplayData from a PipelineOptions instance.
+
+When creating DisplayData, this method will convert the value of any
+item of a non-supported type to its string representation.
+The normal DisplayData.create_from method rejects those items.
+
+Returns:
+  A DisplayData instance with populated items.
+
+Raises:
+  ValueError: If the has_display_data argument is not an instance of
+HasDisplayData.
+"""
+from apache_beam.utils.options import PipelineOptions
+if not isinstance(pipeline_options, PipelineOptions):
+  raise ValueError(
+  'Element of class {}.{} does not subclass PipelineOptions'
+  .format(pipeline_options.__module__,
+  pipeline_options.__class__.__name__))
+
+items = {k: (v if DisplayDataItem._get_value_type(v) is not None
+ else str(v))
+ for k, v in pipeline_options.display_data().items()}
+return cls(pipeline_options._namespace(), items)
+
+  @classmethod
   def create_from(cls, has_display_data):
 """ Creates DisplayData from a HasDisplayData instance.
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c127f7a9/sdks/python/apache_beam/transforms/display_test.py
--
diff --git a/sdks/python/apache_beam/transforms/display_test.py 
b/sdks/python/apache_beam/transforms/display_test.py
index d514065..fc50abe 100644
--- a/sdks/python/apache_beam/transforms/display_test.py
+++ b/sdks/python/apache_beam/transforms/display_test.py
@@ -29,6 +29,7 @@ import apache_beam as beam
 from apache_beam.transforms.display import HasDisplayData
 from apache_beam.transforms.display impor

[1/2] incubator-beam git commit: Closes #1381

2016-11-18 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 4852d2ebf -> bb9c38664


Closes #1381


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/bb9c3866
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/bb9c3866
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/bb9c3866

Branch: refs/heads/master
Commit: bb9c3866421863b257470ec9fb6bcb8328158045
Parents: 4852d2e 5fc68a5
Author: bchambers <bchamb...@google.com>
Authored: Fri Nov 18 09:55:29 2016 -0800
Committer: bchambers <bchamb...@google.com>
Committed: Fri Nov 18 09:55:29 2016 -0800

--
 .../org/apache/beam/sdk/metrics/MetricsContainer.java  | 13 -
 1 file changed, 13 deletions(-)
--

[2/2] incubator-beam git commit: Remove dead code from MetricsContainer

2016-11-18 Thread bchambers

Remove dead code from MetricsContainer


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/5fc68a5c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/5fc68a5c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/5fc68a5c

Branch: refs/heads/master
Commit: 5fc68a5c109fdbe3ed8afb370c8fcee803e9f1d8
Parents: 4852d2e
Author: bchambers <bchamb...@google.com>
Authored: Thu Nov 17 14:40:52 2016 -0800
Committer: bchambers <bchamb...@google.com>
Committed: Fri Nov 18 09:55:29 2016 -0800

--
 .../org/apache/beam/sdk/metrics/MetricsContainer.java  | 13 -
 1 file changed, 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5fc68a5c/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java
index 10032a2..ba5a343 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java
@@ -130,19 +130,6 @@ public class MetricsContainer {
* container.
*/
   public MetricUpdates getCumulative() {
-ImmutableList.Builder<MetricUpdate> counterUpdates = 
ImmutableList.builder();
-for (Map.Entry<MetricName, CounterCell> counter : counters.entries()) {
-  counterUpdates.add(MetricUpdate.create(
-  MetricKey.create(stepName, counter.getKey()), 
counter.getValue().getCumulative()));
-}
-
-ImmutableList.Builder<MetricUpdate> distributionUpdates =
-ImmutableList.builder();
-for (Map.Entry<MetricName, DistributionCell> distribution : 
distributions.entries()) {
-  distributionUpdates.add(MetricUpdate.create(
-  MetricKey.create(stepName, distribution.getKey()),
-  distribution.getValue().getCumulative()));
-}
 return MetricUpdates.create(
 extractCumulatives(counters),
 extractCumulatives(distributions));

[2/2] incubator-beam git commit: Closes #1155

2016-10-21 Thread bchambers

Closes #1155


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/18e5d128
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/18e5d128
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/18e5d128

Branch: refs/heads/master
Commit: 18e5d128b7918f743c57d8c735452b559bea25e8
Parents: 1533511 4869ef1
Author: bchambers <bchamb...@google.com>
Authored: Fri Oct 21 13:27:37 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Fri Oct 21 13:27:37 2016 -0700

--
 .../java/org/apache/beam/runners/dataflow/DataflowRunner.java   | 5 +
 1 file changed, 1 insertion(+), 4 deletions(-)
--

[1/2] incubator-beam git commit: Use DisplayData delegate in Dataflow Read transform override

2016-10-21 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 153351126 -> 18e5d128b


Use DisplayData delegate in Dataflow Read transform override


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/4869ef19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/4869ef19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/4869ef19

Branch: refs/heads/master
Commit: 4869ef19fa70769831dacc2ac332ec0a11b3f9d8
Parents: 1533511
Author: Scott Wegner 
Authored: Fri Oct 21 11:41:41 2016 -0700
Committer: Scott Wegner 
Committed: Fri Oct 21 12:32:42 2016 -0700

--
 .../java/org/apache/beam/runners/dataflow/DataflowRunner.java   | 5 +
 1 file changed, 1 insertion(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4869ef19/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
--
diff --git 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index 7bf270d..89c364e 100644
--- 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -2248,10 +2248,7 @@ public class DataflowRunner extends 
PipelineRunner {
 
   @Override
   public void populateDisplayData(DisplayData.Builder builder) {
-super.populateDisplayData(builder);
-builder
-.add(DisplayData.item("source", source.getClass()))
-.include("source", source);
+builder.delegate(source);
   }
 
   public UnboundedSource getSource() {

[2/3] incubator-beam git commit: Add Display Data 'path' metadata

2016-10-20 Thread bchambers

Add Display Data 'path' metadata

Display Data supports the notion of "sub components", components within
a transform class which can contribute their own display data. We add a
namespace to display data items based on the originating component,
which keeps the display data items unique within the step.

There are instances where a component is included multiple times within
a step. We handle the case of the same instance being shared by simply
ignoring it the second time. However, we don't handle the case of a
separate instance being added of the same class. Currently the separate
instances will add display data with the same namespace and key, causing
a failure.

This can come up for example when infrastructure at different levels
wrap and re-wrap a component. We saw this with a bounded source being
adapted multiple times, Bounded -> Unbounded -> Bounded -> Unbounded.
The BoundedToUnboundedSourceAdapter was included multiple times with
separate instances and caused a failure while populating display data.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ad03d07a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ad03d07a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ad03d07a

Branch: refs/heads/master
Commit: ad03d07ae783f054a31e8b2e14100afff8cdf747
Parents: ff6301b
Author: Scott Wegner <sweg...@google.com>
Authored: Wed Oct 12 14:49:41 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Thu Oct 20 11:10:13 2016 -0700

--
 .../core/UnboundedReadFromBoundedSource.java|   2 +-
 runners/direct-java/pom.xml |   5 -
 .../runners/direct/ForwardingPTransform.java|   2 +-
 .../beam/runners/direct/DirectRunnerTest.java   |  32 --
 .../direct/ForwardingPTransformTest.java|   7 +-
 .../beam/runners/dataflow/DataflowRunner.java   |   5 +-
 .../DataflowUnboundedReadFromBoundedSource.java |   4 +-
 .../sdk/io/BoundedReadFromUnboundedSource.java  |   5 +-
 .../apache/beam/sdk/io/CompressedSource.java|   2 +-
 .../java/org/apache/beam/sdk/io/PubsubIO.java   |   5 +-
 .../main/java/org/apache/beam/sdk/io/Read.java  |   4 +-
 .../main/java/org/apache/beam/sdk/io/Write.java |   6 +-
 .../sdk/options/ProxyInvocationHandler.java | 149 +++---
 .../org/apache/beam/sdk/transforms/Combine.java |  60 +--
 .../apache/beam/sdk/transforms/CombineFns.java  |  33 +-
 .../beam/sdk/transforms/CombineWithContext.java |   3 +-
 .../beam/sdk/transforms/DoFnAdapters.java   |   2 +-
 .../beam/sdk/transforms/FlatMapElements.java|   6 +-
 .../apache/beam/sdk/transforms/MapElements.java |   8 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   |  60 ++-
 .../apache/beam/sdk/transforms/Partition.java   |   2 +-
 .../sdk/transforms/display/DisplayData.java | 518 +--
 .../beam/sdk/transforms/windowing/Window.java   |   2 +-
 .../io/BoundedReadFromUnboundedSourceTest.java  |   4 +-
 .../beam/sdk/io/CompressedSourceTest.java   |   4 +-
 .../java/org/apache/beam/sdk/io/ReadTest.java   |  10 +-
 .../java/org/apache/beam/sdk/io/WriteTest.java  |   6 +-
 .../sdk/options/ProxyInvocationHandlerTest.java |  40 ++
 .../beam/sdk/transforms/CombineFnsTest.java |   7 +-
 .../apache/beam/sdk/transforms/CombineTest.java |   4 +-
 .../apache/beam/sdk/transforms/ParDoTest.java   |   8 +-
 .../transforms/display/DisplayDataMatchers.java | 141 +++--
 .../display/DisplayDataMatchersTest.java|  67 ++-
 .../sdk/transforms/display/DisplayDataTest.java | 367 +
 .../sdk/transforms/windowing/WindowTest.java|   4 +-
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java|   5 -
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java|   2 +-
 .../beam/sdk/io/gcp/datastore/DatastoreV1.java  |   2 +-
 38 files changed, 988 insertions(+), 605 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ad03d07a/runners/core-java/src/main/java/org/apache/beam/runners/core/UnboundedReadFromBoundedSource.java
--
diff --git 
a/runners/core-java/src/main/java/org/apache/beam/runners/core/UnboundedReadFromBoundedSource.java
 
b/runners/core-java/src/main/java/org/apache/beam/runners/core/UnboundedReadFromBoundedSource.java
index 91a1715..2afdcf2 100644
--- 
a/runners/core-java/src/main/java/org/apache/beam/runners/core/UnboundedReadFromBoundedSource.java
+++ 
b/runners/core-java/src/main/java/org/apache/beam/runners/core/UnboundedReadFromBoundedSource.java
@@ -108,7 +108,7 @@ public class UnboundedReadFromBoundedSource extends 
PTransform<PBegin, PColle
 // We explicitly do not register base-class data, instead we use the 
delegate inner source.
 builder
 .add(DisplayData.item("source&

[1/5] incubator-beam git commit: Implement Metrics in the DirectRunner

2016-10-13 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master e969f3d38 -> 3c731707b


Implement Metrics in the DirectRunner


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/834933c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/834933c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/834933c5

Branch: refs/heads/master
Commit: 834933c520997b4f83cf8b04219c2c63dac61e61
Parents: 51fee39
Author: bchambers <bchamb...@google.com>
Authored: Wed Oct 12 10:55:53 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Thu Oct 13 15:29:29 2016 -0700

--
 .../beam/runners/direct/DirectMetrics.java  | 331 +++
 .../beam/runners/direct/DirectRunner.java   |   8 +-
 .../beam/runners/direct/EvaluationContext.java  |  10 +
 .../direct/ExecutorServiceParallelExecutor.java |   1 +
 .../direct/ImmutableListBundleFactory.java  |  10 +
 .../runners/direct/StepTransformResult.java |  49 ++-
 .../beam/runners/direct/TransformExecutor.java  |  35 +-
 .../beam/runners/direct/TransformResult.java|  12 +
 .../beam/runners/direct/DirectMetricsTest.java  | 133 
 .../beam/runners/direct/DirectRunnerTest.java   |  36 ++
 .../runners/direct/TransformExecutorTest.java   |  12 +
 11 files changed, 602 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/834933c5/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java
--
diff --git 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java
 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java
new file mode 100644
index 000..a749a76
--- /dev/null
+++ 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import static java.util.Arrays.asList;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.base.Objects;
+import com.google.common.collect.ImmutableList;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
+import javax.annotation.concurrent.GuardedBy;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.sdk.metrics.DistributionData;
+import org.apache.beam.sdk.metrics.DistributionResult;
+import org.apache.beam.sdk.metrics.MetricKey;
+import org.apache.beam.sdk.metrics.MetricName;
+import org.apache.beam.sdk.metrics.MetricNameFilter;
+import org.apache.beam.sdk.metrics.MetricQueryResults;
+import org.apache.beam.sdk.metrics.MetricResult;
+import org.apache.beam.sdk.metrics.MetricResults;
+import org.apache.beam.sdk.metrics.MetricUpdates;
+import org.apache.beam.sdk.metrics.MetricUpdates.MetricUpdate;
+import org.apache.beam.sdk.metrics.MetricsFilter;
+import org.apache.beam.sdk.metrics.MetricsMap;
+
+/**
+ * Implementation of {@link MetricResults} for the Direct Runner.
+ */
+class DirectMetrics extends MetricResults {
+
+  // TODO: (BEAM-723) Create a shared ExecutorService for maintenance tasks in 
the DirectRunner.
+  private static final ExecutorService COUNTER_COMMITTER = 
Executors.newCachedThreadPool();
+
+  private interface MetricAggregation<UpdateT, ResultT> {
+UpdateT zero();
+UpdateT combine(Iterable updates);
+ResultT extract(UpdateT data);
+  }
+
+  /**
+   * Implementation of a metric in the direct runner.
+   *
+   * @param  The type of raw data received and aggregated across 
updates.
+   * @param  The type of result extracted from the data.
+   */
+  private static class Dire

[5/5] incubator-beam git commit: Closes #1024

2016-10-13 Thread bchambers

Closes #1024


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/3c731707
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/3c731707
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/3c731707

Branch: refs/heads/master
Commit: 3c731707b2d986e7a460907c8f64bbebf1dff714
Parents: e969f3d 834933c
Author: bchambers <bchamb...@google.com>
Authored: Thu Oct 13 15:29:30 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Thu Oct 13 15:29:30 2016 -0700

--
 .../beam/runners/direct/DirectMetrics.java  | 331 +++
 .../beam/runners/direct/DirectRunner.java   |  11 +-
 .../beam/runners/direct/EvaluationContext.java  |  10 +
 .../direct/ExecutorServiceParallelExecutor.java |   1 +
 .../direct/ImmutableListBundleFactory.java  |  10 +
 .../runners/direct/StepTransformResult.java |  49 ++-
 .../beam/runners/direct/TransformExecutor.java  |  35 +-
 .../beam/runners/direct/TransformResult.java|  12 +
 .../beam/runners/direct/DirectMetricsTest.java  | 133 
 .../beam/runners/direct/DirectRunnerTest.java   |  36 ++
 .../runners/direct/TransformExecutorTest.java   |  12 +
 .../beam/runners/flink/FlinkRunnerResult.java   |   6 +
 .../runners/dataflow/DataflowPipelineJob.java   |   7 +
 .../spark/translation/EvaluationContext.java|   6 +
 .../org/apache/beam/sdk/PipelineResult.java |  11 +
 .../beam/sdk/annotations/Experimental.java  |   3 +
 .../org/apache/beam/sdk/metrics/Counter.java|  40 +++
 .../apache/beam/sdk/metrics/CounterCell.java|  76 +
 .../org/apache/beam/sdk/metrics/DirtyState.java |  98 ++
 .../apache/beam/sdk/metrics/Distribution.java   |  30 ++
 .../beam/sdk/metrics/DistributionCell.java  |  58 
 .../beam/sdk/metrics/DistributionData.java  |  59 
 .../beam/sdk/metrics/DistributionResult.java|  42 +++
 .../org/apache/beam/sdk/metrics/Metric.java |  24 ++
 .../org/apache/beam/sdk/metrics/MetricCell.java |  47 +++
 .../org/apache/beam/sdk/metrics/MetricKey.java  |  40 +++
 .../org/apache/beam/sdk/metrics/MetricName.java |  46 +++
 .../beam/sdk/metrics/MetricNameFilter.java  |  60 
 .../beam/sdk/metrics/MetricQueryResults.java|  33 ++
 .../apache/beam/sdk/metrics/MetricResult.java   |  45 +++
 .../apache/beam/sdk/metrics/MetricResults.java  |  34 ++
 .../apache/beam/sdk/metrics/MetricUpdates.java  |  72 
 .../org/apache/beam/sdk/metrics/Metrics.java| 110 ++
 .../beam/sdk/metrics/MetricsContainer.java  | 150 +
 .../beam/sdk/metrics/MetricsEnvironment.java|  85 +
 .../apache/beam/sdk/metrics/MetricsFilter.java  |  86 +
 .../org/apache/beam/sdk/metrics/MetricsMap.java |  86 +
 .../apache/beam/sdk/metrics/package-info.java   |  28 ++
 .../beam/sdk/metrics/CounterCellTest.java   |  55 +++
 .../apache/beam/sdk/metrics/DirtyStateTest.java |  56 
 .../beam/sdk/metrics/DistributionCellTest.java  |  53 +++
 .../apache/beam/sdk/metrics/MetricMatchers.java |  99 ++
 .../beam/sdk/metrics/MetricsContainerTest.java  | 129 
 .../sdk/metrics/MetricsEnvironmentTest.java |  63 
 .../apache/beam/sdk/metrics/MetricsMapTest.java | 103 ++
 .../apache/beam/sdk/metrics/MetricsTest.java|  98 ++
 46 files changed, 2645 insertions(+), 33 deletions(-)
--

[3/5] incubator-beam git commit: Initial Metrics API for Beam Java

2016-10-13 Thread bchambers

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/8524ed95/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsMapTest.java
--
diff --git 
a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsMapTest.java 
b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsMapTest.java
new file mode 100644
index 000..4104f8d
--- /dev/null
+++ 
b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsMapTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.metrics;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.junit.Assert.assertThat;
+
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicLong;
+import org.hamcrest.Description;
+import org.hamcrest.Matcher;
+import org.hamcrest.TypeSafeMatcher;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+/**
+ * Tests for {@link MetricsMap}.
+ */
+@RunWith(JUnit4.class)
+public class MetricsMapTest {
+
+  public MetricsMap metricsMap =
+  new MetricsMap<>(new MetricsMap.Factory() {
+@Override
+public AtomicLong createInstance(String unusedKey) {
+  return new AtomicLong();
+}
+  });
+
+  @Test
+  public void testCreateSeparateInstances() {
+AtomicLong foo = metricsMap.get("foo");
+AtomicLong bar = metricsMap.get("bar");
+
+assertThat(foo, not(sameInstance(bar)));
+  }
+
+  @Test
+  public void testReuseInstances() {
+AtomicLong foo1 = metricsMap.get("foo");
+AtomicLong foo2 = metricsMap.get("foo");
+
+assertThat(foo1, sameInstance(foo2));
+  }
+
+  @Test
+  public void testGet() {
+assertThat(metricsMap.tryGet("foo"), nullValue(AtomicLong.class));
+
+AtomicLong foo = metricsMap.get("foo");
+assertThat(metricsMap.tryGet("foo"), sameInstance(foo));
+  }
+
+  @Test
+  public void testGetEntries() {
+AtomicLong foo = metricsMap.get("foo");
+AtomicLong bar = metricsMap.get("bar");
+assertThat(metricsMap.entries(), containsInAnyOrder(
+hasEntry("foo", foo),
+hasEntry("bar", bar)));
+  }
+
+  private static Matcher> hasEntry(
+  final String key, final AtomicLong value) {
+return new TypeSafeMatcher>() {
+
+  @Override
+  public void describeTo(Description description) {
+description
+.appendText("Map.Entry{key=").appendValue(key)
+.appendText(", value=").appendValue(value)
+.appendText("}");
+  }
+
+  @Override
+  protected boolean matchesSafely(Entry item) {
+return Objects.equals(key, item.getKey())
+&& Objects.equals(value, item.getValue());
+  }
+};
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/8524ed95/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java
--
diff --git 
a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java 
b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java
new file mode 100644
index 000..d11b44d
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is

[2/5] incubator-beam git commit: Add the ability to query metrics on PipelineResult

2016-10-13 Thread bchambers

Add the ability to query metrics on PipelineResult

All runners currently implement this by throwing an
UnsupportedOperationException.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/51fee39b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/51fee39b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/51fee39b

Branch: refs/heads/master
Commit: 51fee39b7bc66d7f60ea2e0ce31e3cb516a89305
Parents: 8524ed9
Author: bchambers <bchamb...@google.com>
Authored: Wed Oct 12 10:55:05 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Thu Oct 13 15:29:29 2016 -0700

--
 .../org/apache/beam/runners/direct/DirectRunner.java |  7 +++
 .../org/apache/beam/runners/flink/FlinkRunnerResult.java |  6 ++
 .../beam/runners/dataflow/DataflowPipelineJob.java   |  7 +++
 .../runners/spark/translation/EvaluationContext.java |  6 ++
 .../main/java/org/apache/beam/sdk/PipelineResult.java| 11 +++
 5 files changed, 37 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/51fee39b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
--
diff --git 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
index a72f7ae..e13046d 100644
--- 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
+++ 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
@@ -42,6 +42,7 @@ import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.io.Write;
+import org.apache.beam.sdk.metrics.MetricResults;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.testing.TestStream;
@@ -380,6 +381,12 @@ public class DirectRunner
   };
 }
 
+@Override
+public MetricResults metrics() {
+  throw new UnsupportedOperationException(
+  "The DirectRunner does not currently support metrics.");
+}
+
 /**
  * Blocks until the {@link Pipeline} execution represented by this
  * {@link DirectPipelineResult} is complete, returning the terminal state.

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/51fee39b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java
--
diff --git 
a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java
 
b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java
index 90bb64d..6b15485 100644
--- 
a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java
+++ 
b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java
@@ -23,6 +23,7 @@ import java.util.Map;
 import org.apache.beam.sdk.AggregatorRetrievalException;
 import org.apache.beam.sdk.AggregatorValues;
 import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.metrics.MetricResults;
 import org.apache.beam.sdk.transforms.Aggregator;
 import org.joda.time.Duration;
 
@@ -86,4 +87,9 @@ public class FlinkRunnerResult implements PipelineResult {
   public State waitUntilFinish(Duration duration) {
 throw new UnsupportedOperationException("FlinkRunnerResult does not 
support waitUntilFinish.");
   }
+
+  @Override
+  public MetricResults metrics() {
+throw new UnsupportedOperationException("The FlinkRunner does not 
currently support metrics.");
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/51fee39b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
--
diff --git 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
index 269b824..bbcf11f 100644
--- 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
+++ 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
@@ -42,6 +42,7 @@ import org.apache.beam.runners.dataflow.util.MonitoringUtil;
 import org.apache.beam.sdk.AggregatorRetrievalException;
 import org.apache.beam.sdk.AggregatorValues;
 import o

[4/5] incubator-beam git commit: Initial Metrics API for Beam Java

2016-10-13 Thread bchambers

Initial Metrics API for Beam Java

This includes a simple Counter metric and a Distribution metric that
reports the SUM, COUNT, MIN, MAX and MEAN of the reported values.

The API is labeled @Experimental since metrics will only be reported
and queryable with the DirectRunner, and the API may change as it is
implemented on other runners.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/8524ed95
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/8524ed95
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/8524ed95

Branch: refs/heads/master
Commit: 8524ed9545f5af4bdeb54601f333549b34eb35aa
Parents: e969f3d
Author: bchambers <bchamb...@google.com>
Authored: Wed Oct 12 10:29:50 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Thu Oct 13 15:29:29 2016 -0700

--
 .../beam/sdk/annotations/Experimental.java  |   3 +
 .../org/apache/beam/sdk/metrics/Counter.java|  40 +
 .../apache/beam/sdk/metrics/CounterCell.java|  76 ++
 .../org/apache/beam/sdk/metrics/DirtyState.java |  98 
 .../apache/beam/sdk/metrics/Distribution.java   |  30 
 .../beam/sdk/metrics/DistributionCell.java  |  58 +++
 .../beam/sdk/metrics/DistributionData.java  |  59 
 .../beam/sdk/metrics/DistributionResult.java|  42 ++
 .../org/apache/beam/sdk/metrics/Metric.java |  24 +++
 .../org/apache/beam/sdk/metrics/MetricCell.java |  47 ++
 .../org/apache/beam/sdk/metrics/MetricKey.java  |  40 +
 .../org/apache/beam/sdk/metrics/MetricName.java |  46 ++
 .../beam/sdk/metrics/MetricNameFilter.java  |  60 
 .../beam/sdk/metrics/MetricQueryResults.java|  33 
 .../apache/beam/sdk/metrics/MetricResult.java   |  45 ++
 .../apache/beam/sdk/metrics/MetricResults.java  |  34 +
 .../apache/beam/sdk/metrics/MetricUpdates.java  |  72 +
 .../org/apache/beam/sdk/metrics/Metrics.java| 110 ++
 .../beam/sdk/metrics/MetricsContainer.java  | 150 +++
 .../beam/sdk/metrics/MetricsEnvironment.java|  85 +++
 .../apache/beam/sdk/metrics/MetricsFilter.java  |  86 +++
 .../org/apache/beam/sdk/metrics/MetricsMap.java |  86 +++
 .../apache/beam/sdk/metrics/package-info.java   |  28 
 .../beam/sdk/metrics/CounterCellTest.java   |  55 +++
 .../apache/beam/sdk/metrics/DirtyStateTest.java |  56 +++
 .../beam/sdk/metrics/DistributionCellTest.java  |  53 +++
 .../apache/beam/sdk/metrics/MetricMatchers.java |  99 
 .../beam/sdk/metrics/MetricsContainerTest.java  | 129 
 .../sdk/metrics/MetricsEnvironmentTest.java |  63 
 .../apache/beam/sdk/metrics/MetricsMapTest.java | 103 +
 .../apache/beam/sdk/metrics/MetricsTest.java|  98 
 31 files changed, 2008 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/8524ed95/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java
index 789f4b2..14d2358 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java
@@ -83,5 +83,8 @@ public @interface Experimental {
  * Do not use: API is unstable and runner support is incomplete.
  */
 SPLITTABLE_DO_FN,
+
+/** Metrics-related experimental APIs. */
+METRICS
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/8524ed95/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/Counter.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/Counter.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/Counter.java
new file mode 100644
index 000..9f48016
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/Counter.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software

[3/4] incubator-beam git commit: [BEAM-65] SplittableDoFn prototype.

2016-10-12 Thread bchambers

[BEAM-65] SplittableDoFn prototype.

Work in progress. Currently only runs in direct runner,
and not ready for any use by real users.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a0a24883
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a0a24883
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a0a24883

Branch: refs/heads/master
Commit: a0a24883737850052f54290255780e868c0b63dc
Parents: a5d1293
Author: Eugene Kirpichov <kirpic...@google.com>
Authored: Thu Aug 11 17:13:53 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Oct 12 17:29:20 2016 -0700

--
 runners/core-java/pom.xml   |   6 +
 .../runners/core/ElementAndRestriction.java |  42 ++
 .../core/ElementAndRestrictionCoder.java|  67 +++
 .../runners/core/GBKIntoKeyedWorkItems.java |  40 ++
 .../beam/runners/core/SplittableParDo.java  | 469 
 .../core/ElementAndRestrictionCoderTest.java| 127 +
 .../beam/runners/core/SplittableParDoTest.java  | 467 
 ...ectGBKIntoKeyedWorkItemsOverrideFactory.java |  66 +++
 .../beam/runners/direct/DirectRunner.java   |   5 +
 .../runners/direct/ParDoOverrideFactory.java|  55 ++
 .../beam/runners/direct/SplittableDoFnTest.java | 225 
 .../beam/sdk/annotations/Experimental.java  |   8 +-
 .../org/apache/beam/sdk/transforms/DoFn.java| 218 +++-
 .../beam/sdk/transforms/DoFnAdapters.java   |  27 +-
 .../apache/beam/sdk/transforms/DoFnTester.java  | 117 ++--
 .../org/apache/beam/sdk/transforms/ParDo.java   |  19 +
 .../sdk/transforms/reflect/DoFnInvoker.java |  26 +-
 .../sdk/transforms/reflect/DoFnInvokers.java| 179 +-
 .../sdk/transforms/reflect/DoFnSignature.java   | 114 +++-
 .../sdk/transforms/reflect/DoFnSignatures.java  | 366 -
 .../splittabledofn/RestrictionTracker.java  |  42 ++
 .../transforms/splittabledofn/package-info.java |  22 +
 .../org/apache/beam/sdk/coders/KvCoderTest.java |  99 ++--
 .../apache/beam/sdk/transforms/ParDoTest.java   |  46 ++
 .../transforms/reflect/DoFnInvokersTest.java| 261 -
 .../DoFnSignaturesProcessElementTest.java   |   4 +-
 .../DoFnSignaturesSplittableDoFnTest.java   | 543 +++
 27 files changed, 3495 insertions(+), 165 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a0a24883/runners/core-java/pom.xml
--
diff --git a/runners/core-java/pom.xml b/runners/core-java/pom.xml
index d958dd2..d84c420 100644
--- a/runners/core-java/pom.xml
+++ b/runners/core-java/pom.xml
@@ -190,6 +190,12 @@
 
 
 
+  com.google.auto.value
+  auto-value
+  provided
+
+
+
   com.google.code.findbugs
   annotations
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a0a24883/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestriction.java
--
diff --git 
a/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestriction.java
 
b/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestriction.java
new file mode 100644
index 000..4a5d0c4
--- /dev/null
+++ 
b/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestriction.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core;
+
+import com.google.auto.value.AutoValue;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.transforms.DoFn;
+
+/**
+ * A tuple of an element and a restriction applied to processing it with a
+ * https://s.apache.org/splittable-do-fn;>splittable {@link DoFn}.
+ */
+@Experimental(Experimental.Kind.SPLITTABLE_DO_FN)
+@AutoValue
+public abstract class ElementAndRestriction<ElementT, RestrictionT> {
+  /** The element t

[2/4] incubator-beam git commit: [BEAM-65] SplittableDoFn prototype.

2016-10-12 Thread bchambers

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a0a24883/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnAdapters.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnAdapters.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnAdapters.java
index 3eee74a..f671a67 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnAdapters.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnAdapters.java
@@ -25,6 +25,7 @@ import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
+import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.WindowingInternals;
@@ -37,8 +38,8 @@ import org.joda.time.Instant;
 /**
  * Utility class containing adapters to/from {@link DoFn} and {@link OldDoFn}.
  *
- * @deprecated This class will go away when we start running {@link DoFn}'s 
directly (using
- * {@link DoFnInvoker}) rather than via {@link OldDoFn}.
+ * @deprecated This class will go away when we start running {@link DoFn}'s 
directly (using {@link
+ * DoFnInvoker}) rather than via {@link OldDoFn}.
  */
 @Deprecated
 public class DoFnAdapters {
@@ -176,6 +177,18 @@ public class DoFnAdapters {
   }
 
   /**
+   * If the fn was created using {@link #toOldDoFn}, returns the original 
{@link DoFn}. Otherwise,
+   * returns {@code null}.
+   */
+  public static  DoFn 
getDoFn(OldDoFn fn) {
+if (fn instanceof SimpleDoFnAdapter) {
+  return ((SimpleDoFnAdapter) fn).fn;
+} else {
+  return null;
+}
+  }
+
+  /**
* Wraps a {@link DoFn} that doesn't require access to {@link BoundedWindow} 
as an {@link
* OldDoFn}.
*/
@@ -324,6 +337,11 @@ public class DoFnAdapters {
 public DoFn.OutputReceiver outputReceiver() {
   throw new UnsupportedOperationException("outputReceiver() exists only 
for testing");
 }
+
+@Override
+public  RestrictionTracker 
restrictionTracker() {
+  throw new UnsupportedOperationException("This is a non-splittable DoFn");
+}
   }
 
   /**
@@ -412,5 +430,10 @@ public class DoFnAdapters {
 public DoFn.OutputReceiver outputReceiver() {
   throw new UnsupportedOperationException("outputReceiver() exists only 
for testing");
 }
+
+@Override
+public  RestrictionTracker 
restrictionTracker() {
+  throw new UnsupportedOperationException("This is a non-splittable DoFn");
+}
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a0a24883/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
index 11a4cbd..302bb02 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
@@ -46,7 +46,9 @@ import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingInternals;
 import org.apache.beam.sdk.util.state.InMemoryStateInternals;
+import org.apache.beam.sdk.util.state.InMemoryTimerInternals;
 import org.apache.beam.sdk.util.state.StateInternals;
+import org.apache.beam.sdk.util.state.TimerCallback;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TimestampedValue;
 import org.apache.beam.sdk.values.TupleTag;
@@ -222,8 +224,11 @@ public class DoFnTester implements 
AutoCloseable {
 if (state == State.UNINITIALIZED) {
   initializeState();
 }
-TestContext context = createContext(fn);
+TestContext context = createContext(fn);
 context.setupDelegateAggregators();
+// State and timer internals are per-bundle.
+stateInternals = InMemoryStateInternals.forKey(new Object());
+timerInternals = new InMemoryTimerInternals();
 try {
   fn.startBundle(context);
 } catch (UserCodeException e) {
@@ -460,6 +465,35 @@ public class DoFnTester implements 
AutoCloseable {
 return extractAggregatorValue(agg.getName(), agg.getCombineFn());
   }
 
+  private static TimerCallback collectInto(final 
List firedTimers) {
+return new TimerCallback() {
+  @Override
+  public void onTimer(TimerInternals.TimerData timer) throws Exception {
+

[1/4] incubator-beam git commit: [BEAM-65] SplittableDoFn prototype.

2016-10-12 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master a5d129361 -> 13b45895e


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a0a24883/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
--
diff --git 
a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
 
b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
index d057765..0bfe2be 100644
--- 
a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
+++ 
b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
@@ -17,14 +17,32 @@
  */
 package org.apache.beam.sdk.transforms.reflect;
 
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.same;
 import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
-
+import static org.mockito.Mockito.when;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.DoFn.GetInitialRestriction;
+import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation;
 import org.apache.beam.sdk.transforms.OldDoFn;
 import 
org.apache.beam.sdk.transforms.reflect.testhelper.DoFnInvokersTestHelper;
+import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowingInternals;
@@ -34,7 +52,9 @@ import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
+import org.mockito.AdditionalAnswers;
 import org.mockito.Mock;
+import org.mockito.Mockito;
 import org.mockito.MockitoAnnotations;
 
 /** Tests for {@link DoFnInvokers}. */
@@ -76,11 +96,16 @@ public class DoFnInvokersTest {
   public WindowingInternals windowingInternals() {
 return mockWindowingInternals;
   }
+
+  @Override
+  public  RestrictionTracker 
restrictionTracker() {
+return null;
+  }
 };
   }
 
-  private void invokeProcessElement(DoFn fn) {
-DoFnInvokers.INSTANCE
+  private ProcessContinuation invokeProcessElement(DoFn fn) {
+return DoFnInvokers.INSTANCE
 .newByteBuddyInvoker(fn)
 .invokeProcessElement(mockContext, extraContextFactory);
   }
@@ -106,9 +131,9 @@ public class DoFnInvokersTest {
   @ProcessElement
   public void processElement(ProcessContext c) throws Exception {}
 }
-MockFn fn = mock(MockFn.class);
-invokeProcessElement(fn);
-verify(fn).processElement(mockContext);
+MockFn mockFn = mock(MockFn.class);
+assertEquals(ProcessContinuation.stop(), invokeProcessElement(mockFn));
+verify(mockFn).processElement(mockContext);
   }
 
   interface InterfaceWithProcessElement {
@@ -128,7 +153,7 @@ public class DoFnInvokersTest {
   public void testDoFnWithProcessElementInterface() throws Exception {
 IdentityUsingInterfaceWithProcessElement fn =
 mock(IdentityUsingInterfaceWithProcessElement.class);
-invokeProcessElement(fn);
+assertEquals(ProcessContinuation.stop(), invokeProcessElement(fn));
 verify(fn).processElement(mockContext);
   }
 
@@ -149,14 +174,14 @@ public class DoFnInvokersTest {
   @Test
   public void testDoFnWithMethodInSuperclass() throws Exception {
 IdentityChildWithoutOverride fn = mock(IdentityChildWithoutOverride.class);
-invokeProcessElement(fn);
+assertEquals(ProcessContinuation.stop(), invokeProcessElement(fn));
 verify(fn).process(mockContext);
   }
 
   @Test
   public void testDoFnWithMethodInSubclass() throws Exception {
 IdentityChildWithOverride fn = mock(IdentityChildWithOverride.class);
-invokeProcessElement(fn);
+assertEquals(ProcessContinuation.stop(), invokeProcessElement(fn));
 verify(fn).process(mockContext);
   }
 
@@ -167,7 +192,7 @@ public class DoFnInvokersTest {
   public void processElement(ProcessContext c, BoundedWindow w) throws 
Exception {}
 }
 MockFn fn = mock(MockFn.class);
-invokeProcessElement(fn);
+assertEquals(ProcessContinuation.stop(), invokeProcessElement(fn));
 verify(fn).processElement(mockContext, mockWindow);
   }
 
@@ -178,7

[4/4] incubator-beam git commit: Closes #896

2016-10-12 Thread bchambers

Closes #896


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/13b45895
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/13b45895
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/13b45895

Branch: refs/heads/master
Commit: 13b45895eb13cd48557472463404bd097d7097d7
Parents: a5d1293 a0a2488
Author: bchambers <bchamb...@google.com>
Authored: Wed Oct 12 17:29:21 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Oct 12 17:29:21 2016 -0700

--
 runners/core-java/pom.xml   |   6 +
 .../runners/core/ElementAndRestriction.java |  42 ++
 .../core/ElementAndRestrictionCoder.java|  67 +++
 .../runners/core/GBKIntoKeyedWorkItems.java |  40 ++
 .../beam/runners/core/SplittableParDo.java  | 469 
 .../core/ElementAndRestrictionCoderTest.java| 127 +
 .../beam/runners/core/SplittableParDoTest.java  | 467 
 ...ectGBKIntoKeyedWorkItemsOverrideFactory.java |  66 +++
 .../beam/runners/direct/DirectRunner.java   |   5 +
 .../runners/direct/ParDoOverrideFactory.java|  55 ++
 .../beam/runners/direct/SplittableDoFnTest.java | 225 
 .../beam/sdk/annotations/Experimental.java  |   8 +-
 .../org/apache/beam/sdk/transforms/DoFn.java| 218 +++-
 .../beam/sdk/transforms/DoFnAdapters.java   |  27 +-
 .../apache/beam/sdk/transforms/DoFnTester.java  | 117 ++--
 .../org/apache/beam/sdk/transforms/ParDo.java   |  19 +
 .../sdk/transforms/reflect/DoFnInvoker.java |  26 +-
 .../sdk/transforms/reflect/DoFnInvokers.java| 179 +-
 .../sdk/transforms/reflect/DoFnSignature.java   | 114 +++-
 .../sdk/transforms/reflect/DoFnSignatures.java  | 366 -
 .../splittabledofn/RestrictionTracker.java  |  42 ++
 .../transforms/splittabledofn/package-info.java |  22 +
 .../org/apache/beam/sdk/coders/KvCoderTest.java |  99 ++--
 .../apache/beam/sdk/transforms/ParDoTest.java   |  46 ++
 .../transforms/reflect/DoFnInvokersTest.java| 261 -
 .../DoFnSignaturesProcessElementTest.java   |   4 +-
 .../DoFnSignaturesSplittableDoFnTest.java   | 543 +++
 27 files changed, 3495 insertions(+), 165 deletions(-)
--

[2/2] incubator-beam git commit: Closes #1029

2016-09-30 Thread bchambers

Closes #1029


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/bc80ee34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/bc80ee34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/bc80ee34

Branch: refs/heads/master
Commit: bc80ee342749768bbfc2a9fda80a4b15680e5898
Parents: b237e2f 3e7c396
Author: bchambers <bchamb...@google.com>
Authored: Fri Sep 30 16:12:47 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Fri Sep 30 16:12:47 2016 -0700

--
 .../beam/runners/dataflow/DataflowRunner.java   |  8 ++
 .../DataflowUnboundedReadFromBoundedSource.java | 16 +++-
 ...aflowUnboundedReadFromBoundedSourceTest.java | 82 
 .../sdk/io/BoundedReadFromUnboundedSource.java  |  6 ++
 .../java/org/apache/beam/sdk/io/ReadTest.java   | 56 -
 .../display/DisplayDataEvaluator.java   |  7 +-
 6 files changed, 168 insertions(+), 7 deletions(-)
--

[1/2] incubator-beam git commit: Enable EncodabilityEnforcement by default

2016-09-30 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 0d229f5a6 -> 7d46698f2


Enable EncodabilityEnforcement by default

This ensures that all elements, rather than only non-null elements, will
have their encodability checked.

Also, only encode elements in EncodabilityEnforcement once.

Instead of checking that all input elements are encodable, ensure all
elements produced by a PTransform can be encoded with the provided
coder. This reduces the number of duplicate checks performed and enables
EncodabilityEnforcement to be attached to Read PTransforms to ensure
that provided coders can encode all elements output by a source.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/cdd88a98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/cdd88a98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/cdd88a98

Branch: refs/heads/master
Commit: cdd88a983fd2937110d52ea4661e256c8f4422f6
Parents: 0d229f5
Author: Thomas Groh <tg...@google.com>
Authored: Tue Aug 23 09:37:07 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Fri Sep 30 11:23:42 2016 -0700

--
 .../beam/runners/direct/DirectOptions.java  |  13 +-
 .../beam/runners/direct/DirectRunner.java   |  16 ++-
 .../direct/EncodabilityEnforcementFactory.java  |  50 +---
 .../beam/runners/direct/DirectRunnerTest.java   |  66 ++
 .../EncodabilityEnforcementFactoryTest.java | 122 +++
 5 files changed, 215 insertions(+), 52 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/cdd88a98/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectOptions.java
--
diff --git 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectOptions.java
 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectOptions.java
index 798fda4..89e1bb8 100644
--- 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectOptions.java
+++ 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectOptions.java
@@ -49,10 +49,17 @@ public interface DirectOptions extends PipelineOptions, 
ApplicationNameOptions {
 
   @Default.Boolean(true)
   @Description(
-  "Controls whether the runner should ensure that all of the elements of 
every "
+  "Controls whether the DirectRunner should ensure that all of the 
elements of every "
   + "PCollection are not mutated. PTransforms are not permitted to 
mutate input elements "
   + "at any point, or output elements after they are output.")
-  boolean isTestImmutability();
+  boolean isEnforceImmutability();
 
-  void setTestImmutability(boolean test);
+  void setEnforceImmutability(boolean test);
+
+  @Default.Boolean(true)
+  @Description(
+  "Controls whether the DirectRunner should ensure that all of the 
elements of every "
+  + "PCollection are encodable. All elements in a PCollection must be 
encodable.")
+  boolean isEnforceEncodability();
+  void setEnforceEncodability(boolean test);
 }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/cdd88a98/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
--
diff --git 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
index a3d20f6..2ec4f08 100644
--- 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
+++ 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
@@ -39,6 +39,7 @@ import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.io.Write;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.runners.PipelineRunner;
@@ -284,21 +285,32 @@ public class DirectRunner
 Collection parDoEnforcements = 
createParDoEnforcements(options);
 enforcements.put(ParDo.Bound.class, parDoEnforcements);
 enforcements.put(ParDo.BoundMulti.class, parDoEnforcements);
+if (options.isEnforceEncodability()) {
+  enforcements.put(
+  Read.Unbounded.class,
+  
ImmutableSet.of(EncodabilityEnforcementFactory.create()));
+  enforcements.put(
+  Read.Bounded.class,
+  
ImmutableSet.of(EncodabilityEnforcementFactory.create()));
+}
 return enforcements.build();

[2/2] incubator-beam git commit: Closes #864

2016-09-30 Thread bchambers

Closes #864


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/7d46698f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/7d46698f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/7d46698f

Branch: refs/heads/master
Commit: 7d46698f2878e41369077c75f9347aafbeff0c5a
Parents: 0d229f5 cdd88a9
Author: bchambers <bchamb...@google.com>
Authored: Fri Sep 30 11:24:22 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Fri Sep 30 11:24:22 2016 -0700

--
 .../beam/runners/direct/DirectOptions.java  |  13 +-
 .../beam/runners/direct/DirectRunner.java   |  16 ++-
 .../direct/EncodabilityEnforcementFactory.java  |  50 +---
 .../beam/runners/direct/DirectRunnerTest.java   |  66 ++
 .../EncodabilityEnforcementFactoryTest.java | 122 +++
 5 files changed, 215 insertions(+), 52 deletions(-)
--

[1/2] incubator-beam git commit: Dedups TimerInternal implementations

2016-09-30 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master e7684b79b -> 0d229f5a6


Dedups TimerInternal implementations

Factors InMemoryTimerInternals out of ReduceFnTester

Uses InMemoryTimerInternals instead of BatchTimerInternals and 
TriggerTester.TestTimerInternals

Previously, there were 3 implementations:
TestTimerInternals in ReduceFnTester
TestTimerInternals in TriggerTester (these two were nearly identical)
BatchTimerInternals (it was a subset of the above)
There were also 2 copies of TestInMemoryStateInternals.

This change deduplicates and reorganizes them:

1. Deduplicates the TestInMemoryStateInternals.
2. Factors out the common timer stuff into InMemoryTimerInternals.
3. TriggerTester's implementation of TestTimerInternals used to
  (unnecessarily) access TestInMemoryStateInternals, presumably due to
  copy-paste. Now it uses the regular InMemoryTimerInternals.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/425c7781
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/425c7781
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/425c7781

Branch: refs/heads/master
Commit: 425c77818fc7bbd919fd1ed10a1c3a10b3bfb920
Parents: e7684b7
Author: Eugene Kirpichov <kirpic...@google.com>
Authored: Wed Sep 28 19:19:12 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Fri Sep 30 11:11:27 2016 -0700

--
 .../beam/runners/core/BatchTimerInternals.java  | 140 ---
 .../GroupAlsoByWindowsViaOutputBufferDoFn.java  |   7 +-
 .../beam/runners/core/ReduceFnRunner.java   |   3 +-
 .../runners/core/BatchTimerInternalsTest.java   | 118 -
 .../beam/runners/core/ReduceFnTester.java   | 248 ++-
 .../sdk/util/state/InMemoryTimerInternals.java  | 235 ++
 .../util/state/TestInMemoryStateInternals.java  |  61 +
 .../beam/sdk/util/state/TimerCallback.java  |  35 +++
 .../org/apache/beam/sdk/util/TriggerTester.java | 206 +--
 .../util/state/InMemoryTimerInternalsTest.java  | 116 +
 10 files changed, 482 insertions(+), 687 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/425c7781/runners/core-java/src/main/java/org/apache/beam/runners/core/BatchTimerInternals.java
--
diff --git 
a/runners/core-java/src/main/java/org/apache/beam/runners/core/BatchTimerInternals.java
 
b/runners/core-java/src/main/java/org/apache/beam/runners/core/BatchTimerInternals.java
deleted file mode 100644
index 829dbde..000
--- 
a/runners/core-java/src/main/java/org/apache/beam/runners/core/BatchTimerInternals.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import static com.google.common.base.Preconditions.checkState;
-
-import com.google.common.base.MoreObjects;
-import java.util.HashSet;
-import java.util.PriorityQueue;
-import java.util.Set;
-import javax.annotation.Nullable;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.TimerInternals;
-
-import org.joda.time.Instant;
-
-/**
- * TimerInternals that uses priority queues to manage the timers that are 
ready to fire.
- */
-public class BatchTimerInternals implements TimerInternals {
-  /** Set of timers that are scheduled used for deduplicating timers. */
-  private Set existingTimers = new HashSet<>();
-
-  // Keep these queues separate so we can advance over them separately.
-  private PriorityQueue watermarkTimers = new PriorityQueue<>(11);
-  private PriorityQueue processingTimers = new PriorityQueue<>(11);
-
-  private Instant inputWatermarkTime;
-  private Instant processingTime;
-
-  private PriorityQueue queue(TimeDomain domain) {
-return TimeDomain.EVENT_TIME.equals(domain) ? watermarkTimers : 
processingTimers;
-  }
-
-  public BatchTimerInterna

[2/2] incubator-beam git commit: Closes #1023

2016-09-30 Thread bchambers

Closes #1023


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/0d229f5a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/0d229f5a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/0d229f5a

Branch: refs/heads/master
Commit: 0d229f5a69059024377202e09eab7d2a2f332a84
Parents: e7684b7 425c778
Author: bchambers <bchamb...@google.com>
Authored: Fri Sep 30 11:12:39 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Fri Sep 30 11:12:39 2016 -0700

--
 .../beam/runners/core/BatchTimerInternals.java  | 140 ---
 .../GroupAlsoByWindowsViaOutputBufferDoFn.java  |   7 +-
 .../beam/runners/core/ReduceFnRunner.java   |   3 +-
 .../runners/core/BatchTimerInternalsTest.java   | 118 -
 .../beam/runners/core/ReduceFnTester.java   | 248 ++-
 .../sdk/util/state/InMemoryTimerInternals.java  | 235 ++
 .../util/state/TestInMemoryStateInternals.java  |  61 +
 .../beam/sdk/util/state/TimerCallback.java  |  35 +++
 .../org/apache/beam/sdk/util/TriggerTester.java | 206 +--
 .../util/state/InMemoryTimerInternalsTest.java  | 116 +
 10 files changed, 482 insertions(+), 687 deletions(-)
--

[2/3] incubator-beam git commit: More changes to DoFn{Signatures, Invokers}

2016-09-12 Thread bchambers

More changes to DoFn{Signatures,Invokers}

In preparation for Splittable DoFn:
* More generic code generation in DoFnInvokers:
  supports methods with return values (thanks @bjchambers).
* Uses AutoValue builder in DoFnSignature.
* Contextual error reporting in DoFnSignatures parsing code.
* Rewrote DoFnInvokers tests to use Mockito.
* Changed DoFnSignatures tests to use local classes
  and an "AnonymousMethod" class for testing analysis of
  single methods.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/75c8bb8d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/75c8bb8d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/75c8bb8d

Branch: refs/heads/master
Commit: 75c8bb8dd3127b4dce63e8359ae27185f246b28c
Parents: 05c6c27
Author: Eugene Kirpichov <kirpic...@google.com>
Authored: Thu Aug 11 17:13:53 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Mon Sep 12 10:08:57 2016 -0700

--
 .../beam/sdk/transforms/DoFnAdapters.java   |   2 +-
 .../sdk/transforms/reflect/DoFnInvoker.java |  20 +-
 .../sdk/transforms/reflect/DoFnInvokers.java| 408 +
 .../sdk/transforms/reflect/DoFnSignature.java   |  71 ++-
 .../sdk/transforms/reflect/DoFnSignatures.java  | 245 +-
 .../beam/sdk/util/common/ReflectHelpers.java|  22 -
 .../transforms/reflect/DoFnInvokersTest.java| 455 ++-
 .../reflect/DoFnInvokersTestHelper.java | 116 -
 .../DoFnSignaturesProcessElementTest.java   | 213 +
 .../transforms/reflect/DoFnSignaturesTest.java  | 269 +--
 .../reflect/DoFnSignaturesTestUtils.java|  64 +++
 .../testhelper/DoFnInvokersTestHelper.java  | 124 +
 12 files changed, 968 insertions(+), 1041 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/75c8bb8d/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnAdapters.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnAdapters.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnAdapters.java
index 4803d77..77a71e9 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnAdapters.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnAdapters.java
@@ -86,7 +86,7 @@ public class DoFnAdapters {
 
 @Override
 public void startBundle(Context c) throws Exception {
-  this.fn.prepareForProcessing();
+  fn.prepareForProcessing();
   invoker.invokeStartBundle(new ContextAdapter<>(fn, c));
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/75c8bb8d/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
index 5818a59..9de6759 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
@@ -26,28 +26,16 @@ import org.apache.beam.sdk.transforms.DoFn;
  * referred to as the bound {@link DoFn}.
  */
 public interface DoFnInvoker<InputT, OutputT> {
-  /**
-   * Invoke the {@link DoFn.Setup} method on the bound {@link DoFn}.
-   */
+  /** Invoke the {@link DoFn.Setup} method on the bound {@link DoFn}. */
   void invokeSetup();
 
-  /**
-   * Invoke the {@link DoFn.StartBundle} method on the bound {@link DoFn}.
-   *
-   * @param c The {@link DoFn.Context} to invoke the fn with.
-   */
+  /** Invoke the {@link DoFn.StartBundle} method on the bound {@link DoFn}. */
   void invokeStartBundle(DoFn<InputT, OutputT>.Context c);
 
-  /**
-   * Invoke the {@link DoFn.FinishBundle} method on the bound {@link DoFn}.
-   *
-   * @param c The {@link DoFn.Context} to invoke the fn with.
-   */
+  /** Invoke the {@link DoFn.FinishBundle} method on the bound {@link DoFn}. */
   void invokeFinishBundle(DoFn<InputT, OutputT>.Context c);
 
-  /**
-   * Invoke the {@link DoFn.Teardown} method on the bound {@link DoFn}.
-   */
+  /** Invoke the {@link DoFn.Teardown} method on the bound {@link DoFn}. */
   void invokeTeardown();
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/75c8bb8d/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
 
b/sd

[1/3] incubator-beam git commit: More changes to DoFn{Signatures, Invokers}

2016-09-12 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 05c6c2749 -> b304d037f


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/75c8bb8d/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
--
diff --git 
a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
 
b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
index 447b993..fc468c9 100644
--- 
a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
+++ 
b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
@@ -17,10 +17,11 @@
  */
 package org.apache.beam.sdk.transforms.reflect;
 
+import static 
org.apache.beam.sdk.transforms.reflect.DoFnSignaturesTestUtils.errors;
+
 import com.google.common.reflect.TypeToken;
-import java.lang.reflect.Method;
-import java.util.List;
 import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignaturesTestUtils.FakeDoFn;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -33,257 +34,21 @@ public class DoFnSignaturesTest {
 
   @Rule public ExpectedException thrown = ExpectedException.none();
 
-  private static class FakeDoFn extends DoFn {}
-
-  @SuppressWarnings({"unused"})
-  private void missingProcessContext() {}
-
-  @Test
-  public void testMissingProcessContext() throws Exception {
-thrown.expect(IllegalArgumentException.class);
-thrown.expectMessage(
-getClass().getName()
-+ "#missingProcessContext() must take a ProcessContext<> as its 
first argument");
-
-DoFnSignatures.analyzeProcessElementMethod(
-TypeToken.of(FakeDoFn.class),
-getClass().getDeclaredMethod("missingProcessContext"),
-TypeToken.of(Integer.class),
-TypeToken.of(String.class));
-  }
-
-  @SuppressWarnings({"unused"})
-  private void badProcessContext(String s) {}
-
-  @Test
-  public void testBadProcessContextType() throws Exception {
-thrown.expect(IllegalArgumentException.class);
-thrown.expectMessage(
-getClass().getName()
-+ "#badProcessContext(String) must take a ProcessContext<> as its 
first argument");
-
-DoFnSignatures.analyzeProcessElementMethod(
-TypeToken.of(FakeDoFn.class),
-getClass().getDeclaredMethod("badProcessContext", String.class),
-TypeToken.of(Integer.class),
-TypeToken.of(String.class));
-  }
-
-  @SuppressWarnings({"unused"})
-  private void badExtraContext(DoFn.Context c, int n) {}
-
   @Test
   public void testBadExtraContext() throws Exception {
 thrown.expect(IllegalArgumentException.class);
-thrown.expectMessage(
-getClass().getName()
-+ "#badExtraContext(Context, int) must have a single argument of 
type Context");
+thrown.expectMessage("Must take a single argument of type Context");
 
 DoFnSignatures.analyzeBundleMethod(
+errors(),
 TypeToken.of(FakeDoFn.class),
-getClass().getDeclaredMethod("badExtraContext", DoFn.Context.class, 
int.class),
+new DoFnSignaturesTestUtils.AnonymousMethod() {
+  void method(DoFn.Context c, int n) {}
+}.getMethod(),
 TypeToken.of(Integer.class),
 TypeToken.of(String.class));
   }
 
-  @SuppressWarnings({"unused"})
-  private void badExtraProcessContext(DoFn.ProcessContext c, 
Integer n) {}
-
-  @Test
-  public void testBadExtraProcessContextType() throws Exception {
-thrown.expect(IllegalArgumentException.class);
-thrown.expectMessage(
-"Integer is not a valid context parameter for method "
-+ getClass().getName()
-+ "#badExtraProcessContext(ProcessContext, Integer)"
-+ ". Should be one of [BoundedWindow]");
-
-DoFnSignatures.analyzeProcessElementMethod(
-TypeToken.of(FakeDoFn.class),
-getClass()
-.getDeclaredMethod("badExtraProcessContext", 
DoFn.ProcessContext.class, Integer.class),
-TypeToken.of(Integer.class),
-TypeToken.of(String.class));
-  }
-
-  @SuppressWarnings("unused")
-  private int badReturnType() {
-return 0;
-  }
-
-  @Test
-  public void testBadReturnType() throws Exception {
-thrown.expect(IllegalArgumentException.class);
-thrown.expectMessage(getClass().getName() + "#badReturnType() must have a 
void return type");
-
-DoFnSignatures.analyzeProcessElementMethod(
-TypeToken.of(FakeDoFn.class),
-getClass().getDeclaredMethod("badReturnType"),
-TypeToken.of(Integer.class),
-TypeToken.of(String.class));
-  }
-
-  @SuppressWarnings("unused")
-  private void goodConcreteTypes(
-  DoFn.ProcessContext c,
-  DoFn.InputProvider input,
-

[2/2] incubator-beam git commit: Remove EvaluationContext as a forApplication Parameter

2016-09-12 Thread bchambers

Remove EvaluationContext as a forApplication Parameter

Instead use it as a paraemmter to the Evaluator Factory. Evaluator
Factories must not be reused across pipelines, as they may be stateful.
Evaluation Contexts are representative of a single execution of a
Pipeline and thus can be passed at construction time.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/c2970aa7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/c2970aa7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/c2970aa7

Branch: refs/heads/master
Commit: c2970aa7cc10e5e2d4bdc9c939a30df686c41ad2
Parents: 643cf63
Author: Thomas Groh <tg...@google.com>
Authored: Thu Sep 8 13:41:52 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Mon Sep 12 09:53:16 2016 -0700

--
 .../direct/BoundedReadEvaluatorFactory.java | 30 
 .../beam/runners/direct/DirectRunner.java   |  2 +-
 .../direct/ExecutorServiceParallelExecutor.java |  1 -
 .../runners/direct/FlattenEvaluatorFactory.java | 15 ++--
 .../GroupAlsoByWindowEvaluatorFactory.java  | 14 ++--
 .../direct/GroupByKeyOnlyEvaluatorFactory.java  | 14 ++--
 .../direct/ParDoMultiEvaluatorFactory.java  | 13 ++--
 .../direct/ParDoSingleEvaluatorFactory.java | 13 ++--
 .../direct/TestStreamEvaluatorFactory.java  | 13 ++--
 .../direct/TransformEvaluatorFactory.java   | 22 +++---
 .../direct/TransformEvaluatorRegistry.java  | 37 --
 .../beam/runners/direct/TransformExecutor.java  |  8 +--
 .../direct/UnboundedReadEvaluatorFactory.java   | 46 ++--
 .../runners/direct/ViewEvaluatorFactory.java| 14 ++--
 .../runners/direct/WindowEvaluatorFactory.java  | 14 ++--
 .../direct/BoundedReadEvaluatorFactoryTest.java | 16 ++---
 .../direct/FlattenEvaluatorFactoryTest.java | 15 ++--
 .../direct/GroupByKeyEvaluatorFactoryTest.java  |  5 +-
 .../GroupByKeyOnlyEvaluatorFactoryTest.java |  4 +-
 .../direct/ParDoMultiEvaluatorFactoryTest.java  | 16 ++---
 .../direct/ParDoSingleEvaluatorFactoryTest.java | 16 ++---
 .../direct/TestStreamEvaluatorFactoryTest.java  | 35 +
 .../runners/direct/TransformExecutorTest.java   | 75 +++-
 .../UnboundedReadEvaluatorFactoryTest.java  | 24 +++
 .../direct/ViewEvaluatorFactoryTest.java|  4 +-
 .../direct/WindowEvaluatorFactoryTest.java  |  6 +-
 26 files changed, 231 insertions(+), 241 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c2970aa7/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
--
diff --git 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
index 2b15ad0..2046d31 100644
--- 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
+++ 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
@@ -45,33 +45,35 @@ final class BoundedReadEvaluatorFactory implements 
TransformEvaluatorFactory {
* retriggered.
*/
   private final ConcurrentMap<AppliedPTransform, Queue>>
-  sourceEvaluators = new ConcurrentHashMap<>();
+  sourceEvaluators;
+  private final EvaluationContext evaluationContext;
+
+  BoundedReadEvaluatorFactory(EvaluationContext evaluationContext) {
+this.evaluationContext = evaluationContext;
+sourceEvaluators = new ConcurrentHashMap<>();
+  }
 
   @SuppressWarnings({"unchecked", "rawtypes"})
   @Override
   @Nullable
   public  TransformEvaluator forApplication(
-  AppliedPTransform application,
-  @Nullable CommittedBundle inputBundle,
-  EvaluationContext evaluationContext)
+  AppliedPTransform application, @Nullable CommittedBundle 
inputBundle)
   throws IOException {
-return getTransformEvaluator((AppliedPTransform) application, 
evaluationContext);
+return getTransformEvaluator((AppliedPTransform) application);
   }
 
   @Override
-  public void cleanup() {
-  }
+  public void cleanup() {}
 
   /**
-   * Get a {@link TransformEvaluator} that produces elements for the provided 
application of
-   * {@link Bounded Read.Bounded}, initializing the queue of evaluators if 
required.
+   * Get a {@link TransformEvaluator} that produces elements for the provided 
application of {@link
+   * Bounded Read.Bounded}, initializing the queue of evaluators if required.
*
* This method is thread-safe, and will only produce new evaluators if no 
other invocation has
* already done so.
*/
   private  T

[1/2] incubator-beam git commit: Closes #937

2016-09-12 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 643cf63d5 -> 05c6c2749


Closes #937


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/05c6c274
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/05c6c274
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/05c6c274

Branch: refs/heads/master
Commit: 05c6c2749f667c9e90569dd2933b17f4385f931c
Parents: 643cf63 c2970aa
Author: bchambers <bchamb...@google.com>
Authored: Mon Sep 12 09:53:16 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Mon Sep 12 09:53:16 2016 -0700

--
 .../direct/BoundedReadEvaluatorFactory.java | 30 
 .../beam/runners/direct/DirectRunner.java   |  2 +-
 .../direct/ExecutorServiceParallelExecutor.java |  1 -
 .../runners/direct/FlattenEvaluatorFactory.java | 15 ++--
 .../GroupAlsoByWindowEvaluatorFactory.java  | 14 ++--
 .../direct/GroupByKeyOnlyEvaluatorFactory.java  | 14 ++--
 .../direct/ParDoMultiEvaluatorFactory.java  | 13 ++--
 .../direct/ParDoSingleEvaluatorFactory.java | 13 ++--
 .../direct/TestStreamEvaluatorFactory.java  | 13 ++--
 .../direct/TransformEvaluatorFactory.java   | 22 +++---
 .../direct/TransformEvaluatorRegistry.java  | 37 --
 .../beam/runners/direct/TransformExecutor.java  |  8 +--
 .../direct/UnboundedReadEvaluatorFactory.java   | 46 ++--
 .../runners/direct/ViewEvaluatorFactory.java| 14 ++--
 .../runners/direct/WindowEvaluatorFactory.java  | 14 ++--
 .../direct/BoundedReadEvaluatorFactoryTest.java | 16 ++---
 .../direct/FlattenEvaluatorFactoryTest.java | 15 ++--
 .../direct/GroupByKeyEvaluatorFactoryTest.java  |  5 +-
 .../GroupByKeyOnlyEvaluatorFactoryTest.java |  4 +-
 .../direct/ParDoMultiEvaluatorFactoryTest.java  | 16 ++---
 .../direct/ParDoSingleEvaluatorFactoryTest.java | 16 ++---
 .../direct/TestStreamEvaluatorFactoryTest.java  | 35 +
 .../runners/direct/TransformExecutorTest.java   | 75 +++-
 .../UnboundedReadEvaluatorFactoryTest.java  | 24 +++
 .../direct/ViewEvaluatorFactoryTest.java|  4 +-
 .../direct/WindowEvaluatorFactoryTest.java  |  6 +-
 26 files changed, 231 insertions(+), 241 deletions(-)
--

[1/2] incubator-beam git commit: Add Latest CombineFn and PTransforms

2016-09-01 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master f65795661 -> 268ec11c6


Add Latest CombineFn and PTransforms

Add DoFnTester support for specifying input timestamps


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/0d3f2f2b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/0d3f2f2b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/0d3f2f2b

Branch: refs/heads/master
Commit: 0d3f2f2b7300f7a03b7f94c927f8160b8cd81798
Parents: f657956
Author: Scott Wegner <sweg...@google.com>
Authored: Thu Aug 18 13:56:34 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Thu Sep 1 13:16:02 2016 -0700

--
 .../apache/beam/sdk/coders/NullableCoder.java   |   7 +
 .../apache/beam/sdk/transforms/DoFnTester.java  |  33 ++-
 .../org/apache/beam/sdk/transforms/Latest.java  | 203 
 .../beam/sdk/values/TimestampedValue.java   |  14 ++
 .../beam/sdk/transforms/DoFnTesterTest.java |  34 ++-
 .../beam/sdk/transforms/LatestFnTests.java  | 233 +++
 .../apache/beam/sdk/transforms/LatestTest.java  | 146 
 .../beam/sdk/values/TimestampedValueTest.java   |  83 +++
 8 files changed, 747 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0d3f2f2b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java
index 44aadbd..9c6c7c0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java
@@ -65,6 +65,13 @@ public class NullableCoder extends StandardCoder {
 this.valueCoder = valueCoder;
   }
 
+  /**
+   * Returns the inner {@link Coder} wrapped by this {@link NullableCoder} 
instance.
+   */
+  public Coder getValueCoder() {
+return valueCoder;
+  }
+
   @Override
   public void encode(@Nullable T value, OutputStream outStream, Context 
context)
   throws IOException, CoderException  {

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0d3f2f2b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
index b867a55..0e018ba 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
@@ -17,6 +17,9 @@
  */
 package org.apache.beam.sdk.transforms;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+
 import com.google.common.base.Function;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableList;
@@ -221,9 +224,26 @@ public class DoFnTester<InputT, OutputT> {
* been finished
*/
   public void processElement(InputT element) throws Exception {
-if (state == State.FINISHED) {
-  throw new IllegalStateException("finishBundle() has already been 
called");
-}
+processTimestampedElement(TimestampedValue.atMinimumTimestamp(element));
+  }
+
+  /**
+   * Calls {@link OldDoFn#processElement} on the {@code OldDoFn} under test, 
in a
+   * context where {@link OldDoFn.ProcessContext#element} returns the
+   * given element and timestamp.
+   *
+   * Will call {@link #startBundle} automatically, if it hasn't
+   * already been called.
+   *
+   * If the input timestamp is {@literal null}, the minimum timestamp will 
be used.
+   *
+   * @throws IllegalStateException if the {@code OldDoFn} under test has 
already
+   * been finished
+   */
+  public void processTimestampedElement(TimestampedValue element) 
throws Exception {
+checkNotNull(element, "Timestamped element cannot be null");
+checkState(state != State.FINISHED, "finishBundle() has already been 
called");
+
 if (state == State.UNSTARTED) {
   startBundle();
 }
@@ -522,10 +542,13 @@ public class DoFnTester<InputT, OutputT> {
 
   private TestProcessContext<InputT, OutputT> createProcessContext(
   OldDoFn<InputT, OutputT> fn,
-  InputT elem) {
+  TimestampedValue elem) {
+WindowedValue windowedValue = 
WindowedValue.timestampedValueInGlobalWindow(
+elem.getValue(), elem.getTimestamp());
+
 return new TestProcessContext<>

[2/2] incubator-beam git commit: Closes #862

2016-09-01 Thread bchambers

Closes #862


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/268ec11c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/268ec11c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/268ec11c

Branch: refs/heads/master
Commit: 268ec11c62f0803e2eef5c5f1715b270d261c3a6
Parents: f657956 0d3f2f2
Author: bchambers <bchamb...@google.com>
Authored: Thu Sep 1 13:16:47 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Thu Sep 1 13:16:47 2016 -0700

--
 .../apache/beam/sdk/coders/NullableCoder.java   |   7 +
 .../apache/beam/sdk/transforms/DoFnTester.java  |  33 ++-
 .../org/apache/beam/sdk/transforms/Latest.java  | 203 
 .../beam/sdk/values/TimestampedValue.java   |  14 ++
 .../beam/sdk/transforms/DoFnTesterTest.java |  34 ++-
 .../beam/sdk/transforms/LatestFnTests.java  | 233 +++
 .../apache/beam/sdk/transforms/LatestTest.java  | 146 
 .../beam/sdk/values/TimestampedValueTest.java   |  83 +++
 8 files changed, 747 insertions(+), 6 deletions(-)
--

[1/2] incubator-beam git commit: Remove empty unused method in TestStreamEvaluatorFactory

2016-08-31 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master a5320607a -> f65795661


Remove empty unused method in TestStreamEvaluatorFactory


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ccfb78ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ccfb78ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ccfb78ea

Branch: refs/heads/master
Commit: ccfb78eac4d9b992d8694ad1f6347f50d80169c1
Parents: a532060
Author: Thomas Groh 
Authored: Wed Aug 31 15:34:21 2016 -0700
Committer: Thomas Groh 
Committed: Wed Aug 31 15:34:21 2016 -0700

--
 .../beam/runners/direct/TestStreamEvaluatorFactory.java   | 7 ---
 1 file changed, 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ccfb78ea/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
--
diff --git 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
index 3dbd886..5fe771c 100644
--- 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
+++ 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
@@ -84,13 +84,6 @@ class TestStreamEvaluatorFactory implements 
TransformEvaluatorFactory {
 .orNull();
   }
 
-  /**
-   * Release the provided {@link Evaluator} after completing an evaluation. 
The next call to {@link
-   * #createEvaluator(AppliedPTransform, EvaluationContext)} with the {@link 
AppliedPTransform} will
-   * return this evaluator.
-   */
-  private void completeEvaluation(Evaluator evaluator) {}
-
   private static class Evaluator implements TransformEvaluator {
 private final AppliedPTransform 
application;
 private final EvaluationContext context;

[2/2] incubator-beam git commit: Closes #910

2016-08-31 Thread bchambers

Closes #910


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/f6579566
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/f6579566
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/f6579566

Branch: refs/heads/master
Commit: f65795661a9d9b20c757791b953bd343b461b582
Parents: a532060 ccfb78e
Author: bchambers <bchamb...@google.com>
Authored: Wed Aug 31 15:54:20 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Aug 31 15:54:20 2016 -0700

--
 .../beam/runners/direct/TestStreamEvaluatorFactory.java   | 7 ---
 1 file changed, 7 deletions(-)
--

[1/2] incubator-beam git commit: Test that multiple instances of TestStream are supported

2016-08-31 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 7dcb4c72c -> a5320607a


Test that multiple instances of TestStream are supported

Add KeyedResourcePool

This interface represents some shared pool of values that may be used by
at most one caller at a time.

Add LockedKeyedResourcePool which has at most one value per key and
at most one user per value at a time.

Use KeyedResourcePool in TestStream


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/89680975
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/89680975
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/89680975

Branch: refs/heads/master
Commit: 89680975b5a89351ccc4bf99a3a6bd8772d87f40
Parents: 7dcb4c7
Author: Thomas Groh <tg...@google.com>
Authored: Tue Aug 30 14:17:50 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Aug 31 15:00:39 2016 -0700

--
 .../beam/runners/direct/KeyedResourcePool.java  |  47 +
 .../runners/direct/LockedKeyedResourcePool.java |  95 +
 .../direct/TestStreamEvaluatorFactory.java  | 141 +++--
 .../direct/LockedKeyedResourcePoolTest.java | 163 +++
 .../direct/TestStreamEvaluatorFactoryTest.java  | 206 +++
 .../apache/beam/sdk/testing/TestStreamTest.java |  29 +++
 6 files changed, 623 insertions(+), 58 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/89680975/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedResourcePool.java
--
diff --git 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedResourcePool.java
 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedResourcePool.java
new file mode 100644
index 000..b976b69
--- /dev/null
+++ 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedResourcePool.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.direct;
+
+import com.google.common.base.Optional;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * A pool of resources associated with specific keys. Implementations enforce 
specific use patterns,
+ * such as limiting the the number of outstanding elements available per key.
+ */
+interface KeyedResourcePool<K, V> {
+  /**
+   * Tries to acquire a value for the provided key, loading it via the 
provided loader if necessary.
+   *
+   * If the returned {@link Optional} contains a value, the caller obtains 
ownership of that
+   * value. The value should be released back to this {@link 
KeyedResourcePool} after the
+   * caller no longer has use of it using {@link #release(Object, Object)}.
+   *
+   * The provided {@link Callable} must not return null; it may 
either return a non-null
+   * value or throw an exception.
+   */
+  Optional tryAcquire(K key, Callable loader) throws ExecutionException;
+
+  /**
+   * Release the provided value, relinquishing ownership of it. Future calls to
+   * {@link #tryAcquire(Object, Callable)} may return the released value.
+   */
+  void release(K key, V value);
+}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/89680975/runners/direct-java/src/main/java/org/apache/beam/runners/direct/LockedKeyedResourcePool.java
--
diff --git 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/LockedKeyedResourcePool.java
 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/LockedKeyedResourcePool.java
new file mode 100644
index 000..8b1e0b1
--- /dev/null
+++ 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/LockedKeyedResourcePool.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed

[2/2] incubator-beam git commit: Closes #907

2016-08-31 Thread bchambers

Closes #907


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a5320607
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a5320607
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a5320607

Branch: refs/heads/master
Commit: a5320607af10dd6b45440384b8afbbc8ad9889b7
Parents: 7dcb4c7 8968097
Author: bchambers <bchamb...@google.com>
Authored: Wed Aug 31 15:01:53 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Aug 31 15:01:53 2016 -0700

--
 .../beam/runners/direct/KeyedResourcePool.java  |  47 +
 .../runners/direct/LockedKeyedResourcePool.java |  95 +
 .../direct/TestStreamEvaluatorFactory.java  | 141 +++--
 .../direct/LockedKeyedResourcePoolTest.java | 163 +++
 .../direct/TestStreamEvaluatorFactoryTest.java  | 206 +++
 .../apache/beam/sdk/testing/TestStreamTest.java |  29 +++
 6 files changed, 623 insertions(+), 58 deletions(-)
--

[1/2] incubator-beam git commit: Correct some accidental renames

2016-08-31 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 98da6e8fb -> 7dcb4c72c


Correct some accidental renames

IDE over-eagerly replaced some occurrences of createAggregator with
createAggregatorForDoFn. This corrects that.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/f70aa49e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/f70aa49e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/f70aa49e

Branch: refs/heads/master
Commit: f70aa49e2babc79a65a339309776837be2a45126
Parents: 98da6e8
Author: bchambers <bchamb...@google.com>
Authored: Wed Aug 3 13:38:43 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Aug 31 10:23:29 2016 -0700

--
 .../main/java/org/apache/beam/sdk/util/DoFnRunnerBase.java   | 2 +-
 .../main/java/org/apache/beam/sdk/transforms/Aggregator.java | 8 
 .../main/java/org/apache/beam/sdk/transforms/DoFnTester.java | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f70aa49e/runners/core-java/src/main/java/org/apache/beam/sdk/util/DoFnRunnerBase.java
--
diff --git 
a/runners/core-java/src/main/java/org/apache/beam/sdk/util/DoFnRunnerBase.java 
b/runners/core-java/src/main/java/org/apache/beam/sdk/util/DoFnRunnerBase.java
index 04a0978..f0cfd74 100644
--- 
a/runners/core-java/src/main/java/org/apache/beam/sdk/util/DoFnRunnerBase.java
+++ 
b/runners/core-java/src/main/java/org/apache/beam/sdk/util/DoFnRunnerBase.java
@@ -344,7 +344,7 @@ public abstract class DoFnRunnerBase<InputT, OutputT> 
implements DoFnRunner Aggregator<AggInputT, AggOutputT> 
createAggregatorInternal(
 String name, CombineFn<AggInputT, ?, AggOutputT> combiner) {
-  checkNotNull(combiner, "Combiner passed to createAggregatorForDoFn 
cannot be null");
+  checkNotNull(combiner, "Combiner passed to createAggregatorInternal 
cannot be null");
   return aggregatorFactory.createAggregatorForDoFn(fn.getClass(), 
stepContext, name, combiner);
 }
   }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f70aa49e/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java
index 67d399f..e8f6247 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java
@@ -25,7 +25,7 @@ import org.apache.beam.sdk.util.ExecutionContext;
  * to be combined across all bundles.
  *
  * Aggregators are created by calling
- * {@link DoFn#createAggregator DoFn.createAggregatorForDoFn},
+ * {@link DoFn#createAggregator DoFn.createAggregator},
  * typically from the {@link DoFn} constructor. Elements can be added to the
  * {@code Aggregator} by calling {@link Aggregator#addValue}.
  *
@@ -41,7 +41,7 @@ import org.apache.beam.sdk.util.ExecutionContext;
  *   private Aggregator<Integer, Integer> myAggregator;
  *
  *   public MyDoFn() {
- * myAggregator = createAggregatorForDoFn("myAggregator", new 
Sum.SumIntegerFn());
+ * myAggregator = createAggregator("myAggregator", new Sum.SumIntegerFn());
  *   }
  *
  *   @ProcessElement
@@ -89,9 +89,9 @@ public interface Aggregator<InputT, OutputT> {
   }
 
   // TODO: Consider the following additional API conveniences:
-  // - In addition to createAggregatorForDoFn(), consider adding 
getAggregator() to
+  // - In addition to createAggregator(), consider adding getAggregator() to
   //   avoid the need to store the aggregator locally in a DoFn, i.e., create
   //   if not already present.
   // - Add a shortcut for the most common aggregator:
-  //   c.createAggregatorForDoFn("name", new Sum.SumIntegerFn()).
+  //   c.createAggregator("name", new Sum.SumIntegerFn()).
 }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f70aa49e/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
index 6801768..b867a55 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
@@ -667,7 +667,7 @@ public class DoFnTester&

[2/2] incubator-beam git commit: Closes #778

2016-08-31 Thread bchambers

Closes #778


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/7dcb4c72
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/7dcb4c72
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/7dcb4c72

Branch: refs/heads/master
Commit: 7dcb4c72cd7e2be0ccaa30424226a3451e903f76
Parents: 98da6e8 f70aa49
Author: bchambers <bchamb...@google.com>
Authored: Wed Aug 31 10:23:30 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Aug 31 10:23:30 2016 -0700

--
 .../main/java/org/apache/beam/sdk/util/DoFnRunnerBase.java   | 2 +-
 .../main/java/org/apache/beam/sdk/transforms/Aggregator.java | 8 
 .../main/java/org/apache/beam/sdk/transforms/DoFnTester.java | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)
--

[1/2] incubator-beam git commit: Closes #756

2016-08-17 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 89367cfb1 -> d93ef2edd


Closes #756


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/d93ef2ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/d93ef2ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/d93ef2ed

Branch: refs/heads/master
Commit: d93ef2edd260a2077bc2ba6abd1ca02abd147a9a
Parents: 89367cf 236945d
Author: bchambers <bchamb...@google.com>
Authored: Wed Aug 17 16:09:01 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Aug 17 16:09:01 2016 -0700

--
 .../org/apache/beam/sdk/transforms/Combine.java | 28 ++--
 .../org/apache/beam/sdk/transforms/Count.java   |  8 +++---
 .../beam/sdk/transforms/FlatMapElements.java|  4 +--
 .../org/apache/beam/sdk/transforms/Flatten.java | 12 -
 .../org/apache/beam/sdk/transforms/Keys.java|  8 +++---
 .../org/apache/beam/sdk/transforms/KvSwap.java  |  9 +++
 .../apache/beam/sdk/transforms/MapElements.java | 16 ---
 .../beam/sdk/transforms/RemoveDuplicates.java   |  8 +++---
 .../org/apache/beam/sdk/transforms/Values.java  |  8 +++---
 .../apache/beam/sdk/transforms/WithKeys.java|  9 +++
 .../beam/sdk/transforms/windowing/Window.java   | 11 
 .../java/org/apache/beam/sdk/PipelineTest.java  | 12 -
 .../java/org/apache/beam/sdk/io/WriteTest.java  |  4 ++-
 .../beam/sdk/transforms/MapElementsTest.java|  8 +++---
 .../org/apache/beam/sdk/io/kafka/KafkaIO.java   | 10 ---
 15 files changed, 81 insertions(+), 74 deletions(-)
--

[3/4] incubator-beam git commit: Rewrites DoFnReflector to go via DoFnSignature

2016-08-17 Thread bchambers

Rewrites DoFnReflector to go via DoFnSignature

DoFnSignature encapsulates type information about a DoFn,
in particular which arguments/features its methods
actually use.

Before this commit, DoFnReflector would parse/verify/generate
code in one go; after this commit, these stages are separated:
DoFnSignature encapsulates all information needed to generate
the code.

Additionally, removes the unnecessary genericity in the
implementation of DoFnReflector's code generation for the
very different methods processElement and start/finishBundle.
The code is simpler if decomposed into utility functions,
rather than attempting a uniform representation for different
methods.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/fbf77f90
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/fbf77f90
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/fbf77f90

Branch: refs/heads/master
Commit: fbf77f90e0391304a580178f99441256526c4b0e
Parents: 4609773
Author: Eugene Kirpichov <kirpic...@google.com>
Authored: Tue Aug 9 17:16:00 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Aug 17 15:43:46 2016 -0700

--
 .../org/apache/beam/sdk/transforms/DoFn.java|   17 +-
 .../beam/sdk/transforms/DoFnAdapters.java   |  281 +
 .../beam/sdk/transforms/DoFnReflector.java  | 1150 --
 .../apache/beam/sdk/transforms/DoFnTester.java  |2 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   |6 +-
 .../sdk/transforms/reflect/DoFnInvoker.java |   61 +
 .../sdk/transforms/reflect/DoFnInvokers.java|  506 
 .../sdk/transforms/reflect/DoFnSignature.java   |  113 ++
 .../sdk/transforms/reflect/DoFnSignatures.java  |  321 +
 .../sdk/transforms/reflect/package-info.java|   23 +
 .../beam/sdk/transforms/DoFnReflectorTest.java  |  822 -
 .../apache/beam/sdk/transforms/FlattenTest.java |4 +-
 .../dofnreflector/DoFnReflectorTestHelper.java  |  116 --
 .../transforms/reflect/DoFnInvokersTest.java|  498 
 .../reflect/DoFnInvokersTestHelper.java |  116 ++
 .../transforms/reflect/DoFnSignaturesTest.java  |  371 ++
 .../transforms/DoFnInvokersBenchmark.java   |  224 
 .../transforms/DoFnReflectorBenchmark.java  |  232 
 18 files changed, 2529 insertions(+), 2334 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/fbf77f90/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
index 80b67af..2348783 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
@@ -26,6 +26,7 @@ import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.transforms.OldDoFn.DelegatingAggregator;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.transforms.display.HasDisplayData;
+import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.values.PCollectionView;
@@ -247,7 +248,7 @@ public abstract class DoFn<InputT, OutputT> implements 
Serializable, HasDisplayD
 
   /
 
-  Map<String, DelegatingAggregator> aggregators = new HashMap<>();
+  protected Map<String, DelegatingAggregator> aggregators = new 
HashMap<>();
 
   /**
* Protects aggregators from being created after initialization.
@@ -283,7 +284,7 @@ public abstract class DoFn<InputT, OutputT> implements 
Serializable, HasDisplayD
   /**
* Interface for runner implementors to provide implementations of extra 
context information.
*
-   * The methods on this interface are called by {@link DoFnReflector} 
before invoking an
+   * The methods on this interface are called by {@link DoFnInvoker} before 
invoking an
* annotated {@link StartBundle}, {@link ProcessElement} or {@link 
FinishBundle} method that
* has indicated it needs the given extra context.
*
@@ -301,23 +302,23 @@ public abstract class DoFn<InputT, OutputT> implements 
Serializable, HasDisplayD
 BoundedWindow window();
 
 /**
- * A placeholder for testing purposes. The return type itself is 
package-private and not
- * implemented.
+ * A placeholder for testing purposes.
  */
 InputProvider inputProvider();
 
 /**
- * A pl

[2/4] incubator-beam git commit: Rewrites DoFnReflector to go via DoFnSignature

2016-08-17 Thread bchambers

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/fbf77f90/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
new file mode 100644
index 000..6730140
--- /dev/null
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms.reflect;
+
+import org.apache.beam.sdk.transforms.DoFn;
+
+import com.google.auto.value.AutoValue;
+
+import java.lang.reflect.Method;
+import java.util.Collections;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+/**
+ * Describes the signature of a {@link DoFn}, in particular, which features it 
uses, which extra
+ * context it requires, types of the input and output elements, etc.
+ *
+ * See https://s.apache.org/a-new-dofn;>A new DoFn.
+ */
+@AutoValue
+public abstract class DoFnSignature {
+  public abstract Class fnClass();
+
+  public abstract ProcessElementMethod processElement();
+
+  @Nullable
+  public abstract BundleMethod startBundle();
+
+  @Nullable
+  public abstract BundleMethod finishBundle();
+
+  @Nullable
+  public abstract LifecycleMethod setup();
+
+  @Nullable
+  public abstract LifecycleMethod teardown();
+
+  static DoFnSignature create(
+  Class fnClass,
+  ProcessElementMethod processElement,
+  @Nullable BundleMethod startBundle,
+  @Nullable BundleMethod finishBundle,
+  @Nullable LifecycleMethod setup,
+  @Nullable LifecycleMethod teardown) {
+return new AutoValue_DoFnSignature(
+fnClass,
+processElement,
+startBundle,
+finishBundle,
+setup,
+teardown);
+  }
+
+  /** Describes a {@link DoFn.ProcessElement} method. */
+  @AutoValue
+  public abstract static class ProcessElementMethod {
+enum Parameter {
+  BOUNDED_WINDOW,
+  INPUT_PROVIDER,
+  OUTPUT_RECEIVER
+}
+
+public abstract Method targetMethod();
+
+public abstract List extraParameters();
+
+static ProcessElementMethod create(Method targetMethod, List 
extraParameters) {
+  return new AutoValue_DoFnSignature_ProcessElementMethod(
+  targetMethod, Collections.unmodifiableList(extraParameters));
+}
+
+/** @return true if the reflected {@link DoFn} uses a Single Window. */
+public boolean usesSingleWindow() {
+  return extraParameters().contains(Parameter.BOUNDED_WINDOW);
+}
+  }
+
+  /** Describes a {@link DoFn.StartBundle} or {@link DoFn.FinishBundle} 
method. */
+  @AutoValue
+  public abstract static class BundleMethod {
+public abstract Method targetMethod();
+
+static BundleMethod create(Method targetMethod) {
+  return new AutoValue_DoFnSignature_BundleMethod(targetMethod);
+}
+  }
+
+  /** Describes a {@link DoFn.Setup} or {@link DoFn.Teardown} method. */
+  @AutoValue
+  public abstract static class LifecycleMethod {
+public abstract Method targetMethod();
+
+static LifecycleMethod create(Method targetMethod) {
+  return new AutoValue_DoFnSignature_LifecycleMethod(targetMethod);
+}
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/fbf77f90/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
new file mode 100644
index 000..80b3b4f
--- /dev/null
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF

[4/4] incubator-beam git commit: Closes #812

2016-08-17 Thread bchambers

Closes #812


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/89367cfb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/89367cfb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/89367cfb

Branch: refs/heads/master
Commit: 89367cfb19ae86d66441970277177512961d3b6a
Parents: 4609773 fbf77f9
Author: bchambers <bchamb...@google.com>
Authored: Wed Aug 17 15:43:47 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Aug 17 15:43:47 2016 -0700

--
 .../org/apache/beam/sdk/transforms/DoFn.java|   17 +-
 .../beam/sdk/transforms/DoFnAdapters.java   |  281 +
 .../beam/sdk/transforms/DoFnReflector.java  | 1150 --
 .../apache/beam/sdk/transforms/DoFnTester.java  |2 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   |6 +-
 .../sdk/transforms/reflect/DoFnInvoker.java |   61 +
 .../sdk/transforms/reflect/DoFnInvokers.java|  506 
 .../sdk/transforms/reflect/DoFnSignature.java   |  113 ++
 .../sdk/transforms/reflect/DoFnSignatures.java  |  321 +
 .../sdk/transforms/reflect/package-info.java|   23 +
 .../beam/sdk/transforms/DoFnReflectorTest.java  |  822 -
 .../apache/beam/sdk/transforms/FlattenTest.java |4 +-
 .../dofnreflector/DoFnReflectorTestHelper.java  |  116 --
 .../transforms/reflect/DoFnInvokersTest.java|  498 
 .../reflect/DoFnInvokersTestHelper.java |  116 ++
 .../transforms/reflect/DoFnSignaturesTest.java  |  371 ++
 .../transforms/DoFnInvokersBenchmark.java   |  224 
 .../transforms/DoFnReflectorBenchmark.java  |  232 
 18 files changed, 2529 insertions(+), 2334 deletions(-)
--

[1/4] incubator-beam git commit: Rewrites DoFnReflector to go via DoFnSignature

2016-08-17 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 46097736b -> 89367cfb1


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/fbf77f90/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
--
diff --git 
a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
 
b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
new file mode 100644
index 000..1a26df2
--- /dev/null
+++ 
b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
@@ -0,0 +1,371 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms.reflect;
+
+import org.apache.beam.sdk.transforms.DoFn;
+
+import com.google.common.reflect.TypeToken;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.lang.reflect.Method;
+import java.util.List;
+
+/** Tests for {@link DoFnSignatures}. */
+@RunWith(JUnit4.class)
+public class DoFnSignaturesTest {
+
+  @Rule public ExpectedException thrown = ExpectedException.none();
+
+  private static class FakeDoFn extends DoFn {}
+
+  @SuppressWarnings({"unused"})
+  private void missingProcessContext() {}
+
+  @Test
+  public void testMissingProcessContext() throws Exception {
+thrown.expect(IllegalArgumentException.class);
+thrown.expectMessage(
+getClass().getName()
++ "#missingProcessContext() must take a ProcessContext<> as its 
first argument");
+
+DoFnSignatures.analyzeProcessElementMethod(
+TypeToken.of(FakeDoFn.class),
+getClass().getDeclaredMethod("missingProcessContext"),
+TypeToken.of(Integer.class),
+TypeToken.of(String.class));
+  }
+
+  @SuppressWarnings({"unused"})
+  private void badProcessContext(String s) {}
+
+  @Test
+  public void testBadProcessContextType() throws Exception {
+thrown.expect(IllegalArgumentException.class);
+thrown.expectMessage(
+getClass().getName()
++ "#badProcessContext(String) must take a ProcessContext<> as its 
first argument");
+
+DoFnSignatures.analyzeProcessElementMethod(
+TypeToken.of(FakeDoFn.class),
+getClass().getDeclaredMethod("badProcessContext", String.class),
+TypeToken.of(Integer.class),
+TypeToken.of(String.class));
+  }
+
+  @SuppressWarnings({"unused"})
+  private void badExtraContext(DoFn.Context c, int n) {}
+
+  @Test
+  public void testBadExtraContext() throws Exception {
+thrown.expect(IllegalArgumentException.class);
+thrown.expectMessage(
+getClass().getName()
++ "#badExtraContext(Context, int) must have a single argument of 
type Context");
+
+DoFnSignatures.analyzeBundleMethod(
+TypeToken.of(FakeDoFn.class),
+getClass().getDeclaredMethod("badExtraContext", DoFn.Context.class, 
int.class),
+TypeToken.of(Integer.class),
+TypeToken.of(String.class));
+  }
+
+  @SuppressWarnings({"unused"})
+  private void badExtraProcessContext(DoFn.ProcessContext c, 
Integer n) {}
+
+  @Test
+  public void testBadExtraProcessContextType() throws Exception {
+thrown.expect(IllegalArgumentException.class);
+thrown.expectMessage(
+"Integer is not a valid context parameter for method "
++ getClass().getName()
++ "#badExtraProcessContext(ProcessContext, Integer)"
++ ". Should be one of [BoundedWindow]");
+
+DoFnSignatures.analyzeProcessElementMethod(
+TypeToken.of(FakeDoFn.class),
+getClass()
+.getDeclaredMethod("badExtraProcessContext", 
DoFn.ProcessContext.class, Integer.class),
+TypeToken.of(Integer.class),
+TypeToken.of(String.class));
+  }
+
+  @SuppressWarnings("unused")
+  private int badReturnType() {
+return 0;
+  }
+
+  @Test
+  public void testBadReturnType() throws Exception {
+thrown.expect(IllegalArgumentException.class);
+

[3/3] incubator-beam git commit: This closes #815

2016-08-11 Thread bchambers

This closes #815


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a0769ad2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a0769ad2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a0769ad2

Branch: refs/heads/master
Commit: a0769ad2a348c1296086b9dc8994e32ba5a06760
Parents: 3a858ee d20a7ad
Author: bchambers <bchamb...@google.com>
Authored: Thu Aug 11 10:28:04 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Thu Aug 11 11:06:56 2016 -0700

--
 .../org/apache/beam/sdk/util/DoFnRunners.java   |   78 --
 .../beam/runners/dataflow/DataflowRunner.java   |4 +-
 .../org/apache/beam/sdk/transforms/Combine.java |   13 -
 .../org/apache/beam/sdk/transforms/Max.java |   27 +-
 .../org/apache/beam/sdk/transforms/Min.java |   28 +-
 .../org/apache/beam/sdk/transforms/Sum.java |   27 +-
 .../apache/beam/sdk/util/CounterAggregator.java |  128 --
 .../apache/beam/sdk/util/common/Counter.java| 1287 --
 .../beam/sdk/util/common/CounterName.java   |  153 ---
 .../beam/sdk/util/common/CounterProvider.java   |   27 -
 .../apache/beam/sdk/util/common/CounterSet.java |  179 ---
 .../util/common/ElementByteSizeObserver.java|   24 +-
 .../beam/sdk/util/CounterAggregatorTest.java|  256 
 .../beam/sdk/util/common/CounterSetTest.java|  227 ---
 .../beam/sdk/util/common/CounterTest.java   |  736 --
 15 files changed, 15 insertions(+), 3179 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a0769ad2/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
--

[1/2] incubator-beam git commit: Demonstrate that the DirectRunner runs per-call

2016-08-10 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 6da92ad94 -> 2a1055dd8


Demonstrate that the DirectRunner runs per-call

Add a field that is modified per output, which should occur twice.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/65f9076d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/65f9076d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/65f9076d

Branch: refs/heads/master
Commit: 65f9076d654be02cbdc07442d008f6c5245d1ab5
Parents: 6da92ad
Author: Thomas Groh <tg...@google.com>
Authored: Wed Aug 10 11:29:38 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Aug 10 15:54:09 2016 -0700

--
 .../apache/beam/runners/direct/DirectRunnerTest.java | 15 +++
 1 file changed, 15 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/65f9076d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
--
diff --git 
a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
 
b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
index 1e73ec0..ddce458 100644
--- 
a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
+++ 
b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.runners.direct;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.fail;
@@ -32,6 +33,7 @@ import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.transforms.Count;
 import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.OldDoFn;
@@ -58,6 +60,7 @@ import java.io.Serializable;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * Tests for basic {@link DirectRunner} functionality.
@@ -109,10 +112,12 @@ public class DirectRunnerTest implements Serializable {
 result.awaitCompletion();
   }
 
+  private static AtomicInteger changed;
   @Test
   public void reusePipelineSucceeds() throws Throwable {
 Pipeline p = getPipeline();
 
+changed = new AtomicInteger(0);
 PCollection<KV<String, Long>> counts =
 p.apply(Create.of("foo", "bar", "foo", "baz", "bar", "foo"))
 .apply(MapElements.via(new SimpleFunction<String, String>() {
@@ -131,6 +136,14 @@ public class DirectRunnerTest implements Serializable {
   }
 }));
 
+counts.apply(ParDo.of(new DoFn<KV<String, Long>, Void>() {
+  @ProcessElement
+  public void updateChanged(ProcessContext c) {
+changed.getAndIncrement();
+  }
+}));
+
+
 PAssert.that(countStrs).containsInAnyOrder("baz: 1", "bar: 2", "foo: 3");
 
 DirectPipelineResult result = ((DirectPipelineResult) p.run());
@@ -138,6 +151,8 @@ public class DirectRunnerTest implements Serializable {
 
 DirectPipelineResult otherResult = ((DirectPipelineResult) p.run());
 otherResult.awaitCompletion();
+
+assertThat("Each element should have been processed twice", changed.get(), 
equalTo(6));
   }
 
   @Test(timeout = 5000L)

[1/2] incubator-beam git commit: Add ElementByteSizeObservable default constructor

2016-08-02 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master f13725ff5 -> d73e614b7


Add ElementByteSizeObservable default constructor

This closes #764


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/d73e614b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/d73e614b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/d73e614b

Branch: refs/heads/master
Commit: d73e614b7bd722773882a0da36bc87f84c499a00
Parents: f13725f aae1b01
Author: bchambers <bchamb...@google.com>
Authored: Tue Aug 2 09:41:26 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Tue Aug 2 09:55:39 2016 -0700

--
 .../beam/sdk/testing/CoderProperties.java   | 65 +++-
 .../util/common/ElementByteSizeObserver.java| 18 +-
 .../beam/sdk/coders/BigDecimalCoderTest.java| 15 +
 .../beam/sdk/coders/BigIntegerCoderTest.java| 14 +
 .../beam/sdk/coders/ByteArrayCoderTest.java |  5 +-
 .../beam/sdk/util/common/CounterTestUtils.java  | 57 -
 6 files changed, 88 insertions(+), 86 deletions(-)
--

[2/2] incubator-beam git commit: Add ElementByteSizeObserver default constructor

2016-08-02 Thread bchambers

Add ElementByteSizeObserver default constructor

This enables constructing an observer without the Counter to simplify
testing and use with other ways of reporting byte size.

Once compatibility issues are resolved, the class will become abstract
with the Counter-specific functionality removed.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/aae1b010
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/aae1b010
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/aae1b010

Branch: refs/heads/master
Commit: aae1b010e9da59dbd68ce24401f15561440f2bee
Parents: f13725f
Author: bchambers <bchamb...@google.com>
Authored: Mon Aug 1 10:37:14 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Tue Aug 2 09:55:39 2016 -0700

--
 .../beam/sdk/testing/CoderProperties.java   | 65 +++-
 .../util/common/ElementByteSizeObserver.java| 18 +-
 .../beam/sdk/coders/BigDecimalCoderTest.java| 15 +
 .../beam/sdk/coders/BigIntegerCoderTest.java| 14 +
 .../beam/sdk/coders/ByteArrayCoderTest.java |  5 +-
 .../beam/sdk/util/common/CounterTestUtils.java  | 57 -
 6 files changed, 88 insertions(+), 86 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/aae1b010/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CoderProperties.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CoderProperties.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CoderProperties.java
index c1fc856..e56f01f 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CoderProperties.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CoderProperties.java
@@ -36,11 +36,13 @@ import org.apache.beam.sdk.util.Serializer;
 import org.apache.beam.sdk.util.Structs;
 import org.apache.beam.sdk.util.UnownedInputStream;
 import org.apache.beam.sdk.util.UnownedOutputStream;
-
+import org.apache.beam.sdk.util.common.ElementByteSizeObserver;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
+import com.google.common.io.ByteStreams;
 import com.google.common.io.CountingInputStream;
+import com.google.common.io.CountingOutputStream;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -362,4 +364,65 @@ public class CoderProperties {
   throws CoderException, IOException {
 return decode(coder, context, encode(coder, context, value));
   }
+
+  /**
+   * A utility method that passes the given (unencoded) elements through
+   * coder's registerByteSizeObserver() and encode() methods, and confirms
+   * they are mutually consistent. This is useful for testing coder
+   * implementations.
+   */
+  public static  void testByteCount(Coder coder, Coder.Context context, 
T[] elements)
+  throws Exception {
+TestElementByteSizeObserver observer = new TestElementByteSizeObserver();
+
+CountingOutputStream os = new 
CountingOutputStream(ByteStreams.nullOutputStream());
+for (T elem : elements) {
+  coder.registerByteSizeObserver(elem, observer, context);
+  coder.encode(elem, os, context);
+  observer.advance();
+}
+long expectedLength = os.getCount();
+
+assertEquals(expectedLength, observer.getSum());
+assertEquals(elements.length, observer.getCount());
+  }
+
+  /**
+   * An {@link ElementByteSizeObserver} that records the observed element 
sizes for testing
+   * purposes.
+   */
+  public static class TestElementByteSizeObserver extends 
ElementByteSizeObserver {
+
+private long currentSum = 0;
+private long count = 0;
+
+@Override
+protected void reportElementSize(long elementByteSize) {
+  count++;
+  currentSum += elementByteSize;
+}
+
+public double getMean() {
+  return ((double) currentSum) / count;
+}
+
+public long getSum() {
+  return currentSum;
+}
+
+public long getCount() {
+  return count;
+}
+
+public void reset() {
+  currentSum = 0;
+  count = 0;
+}
+
+public long getSumAndReset() {
+  long returnValue = currentSum;
+  reset();
+  return returnValue;
+}
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/aae1b010/sdks/java/core/src/main/java/org/apache/beam/sdk/util/common/ElementByteSizeObserver.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/common/ElementByteSizeObserver.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/common/ElementByteSizeObserver.java
i

[1/2] incubator-beam git commit: Implement Aggregators directly for direct-java

2016-07-28 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 9c447510a -> 1df6f5f97


Implement Aggregators directly for direct-java

Previously, we relied on conversion to Counter rather than just
running the specified CombineFn. This aligns the direct runner
more closely with the model.

This PR also parameterizes DoFnRunner on an AggregatorFactory to
implement aggregators, allowing each runner to provide the appropriate
implementation.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/15eb67bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/15eb67bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/15eb67bb

Branch: refs/heads/master
Commit: 15eb67bb06f7cf0f225249810df8b521649f4f23
Parents: 9c44751
Author: bchambers <bchamb...@google.com>
Authored: Tue Jul 26 10:33:40 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Thu Jul 28 17:20:44 2016 -0700

--
 .../apache/beam/sdk/util/DoFnRunnerBase.java|  28 +--
 .../org/apache/beam/sdk/util/DoFnRunners.java   |  92 +-
 .../apache/beam/sdk/util/SimpleDoFnRunner.java  |   7 +-
 .../runners/direct/AggregatorContainer.java | 183 +++
 .../beam/runners/direct/DirectRunner.java   |  14 +-
 .../beam/runners/direct/EvaluationContext.java  |  32 ++--
 .../beam/runners/direct/ParDoEvaluator.java |  16 +-
 .../runners/direct/StepTransformResult.java |  15 +-
 .../beam/runners/direct/TransformResult.java|   9 +-
 .../runners/direct/AggregatorContainerTest.java | 134 ++
 .../beam/runners/direct/DirectRunnerTest.java   |   4 -
 .../runners/direct/EvaluationContextTest.java   |  35 ++--
 .../beam/runners/direct/ParDoEvaluatorTest.java |  11 +-
 .../direct/ParDoMultiEvaluatorFactoryTest.java  |  31 ++--
 .../direct/ParDoSingleEvaluatorFactoryTest.java |  31 ++--
 .../apache/beam/sdk/transforms/Aggregator.java  |  24 ++-
 .../apache/beam/sdk/transforms/DoFnTester.java  |   7 +-
 .../apache/beam/sdk/util/CounterAggregator.java |  35 +++-
 .../apache/beam/sdk/transforms/DoFnTest.java|  35 
 19 files changed, 592 insertions(+), 151 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/15eb67bb/runners/core-java/src/main/java/org/apache/beam/sdk/util/DoFnRunnerBase.java
--
diff --git 
a/runners/core-java/src/main/java/org/apache/beam/sdk/util/DoFnRunnerBase.java 
b/runners/core-java/src/main/java/org/apache/beam/sdk/util/DoFnRunnerBase.java
index e267a31..2696020 100644
--- 
a/runners/core-java/src/main/java/org/apache/beam/sdk/util/DoFnRunnerBase.java
+++ 
b/runners/core-java/src/main/java/org/apache/beam/sdk/util/DoFnRunnerBase.java
@@ -23,6 +23,7 @@ import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.IterableCoder;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.Aggregator;
+import org.apache.beam.sdk.transforms.Aggregator.AggregatorFactory;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.RequiresWindowAccess;
@@ -33,19 +34,15 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.DoFnRunners.OutputManager;
 import org.apache.beam.sdk.util.ExecutionContext.StepContext;
-import org.apache.beam.sdk.util.common.CounterSet;
 import org.apache.beam.sdk.util.state.StateInternals;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
-
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
-
 import org.joda.time.Instant;
 import org.joda.time.format.PeriodFormat;
-
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
@@ -75,7 +72,7 @@ public abstract class DoFnRunnerBase<InputT, OutputT> 
implements DoFnRunner mainOutputTag,
   List<TupleTag> sideOutputTags,
   StepContext stepContext,
-  CounterSet.AddCounterMutator addCounterMutator,
+  AggregatorFactory aggregatorFactory,
   WindowingStrategy windowingStrategy) {
 this.fn = fn;
 this.context = new DoFnContext<>(
@@ -86,13 +83,13 @@ public abstract class DoFnRunnerBase<InputT, OutputT> 
implements DoFnRunner 
implements DoFnRunner mainOutputTag;
 final StepContext stepContext;
-final CounterSet.AddCounterMutator addCounterMutator;
+final AggregatorFactory aggregatorFactory;
 final WindowFn windowFn;
 
 /**
@@ -196,7 +193,7 @@ public abstract class DoFnRunnerBase&

[2/2] incubator-beam git commit: Implement aggregators directly in the direct runner

2016-07-28 Thread bchambers

Implement aggregators directly in the direct runner

This closes #735


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/1df6f5f9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/1df6f5f9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/1df6f5f9

Branch: refs/heads/master
Commit: 1df6f5f976108aa584abbb8a9412578e1f6a621b
Parents: 9c44751 15eb67b
Author: bchambers <bchamb...@google.com>
Authored: Thu Jul 28 17:22:03 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Thu Jul 28 17:22:03 2016 -0700

--
 .../apache/beam/sdk/util/DoFnRunnerBase.java|  28 +--
 .../org/apache/beam/sdk/util/DoFnRunners.java   |  92 +-
 .../apache/beam/sdk/util/SimpleDoFnRunner.java  |   7 +-
 .../runners/direct/AggregatorContainer.java | 183 +++
 .../beam/runners/direct/DirectRunner.java   |  14 +-
 .../beam/runners/direct/EvaluationContext.java  |  32 ++--
 .../beam/runners/direct/ParDoEvaluator.java |  16 +-
 .../runners/direct/StepTransformResult.java |  15 +-
 .../beam/runners/direct/TransformResult.java|   9 +-
 .../runners/direct/AggregatorContainerTest.java | 134 ++
 .../beam/runners/direct/DirectRunnerTest.java   |   4 -
 .../runners/direct/EvaluationContextTest.java   |  35 ++--
 .../beam/runners/direct/ParDoEvaluatorTest.java |  11 +-
 .../direct/ParDoMultiEvaluatorFactoryTest.java  |  31 ++--
 .../direct/ParDoSingleEvaluatorFactoryTest.java |  31 ++--
 .../apache/beam/sdk/transforms/Aggregator.java  |  24 ++-
 .../apache/beam/sdk/transforms/DoFnTester.java  |   7 +-
 .../apache/beam/sdk/util/CounterAggregator.java |  35 +++-
 .../apache/beam/sdk/transforms/DoFnTest.java|  35 
 19 files changed, 592 insertions(+), 151 deletions(-)
--

[1/3] incubator-beam git commit: Remove many definitions of named methods

2016-06-27 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 4f580f5f1 -> 9abd0926a


Remove many definitions of named methods

Specifically, remove the occurrences in:
  - Window
  - AvroIO
  - PubsubIO
  - TextIO
  - BigQueryIO
  - Read


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/fc52a102
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/fc52a102
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/fc52a102

Branch: refs/heads/master
Commit: fc52a10259cd045f4b55ec59b2ae87c02c926ed4
Parents: 5719535
Author: Ben Chambers 
Authored: Thu Jun 23 17:55:24 2016 -0700
Committer: Ben Chambers 
Committed: Sun Jun 26 10:06:35 2016 -0700

--
 .../java/org/apache/beam/sdk/io/AvroIO.java | 53 ---
 .../java/org/apache/beam/sdk/io/BigQueryIO.java | 42 +--
 .../java/org/apache/beam/sdk/io/PubsubIO.java   | 35 +
 .../main/java/org/apache/beam/sdk/io/Read.java  | 29 +--
 .../java/org/apache/beam/sdk/io/TextIO.java | 55 
 .../org/apache/beam/sdk/io/package-info.java|  6 +--
 .../beam/sdk/transforms/windowing/Window.java   | 42 ---
 7 files changed, 25 insertions(+), 237 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/fc52a102/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
--
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
index 4b40c01..604051b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
@@ -55,9 +55,7 @@ import javax.annotation.Nullable;
  * {@link AvroIO.Read}, specifying {@link AvroIO.Read#from} to specify
  * the path of the file(s) to read from (e.g., a local filename or
  * filename pattern if running locally, or a Google Cloud Storage
- * filename or filename pattern of the form
- * {@code "gs:///"}), and optionally
- * {@link AvroIO.Read#named} to specify the name of the pipeline step.
+ * filename or filename pattern of the form {@code 
"gs:///"}).
  *
  * It is required to specify {@link AvroIO.Read#withSchema}. To
  * read specific records, such as Avro-generated classes, provide an
@@ -73,15 +71,15 @@ import javax.annotation.Nullable;
  * // A simple Read of a local file (only runs locally):
  * PCollection records =
  * p.apply(AvroIO.Read.from("/path/to/file.avro")
- *.withSchema(AvroAutoGenClass.class));
+ * .withSchema(AvroAutoGenClass.class));
  *
  * // A Read from a GCS file (runs locally and via the Google Cloud
  * // Dataflow service):
  * Schema schema = new Schema.Parser().parse(new File("schema.avsc"));
  * PCollection records =
- * p.apply(AvroIO.Read.named("ReadFromAvro")
- *.from("gs://my_bucket/path/to/records-*.avro")
- *.withSchema(schema));
+ * p.apply(AvroIO.Read
+ *.from("gs://my_bucket/path/to/records-*.avro")
+ *.withSchema(schema));
  * } 
  *
  * To write a {@link PCollection} to one or more Avro files, use
@@ -110,10 +108,10 @@ import javax.annotation.Nullable;
  * // Dataflow service):
  * Schema schema = new Schema.Parser().parse(new File("schema.avsc"));
  * PCollection records = ...;
- * records.apply(AvroIO.Write.named("WriteToAvro")
- *   .to("gs://my_bucket/path/to/numbers")
- *   .withSchema(schema)
- *   .withSuffix(".avro"));
+ * records.apply("WriteToAvro", AvroIO.Write
+ * .to("gs://my_bucket/path/to/numbers")
+ * .withSchema(schema)
+ * .withSuffix(".avro"));
  * } 
  *
  * Permissions
@@ -128,12 +126,6 @@ public class AvroIO {
* the decoding of each record.
*/
   public static class Read {
-/**
- * Returns a {@link PTransform} with the given step name.
- */
-public static Bound named(String name) {
-  return new Bound<>(GenericRecord.class).named(name);
-}
 
 /**
  * Returns a {@link PTransform} that reads from the file(s)
@@ -223,16 +215,6 @@ public class AvroIO {
 
   /**
* Returns a new {@link PTransform} that's like this one but
-   * with the given step name.
-   *
-   * Does not modify this object.
-   */
-  public Bound named(String name) {
-return new Bound<>(name, filepattern, type, schema, validate);
-  }
-
-  /**
-   * Returns a new {@link PTransform} that's like this one but
* that reads from the file(s) with the given name or pattern.
* (See

[3/3] incubator-beam git commit: This closes #529

2016-06-27 Thread bchambers

This closes #529


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/9abd0926
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/9abd0926
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/9abd0926

Branch: refs/heads/master
Commit: 9abd0926ac471c178eb30eeb01c964f9ecbd9cce
Parents: 4f580f5 fc52a10
Author: bchambers <bchamb...@google.com>
Authored: Mon Jun 27 10:29:31 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Mon Jun 27 10:29:31 2016 -0700

--
 .../beam/examples/DebuggingWordCount.java   |   2 +-
 .../org/apache/beam/examples/WordCount.java |   4 +-
 .../apache/beam/examples/complete/TfIdf.java|   3 +-
 .../examples/complete/TopWikipediaSessions.java |   2 +-
 .../examples/cookbook/DatastoreWordCount.java   |   2 +-
 .../beam/examples/cookbook/DeDupExample.java|   5 +-
 .../beam/examples/cookbook/TriggerExample.java  |   4 +-
 .../beam/examples/complete/game/GameStats.java  |  28 ++-
 .../examples/complete/game/HourlyTeamScore.java |   5 +-
 .../examples/complete/game/LeaderBoard.java |   8 +-
 .../beam/runners/flink/examples/TFIDF.java  |   3 +-
 .../beam/runners/flink/examples/WordCount.java  |   4 +-
 .../flink/examples/streaming/AutoComplete.java  |   9 +-
 .../flink/examples/streaming/JoinExamples.java  |  13 +-
 .../KafkaWindowedWordCountExample.java  |   2 +-
 .../examples/streaming/WindowedWordCount.java   |   3 +-
 .../beam/runners/dataflow/DataflowRunner.java   |   2 +-
 .../DataflowPipelineTranslatorTest.java |   6 +-
 .../runners/dataflow/DataflowRunnerTest.java|  18 +-
 .../beam/runners/spark/SimpleWordCountTest.java |   3 +-
 .../java/org/apache/beam/sdk/io/AvroIO.java |  53 +
 .../java/org/apache/beam/sdk/io/BigQueryIO.java |  43 +
 .../java/org/apache/beam/sdk/io/PubsubIO.java   |  35 +---
 .../main/java/org/apache/beam/sdk/io/Read.java  |  29 +--
 .../java/org/apache/beam/sdk/io/TextIO.java |  55 +-
 .../org/apache/beam/sdk/io/package-info.java|   6 +-
 .../beam/sdk/transforms/windowing/Window.java   |  42 
 .../beam/sdk/io/AvroIOGeneratedClassTest.java   | 192 +--
 .../java/org/apache/beam/sdk/io/AvroIOTest.java |   5 +-
 .../org/apache/beam/sdk/io/BigQueryIOTest.java  |  82 +++-
 .../apache/beam/sdk/io/FileBasedSourceTest.java |   5 +-
 .../org/apache/beam/sdk/io/PubsubIOTest.java|   4 -
 .../java/org/apache/beam/sdk/io/TextIOTest.java |  37 +---
 .../org/apache/beam/sdk/io/XmlSourceTest.java   |  19 +-
 .../beam/sdk/runners/TransformTreeTest.java |   4 +-
 .../sdk/transforms/windowing/WindowTest.java|   6 +-
 .../sdk/transforms/windowing/WindowingTest.java |   2 +-
 .../src/main/java/DebuggingWordCount.java   |   2 +-
 .../src/main/java/WordCount.java|   4 +-
 39 files changed, 183 insertions(+), 568 deletions(-)
--

[2/3] incubator-beam git commit: Remove many uses of .named methods

2016-06-27 Thread bchambers

Remove many uses of .named methods

Specifically, remove uses of:
  - Window.named
  - AvroIO.named
  - PubSubIO.named
  - TextIO.named
  - BigQueryIO.named
  - Read.named


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/57195358
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/57195358
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/57195358

Branch: refs/heads/master
Commit: 57195358592548e6f7e05bc8e4e292b126a726c5
Parents: 4f580f5
Author: Ben Chambers 
Authored: Thu Jun 23 22:27:05 2016 -0700
Committer: Ben Chambers 
Committed: Sun Jun 26 10:06:35 2016 -0700

--
 .../beam/examples/DebuggingWordCount.java   |   2 +-
 .../org/apache/beam/examples/WordCount.java |   4 +-
 .../apache/beam/examples/complete/TfIdf.java|   3 +-
 .../examples/complete/TopWikipediaSessions.java |   2 +-
 .../examples/cookbook/DatastoreWordCount.java   |   2 +-
 .../beam/examples/cookbook/DeDupExample.java|   5 +-
 .../beam/examples/cookbook/TriggerExample.java  |   4 +-
 .../beam/examples/complete/game/GameStats.java  |  28 ++-
 .../examples/complete/game/HourlyTeamScore.java |   5 +-
 .../examples/complete/game/LeaderBoard.java |   8 +-
 .../beam/runners/flink/examples/TFIDF.java  |   3 +-
 .../beam/runners/flink/examples/WordCount.java  |   4 +-
 .../flink/examples/streaming/AutoComplete.java  |   9 +-
 .../flink/examples/streaming/JoinExamples.java  |  13 +-
 .../KafkaWindowedWordCountExample.java  |   2 +-
 .../examples/streaming/WindowedWordCount.java   |   3 +-
 .../beam/runners/dataflow/DataflowRunner.java   |   2 +-
 .../DataflowPipelineTranslatorTest.java |   6 +-
 .../runners/dataflow/DataflowRunnerTest.java|  18 +-
 .../beam/runners/spark/SimpleWordCountTest.java |   3 +-
 .../java/org/apache/beam/sdk/io/BigQueryIO.java |   1 -
 .../beam/sdk/io/AvroIOGeneratedClassTest.java   | 192 +--
 .../java/org/apache/beam/sdk/io/AvroIOTest.java |   5 +-
 .../org/apache/beam/sdk/io/BigQueryIOTest.java  |  82 +++-
 .../apache/beam/sdk/io/FileBasedSourceTest.java |   5 +-
 .../org/apache/beam/sdk/io/PubsubIOTest.java|   4 -
 .../java/org/apache/beam/sdk/io/TextIOTest.java |  37 +---
 .../org/apache/beam/sdk/io/XmlSourceTest.java   |  19 +-
 .../beam/sdk/runners/TransformTreeTest.java |   4 +-
 .../sdk/transforms/windowing/WindowTest.java|   6 +-
 .../sdk/transforms/windowing/WindowingTest.java |   2 +-
 .../src/main/java/DebuggingWordCount.java   |   2 +-
 .../src/main/java/WordCount.java|   4 +-
 33 files changed, 158 insertions(+), 331 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/57195358/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java
--
diff --git 
a/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java 
b/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java
index 85823c2..8d85d44 100644
--- 
a/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java
+++ 
b/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java
@@ -173,7 +173,7 @@ public class DebuggingWordCount {
 Pipeline p = Pipeline.create(options);
 
 PCollection> filteredWords =
-p.apply(TextIO.Read.named("ReadLines").from(options.getInputFile()))
+p.apply("ReadLines", TextIO.Read.from(options.getInputFile()))
  .apply(new WordCount.CountWords())
  .apply(ParDo.of(new FilterTextFn(options.getFilterPattern(;
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/57195358/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
--
diff --git 
a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java 
b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
index cf6c45a..af16c44 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
@@ -205,10 +205,10 @@ public class WordCount {
 
 // Concepts #2 and #3: Our pipeline applies the composite CountWords 
transform, and passes the
 // static FormatAsTextFn() to the ParDo transform.
-p.apply(TextIO.Read.named("ReadLines").from(options.getInputFile()))
+p.apply("ReadLines", TextIO.Read.from(options.getInputFile()))
  .apply(new CountWords())
  .apply(MapElements.via(new FormatAsTextFn()))
- .apply(TextIO.Write.named("WriteCounts").to(options.getOutput()));
+ .apply("WriteCounts", TextIO.Write.to(options.getOutput()));

[3/3] incubator-beam git commit: This closes #522

2016-06-24 Thread bchambers

This closes #522


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/41faee4f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/41faee4f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/41faee4f

Branch: refs/heads/master
Commit: 41faee4f9cbe29ad6a80742789a0004abe59e684
Parents: 7745b92 4f050bf
Author: bchambers <bchamb...@google.com>
Authored: Thu Jun 23 22:55:46 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Thu Jun 23 22:55:46 2016 -0700

--
 .../apache/beam/examples/MinimalWordCount.java  |  2 +-
 .../beam/examples/complete/AutoComplete.java|  4 +-
 .../apache/beam/examples/complete/TfIdf.java| 13 ++--
 .../examples/complete/TopWikipediaSessions.java |  7 +-
 .../beam/examples/cookbook/FilterExamples.java  |  3 +-
 .../beam/examples/cookbook/JoinExamples.java|  4 +-
 .../beam/examples/cookbook/TriggerExample.java  |  2 +-
 .../beam/examples/complete/game/GameStats.java  |  7 +-
 .../examples/complete/game/HourlyTeamScore.java |  4 +-
 .../examples/complete/game/LeaderBoard.java |  4 +-
 .../beam/examples/complete/game/UserScore.java  |  2 +-
 .../complete/game/utils/WriteToBigQuery.java|  2 +-
 .../game/utils/WriteWindowedToBigQuery.java |  2 +-
 .../complete/game/HourlyTeamScoreTest.java  |  2 +-
 .../beam/runners/flink/examples/TFIDF.java  | 48 +++---
 .../flink/examples/streaming/AutoComplete.java  |  4 +-
 .../flink/examples/streaming/JoinExamples.java  |  4 +-
 .../beam/runners/dataflow/DataflowRunner.java   |  5 +-
 .../dataflow/internal/AssignWindows.java|  2 +-
 .../DataflowPipelineTranslatorTest.java |  6 +-
 .../sdk/io/BoundedReadFromUnboundedSource.java  |  3 +-
 .../apache/beam/sdk/io/PubsubUnboundedSink.java | 24 +++
 .../beam/sdk/io/PubsubUnboundedSource.java  |  5 +-
 .../org/apache/beam/sdk/testing/PAssert.java|  6 +-
 .../org/apache/beam/sdk/transforms/Combine.java | 48 +++---
 .../org/apache/beam/sdk/transforms/Count.java   |  2 +-
 .../beam/sdk/transforms/FlatMapElements.java|  2 +-
 .../org/apache/beam/sdk/transforms/Flatten.java |  2 +-
 .../org/apache/beam/sdk/transforms/Keys.java| 13 ++--
 .../org/apache/beam/sdk/transforms/KvSwap.java  | 15 ++---
 .../apache/beam/sdk/transforms/MapElements.java |  2 +-
 .../apache/beam/sdk/transforms/PTransform.java  |  7 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   | 70 +---
 .../beam/sdk/transforms/RemoveDuplicates.java   | 13 ++--
 .../org/apache/beam/sdk/transforms/Values.java  | 13 ++--
 .../apache/beam/sdk/transforms/WithKeys.java| 15 ++---
 .../beam/sdk/transforms/WithTimestamps.java |  2 +-
 .../beam/sdk/transforms/join/CoGroupByKey.java  | 10 ++-
 .../beam/sdk/transforms/windowing/Window.java   | 21 +++---
 .../org/apache/beam/sdk/util/Reshuffle.java |  3 +-
 .../apache/beam/sdk/util/ValueWithRecordId.java | 19 ++
 .../apache/beam/sdk/transforms/ParDoTest.java   | 27 ++--
 .../sdk/transforms/windowing/WindowingTest.java |  8 +--
 .../src/main/java/MinimalWordCount.java |  4 +-
 44 files changed, 180 insertions(+), 281 deletions(-)
--

[2/3] incubator-beam git commit: Remove uses of ParDo.named

2016-06-24 Thread bchambers

Remove uses of ParDo.named

Also removed a few other uses of .named() methods that were nearby
use-sites being cleaned up.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/d7f2810a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/d7f2810a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/d7f2810a

Branch: refs/heads/master
Commit: d7f2810ad24e805793d6d43a639a48845aac4e48
Parents: 7745b92
Author: Ben Chambers <bchamb...@google.com>
Authored: Thu Jun 23 09:53:40 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Thu Jun 23 22:55:43 2016 -0700

--
 .../apache/beam/examples/MinimalWordCount.java  |  2 +-
 .../beam/examples/complete/AutoComplete.java|  4 +-
 .../apache/beam/examples/complete/TfIdf.java| 13 +++---
 .../examples/complete/TopWikipediaSessions.java |  7 ++-
 .../beam/examples/cookbook/FilterExamples.java  |  3 +-
 .../beam/examples/cookbook/JoinExamples.java|  4 +-
 .../beam/examples/cookbook/TriggerExample.java  |  2 +-
 .../beam/examples/complete/game/GameStats.java  |  7 ++-
 .../examples/complete/game/HourlyTeamScore.java |  4 +-
 .../examples/complete/game/LeaderBoard.java |  4 +-
 .../beam/examples/complete/game/UserScore.java  |  2 +-
 .../complete/game/utils/WriteToBigQuery.java|  2 +-
 .../game/utils/WriteWindowedToBigQuery.java |  2 +-
 .../complete/game/HourlyTeamScoreTest.java  |  2 +-
 .../beam/runners/flink/examples/TFIDF.java  | 48 ++--
 .../flink/examples/streaming/AutoComplete.java  |  4 +-
 .../flink/examples/streaming/JoinExamples.java  |  4 +-
 .../beam/runners/dataflow/DataflowRunner.java   |  5 +-
 .../dataflow/internal/AssignWindows.java|  2 +-
 .../DataflowPipelineTranslatorTest.java |  6 +--
 .../sdk/io/BoundedReadFromUnboundedSource.java  |  3 +-
 .../apache/beam/sdk/io/PubsubUnboundedSink.java | 24 +-
 .../beam/sdk/io/PubsubUnboundedSource.java  |  5 +-
 .../org/apache/beam/sdk/testing/PAssert.java|  6 +--
 .../org/apache/beam/sdk/transforms/Combine.java | 48 ++--
 .../org/apache/beam/sdk/transforms/Count.java   |  2 +-
 .../beam/sdk/transforms/FlatMapElements.java|  2 +-
 .../org/apache/beam/sdk/transforms/Flatten.java |  2 +-
 .../org/apache/beam/sdk/transforms/Keys.java| 13 +++---
 .../org/apache/beam/sdk/transforms/KvSwap.java  | 15 +++---
 .../apache/beam/sdk/transforms/MapElements.java |  2 +-
 .../apache/beam/sdk/transforms/PTransform.java  |  7 ++-
 .../beam/sdk/transforms/RemoveDuplicates.java   | 13 +++---
 .../org/apache/beam/sdk/transforms/Values.java  | 13 +++---
 .../apache/beam/sdk/transforms/WithKeys.java| 15 +++---
 .../beam/sdk/transforms/WithTimestamps.java |  2 +-
 .../beam/sdk/transforms/join/CoGroupByKey.java  | 10 ++--
 .../beam/sdk/transforms/windowing/Window.java   | 21 -
 .../org/apache/beam/sdk/util/Reshuffle.java |  3 +-
 .../apache/beam/sdk/util/ValueWithRecordId.java | 19 +++-
 .../apache/beam/sdk/transforms/ParDoTest.java   | 27 +++
 .../sdk/transforms/windowing/WindowingTest.java |  8 ++--
 .../src/main/java/MinimalWordCount.java |  4 +-
 43 files changed, 177 insertions(+), 214 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d7f2810a/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java
--
diff --git 
a/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java 
b/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java
index 355a1ff..2c67609 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java
@@ -92,7 +92,7 @@ public class MinimalWordCount {
  // DoFn (defined in-line) on each element that tokenizes the text line 
into individual words.
  // The ParDo returns a PCollection, where each element is an 
individual word in
  // Shakespeare's collected texts.
- .apply(ParDo.named("ExtractWords").of(new DoFn<String, String>() {
+ .apply("ExtractWords", ParDo.of(new DoFn<String, String>() {
@Override
public void processElement(ProcessContext c) {
  for (String word : c.element().split("[^a-zA-Z']+")) {

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d7f2810a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
--
diff --git 
a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
 
b/example

[1/3] incubator-beam git commit: Remove ParDo.named on all variants

2016-06-24 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 7745b921f -> 41faee4f9


Remove ParDo.named on all variants


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/4f050bf5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/4f050bf5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/4f050bf5

Branch: refs/heads/master
Commit: 4f050bf56832b1cdeda7af1a58589a1ed952a41a
Parents: d7f2810
Author: Ben Chambers <bchamb...@google.com>
Authored: Thu Jun 23 09:57:55 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Thu Jun 23 22:55:43 2016 -0700

--
 .../org/apache/beam/sdk/transforms/ParDo.java   | 70 +---
 1 file changed, 3 insertions(+), 67 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4f050bf5/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
index cb7d372..16dfcac 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
@@ -123,16 +123,13 @@ import java.util.List;
  * a unique name - which may not be stable across pipeline revision -
  * will be generated, based on the transform name.
  *
- * If a {@link ParDo} is applied exactly once inlined, then
- * it can be given a name via {@link #named}. For example:
+ * For example:
  *
  *  {@code
  * PCollection words =
- * lines.apply(ParDo.named("ExtractWords")
- *  .of(new DoFn<String, String>() { ... }));
+ * lines.apply("ExtractWords", ParDo.of(new DoFn<String, String>() { ... 
}));
  * PCollection wordLengths =
- * words.apply(ParDo.named("ComputeWordLengths")
- *  .of(new DoFn<String, Integer>() { ... }));
+ * words.apply("ComputeWordLengths", ParDo.of(new DoFn<String, Integer>() 
{ ... }));
  * } 
  *
  * Side Inputs
@@ -437,21 +434,6 @@ import java.util.List;
 public class ParDo {
 
   /**
-   * Creates a {@link ParDo} {@link PTransform} with the given name.
-   *
-   * See the discussion of naming above for more explanation.
-   *
-   * The resulting {@link PTransform} is incomplete, and its
-   * input/output types are not yet bound. Use
-   * {@link ParDo.Unbound#of} to specify the {@link DoFn} to
-   * invoke, which will also bind the input/output types of this
-   * {@link PTransform}.
-   */
-  public static Unbound named(String name) {
-return new Unbound().named(name);
-  }
-
-  /**
* Creates a {@link ParDo} {@link PTransform} with the given
* side inputs.
*
@@ -588,17 +570,6 @@ public class ParDo {
 
 /**
  * Returns a new {@link ParDo} transform that's like this
- * transform but with the specified name. Does not modify this
- * transform. The resulting transform is still incomplete.
- *
- * See the discussion of naming above for more explanation.
- */
-public Unbound named(String name) {
-  return new Unbound(name, sideInputs);
-}
-
-/**
- * Returns a new {@link ParDo} transform that's like this
  * transform but with the specified additional side inputs.
  * Does not modify this transform. The resulting transform is
  * still incomplete.
@@ -703,17 +674,6 @@ public class ParDo {
 
 /**
  * Returns a new {@link ParDo} {@link PTransform} that's like this
- * {@link PTransform} but with the specified name. Does not
- * modify this {@link PTransform}.
- *
- * See the discussion of Naming above for more explanation.
- */
-public Bound<InputT, OutputT> named(String name) {
-  return new Bound<>(name, sideInputs, fn, fnClass);
-}
-
-/**
- * Returns a new {@link ParDo} {@link PTransform} that's like this
  * {@link PTransform} but with the specified additional side inputs. Does 
not
  * modify this {@link PTransform}.
  *
@@ -833,18 +793,6 @@ public class ParDo {
 
 /**
  * Returns a new multi-output {@link ParDo} transform that's like
- * this transform but with the specified name. Does not modify
- * this transform. The resulting transform is still incomplete.
- *
- * See the discussion of Naming above for more explanation.
- */
-public UnboundMulti named(String name) {
-  return new UnboundMulti<>(
-  name, sideInputs, mainOutputTag, sideOutputTags);
-}
-
-/**
- * Returns a new multi-output {@link ParDo} transform that's like
  * this tr

[1/2] incubator-beam git commit: [BEAM-359] Treat erased type variables as non-deterministic in AvroCoder

2016-06-22 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 9bcf9b0a7 -> 8949ec315


[BEAM-359] Treat erased type variables as non-deterministic in AvroCoder

Previously, loss of type information due to erasure would lead to an
IllegalArgumentException in the constructor. Now, the coder is created
and usable but treated as non-deterministic.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/9c8dc4fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/9c8dc4fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/9c8dc4fe

Branch: refs/heads/master
Commit: 9c8dc4fe920618253b425f0d998f8d63552ec358
Parents: 9bcf9b0
Author: bchambers <bchamb...@google.com>
Authored: Tue Jun 21 13:42:33 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Jun 22 14:33:31 2016 -0700

--
 .../org/apache/beam/sdk/coders/AvroCoder.java   | 10 +---
 .../apache/beam/sdk/coders/AvroCoderTest.java   | 26 
 2 files changed, 33 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9c8dc4fe/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java
index 3b93ec3..00c1cbc 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java
@@ -475,6 +475,10 @@ public class AvroCoder extends StandardCoder {
   checkMap(context, type, schema);
   break;
 case RECORD:
+  if (!(type.getType() instanceof Class)) {
+reportError(context, "Cannot determine type from generic %s due to 
erasure", type);
+return;
+  }
   checkRecord(type, schema);
   break;
 case UNION:
@@ -695,7 +699,8 @@ public class AvroCoder extends StandardCoder {
  * Extract a field from a class. We need to look at the declared fields so 
that we can
  * see private fields. We may need to walk up to the parent to get classes 
from the parent.
  */
-private static Field getField(Class clazz, String name) {
+private static Field getField(Class originalClazz, String name) {
+  Class clazz = originalClazz;
   while (clazz != null) {
 for (Field field : clazz.getDeclaredFields()) {
   AvroName avroName = field.getAnnotation(AvroName.class);
@@ -708,8 +713,7 @@ public class AvroCoder extends StandardCoder {
 clazz = clazz.getSuperclass();
   }
 
-  throw new IllegalArgumentException(
-  "Unable to get field " + name + " from class " + clazz);
+  throw new IllegalArgumentException("Unable to get field " + name + " 
from " + originalClazz);
 }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9c8dc4fe/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/AvroCoderTest.java
--
diff --git 
a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/AvroCoderTest.java 
b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/AvroCoderTest.java
index 8f28cc4..207bfdd 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/AvroCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/AvroCoderTest.java
@@ -755,4 +755,30 @@ public class AvroCoderTest {
   return Objects.hash(getClass(), onlySomeTypesAllowed);
 }
   }
+
+  @Test
+  public void testAvroCoderForGenerics() throws Exception {
+Schema fooSchema = AvroCoder.of(Foo.class).getSchema();
+Schema schema = new Schema.Parser().parse("{"
++ "\"type\":\"record\","
++ "\"name\":\"SomeGeneric\","
++ "\"namespace\":\"ns\","
++ "\"fields\":["
++ "  {\"name\":\"foo\", \"type\":" + fooSchema.toString() + "}"
++ "]}");
+@SuppressWarnings("rawtypes")
+AvroCoder coder = AvroCoder.of(SomeGeneric.class, schema);
+
+assertNonDeterministic(coder,
+reasonField(SomeGeneric.class, "foo", "erasure"));
+  }
+
+  private static class SomeGeneric {
+@SuppressWarnings("unused")
+private T foo;
+  }
+  private static class Foo {
+@SuppressWarnings("unused")
+String id;
+  }
 }

[2/2] incubator-beam git commit: This closes #513

2016-06-22 Thread bchambers

This closes #513


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/8949ec31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/8949ec31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/8949ec31

Branch: refs/heads/master
Commit: 8949ec3159d552be108819d7951b649c00271dd7
Parents: 9bcf9b0 9c8dc4f
Author: bchambers <bchamb...@google.com>
Authored: Wed Jun 22 14:35:28 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Jun 22 14:35:28 2016 -0700

--
 .../org/apache/beam/sdk/coders/AvroCoder.java   | 10 +---
 .../apache/beam/sdk/coders/AvroCoderTest.java   | 26 
 2 files changed, 33 insertions(+), 3 deletions(-)
--

[2/2] incubator-beam git commit: This closes #512

2016-06-21 Thread bchambers

This closes #512


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/4840f527
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/4840f527
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/4840f527

Branch: refs/heads/python-sdk
Commit: 4840f5275c569c190088179cb5dee6380ae641c9
Parents: f166b16 0608ad9
Author: bchambers <bchamb...@google.com>
Authored: Tue Jun 21 15:20:27 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Tue Jun 21 15:20:27 2016 -0700

--
 sdks/python/apache_beam/transforms/ptransform_test.py | 13 +++--
 1 file changed, 7 insertions(+), 6 deletions(-)
--

[1/2] incubator-beam git commit: Fix AutoComplete example streaming configuration

2016-06-13 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 915bc198d -> 522762452


Fix AutoComplete example streaming configuration


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/e60e3e6d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/e60e3e6d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/e60e3e6d

Branch: refs/heads/master
Commit: e60e3e6de85a022d4669333c544b0b86145d0d55
Parents: 915bc19
Author: Scott Wegner <sweg...@google.com>
Authored: Mon Jun 13 11:05:00 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Mon Jun 13 17:17:22 2016 -0700

--
 .../java/org/apache/beam/examples/complete/AutoComplete.java   | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e60e3e6d/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
--
diff --git 
a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
 
b/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
index ac8384b..d725e0a 100644
--- 
a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
+++ 
b/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
@@ -31,6 +31,7 @@ import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.options.Validation;
 import org.apache.beam.sdk.transforms.Count;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.Filter;
@@ -416,6 +417,7 @@ public class AutoComplete {
*/
   private static interface Options extends ExamplePubsubTopicOptions, 
ExampleBigQueryTableOptions {
 @Description("Input text file")
+@Validation.Required
 String getInputFile();
 void setInputFile(String value);
 
@@ -494,7 +496,9 @@ public class AutoComplete {
.to(tableRef)
.withSchema(FormatForBigquery.getSchema())

.withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)
-   
.withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE));
+   .withWriteDisposition(options.isStreaming()
+   ? BigQueryIO.Write.WriteDisposition.WRITE_APPEND
+   : BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE));
 }
 
 // Run the pipeline.

[2/2] incubator-beam git commit: This closes #445

2016-06-13 Thread bchambers

This closes #445


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/52276245
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/52276245
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/52276245

Branch: refs/heads/master
Commit: 5227624526334ef18dc2109978f309af2a7d3521
Parents: 915bc19 e60e3e6
Author: bchambers <bchamb...@google.com>
Authored: Mon Jun 13 17:17:27 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Mon Jun 13 17:17:27 2016 -0700

--
 .../java/org/apache/beam/examples/complete/AutoComplete.java   | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)
--

[1/2] incubator-beam git commit: Make example AddTimestampFn range deterministic

2016-06-13 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 9e0d7d650 -> e1b305ea5


Make example AddTimestampFn range deterministic

The timestamps added in the WindowedWordCount example are currently
based on when the bundles are executed, which makes the min/max bounds
non-deterministic. This change makes the range  based on the
construction time.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/335202a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/335202a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/335202a0

Branch: refs/heads/master
Commit: 335202a033ced6f30f1b0e5df9da047241abc750
Parents: 9e0d7d6
Author: Scott Wegner <sweg...@google.com>
Authored: Thu Jun 9 11:31:23 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Mon Jun 13 11:21:25 2016 -0700

--
 .../java/org/apache/beam/examples/WindowedWordCount.java | 11 ---
 1 file changed, 8 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/335202a0/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java
--
diff --git 
a/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java 
b/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java
index 4d019bb..9ca26bf 100644
--- 
a/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java
+++ 
b/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java
@@ -124,13 +124,18 @@ public class WindowedWordCount {
* 2-hour period.
*/
   static class AddTimestampFn extends DoFn<String, String> {
-private static final long RAND_RANGE = 720; // 2 hours in ms
+private static final Duration RAND_RANGE = Duration.standardHours(2);
+private final Instant minTimestamp;
+
+AddTimestampFn() {
+  this.minTimestamp = new Instant(System.currentTimeMillis());
+}
 
 @Override
 public void processElement(ProcessContext c) {
   // Generate a timestamp that falls somewhere in the past two hours.
-  long randomTimestamp = System.currentTimeMillis()
-- (int) (Math.random() * RAND_RANGE);
+  long randMillis = (long) (Math.random() * RAND_RANGE.getMillis());
+  Instant randomTimestamp = minTimestamp.plus(randMillis);
   /**
* Concept #2: Set the data element with that timestamp.
*/

[2/2] incubator-beam git commit: This closes #441

2016-06-13 Thread bchambers

This closes #441


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/e1b305ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/e1b305ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/e1b305ea

Branch: refs/heads/master
Commit: e1b305ea5d48e0f5f71ad0030c20c492b0f95c30
Parents: 9e0d7d6 335202a
Author: bchambers <bchamb...@google.com>
Authored: Mon Jun 13 11:21:31 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Mon Jun 13 11:21:31 2016 -0700

--
 .../java/org/apache/beam/examples/WindowedWordCount.java | 11 ---
 1 file changed, 8 insertions(+), 3 deletions(-)
--

[2/2] incubator-beam git commit: This closes #399

2016-06-06 Thread bchambers

This closes #399


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/c0b67ab1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/c0b67ab1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/c0b67ab1

Branch: refs/heads/master
Commit: c0b67ab1286893b9030d1ba5096b4ff3ab15f863
Parents: c2146b9 ebb6932
Author: bchambers <bchamb...@google.com>
Authored: Mon Jun 6 10:23:21 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Mon Jun 6 10:23:21 2016 -0700

--
 .../runners/direct/ExecutorServiceParallelExecutor.java  | 11 ---
 1 file changed, 8 insertions(+), 3 deletions(-)
--

[1/2] incubator-beam git commit: Return from awaitCompletion if Already Done

2016-06-06 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master c2146b9f9 -> c0b67ab12


Return from awaitCompletion if Already Done

This ensures that a call to ExecutorService#awaitCompletion returns immediately
if there are no visible updates and the executor has completed. If the executor 
is
in this state, no additional visible updates will be published and the call 
will hang.

This sequence generally will not happen, as calls via InProcessPipelineResult
return if the state is already terminal, but this ensures that parallel calls to
awaitCompletion do not hang one calling thread.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ebb69320
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ebb69320
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ebb69320

Branch: refs/heads/master
Commit: ebb69320037fda48df5e6ca24ecc9415a4b2acb4
Parents: c2146b9
Author: Thomas Groh <tg...@users.noreply.github.com>
Authored: Mon May 30 11:04:15 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Mon Jun 6 10:09:47 2016 -0700

--
 .../runners/direct/ExecutorServiceParallelExecutor.java  | 11 ---
 1 file changed, 8 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ebb69320/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
--
diff --git 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
index a627125..3129145 100644
--- 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
+++ 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
@@ -200,11 +200,16 @@ final class ExecutorServiceParallelExecutor implements 
InProcessExecutor {
   public void awaitCompletion() throws Throwable {
 VisibleExecutorUpdate update;
 do {
-  update = visibleUpdates.take();
-  if (update.throwable.isPresent()) {
+  // Get an update; don't block forever if another thread has handled it
+  update = visibleUpdates.poll(2L, TimeUnit.SECONDS);
+  if (update == null && executorService.isShutdown()) {
+// there are no updates to process and no updates will ever be 
published because the
+// executor is shutdown
+return;
+  } else if (update != null && update.throwable.isPresent()) {
 throw update.throwable.get();
   }
-} while (!update.isDone());
+} while (update == null || !update.isDone());
 executorService.shutdown();
   }

[2/2] incubator-beam git commit: This closes #390

2016-05-25 Thread bchambers

This closes #390


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a3fc40aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a3fc40aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a3fc40aa

Branch: refs/heads/master
Commit: a3fc40aa368de8bf5a775096abf0d392208f7061
Parents: 78c8c52 7abedfa
Author: bchambers <bchamb...@google.com>
Authored: Wed May 25 12:47:51 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed May 25 12:47:51 2016 -0700

--
 .../apache/beam/runners/direct/InProcessPipelineRunnerTest.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--

[1/2] incubator-beam git commit: Mark test PipelineOptions interface public

2016-05-25 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 78c8c528e -> a3fc40aa3


Mark test PipelineOptions interface public


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/7abedfae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/7abedfae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/7abedfae

Branch: refs/heads/master
Commit: 7abedfae13319c8e85bfc6a7f3fef4c7edeae9dd
Parents: 78c8c52
Author: Scott Wegner 
Authored: Wed May 25 11:40:45 2016 -0700
Committer: Scott Wegner 
Committed: Wed May 25 11:40:45 2016 -0700

--
 .../apache/beam/runners/direct/InProcessPipelineRunnerTest.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/7abedfae/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessPipelineRunnerTest.java
--
diff --git 
a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessPipelineRunnerTest.java
 
b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessPipelineRunnerTest.java
index e403019..5a92ce3 100644
--- 
a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessPipelineRunnerTest.java
+++ 
b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessPipelineRunnerTest.java
@@ -125,7 +125,8 @@ public class InProcessPipelineRunnerTest implements 
Serializable {
 p.run();
   }
 
-  interface ObjectPipelineOptions extends PipelineOptions {
+  /** {@link PipelineOptions} to inject bad object implementations. */
+  public interface ObjectPipelineOptions extends PipelineOptions {
 Object getValue();
 void setValue(Object value);
   }

[1/2] incubator-beam git commit: Evaluate display data from InProcessPipelineRunner

2016-05-24 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 5535fc3fd -> ffed1d479


Evaluate display data from InProcessPipelineRunner

Display data can be added to any PTransform to be used
for display from any runner. Runners are not required to
consume display data, and currently many don't.

This changes InProcessRunner to consumer display data (and then
discard it) in order to validate that display data is properly
implemented on transforms within a pipeline. Exceptions thrown
within HasDisplayData implementations will cause Pipeline.run()
to fail.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ac98c4a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ac98c4a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ac98c4a0

Branch: refs/heads/master
Commit: ac98c4a01f7e4b2bba3e99271b66e141e7555de8
Parents: 5535fc3
Author: Scott Wegner <sweg...@google.com>
Authored: Mon May 23 14:29:33 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Tue May 24 14:11:44 2016 -0700

--
 .../runners/direct/DisplayDataValidator.java| 67 
 .../runners/direct/InProcessPipelineRunner.java |  2 +
 .../direct/InProcessPipelineRunnerTest.java | 63 ++
 .../sdk/options/ProxyInvocationHandler.java |  7 +-
 .../sdk/transforms/display/DisplayData.java | 14 +++-
 .../sdk/transforms/display/DisplayDataTest.java | 19 ++
 6 files changed, 168 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ac98c4a0/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DisplayDataValidator.java
--
diff --git 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DisplayDataValidator.java
 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DisplayDataValidator.java
new file mode 100644
index 000..e09fe62
--- /dev/null
+++ 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DisplayDataValidator.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.runners.TransformTreeNode;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.HasDisplayData;
+
+/**
+ * Validate correct implementation of {@link DisplayData} by evaluating
+ * {@link HasDisplayData#populateDisplayData(DisplayData.Builder)} during 
pipeline construction.
+ */
+class DisplayDataValidator {
+  // Do not instantiate
+  private DisplayDataValidator() {}
+
+  static void validatePipeline(Pipeline pipeline) {
+validateOptions(pipeline);
+validateTransforms(pipeline);
+  }
+
+  private static void validateOptions(Pipeline pipeline) {
+evaluateDisplayData(pipeline.getOptions());
+  }
+
+  private static void validateTransforms(Pipeline pipeline) {
+pipeline.traverseTopologically(Visitor.INSTANCE);
+  }
+
+  private static void evaluateDisplayData(HasDisplayData component) {
+DisplayData.from(component);
+  }
+
+  private static class Visitor extends Pipeline.PipelineVisitor.Defaults {
+private static final Visitor INSTANCE = new Visitor();
+
+@Override
+public CompositeBehavior enterCompositeTransform(TransformTreeNode node) {
+  if (!node.isRootNode()) {
+evaluateDisplayData(node.getTransform());
+  }
+
+  return CompositeBehavior.ENTER_TRANSFORM;
+}
+
+@Override
+public void visitPrimitiveTransform(TransformTreeNode node) {
+  evaluateDisplayData(node.getTransform());
+}
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ac98c4a0/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessPipelineRunner.java
--
diff --git 
a/runners/direct-java/src/main/

[2/2] incubator-beam git commit: This closes #375

2016-05-24 Thread bchambers

This closes #375


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ffed1d47
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ffed1d47
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ffed1d47

Branch: refs/heads/master
Commit: ffed1d4791f8260a60d4535d754e7e5cd69157d7
Parents: 5535fc3 ac98c4a
Author: bchambers <bchamb...@google.com>
Authored: Tue May 24 14:11:48 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Tue May 24 14:11:48 2016 -0700

--
 .../runners/direct/DisplayDataValidator.java| 67 
 .../runners/direct/InProcessPipelineRunner.java |  2 +
 .../direct/InProcessPipelineRunnerTest.java | 63 ++
 .../sdk/options/ProxyInvocationHandler.java |  7 +-
 .../sdk/transforms/display/DisplayData.java | 14 +++-
 .../sdk/transforms/display/DisplayDataTest.java | 19 ++
 6 files changed, 168 insertions(+), 4 deletions(-)
--

[1/2] incubator-beam git commit: Fix bug in PipelineOptions DisplayData serialization

2016-05-20 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 4755c5a78 -> bb086b8d3


Fix bug in PipelineOptions DisplayData serialization

PipelineOptions has been improved to generate display data
to be consumed by a runner and used for display. However, there
was a bug in the ProxyInvocationHandler implementation of
PipelineOptions display data which was causing NullPointerExceptions
when generated display data from PipelineOptions previously
deserialized from JSON.

This change also makes our error handling for display data exceptions
consistent across the Dataflow runner: exceptions thrown during
display data population will propogate out and cause the pipeline to
fail. This is consistent with other user code which may throw
exceptions at pipeline construction time.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/1e669c44
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/1e669c44
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/1e669c44

Branch: refs/heads/master
Commit: 1e669c44c9d2448b55f5bdba3dcff1831b2cd8b4
Parents: 4755c5a
Author: Scott Wegner 
Authored: Thu May 19 09:17:37 2016 -0700
Committer: Scott Wegner 
Committed: Fri May 20 17:20:29 2016 -0700

--
 .../dataflow/DataflowPipelineTranslator.java| 50 +--
 .../DataflowPipelineTranslatorTest.java | 63 --
 .../sdk/options/ProxyInvocationHandler.java |  4 +-
 .../sdk/transforms/display/DisplayData.java | 14 +--
 .../sdk/options/ProxyInvocationHandlerTest.java |  6 +-
 .../sdk/transforms/display/DisplayDataTest.java | 91 ++--
 6 files changed, 83 insertions(+), 145 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/1e669c44/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
--
diff --git 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
index 7f67393..f5fefc0 100644
--- 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
+++ 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
@@ -29,7 +29,6 @@ import static org.apache.beam.sdk.util.Structs.addString;
 import static org.apache.beam.sdk.util.Structs.getString;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
 
 import 
org.apache.beam.runners.dataflow.DataflowPipelineRunner.GroupByKeyAndSortValuesOnly;
 import org.apache.beam.runners.dataflow.internal.ReadTranslator;
@@ -87,8 +86,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -726,18 +723,7 @@ public class DataflowPipelineTranslator {
 }
 
 private void addDisplayData(String stepName, HasDisplayData 
hasDisplayData) {
-  DisplayData displayData;
-  try {
-displayData = DisplayData.from(hasDisplayData);
-  } catch (Exception e) {
-String msg = String.format("Exception thrown while collecting display 
data for step: %s. "
-+ "Display data will be not be available for this step.", 
stepName);
-DisplayDataException displayDataException = new 
DisplayDataException(msg, e);
-LOG.warn(msg, displayDataException);
-
-displayData = displayDataException.asDisplayData();
-  }
-
+  DisplayData displayData = DisplayData.from(hasDisplayData);
   List> list = MAPPER.convertValue(displayData, 
List.class);
   addList(getProperties(), PropertyNames.DISPLAY_DATA, list);
 }
@@ -1056,38 +1042,4 @@ public class DataflowPipelineTranslator {
   context.addOutput(tag.getId(), output);
 }
   }
-
-  /**
-   * Wraps exceptions thrown while collecting {@link DisplayData} for the 
Dataflow pipeline runner.
-   */
-  static class DisplayDataException extends Exception implements 
HasDisplayData {
-public DisplayDataException(String message, Throwable cause) {
-  super(checkNotNull(message), checkNotNull(cause));
-}
-
-/**
- * Retrieve a display data representation of the exception, which can be 
submitted to
- * the service in place of the actual display data.
- */
-public DisplayData asDisplayData() {
-  return DisplayData.from(this);
-

[2/2] incubator-beam git commit: This closes #355

2016-05-20 Thread bchambers

This closes #355


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/bb086b8d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/bb086b8d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/bb086b8d

Branch: refs/heads/master
Commit: bb086b8d367e2c360d965659b094035cf01d9959
Parents: 4755c5a 1e669c4
Author: bchambers <bchamb...@google.com>
Authored: Fri May 20 17:51:15 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Fri May 20 17:51:15 2016 -0700

--
 .../dataflow/DataflowPipelineTranslator.java| 50 +--
 .../DataflowPipelineTranslatorTest.java | 63 --
 .../sdk/options/ProxyInvocationHandler.java |  4 +-
 .../sdk/transforms/display/DisplayData.java | 14 +--
 .../sdk/options/ProxyInvocationHandlerTest.java |  6 +-
 .../sdk/transforms/display/DisplayDataTest.java | 91 ++--
 6 files changed, 83 insertions(+), 145 deletions(-)
--

[1/2] incubator-beam git commit: Add labels for SDK display data

2016-05-16 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 0f2e6f37e -> 638cc2071


Add labels for SDK display data


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/62f05d8b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/62f05d8b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/62f05d8b

Branch: refs/heads/master
Commit: 62f05d8b2513fbe7bb5b2ac181f9550634317792
Parents: 0f2e6f3
Author: Scott Wegner <sweg...@google.com>
Authored: Wed May 11 14:17:10 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Mon May 16 12:44:14 2016 -0700

--
 .../DataflowPipelineTranslatorTest.java |  2 +
 .../java/org/apache/beam/sdk/io/AvroIO.java | 28 +
 .../java/org/apache/beam/sdk/io/BigQueryIO.java | 43 +---
 .../sdk/io/BoundedReadFromUnboundedSource.java  |  9 ++--
 .../apache/beam/sdk/io/CompressedSource.java|  9 ++--
 .../org/apache/beam/sdk/io/CountingInput.java   | 12 --
 .../org/apache/beam/sdk/io/DatastoreIO.java | 18 +---
 .../org/apache/beam/sdk/io/FileBasedSink.java   |  3 +-
 .../org/apache/beam/sdk/io/FileBasedSource.java |  3 +-
 .../apache/beam/sdk/io/OffsetBasedSource.java   |  9 ++--
 .../java/org/apache/beam/sdk/io/PubsubIO.java   | 42 +++
 .../main/java/org/apache/beam/sdk/io/Read.java  |  6 ++-
 .../java/org/apache/beam/sdk/io/TextIO.java | 25 
 .../main/java/org/apache/beam/sdk/io/Write.java |  3 +-
 .../java/org/apache/beam/sdk/io/XmlSink.java|  6 ++-
 .../java/org/apache/beam/sdk/io/XmlSource.java  |  9 ++--
 .../sdk/transforms/ApproximateQuantiles.java|  6 ++-
 .../beam/sdk/transforms/ApproximateUnique.java  |  6 ++-
 .../org/apache/beam/sdk/transforms/Combine.java | 15 ---
 .../apache/beam/sdk/transforms/CombineFns.java  |  3 +-
 .../org/apache/beam/sdk/transforms/Filter.java  |  3 +-
 .../apache/beam/sdk/transforms/GroupByKey.java  |  3 +-
 .../transforms/IntraBundleParallelization.java  |  6 ++-
 .../apache/beam/sdk/transforms/MapElements.java |  3 +-
 .../org/apache/beam/sdk/transforms/Max.java |  3 +-
 .../org/apache/beam/sdk/transforms/Min.java |  3 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   |  3 +-
 .../apache/beam/sdk/transforms/Partition.java   |  6 ++-
 .../org/apache/beam/sdk/transforms/Sample.java  |  6 ++-
 .../org/apache/beam/sdk/transforms/Top.java |  6 ++-
 .../transforms/windowing/CalendarWindows.java   | 18 +---
 .../sdk/transforms/windowing/FixedWindows.java  |  6 ++-
 .../beam/sdk/transforms/windowing/Sessions.java |  3 +-
 .../transforms/windowing/SlidingWindows.java|  9 ++--
 .../beam/sdk/transforms/windowing/Window.java   | 18 +---
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java| 15 ---
 36 files changed, 243 insertions(+), 125 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/62f05d8b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
--
diff --git 
a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
 
b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
index ed7e67d..58c6f75 100644
--- 
a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
+++ 
b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
@@ -936,6 +936,7 @@ public class DataflowPipelineTranslatorTest implements 
Serializable {
 .build(),
 ImmutableMap.<String, Object>builder()
 .put("key", "fn")
+.put("label", "Transform Function")
 .put("type", "JAVA_CLASS")
 .put("value", fn1.getClass().getName())
 .put("shortValue", fn1.getClass().getSimpleName())
@@ -955,6 +956,7 @@ public class DataflowPipelineTranslatorTest implements 
Serializable {
 ImmutableSet<ImmutableMap<String, Object>> expectedFn2DisplayData = 
ImmutableSet.of(
 ImmutableMap.<String, Object>builder()
 .put("key", "fn")
+.put("label", "Transform Function")
 .put("type", "JAVA_CLASS")
 .put("value", fn2.getClass().getName())
 .put("shortValue", fn2.getClass().getSimpleName())

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/62f05d8b/sdks/java/core/src/main/java/org/a

[2/2] incubator-beam git commit: This closes #338

2016-05-16 Thread bchambers

This closes #338


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/638cc207
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/638cc207
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/638cc207

Branch: refs/heads/master
Commit: 638cc20719cb314d9e6b29fa79d45edcf4586dce
Parents: 0f2e6f3 62f05d8
Author: bchambers <bchamb...@google.com>
Authored: Mon May 16 12:44:19 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Mon May 16 12:44:19 2016 -0700

--
 .../DataflowPipelineTranslatorTest.java |  2 +
 .../java/org/apache/beam/sdk/io/AvroIO.java | 28 +
 .../java/org/apache/beam/sdk/io/BigQueryIO.java | 43 +---
 .../sdk/io/BoundedReadFromUnboundedSource.java  |  9 ++--
 .../apache/beam/sdk/io/CompressedSource.java|  9 ++--
 .../org/apache/beam/sdk/io/CountingInput.java   | 12 --
 .../org/apache/beam/sdk/io/DatastoreIO.java | 18 +---
 .../org/apache/beam/sdk/io/FileBasedSink.java   |  3 +-
 .../org/apache/beam/sdk/io/FileBasedSource.java |  3 +-
 .../apache/beam/sdk/io/OffsetBasedSource.java   |  9 ++--
 .../java/org/apache/beam/sdk/io/PubsubIO.java   | 42 +++
 .../main/java/org/apache/beam/sdk/io/Read.java  |  6 ++-
 .../java/org/apache/beam/sdk/io/TextIO.java | 25 
 .../main/java/org/apache/beam/sdk/io/Write.java |  3 +-
 .../java/org/apache/beam/sdk/io/XmlSink.java|  6 ++-
 .../java/org/apache/beam/sdk/io/XmlSource.java  |  9 ++--
 .../sdk/transforms/ApproximateQuantiles.java|  6 ++-
 .../beam/sdk/transforms/ApproximateUnique.java  |  6 ++-
 .../org/apache/beam/sdk/transforms/Combine.java | 15 ---
 .../apache/beam/sdk/transforms/CombineFns.java  |  3 +-
 .../org/apache/beam/sdk/transforms/Filter.java  |  3 +-
 .../apache/beam/sdk/transforms/GroupByKey.java  |  3 +-
 .../transforms/IntraBundleParallelization.java  |  6 ++-
 .../apache/beam/sdk/transforms/MapElements.java |  3 +-
 .../org/apache/beam/sdk/transforms/Max.java |  3 +-
 .../org/apache/beam/sdk/transforms/Min.java |  3 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   |  3 +-
 .../apache/beam/sdk/transforms/Partition.java   |  6 ++-
 .../org/apache/beam/sdk/transforms/Sample.java  |  6 ++-
 .../org/apache/beam/sdk/transforms/Top.java |  6 ++-
 .../transforms/windowing/CalendarWindows.java   | 18 +---
 .../sdk/transforms/windowing/FixedWindows.java  |  6 ++-
 .../beam/sdk/transforms/windowing/Sessions.java |  3 +-
 .../transforms/windowing/SlidingWindows.java|  9 ++--
 .../beam/sdk/transforms/windowing/Window.java   | 18 +---
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java| 15 ---
 36 files changed, 243 insertions(+), 125 deletions(-)
--

[1/2] incubator-beam git commit: Validate display data on transform primitives

2016-05-16 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 9b5cf3247 -> c0efe568e


Validate display data on transform primitives


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/6269b8fa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/6269b8fa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/6269b8fa

Branch: refs/heads/master
Commit: 6269b8fa06daa4608f5ce5b11054cca58747a229
Parents: 9b5cf32
Author: Scott Wegner 
Authored: Mon May 9 13:31:39 2016 -0700
Committer: Scott Wegner 
Committed: Mon May 16 09:41:21 2016 -0700

--
 runners/google-cloud-dataflow-java/pom.xml  | 10 +++
 .../runners/dataflow/io/DataflowAvroIOTest.java | 69 
 .../dataflow/io/DataflowBigQueryIOTest.java | 29 +++-
 .../dataflow/io/DataflowDatastoreIOTest.java| 63 ++
 .../dataflow/io/DataflowPubsubIOTest.java   | 62 ++
 .../runners/dataflow/io/DataflowTextIOTest.java | 15 -
 .../DataflowDisplayDataEvaluator.java   |  3 +-
 .../java/org/apache/beam/sdk/io/BigQueryIO.java | 11 
 .../java/org/apache/beam/sdk/io/PubsubIO.java   |  5 ++
 .../org/apache/beam/sdk/io/BigQueryIOTest.java  |  6 +-
 .../beam/sdk/io/CompressedSourceTest.java   |  3 +-
 .../sdk/options/ProxyInvocationHandlerTest.java |  6 +-
 .../sdk/transforms/ApproximateUniqueTest.java   |  3 +-
 .../display/DisplayDataEvaluator.java   | 28 
 .../display/DisplayDataEvaluatorTest.java   | 24 ++-
 .../transforms/display/DisplayDataMatchers.java |  8 +++
 .../display/DisplayDataMatchersTest.java|  3 +-
 .../sdk/transforms/windowing/WindowTest.java|  4 +-
 .../sdk/io/gcp/bigtable/BigtableIOTest.java |  9 +--
 19 files changed, 316 insertions(+), 45 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/6269b8fa/runners/google-cloud-dataflow-java/pom.xml
--
diff --git a/runners/google-cloud-dataflow-java/pom.xml 
b/runners/google-cloud-dataflow-java/pom.xml
index a1d48ce..f7f1d80 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -288,11 +288,21 @@
 
 
 
+  org.apache.avro
+  avro
+
+
+
   com.google.api-client
   google-api-client
 
 
 
+  com.google.apis
+  google-api-services-datastore-protobuf
+
+
+
   com.google.oauth-client
   google-oauth-client
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/6269b8fa/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowAvroIOTest.java
--
diff --git 
a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowAvroIOTest.java
 
b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowAvroIOTest.java
new file mode 100644
index 000..614affb
--- /dev/null
+++ 
b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowAvroIOTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.dataflow.io;
+
+import static 
org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+
+import static org.hamcrest.Matchers.hasItem;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.runners.dataflow.DataflowPipelineRunner;
+import 
org.apache.beam.runners.dataflow.transforms.DataflowDisplayDataEvaluator;
+import org.apache.beam.sdk.io.AvroIO;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
+
+import org.apache.avro.Schema;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Set;
+
+/**
+ * {@link

[2/2] incubator-beam git commit: This closes #325

2016-05-16 Thread bchambers

This closes #325


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/c0efe568
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/c0efe568
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/c0efe568

Branch: refs/heads/master
Commit: c0efe568e5291298c1394016a12e7979b37afc44
Parents: 9b5cf32 6269b8f
Author: bchambers <bchamb...@google.com>
Authored: Mon May 16 09:55:52 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Mon May 16 09:55:52 2016 -0700

--
 runners/google-cloud-dataflow-java/pom.xml  | 10 +++
 .../runners/dataflow/io/DataflowAvroIOTest.java | 69 
 .../dataflow/io/DataflowBigQueryIOTest.java | 29 +++-
 .../dataflow/io/DataflowDatastoreIOTest.java| 63 ++
 .../dataflow/io/DataflowPubsubIOTest.java   | 62 ++
 .../runners/dataflow/io/DataflowTextIOTest.java | 15 -
 .../DataflowDisplayDataEvaluator.java   |  3 +-
 .../java/org/apache/beam/sdk/io/BigQueryIO.java | 11 
 .../java/org/apache/beam/sdk/io/PubsubIO.java   |  5 ++
 .../org/apache/beam/sdk/io/BigQueryIOTest.java  |  6 +-
 .../beam/sdk/io/CompressedSourceTest.java   |  3 +-
 .../sdk/options/ProxyInvocationHandlerTest.java |  6 +-
 .../sdk/transforms/ApproximateUniqueTest.java   |  3 +-
 .../display/DisplayDataEvaluator.java   | 28 
 .../display/DisplayDataEvaluatorTest.java   | 24 ++-
 .../transforms/display/DisplayDataMatchers.java |  8 +++
 .../display/DisplayDataMatchersTest.java|  3 +-
 .../sdk/transforms/windowing/WindowTest.java|  4 +-
 .../sdk/io/gcp/bigtable/BigtableIOTest.java |  9 +--
 19 files changed, 316 insertions(+), 45 deletions(-)
--

[3/9] incubator-beam git commit: Add display data for MapElements transform

2016-05-11 Thread bchambers

Add display data for MapElements transform


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/0dab643a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/0dab643a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/0dab643a

Branch: refs/heads/master
Commit: 0dab643ae3c7d275016f8b11e406af5d15e7f388
Parents: 9f6c27c
Author: Scott Wegner <sweg...@google.com>
Authored: Tue May 3 11:16:51 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700

--
 .../transforms/DataflowMapElementsTest.java | 55 
 .../apache/beam/sdk/transforms/MapElements.java | 12 +
 .../beam/sdk/transforms/MapElementsTest.java| 36 -
 3 files changed, 101 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dab643a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowMapElementsTest.java
--
diff --git 
a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowMapElementsTest.java
 
b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowMapElementsTest.java
new file mode 100644
index 000..8a5e67d
--- /dev/null
+++ 
b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowMapElementsTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.dataflow.transforms;
+
+import static 
org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+
+import static org.hamcrest.Matchers.hasItem;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
+
+import org.junit.Test;
+
+import java.io.Serializable;
+import java.util.Set;
+
+/**
+ * Unit tests for Dataflow usage of {@link MapElements} transforms.
+ */
+public class DataflowMapElementsTest implements Serializable {
+  @Test
+  public void testPrimitiveDisplayData() {
+SimpleFunction mapFn = new SimpleFunction<Integer, Integer>() {
+  @Override
+  public Integer apply(Integer input) {
+return input;
+  }
+};
+
+MapElements map = MapElements.via(mapFn);
+DisplayDataEvaluator evaluator = DataflowDisplayDataEvaluator.create();
+
+Set displayData = 
evaluator.displayDataForPrimitiveTransforms(map);
+assertThat("MapElements should include the mapFn in its primitive display 
data",
+displayData, hasItem(hasDisplayItem("mapFn", mapFn.getClass(;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dab643a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java
index 47c2f5d..29d1dde 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.sdk.transforms;
 
+import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TypeDescriptor;
 
@@ -108,6 +109,17 @@ extends PTransform<PCollection, 
PCollection> {
   public void processElement(ProcessContext c) {
 c.output(fn.apply(c.element()));
   }
+
+  @Override
+  public void populateDisplayData(DisplayData.Builder

[2/9] incubator-beam git commit: Exclude default Window.allowedLateness display data

2016-05-11 Thread bchambers

Exclude default Window.allowedLateness display data


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/d9355c28
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/d9355c28
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/d9355c28

Branch: refs/heads/master
Commit: d9355c286fd7f3e6f7069b34172c32554b57bac8
Parents: ea8019e
Author: Scott Wegner <sweg...@google.com>
Authored: Fri May 6 16:03:12 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700

--
 .../org/apache/beam/sdk/transforms/windowing/Window.java| 9 +++--
 .../apache/beam/sdk/transforms/windowing/WindowTest.java| 6 --
 2 files changed, 11 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d9355c28/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
index e9b3437..b9dd451 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
@@ -601,8 +601,13 @@ public class Window {
   super.populateDisplayData(builder);
   builder
   .add(DisplayData.item("windowFn", windowFn.getClass()))
-  .include(windowFn)
-  .addIfNotNull(DisplayData.item("allowedLateness", allowedLateness));
+  .include(windowFn);
+
+  if (allowedLateness != null) {
+builder.addIfNotDefault(DisplayData.item("allowedLateness", 
allowedLateness),
+Duration.millis(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()));
+
+  }
 
   if (trigger != null && !(trigger instanceof DefaultTrigger)) {
 builder.add(DisplayData.item("trigger", trigger.toString()));

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d9355c28/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
--
diff --git 
a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
 
b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
index 8ad590d..885f549 100644
--- 
a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
+++ 
b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
@@ -285,11 +285,13 @@ public class WindowTest implements Serializable {
   }
 
   @Test
-  public void testDisplayDataExcludesDefaultTrigger() {
+  public void testDisplayDataExcludesDefaults() {
 Window.Bound window = Window.into(new GlobalWindows())
-.triggering(DefaultTrigger.of());
+.triggering(DefaultTrigger.of())
+
.withAllowedLateness(Duration.millis(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()));
 
 DisplayData data = DisplayData.from(window);
 assertThat(data, not(hasDisplayItem(hasKey("trigger";
+assertThat(data, not(hasDisplayItem(hasKey("allowedLateness";
   }
 }

[6/9] incubator-beam git commit: Fix Write transform primitive display data

2016-05-11 Thread bchambers

Fix Write transform primitive display data


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ea8019ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ea8019ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ea8019ec

Branch: refs/heads/master
Commit: ea8019eced55023e060fd6472b8d69d15bf73839
Parents: 007526f
Author: Scott Wegner <sweg...@google.com>
Authored: Fri May 6 11:40:47 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700

--
 .../runners/dataflow/io/DataflowTextIOTest.java | 23 +++-
 .../main/java/org/apache/beam/sdk/io/Write.java |  5 +
 2 files changed, 27 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ea8019ec/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java
--
diff --git 
a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java
 
b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java
index bfc99e8..8ff7d0e 100644
--- 
a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java
+++ 
b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java
@@ -17,11 +17,21 @@
  */
 package org.apache.beam.runners.dataflow.io;
 
+import static 
org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+import static 
org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasValue;
+
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.startsWith;
+import static org.junit.Assert.assertThat;
+
 import org.apache.beam.runners.dataflow.DataflowPipelineRunner;
 import org.apache.beam.runners.dataflow.testing.TestDataflowPipelineOptions;
+import 
org.apache.beam.runners.dataflow.transforms.DataflowDisplayDataEvaluator;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
 import org.apache.beam.sdk.util.GcsUtil;
 import org.apache.beam.sdk.util.TestCredential;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
@@ -41,13 +51,13 @@ import java.nio.channels.SeekableByteChannel;
 import java.nio.file.Files;
 import java.nio.file.StandardOpenOption;
 import java.util.List;
+import java.util.Set;
 
 /**
  * {@link DataflowPipelineRunner} specific tests for TextIO Read and Write 
transforms.
  */
 @RunWith(JUnit4.class)
 public class DataflowTextIOTest {
-
   private TestDataflowPipelineOptions buildTestPipelineOptions() {
 TestDataflowPipelineOptions options =
 PipelineOptionsFactory.as(TestDataflowPipelineOptions.class);
@@ -116,4 +126,15 @@ public class DataflowTextIOTest {
   private void applyRead(Pipeline pipeline, String path) {
 pipeline.apply("Read(" + path + ")", TextIO.Read.from(path));
   }
+
+  @Test
+  public void testPrimitiveDisplayData() {
+DisplayDataEvaluator evaluator = DataflowDisplayDataEvaluator.create();
+
+TextIO.Write.Bound write = TextIO.Write.to("foobar");
+
+Set displayData = 
evaluator.displayDataForPrimitiveTransforms(write);
+assertThat("TextIO.Write should include the file prefix in its primitive 
display data",
+displayData, hasItem(hasDisplayItem(hasValue(startsWith("foobar");
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ea8019ec/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java
--
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java
index 9cb026a..0f2dbf8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java
@@ -204,6 +204,11 @@ public class Write {
 c.output(result);
   }
 }
+
+@Override
+public void populateDisplayData(DisplayData.Builder builder) {
+  Write.Bound.this.populateDisplayData(builder);
+}
   }).withSideInputs(writeOperationView))
   .setCoder(writeOperation.getWriterResultCoder());

[5/9] incubator-beam git commit: Exclude JsonIgnore PipelineOptions from DisplayData

2016-05-11 Thread bchambers

Exclude JsonIgnore PipelineOptions from DisplayData


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/43085f7e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/43085f7e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/43085f7e

Branch: refs/heads/master
Commit: 43085f7e53e693202ab91768a7f3688fc54f24ee
Parents: 0dab643
Author: Scott Wegner <sweg...@google.com>
Authored: Tue May 3 14:10:59 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700

--
 .../beam/sdk/options/PipelineOptionSpec.java| 61 +---
 .../sdk/options/ProxyInvocationHandler.java | 10 
 .../options/PipelineOptionsReflectorTest.java   | 32 ++
 .../sdk/options/ProxyInvocationHandlerTest.java |  9 +++
 4 files changed, 65 insertions(+), 47 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/43085f7e/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java
index 71f9d46..9a88f70 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java
@@ -17,73 +17,40 @@
  */
 package org.apache.beam.sdk.options;
 
-import com.google.common.base.MoreObjects;
-import com.google.common.base.Objects;
+import com.google.auto.value.AutoValue;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
 
 import java.lang.reflect.Method;
 
 /**
  * For internal use. Specification for an option defined in a {@link 
PipelineOptions} interface.
  */
-class PipelineOptionSpec {
-  private final Class clazz;
-  private final String name;
-  private final Method getter;
-
+@AutoValue
+abstract class PipelineOptionSpec {
   static PipelineOptionSpec of(Class clazz, String 
name, Method getter) {
-return new PipelineOptionSpec(clazz, name, getter);
-  }
-
-  private PipelineOptionSpec(Class clazz, String 
name, Method getter) {
-this.clazz = clazz;
-this.name = name;
-this.getter = getter;
+return new AutoValue_PipelineOptionSpec(clazz, name, getter);
   }
 
   /**
* The {@link PipelineOptions} interface which defines this {@link 
PipelineOptionSpec}.
*/
-  Class getDefiningInterface() {
-return clazz;
-  }
+  abstract Class getDefiningInterface();
 
   /**
* Name of the property.
*/
-  String getName() {
-return name;
-  }
+  abstract String getName();
 
   /**
* The getter method for this property.
*/
-  Method getGetterMethod() {
-return getter;
-  }
+  abstract Method getGetterMethod();
 
-  @Override
-  public String toString() {
-return MoreObjects.toStringHelper(this)
-.add("definingInterface", getDefiningInterface())
-.add("name", getName())
-.add("getterMethod", getGetterMethod())
-.toString();
-  }
-
-  @Override
-  public int hashCode() {
-return Objects.hashCode(getDefiningInterface(), getName(), 
getGetterMethod());
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-if (!(obj instanceof PipelineOptionSpec)) {
-  return false;
-}
-
-PipelineOptionSpec that = (PipelineOptionSpec) obj;
-return Objects.equal(this.getDefiningInterface(), 
that.getDefiningInterface())
-&& Objects.equal(this.getName(), that.getName())
-&& Objects.equal(this.getGetterMethod(), that.getGetterMethod());
+  /**
+   * Whether the option should be serialized. Uses the {@link JsonIgnore} 
annotation.
+   */
+  boolean shouldSerialize() {
+return !getGetterMethod().isAnnotationPresent(JsonIgnore.class);
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/43085f7e/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java
index 745549f..159eb5b 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java
@@ -281,6 +281,12 @@ class ProxyInvocationHandler implements InvocationHandler {
   HashSet specs = new 
HashSet<>(optionsMap.get(option.getKey()));
 
   for (PipelineOptionSpec optionSpec : specs) {
+

[1/2] incubator-beam git commit: Remove ClassForDisplay helper type

2016-05-02 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master 659cf2ee0 -> 1262819ac


Remove ClassForDisplay helper type

- Refactor Combine display data to not use ClassForDisplay
- Refactor DoFnReflector.SimpleDoFnAdapter to not use display data namespace 
override
- Add test case for display data multi-level namespace overrides


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/5eafdee5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/5eafdee5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/5eafdee5

Branch: refs/heads/master
Commit: 5eafdee53cc49938921c5fbdf2bb81483f8773af
Parents: 659cf2e
Author: Scott Wegner <sweg...@google.com>
Authored: Wed Apr 27 13:08:49 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Mon May 2 13:10:10 2016 -0700

--
 .../org/apache/beam/sdk/transforms/Combine.java | 158 ++-
 .../beam/sdk/transforms/DoFnReflector.java  |   2 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   |   2 +-
 .../sdk/transforms/display/ClassForDisplay.java |  94 ---
 .../sdk/transforms/display/DisplayData.java |  78 -
 .../sdk/transforms/display/HasDisplayData.java  |  55 ---
 .../apache/beam/sdk/transforms/CombineTest.java |  16 ++
 .../transforms/display/ClassForDisplayTest.java |  66 
 .../transforms/display/DisplayDataMatchers.java |  14 ++
 .../sdk/transforms/display/DisplayDataTest.java |  83 ++
 .../display/ClassForDisplayJava8Test.java   |  45 --
 11 files changed, 234 insertions(+), 379 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5eafdee5/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
index 68fc1cf..f3a7c03 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
@@ -36,7 +36,6 @@ import 
org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
 import org.apache.beam.sdk.transforms.CombineWithContext.Context;
 import 
org.apache.beam.sdk.transforms.CombineWithContext.KeyedCombineFnWithContext;
 import 
org.apache.beam.sdk.transforms.CombineWithContext.RequiresContextInternal;
-import org.apache.beam.sdk.transforms.display.ClassForDisplay;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.transforms.display.HasDisplayData;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
@@ -103,7 +102,7 @@ public class Combine {
*/
   public static  Globally<V, V> globally(
   SerializableFunction<Iterable, V> combiner) {
-return globally(IterableCombineFn.of(combiner), 
ClassForDisplay.fromInstance(combiner));
+return globally(IterableCombineFn.of(combiner), 
displayDataForFn(combiner));
   }
 
 
@@ -124,12 +123,17 @@ public class Combine {
*/
   public static <InputT, OutputT> Globally<InputT, OutputT> globally(
   GlobalCombineFn fn) {
-return globally(fn, ClassForDisplay.fromInstance(fn));
+return globally(fn, displayDataForFn(fn));
+  }
+
+  private static  DisplayData.Item> displayDataForFn(T 
fn) {
+return DisplayData.item("combineFn", fn.getClass());
   }
 
   private static <InputT, OutputT> Globally<InputT, OutputT> globally(
-  GlobalCombineFn fn, ClassForDisplay 
fnClass) {
-return new Globally<>(fn, fnClass, true, 0);
+  GlobalCombineFn fn,
+  DisplayData.Item> fnDisplayData) {
+return new Globally<>(fn, fnDisplayData, true, 0);
   }
 
   /**
@@ -150,7 +154,7 @@ public class Combine {
*/
   public static <K, V> PerKey<K, V, V> perKey(
   SerializableFunction<Iterable, V> fn) {
-return perKey(IterableCombineFn.of(fn).asKeyedFn(), 
ClassForDisplay.fromInstance(fn));
+return perKey(IterableCombineFn.of(fn).asKeyedFn(), 
displayDataForFn(fn));
   }
 
   /**
@@ -171,7 +175,7 @@ public class Combine {
*/
   public static <K, InputT, OutputT> PerKey<K, InputT, OutputT> perKey(
   GlobalCombineFn fn) {
-return perKey(fn.asKeyedFn(), ClassForDisplay.fromInstance(fn));
+return perKey(fn.asKeyedFn(), displayDataForFn(fn));
   }
 
   /**
@@ -192,12 +196,13 @@ public class Combine {
*/
   public static <K, InputT, OutputT> PerKey<K, InputT, OutputT> perKey(
   PerKeyCombineFn fn) {
-return perKey(fn, ClassForDisplay.fromInstance(fn));
+return perKey(fn, displayDataForFn(fn));
   }

[2/2] incubator-beam git commit: This closes #264

2016-05-02 Thread bchambers

This closes #264


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/659cf2ee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/659cf2ee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/659cf2ee

Branch: refs/heads/master
Commit: 659cf2ee0b32a4e49fac50999e886a8f052a3fb7
Parents: b9116ac 367f3ac
Author: bchambers <bchamb...@google.com>
Authored: Mon May 2 11:04:13 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Mon May 2 11:04:13 2016 -0700

--
 .../direct/InMemoryWatermarkManager.java| 34 +---
 1 file changed, 23 insertions(+), 11 deletions(-)
--

[1/2] incubator-beam git commit: Fix update sequence in InMemoryWatermarkManager

2016-05-02 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master b9116ac42 -> 659cf2ee0


Fix update sequence in InMemoryWatermarkManager

Because the WatermarkManager is not synchronized within the call to
updatePending, the sequence in which pending queues are updated must be
in such a manner as to add additional restrictions, then remove any
restrictions which no longer apply. This ensures any intermediate read
will see at worst a more restricted watermark than the actual watermark.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/367f3aca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/367f3aca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/367f3aca

Branch: refs/heads/master
Commit: 367f3acac60fa421b797009eeb131f8c47d75f1d
Parents: b9116ac
Author: Thomas Groh <tg...@google.com>
Authored: Thu Apr 28 17:38:50 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Mon May 2 11:03:52 2016 -0700

--
 .../direct/InMemoryWatermarkManager.java| 34 +---
 1 file changed, 23 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/367f3aca/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InMemoryWatermarkManager.java
--
diff --git 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InMemoryWatermarkManager.java
 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InMemoryWatermarkManager.java
index 07b6bb4..769457a 100644
--- 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InMemoryWatermarkManager.java
+++ 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InMemoryWatermarkManager.java
@@ -469,9 +469,6 @@ public class InMemoryWatermarkManager {
 }
 
 private synchronized void updateTimers(TimerUpdate update) {
-  for (TimerData completedTimer : update.completedTimers) {
-pendingTimers.remove(completedTimer);
-  }
   Map<TimeDomain, NavigableSet> timerMap = timerMap(update.key);
   for (TimerData addedTimer : update.setTimers) {
 NavigableSet timerQueue = 
timerMap.get(addedTimer.getDomain());
@@ -479,6 +476,10 @@ public class InMemoryWatermarkManager {
   timerQueue.add(addedTimer);
 }
   }
+
+  for (TimerData completedTimer : update.completedTimers) {
+pendingTimers.remove(completedTimer);
+  }
   for (TimerData deletedTimer : update.deletedTimers) {
 NavigableSet timerQueue = 
timerMap.get(deletedTimer.getDomain());
 if (timerQueue != null) {
@@ -832,10 +833,16 @@ public class InMemoryWatermarkManager {
   }
 
   /**
-   * Removes all of the completed Timers from the collection of pending 
timers, adds all new timers,
-   * and removes all deleted timers. Removes all elements consumed by the 
input bundle from the
-   * {@link PTransform PTransforms} collection of pending elements, and adds 
all elements produced
-   * by the {@link PTransform} to the pending queue of each consumer.
+   * First adds all produced elements to the queue of pending elements for 
each consumer, then adds
+   * all pending timers to the collection of pending timers, then removes all 
completed and deleted
+   * timers from the collection of pending timers, then removes all completed 
elements from the
+   * pending queue of the transform.
+   *
+   * It is required that all newly pending elements are added to the queue 
of pending elements
+   * for each consumer prior to the completed elements being removed, as doing 
otherwise could cause
+   * a Watermark to appear in a state in which the upstream (completed) 
element does not hold the
+   * watermark but the element it produced is not yet pending. This can cause 
the watermark to
+   * erroneously advance.
*/
   private void updatePending(
   CommittedBundle input,
@@ -843,17 +850,22 @@ public class InMemoryWatermarkManager {
   TimerUpdate timerUpdate,
   Iterable> outputs) {
 TransformWatermarks completedTransform = 
transformToWatermarks.get(transform);
-completedTransform.updateTimers(timerUpdate);
-if (input != null) {
-  completedTransform.removePending(input);
-}
 
+// Newly pending elements must be added before completed elements are 
removed, as the two
+// do not share a Mutex within this call and thus can be interleaved with 
external calls to
+// refresh.
 for (CommittedBundle bundle : outputs) {
   for (AppliedPTransform consumer : 
consumers.get(bundle.getPCollection())) {
 TransformWatermarks watermarks = transformToWatermarks.get(consumer);
 wate

[3/3] incubator-beam git commit: This closes #247

2016-04-27 Thread bchambers

This closes #247


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/81bf4d98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/81bf4d98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/81bf4d98

Branch: refs/heads/master
Commit: 81bf4d981a7d5d86955c02003fb615c4985552bb
Parents: a9387fc 47c7abc
Author: bchambers <bchamb...@google.com>
Authored: Wed Apr 27 16:44:28 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Apr 27 16:44:28 2016 -0700

--
 .../DataflowPipelineTranslatorTest.java |  10 +-
 .../java/org/apache/beam/sdk/io/AvroIO.java |  18 +-
 .../java/org/apache/beam/sdk/io/BigQueryIO.java |  20 +-
 .../sdk/io/BoundedReadFromUnboundedSource.java  |   6 +-
 .../apache/beam/sdk/io/CompressedSource.java|   6 +-
 .../org/apache/beam/sdk/io/CountingInput.java   |   8 +-
 .../org/apache/beam/sdk/io/DatastoreIO.java |  12 +-
 .../org/apache/beam/sdk/io/FileBasedSink.java   |   2 +-
 .../org/apache/beam/sdk/io/FileBasedSource.java |   2 +-
 .../apache/beam/sdk/io/OffsetBasedSource.java   |   6 +-
 .../java/org/apache/beam/sdk/io/PubsubIO.java   |  18 +-
 .../main/java/org/apache/beam/sdk/io/Read.java  |   4 +-
 .../java/org/apache/beam/sdk/io/TextIO.java |  18 +-
 .../main/java/org/apache/beam/sdk/io/Write.java |   2 +-
 .../java/org/apache/beam/sdk/io/XmlSink.java|   4 +-
 .../java/org/apache/beam/sdk/io/XmlSource.java  |   6 +-
 .../sdk/options/ProxyInvocationHandler.java |  22 +-
 .../sdk/transforms/ApproximateQuantiles.java|   4 +-
 .../beam/sdk/transforms/ApproximateUnique.java  |   4 +-
 .../org/apache/beam/sdk/transforms/Combine.java |  10 +-
 .../apache/beam/sdk/transforms/CombineFns.java  |   2 +-
 .../org/apache/beam/sdk/transforms/Filter.java  |   2 +-
 .../apache/beam/sdk/transforms/GroupByKey.java  |   2 +-
 .../transforms/IntraBundleParallelization.java  |   4 +-
 .../org/apache/beam/sdk/transforms/Max.java |   2 +-
 .../org/apache/beam/sdk/transforms/Min.java |   2 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   |   2 +-
 .../apache/beam/sdk/transforms/Partition.java   |   4 +-
 .../org/apache/beam/sdk/transforms/Sample.java  |   4 +-
 .../org/apache/beam/sdk/transforms/Top.java |   4 +-
 .../sdk/transforms/display/ClassForDisplay.java |   7 +-
 .../sdk/transforms/display/DisplayData.java | 745 ++-
 .../sdk/transforms/display/HasDisplayData.java  |  12 +-
 .../transforms/windowing/CalendarWindows.java   |  15 +-
 .../sdk/transforms/windowing/FixedWindows.java  |   4 +-
 .../beam/sdk/transforms/windowing/Sessions.java |   2 +-
 .../transforms/windowing/SlidingWindows.java|   6 +-
 .../beam/sdk/transforms/windowing/Window.java   |  12 +-
 .../io/BoundedReadFromUnboundedSourceTest.java  |   2 +-
 .../beam/sdk/io/CompressedSourceTest.java   |   2 +-
 .../java/org/apache/beam/sdk/io/ReadTest.java   |   4 +-
 .../java/org/apache/beam/sdk/io/WriteTest.java  |   2 +-
 .../beam/sdk/transforms/CombineFnsTest.java |   4 +-
 .../apache/beam/sdk/transforms/CombineTest.java |   2 +-
 .../IntraBundleParallelizationTest.java |   2 +-
 .../apache/beam/sdk/transforms/ParDoTest.java   |   6 +-
 .../transforms/display/DisplayDataMatchers.java |  62 +-
 .../display/DisplayDataMatchersTest.java|  17 +-
 .../sdk/transforms/display/DisplayDataTest.java | 289 +++
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java|  10 +-
 50 files changed, 601 insertions(+), 814 deletions(-)
--

[2/3] incubator-beam git commit: Reduce overloaded methods in DisplayData

2016-04-27 Thread bchambers

Reduce overloaded methods in DisplayData

Introduce `DisplayData#Item` which represents a single item of
`DisplayData`. Reduce the number of methods on `DisplayData#Builder` to
just deal with adding items, rather than needing per-type overrides.

Update all providers of `DisplayData` to use the new API.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/47c7abca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/47c7abca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/47c7abca

Branch: refs/heads/master
Commit: 47c7abcacdd8ee1c14c2c8d94a263f96dfe5bf8d
Parents: a9387fc
Author: Scott Wegner <sweg...@google.com>
Authored: Mon Apr 25 16:04:38 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Apr 27 16:44:17 2016 -0700

--
 .../DataflowPipelineTranslatorTest.java |  10 +-
 .../java/org/apache/beam/sdk/io/AvroIO.java |  18 +-
 .../java/org/apache/beam/sdk/io/BigQueryIO.java |  20 +-
 .../sdk/io/BoundedReadFromUnboundedSource.java  |   6 +-
 .../apache/beam/sdk/io/CompressedSource.java|   6 +-
 .../org/apache/beam/sdk/io/CountingInput.java   |   8 +-
 .../org/apache/beam/sdk/io/DatastoreIO.java |  12 +-
 .../org/apache/beam/sdk/io/FileBasedSink.java   |   2 +-
 .../org/apache/beam/sdk/io/FileBasedSource.java |   2 +-
 .../apache/beam/sdk/io/OffsetBasedSource.java   |   6 +-
 .../java/org/apache/beam/sdk/io/PubsubIO.java   |  18 +-
 .../main/java/org/apache/beam/sdk/io/Read.java  |   4 +-
 .../java/org/apache/beam/sdk/io/TextIO.java |  18 +-
 .../main/java/org/apache/beam/sdk/io/Write.java |   2 +-
 .../java/org/apache/beam/sdk/io/XmlSink.java|   4 +-
 .../java/org/apache/beam/sdk/io/XmlSource.java  |   6 +-
 .../sdk/options/ProxyInvocationHandler.java |  22 +-
 .../sdk/transforms/ApproximateQuantiles.java|   4 +-
 .../beam/sdk/transforms/ApproximateUnique.java  |   4 +-
 .../org/apache/beam/sdk/transforms/Combine.java |  10 +-
 .../apache/beam/sdk/transforms/CombineFns.java  |   2 +-
 .../org/apache/beam/sdk/transforms/Filter.java  |   2 +-
 .../apache/beam/sdk/transforms/GroupByKey.java  |   2 +-
 .../transforms/IntraBundleParallelization.java  |   4 +-
 .../org/apache/beam/sdk/transforms/Max.java |   2 +-
 .../org/apache/beam/sdk/transforms/Min.java |   2 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   |   2 +-
 .../apache/beam/sdk/transforms/Partition.java   |   4 +-
 .../org/apache/beam/sdk/transforms/Sample.java  |   4 +-
 .../org/apache/beam/sdk/transforms/Top.java |   4 +-
 .../sdk/transforms/display/ClassForDisplay.java |   7 +-
 .../sdk/transforms/display/DisplayData.java | 745 ++-
 .../sdk/transforms/display/HasDisplayData.java  |  12 +-
 .../transforms/windowing/CalendarWindows.java   |  15 +-
 .../sdk/transforms/windowing/FixedWindows.java  |   4 +-
 .../beam/sdk/transforms/windowing/Sessions.java |   2 +-
 .../transforms/windowing/SlidingWindows.java|   6 +-
 .../beam/sdk/transforms/windowing/Window.java   |  12 +-
 .../io/BoundedReadFromUnboundedSourceTest.java  |   2 +-
 .../beam/sdk/io/CompressedSourceTest.java   |   2 +-
 .../java/org/apache/beam/sdk/io/ReadTest.java   |   4 +-
 .../java/org/apache/beam/sdk/io/WriteTest.java  |   2 +-
 .../beam/sdk/transforms/CombineFnsTest.java |   4 +-
 .../apache/beam/sdk/transforms/CombineTest.java |   2 +-
 .../IntraBundleParallelizationTest.java |   2 +-
 .../apache/beam/sdk/transforms/ParDoTest.java   |   6 +-
 .../transforms/display/DisplayDataMatchers.java |  62 +-
 .../display/DisplayDataMatchersTest.java|  17 +-
 .../sdk/transforms/display/DisplayDataTest.java | 289 +++
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java|  10 +-
 50 files changed, 601 insertions(+), 814 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/47c7abca/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
--
diff --git 
a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
 
b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
index 3a39e41..58b38d2 100644
--- 
a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
+++ 
b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
@@ -874,10 +874,10 @@ public class DataflowPipelineTranslatorTest implements 
Serializable {
   @Override
   public void populateDisplayData(DisplayData.Builder builder) {
 builder
-

[1/3] incubator-beam git commit: Refactor CommittedBundle in InProcessBundleFactory

2016-04-27 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master e3105c8e1 -> 6914f2a1d


Refactor CommittedBundle in InProcessBundleFactory

Move to a static nested class.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/e4e7d2e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/e4e7d2e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/e4e7d2e4

Branch: refs/heads/master
Commit: e4e7d2e4fb9da7248df7cde3877e3e6b88467dae
Parents: e3105c8
Author: Thomas Groh <tg...@google.com>
Authored: Mon Apr 25 09:08:09 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Apr 27 10:13:15 2016 -0700

--
 .../inprocess/InProcessBundleFactory.java   | 82 
 1 file changed, 50 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e4e7d2e4/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/inprocess/InProcessBundleFactory.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/inprocess/InProcessBundleFactory.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/inprocess/InProcessBundleFactory.java
index e39d02e..788fde1 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/inprocess/InProcessBundleFactory.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/inprocess/InProcessBundleFactory.java
@@ -100,38 +100,56 @@ class InProcessBundleFactory implements BundleFactory {
   checkState(!committed, "Can't commit already committed bundle %s", this);
   committed = true;
   final Iterable<WindowedValue> committedElements = elements.build();
-  return new CommittedBundle() {
-@Override
-@Nullable
-public Object getKey() {
-  return key;
-}
-
-@Override
-public Iterable<WindowedValue> getElements() {
-  return committedElements;
-}
-
-@Override
-public PCollection getPCollection() {
-  return pcollection;
-}
-
-@Override
-public Instant getSynchronizedProcessingOutputWatermark() {
-  return synchronizedCompletionTime;
-}
-
-@Override
-public String toString() {
-  return MoreObjects.toStringHelper(this)
-  .omitNullValues()
-  .add("pcollection", pcollection)
-  .add("key", key)
-  .add("elements", committedElements)
-  .toString();
-}
-  };
+  return new CommittedInProcessBundle<>(
+  pcollection, key, committedElements, synchronizedCompletionTime);
+}
+  }
+  private static class CommittedInProcessBundle implements 
CommittedBundle {
+public CommittedInProcessBundle(
+PCollection pcollection,
+Object key,
+Iterable<WindowedValue> committedElements,
+Instant synchronizedCompletionTime) {
+  this.pcollection = pcollection;
+  this.key = key;
+  this.committedElements = committedElements;
+  this.synchronizedCompletionTime = synchronizedCompletionTime;
+}
+
+private final PCollection pcollection;
+private final Object key;
+private final Iterable<WindowedValue> committedElements;
+private final Instant synchronizedCompletionTime;
+
+@Override
+@Nullable
+public Object getKey() {
+  return key;
+}
+
+@Override
+public Iterable<WindowedValue> getElements() {
+  return committedElements;
+}
+
+@Override
+public PCollection getPCollection() {
+  return pcollection;
+}
+
+@Override
+public Instant getSynchronizedProcessingOutputWatermark() {
+  return synchronizedCompletionTime;
+}
+
+@Override
+public String toString() {
+  return MoreObjects.toStringHelper(this)
+  .omitNullValues()
+  .add("pcollection", pcollection)
+  .add("key", key)
+  .add("elements", committedElements)
+  .toString();
 }
   }
 }

[2/3] incubator-beam git commit: Add withElements to CommittedBundle

2016-04-27 Thread bchambers

Add withElements to CommittedBundle

The unprocessed elements of a partially completed bundle must be
placed in a new bundle to be processed at a later time. The bundle in
which they are processed should also have identical properties to the
bundle which the elements were initially present in.

withElements provides a simple way to create a "copy" of a bundle that
contains different elements.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/b108c66e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/b108c66e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/b108c66e

Branch: refs/heads/master
Commit: b108c66e641f888a3f6732e581b860783dc3daf8
Parents: e4e7d2e
Author: Thomas Groh <tg...@google.com>
Authored: Mon Apr 25 09:09:15 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Apr 27 10:13:16 2016 -0700

--
 .../inprocess/InProcessBundleFactory.java   |  7 
 .../inprocess/InProcessPipelineRunner.java  | 13 +++
 .../inprocess/InProcessBundleFactoryTest.java   | 40 +++-
 3 files changed, 58 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/b108c66e/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/inprocess/InProcessBundleFactory.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/inprocess/InProcessBundleFactory.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/inprocess/InProcessBundleFactory.java
index 788fde1..58bc4fb 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/inprocess/InProcessBundleFactory.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/inprocess/InProcessBundleFactory.java
@@ -104,6 +104,7 @@ class InProcessBundleFactory implements BundleFactory {
   pcollection, key, committedElements, synchronizedCompletionTime);
 }
   }
+
   private static class CommittedInProcessBundle implements 
CommittedBundle {
 public CommittedInProcessBundle(
 PCollection pcollection,
@@ -151,5 +152,11 @@ class InProcessBundleFactory implements BundleFactory {
   .add("elements", committedElements)
   .toString();
 }
+
+@Override
+public CommittedBundle withElements(Iterable<WindowedValue> 
elements) {
+  return new CommittedInProcessBundle<>(
+  pcollection, key, ImmutableList.copyOf(elements), 
synchronizedCompletionTime);
+}
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/b108c66e/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/inprocess/InProcessPipelineRunner.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/inprocess/InProcessPipelineRunner.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/inprocess/InProcessPipelineRunner.java
index 7c28238..7897f2e 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/inprocess/InProcessPipelineRunner.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/inprocess/InProcessPipelineRunner.java
@@ -154,6 +154,19 @@ public class InProcessPipelineRunner
  * timers that fired to produce this bundle.
  */
 Instant getSynchronizedProcessingOutputWatermark();
+
+/**
+ * Return a new {@link CommittedBundle} that is like this one, except 
calls to
+ * {@link #getElements()} will return the provided elements. This bundle 
is unchanged.
+ *
+ * 
+ * The value of the {@link #getSynchronizedProcessingOutputWatermark() 
synchronized processing
+ * output watermark} of the returned {@link CommittedBundle} is equal to 
the value returned from
+ * the current bundle. This is used to ensure a {@link PTransform} that 
could not complete
+ * processing on input elements properly holds the synchronized processing 
time to the
+ * appropriate value.
+ */
+CommittedBundle withElements(Iterable<WindowedValue> elements);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/b108c66e/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/inprocess/InProcessBundleFactoryTest.java
--
diff --git 
a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/inprocess/InProcessBundleFactoryTest.java
 
b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/inprocess/InProcessBundleFactoryTest.java
index 9adb6f9..3c7c2c6 100644
--- 
a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/inprocess/InProcessBundleFactoryTest.java
+++ 
b/sdks/java/core/s

[3/3] incubator-beam git commit: This closes #231

2016-04-27 Thread bchambers

This closes #231


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/6914f2a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/6914f2a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/6914f2a1

Branch: refs/heads/master
Commit: 6914f2a1d44ca810918592f45e63e09982101f32
Parents: e3105c8 b108c66
Author: bchambers <bchamb...@google.com>
Authored: Wed Apr 27 10:13:21 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Wed Apr 27 10:13:21 2016 -0700

--
 .../inprocess/InProcessBundleFactory.java   | 89 +---
 .../inprocess/InProcessPipelineRunner.java  | 13 +++
 .../inprocess/InProcessBundleFactoryTest.java   | 40 -
 3 files changed, 108 insertions(+), 34 deletions(-)
--

[4/4] incubator-beam git commit: This closes #241

2016-04-26 Thread bchambers

This closes #241


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/7dc1a404
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/7dc1a404
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/7dc1a404

Branch: refs/heads/master
Commit: 7dc1a4047c8a145377c26ed73163a28f7b8835dd
Parents: d299e2c c6690c1
Author: bchambers <bchamb...@google.com>
Authored: Tue Apr 26 09:39:06 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Tue Apr 26 09:39:06 2016 -0700

--
 .../org/apache/beam/sdk/io/DatastoreIO.java |  1 +
 .../main/java/org/apache/beam/sdk/io/Read.java  |  2 +
 .../main/java/org/apache/beam/sdk/io/Sink.java  |  2 +-
 .../java/org/apache/beam/sdk/io/Source.java |  2 +-
 .../main/java/org/apache/beam/sdk/io/Write.java |  1 +
 .../sdk/transforms/ApproximateQuantiles.java|  1 +
 .../beam/sdk/transforms/ApproximateUnique.java  |  2 +
 .../org/apache/beam/sdk/transforms/Combine.java | 11 
 .../beam/sdk/transforms/CombineFnBase.java  |  4 +-
 .../apache/beam/sdk/transforms/CombineFns.java  |  4 ++
 .../beam/sdk/transforms/CombineWithContext.java |  1 +
 .../org/apache/beam/sdk/transforms/DoFn.java|  2 +-
 .../beam/sdk/transforms/DoFnWithContext.java|  2 +-
 .../org/apache/beam/sdk/transforms/Filter.java  |  4 ++
 .../apache/beam/sdk/transforms/GroupByKey.java  |  1 +
 .../transforms/IntraBundleParallelization.java  |  1 +
 .../org/apache/beam/sdk/transforms/Max.java |  1 +
 .../org/apache/beam/sdk/transforms/Min.java |  1 +
 .../apache/beam/sdk/transforms/PTransform.java  |  2 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   |  4 +-
 .../apache/beam/sdk/transforms/Partition.java   |  2 +
 .../org/apache/beam/sdk/transforms/Sample.java  |  2 +
 .../org/apache/beam/sdk/transforms/Top.java |  1 +
 .../sdk/transforms/display/ClassForDisplay.java |  4 +-
 .../sdk/transforms/display/DisplayData.java | 58 ++--
 .../sdk/transforms/display/HasDisplayData.java  |  8 +--
 .../transforms/windowing/CalendarWindows.java   |  6 ++
 .../sdk/transforms/windowing/FixedWindows.java  |  1 +
 .../beam/sdk/transforms/windowing/Sessions.java |  1 +
 .../transforms/windowing/SlidingWindows.java|  1 +
 .../beam/sdk/transforms/windowing/Window.java   |  1 +
 .../beam/sdk/transforms/windowing/WindowFn.java |  2 +-
 .../org/apache/beam/sdk/util/CombineFnUtil.java |  1 +
 .../io/BoundedReadFromUnboundedSourceTest.java  |  5 +-
 .../beam/sdk/io/CompressedSourceTest.java   |  5 +-
 .../java/org/apache/beam/sdk/io/ReadTest.java   |  6 +-
 .../java/org/apache/beam/sdk/io/WriteTest.java  |  5 +-
 .../beam/sdk/transforms/CombineFnsTest.java |  6 +-
 .../apache/beam/sdk/transforms/CombineTest.java |  4 +-
 .../IntraBundleParallelizationTest.java |  4 +-
 .../apache/beam/sdk/transforms/ParDoTest.java   |  8 +--
 .../transforms/display/DisplayDataMatchers.java | 14 ++---
 .../display/DisplayDataMatchersTest.java|  4 +-
 .../sdk/transforms/display/DisplayDataTest.java |  6 +-
 .../sdk/transforms/windowing/WindowTest.java|  4 +-
 45 files changed, 130 insertions(+), 78 deletions(-)
--

[3/4] incubator-beam git commit: Rename DisplayDataMatchers.includes to includesDisplayData to clarify usage

2016-04-26 Thread bchambers

Rename DisplayDataMatchers.includes to includesDisplayData to clarify usage


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/582befdc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/582befdc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/582befdc

Branch: refs/heads/master
Commit: 582befdc8aed6dc62bf1d205749d7f46d7894f97
Parents: d299e2c
Author: Scott Wegner <sweg...@google.com>
Authored: Mon Apr 25 09:31:13 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Tue Apr 26 09:39:01 2016 -0700

--
 .../sdk/io/BoundedReadFromUnboundedSourceTest.java|  5 +++--
 .../org/apache/beam/sdk/io/CompressedSourceTest.java  |  5 +++--
 .../test/java/org/apache/beam/sdk/io/ReadTest.java|  6 +++---
 .../test/java/org/apache/beam/sdk/io/WriteTest.java   |  5 +++--
 .../apache/beam/sdk/transforms/CombineFnsTest.java|  6 +++---
 .../org/apache/beam/sdk/transforms/CombineTest.java   |  4 ++--
 .../transforms/IntraBundleParallelizationTest.java|  4 ++--
 .../org/apache/beam/sdk/transforms/ParDoTest.java |  8 
 .../sdk/transforms/display/DisplayDataMatchers.java   | 14 +++---
 .../transforms/display/DisplayDataMatchersTest.java   |  4 ++--
 .../beam/sdk/transforms/display/DisplayDataTest.java  |  6 +++---
 .../beam/sdk/transforms/windowing/WindowTest.java |  4 ++--
 12 files changed, 37 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/582befdc/sdks/java/core/src/test/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSourceTest.java
--
diff --git 
a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSourceTest.java
 
b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSourceTest.java
index 30f3eca..f7949e7 100644
--- 
a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSourceTest.java
+++ 
b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSourceTest.java
@@ -17,7 +17,8 @@
  */
 package org.apache.beam.sdk.io;
 
-import static 
org.apache.beam.sdk.transforms.display.DisplayDataMatchers.includes;
+import static 
org.apache.beam.sdk.transforms.display.DisplayDataMatchers.includesDisplayDataFrom;
+
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
@@ -79,7 +80,7 @@ public class BoundedReadFromUnboundedSourceTest implements 
Serializable{
 };
 
 BoundedReadFromUnboundedSource<KV<Integer, Integer>> read = 
Read.from(src).withMaxNumRecords(5);
-assertThat(DisplayData.from(read), includes(src));
+assertThat(DisplayData.from(read), includesDisplayDataFrom(src));
   }
 
   private static class Checker

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/582befdc/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
--
diff --git 
a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java 
b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
index 4ad8904..36ef4b3 100644
--- 
a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
+++ 
b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
@@ -19,7 +19,8 @@ package org.apache.beam.sdk.io;
 
 import static 
org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
 import static 
org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasKey;
-import static 
org.apache.beam.sdk.transforms.display.DisplayDataMatchers.includes;
+import static 
org.apache.beam.sdk.transforms.display.DisplayDataMatchers.includesDisplayDataFrom;
+
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.junit.Assert.assertFalse;
@@ -355,7 +356,7 @@ public class CompressedSourceTest {
 assertThat(compressedSourceDisplayData, 
hasDisplayItem(hasKey("compressionMode")));
 assertThat(gzipDisplayData, hasDisplayItem("compressionMode", 
CompressionMode.GZIP.toString()));
 assertThat(compressedSourceDisplayData, hasDisplayItem("source", 
inputSource.getClass()));
-assertThat(compressedSourceDisplayData, includes(inputSource));
+assertThat(compressedSourceDisplayData, 
includesDisplayDataFrom(inputSource));
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/582befdc/sdks/java/core/src/test/java/org/apache/beam/sdk/io/ReadTest.java
--
diff --git

[2/4] incubator-beam git commit: Normalize comments to refer to 'display data' rather than 'display metadata'

2016-04-26 Thread bchambers

Normalize comments to refer to 'display data' rather than 'display metadata'


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/9d45a4a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/9d45a4a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/9d45a4a1

Branch: refs/heads/master
Commit: 9d45a4a11dcf66434c3d434d31900487471dcbe3
Parents: 582befd
Author: Scott Wegner <sweg...@google.com>
Authored: Mon Apr 25 09:34:58 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Tue Apr 26 09:39:01 2016 -0700

--
 .../main/java/org/apache/beam/sdk/io/Sink.java  |  2 +-
 .../java/org/apache/beam/sdk/io/Source.java |  2 +-
 .../beam/sdk/transforms/CombineFnBase.java  |  4 +-
 .../org/apache/beam/sdk/transforms/DoFn.java|  2 +-
 .../beam/sdk/transforms/DoFnWithContext.java|  2 +-
 .../apache/beam/sdk/transforms/PTransform.java  |  2 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   |  2 +-
 .../sdk/transforms/display/ClassForDisplay.java |  4 +-
 .../sdk/transforms/display/DisplayData.java | 58 ++--
 .../sdk/transforms/display/HasDisplayData.java  |  8 +--
 .../beam/sdk/transforms/windowing/WindowFn.java |  2 +-
 11 files changed, 44 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9d45a4a1/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Sink.java
--
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Sink.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Sink.java
index 8b6b637..20b1631 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Sink.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Sink.java
@@ -139,7 +139,7 @@ public abstract class Sink implements Serializable, 
HasDisplayData {
* {@inheritDoc}
*
* By default, does not register any display data. Implementors may 
override this method
-   * to provide their own display metadata.
+   * to provide their own display data.
*/
   @Override
   public void populateDisplayData(DisplayData.Builder builder) {}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9d45a4a1/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Source.java
--
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Source.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Source.java
index 2ab0d4e..b8902f9 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Source.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Source.java
@@ -72,7 +72,7 @@ public abstract class Source implements Serializable, 
HasDisplayData {
* {@inheritDoc}
*
* By default, does not register any display data. Implementors may 
override this method
-   * to provide their own display metadata.
+   * to provide their own display data.
*/
   @Override
   public void populateDisplayData(DisplayData.Builder builder) {}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9d45a4a1/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFnBase.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFnBase.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFnBase.java
index 1b64bb2..c73ba54 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFnBase.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFnBase.java
@@ -225,7 +225,7 @@ public class CombineFnBase {
  * {@inheritDoc}
  *
  * By default, does not register any display data. Implementors may 
override this method
- * to provide their own display metadata.
+ * to provide their own display data.
  */
 @Override
 public void populateDisplayData(DisplayData.Builder builder) {
@@ -300,7 +300,7 @@ public class CombineFnBase {
  * {@inheritDoc}
  *
  * By default, does not register any display data. Implementors may 
override this method
- * to provide their own display metadata.
+ * to provide their own display data.
  */
 @Override
 public void populateDisplayData(DisplayData.Builder builder) {

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9d45a4a1/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
index 04f272d..6d5d1ed 100644
--- a/sdks/j

[1/4] incubator-beam git commit: Add super.populateDisplayData() to standard implementations. The current super implementation is a no-op, but this is the recommended way to implement the pattern

2016-04-26 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master d299e2c25 -> 7dc1a4047


Add super.populateDisplayData() to standard implementations. The current super 
implementation is a no-op, but this is the recommended way to implement the 
pattern


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/c6690c18
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/c6690c18
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/c6690c18

Branch: refs/heads/master
Commit: c6690c18a64ec6c7971712ab43807b13c1849571
Parents: 9d45a4a
Author: Scott Wegner <sweg...@google.com>
Authored: Mon Apr 25 09:41:07 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Tue Apr 26 09:39:01 2016 -0700

--
 .../main/java/org/apache/beam/sdk/io/DatastoreIO.java|  1 +
 .../core/src/main/java/org/apache/beam/sdk/io/Read.java  |  2 ++
 .../core/src/main/java/org/apache/beam/sdk/io/Write.java |  1 +
 .../apache/beam/sdk/transforms/ApproximateQuantiles.java |  1 +
 .../apache/beam/sdk/transforms/ApproximateUnique.java|  2 ++
 .../java/org/apache/beam/sdk/transforms/Combine.java | 11 +++
 .../java/org/apache/beam/sdk/transforms/CombineFns.java  |  4 
 .../apache/beam/sdk/transforms/CombineWithContext.java   |  1 +
 .../main/java/org/apache/beam/sdk/transforms/Filter.java |  4 
 .../java/org/apache/beam/sdk/transforms/GroupByKey.java  |  1 +
 .../beam/sdk/transforms/IntraBundleParallelization.java  |  1 +
 .../main/java/org/apache/beam/sdk/transforms/Max.java|  1 +
 .../main/java/org/apache/beam/sdk/transforms/Min.java|  1 +
 .../main/java/org/apache/beam/sdk/transforms/ParDo.java  |  2 ++
 .../java/org/apache/beam/sdk/transforms/Partition.java   |  2 ++
 .../main/java/org/apache/beam/sdk/transforms/Sample.java |  2 ++
 .../main/java/org/apache/beam/sdk/transforms/Top.java|  1 +
 .../beam/sdk/transforms/windowing/CalendarWindows.java   |  6 ++
 .../beam/sdk/transforms/windowing/FixedWindows.java  |  1 +
 .../apache/beam/sdk/transforms/windowing/Sessions.java   |  1 +
 .../beam/sdk/transforms/windowing/SlidingWindows.java|  1 +
 .../org/apache/beam/sdk/transforms/windowing/Window.java |  1 +
 .../java/org/apache/beam/sdk/util/CombineFnUtil.java |  1 +
 23 files changed, 49 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c6690c18/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DatastoreIO.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DatastoreIO.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DatastoreIO.java
index 4348950..c265659 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DatastoreIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DatastoreIO.java
@@ -604,6 +604,7 @@ public class DatastoreIO {
 
 @Override
 public void populateDisplayData(DisplayData.Builder builder) {
+  super.populateDisplayData(builder);
   builder
   .addIfNotDefault("host", host, DEFAULT_HOST)
   .addIfNotNull("dataset", datasetId);

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c6690c18/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java
--
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java
index 1f41e5c..05b70ac 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java
@@ -147,6 +147,7 @@ public class Read {
 
 @Override
 public void populateDisplayData(DisplayData.Builder builder) {
+  super.populateDisplayData(builder);
   builder
   .add("source", source.getClass())
   .include(source);
@@ -261,6 +262,7 @@ public class Read {
 
 @Override
 public void populateDisplayData(DisplayData.Builder builder) {
+  super.populateDisplayData(builder);
   builder
   .add("source", source.getClass())
   .include(source);

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c6690c18/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java
--
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java
index b8fa259..a7d182d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java
@@ -79,6 +79,7 @@ public class Write

[1/2] incubator-beam git commit: Add DirtyBit to represent whether Counters have been committed.

2016-04-26 Thread bchambers

Repository: incubator-beam
Updated Branches:
  refs/heads/master c59ca3868 -> d299e2c25


Add DirtyBit to represent whether Counters have been committed.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/72cfa709
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/72cfa709
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/72cfa709

Branch: refs/heads/master
Commit: 72cfa709689496f02cbe00fdc3f281e45fa11b23
Parents: c59ca38
Author: Pei He <pe...@google.com>
Authored: Tue Apr 19 18:21:40 2016 -0700
Committer: bchambers <bchamb...@google.com>
Committed: Tue Apr 26 08:24:30 2016 -0700

--
 .../apache/beam/sdk/util/common/Counter.java| 459 ---
 .../beam/sdk/util/common/CounterTest.java   | 146 +-
 2 files changed, 448 insertions(+), 157 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/72cfa709/sdks/java/core/src/main/java/org/apache/beam/sdk/util/common/Counter.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/common/Counter.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/common/Counter.java
index 6024576..9f9b0c1 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/common/Counter.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/common/Counter.java
@@ -25,6 +25,7 @@ import static 
com.google.common.base.Preconditions.checkArgument;
 
 import org.apache.beam.sdk.values.TypeDescriptor;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.util.concurrent.AtomicDouble;
 
 import java.util.Objects;
@@ -44,6 +45,14 @@ import javax.annotation.Nullable;
  * Counters compare using value equality of their name, kind, and
  * cumulative value.  Equal counters should have equal toString()s.
  *
+ * After all possible mutations have completed, the reader should check
+ * {@link #isDirty} for every counter, otherwise updates may be lost.
+ *
+ * A counter may become dirty without a corresponding update to the value.
+ * This generally will occur when the calls to {@code addValue()}, {@code 
committing()},
+ * and {@code committed()} are interleaved such that the value is updated
+ * between the calls to committing and the read of the value.
+ *
  * @param  the type of values aggregated by this counter
  */
 public abstract class Counter {
@@ -295,6 +304,76 @@ public abstract class Counter {
   public abstract CounterMean getMean();
 
   /**
+   * Represents whether counters' values have been committed to the backend.
+   *
+   * Runners can use this information to optimize counters updates.
+   * For example, if counters are committed, runners may choose to skip the 
updates.
+   *
+   * Counters' state transition table:
+   * {@code
+   * Action\Current State COMMITTEDDIRTYCOMMITTING
+   * addValue()   DIRTYDIRTYDIRTY
+   * committing() None COMMITTING   None
+   * committed()  None None COMMITTED
+   * }
+   */
+  @VisibleForTesting
+  enum CommitState {
+/**
+ * There are no local updates that haven't been committed to the backend.
+ */
+COMMITTED,
+/**
+ * There are local updates that haven't been committed to the backend.
+ */
+DIRTY,
+/**
+ * Local updates are committing to the backend, but are still pending.
+ */
+COMMITTING,
+  }
+
+  /**
+   * Returns if the counter contains non-committed aggregate.
+   */
+  public boolean isDirty() {
+return commitState.get() != CommitState.COMMITTED;
+  }
+
+  /**
+   * Changes the counter from {@code CommitState.DIRTY} to {@code 
CommitState.COMMITTING}.
+   *
+   * @return true if successful. False return indicates that the commit state
+   * was not in {@code CommitState.DIRTY}.
+   */
+  public boolean committing() {
+return commitState.compareAndSet(CommitState.DIRTY, 
CommitState.COMMITTING);
+  }
+
+  /**
+   * Changes the counter from {@code CommitState.COMMITTING} to {@code 
CommitState.COMMITTED}.
+   *
+   * @return true if successful.
+   *
+   * False return indicates that the counter was updated while the 
committing is pending.
+   * That counter update might or might not has been committed. The {@code 
commitState} has to
+   * stay in {@code CommitState.DIRTY}.
+   */
+  public boolean committed() {
+return commitState.compareAndSet(CommitState.COMMITTING, 
CommitState.COMMITTED);
+  }
+
+  /**
+   * Sets the counter to {@code CommitState.DIRTY}.
+   *
+   * Must be called at the end of {@link #addValue}, {@link #resetToValue},
+   * {@link #resetMeanToValue}, and {@link #merge}.
+   */
+  protecte

1 2 >

1 - 100 of 139 matches

Mail list logo