Jenkins build is still unstable: beam_PostCommit_Java_MavenInstall #2092

[Apache Jenkins Server]

See 

Jenkins build is still unstable: beam_PostCommit_Java_RunnableOnService_Dataflow #1810

[Apache Jenkins Server]

See 

Jenkins build is still unstable: beam_PostCommit_Java_RunnableOnService_Spark #407

[Apache Jenkins Server]

See 

Jenkins build is still unstable: beam_PostCommit_Java_RunnableOnService_Dataflow #1810

[Apache Jenkins Server]

See 

Jenkins build is still unstable: beam_PostCommit_Java_RunnableOnService_Spark #407

[Apache Jenkins Server]

See 

[2/3] incubator-beam git commit: Better comments and cleanup

[lcwik]

Better comments and cleanup


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a0a1fea3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a0a1fea3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a0a1fea3

Branch: refs/heads/master
Commit: a0a1fea366849dec8c5dba868a71a1ccdce902eb
Parents: ae1d2a3
Author: Vikas Kedigehalli 
Authored: Tue Dec 13 14:16:16 2016 -0800
Committer: Luke Cwik 
Committed: Tue Dec 13 22:37:23 2016 -0800

--
 .../sdk/options/PipelineOptionsFactory.java | 50 ++--
 .../sdk/options/PipelineOptionsFactoryTest.java |  1 -
 2 files changed, 26 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a0a1fea3/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
index 2d013fd..42e1092 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
@@ -1652,9 +1652,11 @@ public class PipelineOptionsFactory {
 return convertedOptions;
   }
 
+
   /**
-   * Returns true if the given type is a SIMPLE_TYPES, enum or any of these 
types in a
-   * parameterized ValueProvider.
+   * Returns true if the given type is one of {@code SIMPLE_TYPES} or an enum, 
or if the given type
+   * is a {@link ValueProvider ValueProviderT} and {@code T} is one of 
{@code SIMPLE_TYPES}
+   * or an enum.
*/
   private static boolean isSimpleType(Class type, JavaType genericType) {
 Class unwrappedType = type.equals(ValueProvider.class)
@@ -1663,23 +1665,23 @@ public class PipelineOptionsFactory {
   }
 
   /**
-   * Returns true if the given type is a any Array or Collection of 
SIMPLE_TYPES or enum, or
-   * any of these types in a parameterized ValueProvider.
+   * Returns true if the given type is an array or {@link Collection} of 
{@code SIMPLE_TYPES} or
+   * enums, or if the given type is a {@link ValueProvider 
ValueProviderT} and {@code T} is
+   * an array or {@link Collection} of {@code SIMPLE_TYPES} or enums.
*/
   private static boolean isCollectionOrArrayOfAllowedTypes(Class type, 
JavaType genericType) {
-Class containerType = type.equals(ValueProvider.class)
-? genericType.containedType(0).getRawClass() : type;
+JavaType containerType = type.equals(ValueProvider.class)
+? genericType.containedType(0) : genericType;
 
 // Check if it is an array of simple types or enum.
-if (containerType.isArray() && 
(SIMPLE_TYPES.contains(containerType.getComponentType())
-|| containerType.getComponentType().isEnum())) {
+if (containerType.getRawClass().isArray()
+&& 
(SIMPLE_TYPES.contains(containerType.getRawClass().getComponentType())
+|| containerType.getRawClass().getComponentType().isEnum())) {
 return true;
 }
 // Check if it is Collection of simple types or enum.
-if (Collection.class.isAssignableFrom(containerType)) {
-  JavaType innerType = type.equals(ValueProvider.class)
-  ? genericType.containedType(0).containedType(0)
-  : genericType.containedType(0);
+if (Collection.class.isAssignableFrom(containerType.getRawClass())) {
+  JavaType innerType = containerType.containedType(0);
   // Note that raw types are allowed, hence the null check.
   if (innerType == null || SIMPLE_TYPES.contains(innerType.getRawClass())
   || innerType.getRawClass().isEnum()) {
@@ -1692,8 +1694,10 @@ public class PipelineOptionsFactory {
   /**
* Ensures that empty string value is allowed for a given type.
*
-   * Empty strings are only allowed for String, String Array, Collection of 
Strings or any of
-   * these types in a parameterized ValueProvider.
+   * Empty strings are only allowed for {@link String}, {@link String 
String[]},
+   * {@link Collection CollectionString}, or {@link ValueProvider 
ValueProviderT}
+   * and {@code T} is of type {@link String}, {@link String String[]},
+   * {@link Collection CollectionString}.
*
* @param type class object for the type under check.
* @param genericType complete type information for the type under check.
@@ -1701,16 +1705,14 @@ public class PipelineOptionsFactory {
*/
   private static void checkEmptyStringAllowed(Class type, JavaType 
genericType,
   String genericTypeName) {
-Class unwrappedType = type.equals(ValueProvider.class)
-? 

[jira] [Commented] (BEAM-1136) Empty string value should be allowed for ValueProvider

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-1136?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15747477#comment-15747477
 ] 

ASF GitHub Bot commented on BEAM-1136:
--

Github user asfgit closed the pull request at:

https://github.com/apache/incubator-beam/pull/1580


> Empty string value should be allowed for ValueProvider
> --
>
> Key: BEAM-1136
> URL: https://issues.apache.org/jira/browse/BEAM-1136
> Project: Beam
>  Issue Type: Bug
>  Components: sdk-java-core
>Reporter: Vikas Kedigehalli
>Assignee: Vikas Kedigehalli
>Priority: Minor
>




--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[3/3] incubator-beam git commit: [BEAM-1136, BEAM-1137] Allow empty string value for ValueProvider types.

[lcwik]

[BEAM-1136, BEAM-1137] Allow empty string value for ValueProvider types.

This closes #1580


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/5a51ace8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/5a51ace8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/5a51ace8

Branch: refs/heads/master
Commit: 5a51ace8d44f7e8b75566c806796a31c9bf4f7e7
Parents: f516627 a0a1fea
Author: Luke Cwik 
Authored: Tue Dec 13 22:37:55 2016 -0800
Committer: Luke Cwik 
Committed: Tue Dec 13 22:37:55 2016 -0800

--
 .../sdk/options/PipelineOptionsFactory.java | 110 ++---
 .../sdk/options/PipelineOptionsFactoryTest.java | 222 ---
 2 files changed, 279 insertions(+), 53 deletions(-)
--

Jenkins build is still unstable: beam_PostCommit_Java_RunnableOnService_Spark #406

[Apache Jenkins Server]

See 

Jenkins build is still unstable: beam_PostCommit_Java_RunnableOnService_Spark #406

[Apache Jenkins Server]

See 

[jira] [Closed] (BEAM-1005) Autogenerate example archetypes as part of build process

[Luke Cwik (JIRA)]

 [ 
https://issues.apache.org/jira/browse/BEAM-1005?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Luke Cwik closed BEAM-1005.
---
   Resolution: Duplicate
Fix Version/s: Not applicable

Duplicate of BEAM-1004

> Autogenerate example archetypes as part of build process
> 
>
> Key: BEAM-1005
> URL: https://issues.apache.org/jira/browse/BEAM-1005
> Project: Beam
>  Issue Type: Improvement
>  Components: sdk-java-extensions
>Reporter: Kenneth Knowles
> Fix For: Not applicable
>
>
> Previously, the maven archetypes were manually curated. Recently, the 
> generation of the content for the example archetype was automated, and 
> another Java 8 example archetype created. The generated content is currently 
> checked into source control, but should be instead generated as part of the 
> build process.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Jenkins build is still unstable: beam_PostCommit_Java_RunnableOnService_Spark #405

[Apache Jenkins Server]

See 

Jenkins build is still unstable: beam_PostCommit_Java_RunnableOnService_Spark #405

[Apache Jenkins Server]

See 

[jira] [Commented] (BEAM-27) Add user-ready API for interacting with timers

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-27?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15747289#comment-15747289
 ] 

ASF GitHub Bot commented on BEAM-27:


GitHub user kennknowles opened a pull request:

https://github.com/apache/incubator-beam/pull/1612

[BEAM-27] Support timer setting and receiving in SimpleDoFnRunner

Be sure to do all of the following to help us incorporate your contribution
quickly and easily:

 - [x] Make sure the PR title is formatted like:
   `[BEAM-] Description of pull request`
 - [ ] Make sure tests pass via `mvn clean verify`. (Even better, enable
   Travis-CI on your fork and ensure the whole test matrix passes).
 - [x] Replace `` in the title with the actual Jira issue
   number, if there is one.
 - [x] If this contribution is large, please file an Apache
   [Individual Contributor License 
Agreement](https://www.apache.org/licenses/icla.txt).

---

These are a couple commits peeled off that should enable each runner to 
begin fulfilling the timer API.

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/kennknowles/incubator-beam 
SimpleDoFnRunner-timers

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-beam/pull/1612.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1612


commit 95f954de00d20b27e127e41a0118a305e2cd8a94
Author: Kenneth Knowles 
Date:   2016-12-08T04:09:06Z

Make TimerSpec and StateSpec fields accessible

commit 19ccebe6170b24af0067c7a04483d8e9156a0ef5
Author: Kenneth Knowles 
Date:   2016-11-23T22:21:40Z

Add timer support to DoFnRunner(s)




> Add user-ready API for interacting with timers
> --
>
> Key: BEAM-27
> URL: https://issues.apache.org/jira/browse/BEAM-27
> Project: Beam
>  Issue Type: New Feature
>  Components: sdk-java-core
>Reporter: Kenneth Knowles
>Assignee: Kenneth Knowles
>
> Pipeline authors will benefit from a different factorization of interaction 
> with underlying timers. The current APIs are targeted at runner implementers.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[GitHub] incubator-beam pull request #1612: [BEAM-27] Support timer setting and recei...

[kennknowles]

GitHub user kennknowles opened a pull request:

https://github.com/apache/incubator-beam/pull/1612

[BEAM-27] Support timer setting and receiving in SimpleDoFnRunner

Be sure to do all of the following to help us incorporate your contribution
quickly and easily:

 - [x] Make sure the PR title is formatted like:
   `[BEAM-] Description of pull request`
 - [ ] Make sure tests pass via `mvn clean verify`. (Even better, enable
   Travis-CI on your fork and ensure the whole test matrix passes).
 - [x] Replace `` in the title with the actual Jira issue
   number, if there is one.
 - [x] If this contribution is large, please file an Apache
   [Individual Contributor License 
Agreement](https://www.apache.org/licenses/icla.txt).

---

These are a couple commits peeled off that should enable each runner to 
begin fulfilling the timer API.

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/kennknowles/incubator-beam 
SimpleDoFnRunner-timers

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-beam/pull/1612.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1612


commit 95f954de00d20b27e127e41a0118a305e2cd8a94
Author: Kenneth Knowles 
Date:   2016-12-08T04:09:06Z

Make TimerSpec and StateSpec fields accessible

commit 19ccebe6170b24af0067c7a04483d8e9156a0ef5
Author: Kenneth Knowles 
Date:   2016-11-23T22:21:40Z

Add timer support to DoFnRunner(s)




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

Jenkins build is still unstable: beam_PostCommit_Java_MavenInstall #2090

[Apache Jenkins Server]

See 

Jenkins build is still unstable: beam_PostCommit_Java_RunnableOnService_Dataflow #1809

[Apache Jenkins Server]

See 

Jenkins build is still unstable: beam_PostCommit_Java_MavenInstall #2089

[Apache Jenkins Server]

See 

Jenkins build became unstable: beam_PostCommit_Java_RunnableOnService_Spark #404

[Apache Jenkins Server]

See 

Jenkins build became unstable: beam_PostCommit_Java_RunnableOnService_Spark #404

[Apache Jenkins Server]

See 

Jenkins build is unstable: beam_PostCommit_Java_MavenInstall #2088

[Apache Jenkins Server]

See 

[jira] [Commented] (BEAM-967) Add Jenkins postcommit for Apex runner RunnableOnService tests

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-967?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15747097#comment-15747097
 ] 

ASF GitHub Bot commented on BEAM-967:
-

Github user asfgit closed the pull request at:

https://github.com/apache/incubator-beam/pull/1598


> Add Jenkins postcommit for Apex runner RunnableOnService tests
> --
>
> Key: BEAM-967
> URL: https://issues.apache.org/jira/browse/BEAM-967
> Project: Beam
>  Issue Type: Bug
>  Components: runner-apex, testing
>Reporter: Kenneth Knowles
>Assignee: Jason Kuster
>
> During development of the Apex runner, all RunnableOnService tests were run 
> in precommit. Now that it has been merged, they are not. It needs the usual 
> treatment that we give runners.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[GitHub] incubator-beam pull request #1598: [BEAM-967] Add Jenkins postcommit for Run...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/incubator-beam/pull/1598


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[2/2] incubator-beam git commit: This closes #1598

[kenn]

This closes #1598


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/f516627e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/f516627e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/f516627e

Branch: refs/heads/master
Commit: f516627e0d5362cb9924d868b0863742446b5425
Parents: 44b4eba c1086f3
Author: Kenneth Knowles 
Authored: Tue Dec 13 19:27:11 2016 -0800
Committer: Kenneth Knowles 
Committed: Tue Dec 13 19:27:11 2016 -0800

--
 ...ostCommit_Java_RunnableOnService_Apex.groovy | 41 
 1 file changed, 41 insertions(+)
--

Jenkins build is still unstable: beam_PostCommit_Java_RunnableOnService_Dataflow #1808

[Apache Jenkins Server]

See 

Jenkins build is still unstable: beam_PostCommit_Java_RunnableOnService_Dataflow #1808

[Apache Jenkins Server]

See 

[2/3] incubator-beam git commit: [BEAM-807] Replace OldDoFn with DoFn.

[kenn]

[BEAM-807] Replace OldDoFn with DoFn.

Add a custom AssignWindows implementation.

Setup and teardown DoFn.

Add implementation for GroupAlsoByWindow via flatMap.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/4ffed3e0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/4ffed3e0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/4ffed3e0

Branch: refs/heads/master
Commit: 4ffed3e09a2f0ec3583098f6cfd53a2ddcc6f8c2
Parents: 2be9a15
Author: Sela 
Authored: Sun Dec 11 14:32:49 2016 +0200
Committer: Sela 
Committed: Tue Dec 13 10:05:18 2016 +0200

--
 .../beam/runners/spark/examples/WordCount.java  |   6 +-
 .../runners/spark/translation/DoFnFunction.java |   2 +-
 .../translation/GroupCombineFunctions.java  |  23 +-
 .../spark/translation/MultiDoFnFunction.java|   2 +-
 .../spark/translation/SparkAssignWindowFn.java  |  69 ++
 .../translation/SparkGroupAlsoByWindowFn.java   | 214 +++
 .../spark/translation/SparkProcessContext.java  |  10 +
 .../spark/translation/TransformTranslator.java  |  31 +--
 .../streaming/StreamingTransformTranslator.java |  35 ++-
 .../streaming/utils/PAssertStreaming.java   |  26 +--
 10 files changed, 345 insertions(+), 73 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4ffed3e0/runners/spark/src/main/java/org/apache/beam/runners/spark/examples/WordCount.java
--
diff --git 
a/runners/spark/src/main/java/org/apache/beam/runners/spark/examples/WordCount.java
 
b/runners/spark/src/main/java/org/apache/beam/runners/spark/examples/WordCount.java
index b2672b5..1252d12 100644
--- 
a/runners/spark/src/main/java/org/apache/beam/runners/spark/examples/WordCount.java
+++ 
b/runners/spark/src/main/java/org/apache/beam/runners/spark/examples/WordCount.java
@@ -25,8 +25,8 @@ import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.transforms.Aggregator;
 import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.MapElements;
-import org.apache.beam.sdk.transforms.OldDoFn;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.SimpleFunction;
@@ -44,11 +44,11 @@ public class WordCount {
* of-line. This DoFn tokenizes lines of text into individual words; we pass 
it to a ParDo in the
* pipeline.
*/
-  static class ExtractWordsFn extends OldDoFn {
+  static class ExtractWordsFn extends DoFn {
 private final Aggregator emptyLines =
 createAggregator("emptyLines", new Sum.SumLongFn());
 
-@Override
+@ProcessElement
 public void processElement(ProcessContext c) {
   if (c.element().trim().isEmpty()) {
 emptyLines.addValue(1L);

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4ffed3e0/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnFunction.java
--
diff --git 
a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnFunction.java
 
b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnFunction.java
index 4c49a7f..6a641b5 100644
--- 
a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnFunction.java
+++ 
b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnFunction.java
@@ -93,7 +93,7 @@ public class DoFnFunction
 windowingStrategy
 );
 
-return new SparkProcessContext<>(doFnRunner, 
outputManager).processPartition(iter);
+return new SparkProcessContext<>(doFn, doFnRunner, 
outputManager).processPartition(iter);
   }
 
   private class DoFnOutputManager

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4ffed3e0/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
--
diff --git 
a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
 
b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
index 421b1b0..4875b0c 100644
--- 
a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
+++ 
b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
@@ -18,11 +18,9 @@
 
 package org.apache.beam.runners.spark.translation;
 
-
 import 

[GitHub] incubator-beam pull request #1578: [BEAM-757, BEAM-807] new DoFn via SimpleD...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/incubator-beam/pull/1578


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[jira] [Commented] (BEAM-757) The SparkRunner should utilize the SDK's DoFnRunner instead of writing it's own.

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-757?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15747054#comment-15747054
 ] 

ASF GitHub Bot commented on BEAM-757:
-

Github user asfgit closed the pull request at:

https://github.com/apache/incubator-beam/pull/1578


> The SparkRunner should utilize the SDK's DoFnRunner instead of writing it's 
> own.
> 
>
> Key: BEAM-757
> URL: https://issues.apache.org/jira/browse/BEAM-757
> Project: Beam
>  Issue Type: Improvement
>  Components: runner-spark
>Reporter: Amit Sela
>Assignee: Amit Sela
>
> The SDK now provides DoFnRunner implementations, and so to avoid maintaining 
> against the SDK, the runner should leverage the runner API instead.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[3/3] incubator-beam git commit: This closes #1578

[kenn]

This closes #1578


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/44b4eba5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/44b4eba5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/44b4eba5

Branch: refs/heads/master
Commit: 44b4eba51fad2fd4c33ad9467ac766d4b433f852
Parents: ce3aa65 4ffed3e
Author: Kenneth Knowles 
Authored: Tue Dec 13 18:54:51 2016 -0800
Committer: Kenneth Knowles 
Committed: Tue Dec 13 18:54:51 2016 -0800

--
 .../spark/aggregators/SparkAggregators.java |  30 +-
 .../beam/runners/spark/examples/WordCount.java  |   6 +-
 .../runners/spark/translation/DoFnFunction.java | 110 +++---
 .../translation/GroupCombineFunctions.java  |  23 +-
 .../spark/translation/MultiDoFnFunction.java| 135 ---
 .../spark/translation/SparkAssignWindowFn.java  |  69 
 .../translation/SparkGroupAlsoByWindowFn.java   | 214 +++
 .../spark/translation/SparkProcessContext.java  | 385 ---
 .../spark/translation/TransformTranslator.java  |  31 +-
 .../streaming/StreamingTransformTranslator.java |  35 +-
 .../streaming/utils/PAssertStreaming.java   |  26 +-
 11 files changed, 574 insertions(+), 490 deletions(-)
--

[jira] [Resolved] (BEAM-1152) Java-Maven Archetypes-Starter Build Failed

[Daniel Halperin (JIRA)]

 [ 
https://issues.apache.org/jira/browse/BEAM-1152?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Daniel Halperin resolved BEAM-1152.
---
   Resolution: Fixed
Fix Version/s: Not applicable

> Java-Maven Archetypes-Starter Build Failed
> --
>
> Key: BEAM-1152
> URL: https://issues.apache.org/jira/browse/BEAM-1152
> Project: Beam
>  Issue Type: Bug
>  Components: testing
>Reporter: Mark Liu
>Assignee: Davor Bonaci
> Fix For: Not applicable
>
>
> Apache Beam :: SDKs :: Java :: Maven Archetypes :: Starter keep failed breaks 
> Jenkins precommit.
> https://builds.apache.org/job/beam_PreCommit_Java_MavenInstall/5867/



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Commented] (BEAM-1152) Java-Maven Archetypes-Starter Build Failed

[Daniel Halperin (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-1152?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15747040#comment-15747040
 ] 

Daniel Halperin commented on BEAM-1152:
---

Thanks Mark! This is fixed by the recent 
https://github.com/apache/incubator-beam/pull/1610

> Java-Maven Archetypes-Starter Build Failed
> --
>
> Key: BEAM-1152
> URL: https://issues.apache.org/jira/browse/BEAM-1152
> Project: Beam
>  Issue Type: Bug
>  Components: testing
>Reporter: Mark Liu
>Assignee: Davor Bonaci
> Fix For: Not applicable
>
>
> Apache Beam :: SDKs :: Java :: Maven Archetypes :: Starter keep failed breaks 
> Jenkins precommit.
> https://builds.apache.org/job/beam_PreCommit_Java_MavenInstall/5867/



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Updated] (BEAM-1152) Java-Maven Archetypes-Starter Build Failed

[Daniel Halperin (JIRA)]

 [ 
https://issues.apache.org/jira/browse/BEAM-1152?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Daniel Halperin updated BEAM-1152:
--
Component/s: (was: build-system)

> Java-Maven Archetypes-Starter Build Failed
> --
>
> Key: BEAM-1152
> URL: https://issues.apache.org/jira/browse/BEAM-1152
> Project: Beam
>  Issue Type: Bug
>  Components: testing
>Reporter: Mark Liu
>Assignee: Davor Bonaci
> Fix For: Not applicable
>
>
> Apache Beam :: SDKs :: Java :: Maven Archetypes :: Starter keep failed breaks 
> Jenkins precommit.
> https://builds.apache.org/job/beam_PreCommit_Java_MavenInstall/5867/



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[GitHub] incubator-beam pull request #1610: [BUILD BREAK] starter: fix typo in pom.xm...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/incubator-beam/pull/1610


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[1/2] incubator-beam git commit: starter: fix typo in pom.xml

[dhalperi]

Repository: incubator-beam
Updated Branches:
  refs/heads/master b742a2c09 -> ce3aa657a


starter: fix typo in pom.xml

Manual edit introduced in 
https://github.com/apache/incubator-beam/commit/25215889381f7da61766054af68c84ffed4c0c71


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/60c33dd5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/60c33dd5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/60c33dd5

Branch: refs/heads/master
Commit: 60c33dd5a40a80e2d782cddbbd6940b96f34d975
Parents: b742a2c
Author: Dan Halperin 
Authored: Tue Dec 13 17:27:07 2016 -0800
Committer: Dan Halperin 
Committed: Tue Dec 13 17:27:07 2016 -0800

--
 .../starter/src/main/resources/archetype-resources/pom.xml| 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/60c33dd5/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
--
diff --git 
a/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
 
b/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
index c59ffee..45aa1f8 100644
--- 
a/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
+++ 
b/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
@@ -1,4 +1,5 @@
- 

[jira] [Commented] (BEAM-1153) GcsUtil needs to set timeout and retry explicitly in BatchRequest.

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-1153?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15746998#comment-15746998
 ] 

ASF GitHub Bot commented on BEAM-1153:
--

GitHub user peihe opened a pull request:

https://github.com/apache/incubator-beam/pull/1611

[BEAM-1153] GcsUtil: use non-batch API for single file size requests.

Be sure to do all of the following to help us incorporate your contribution
quickly and easily:

 - [ ] Make sure the PR title is formatted like:
   `[BEAM-] Description of pull request`
 - [ ] Make sure tests pass via `mvn clean verify`. (Even better, enable
   Travis-CI on your fork and ensure the whole test matrix passes).
 - [ ] Replace `` in the title with the actual Jira issue
   number, if there is one.
 - [ ] If this contribution is large, please file an Apache
   [Individual Contributor License 
Agreement](https://www.apache.org/licenses/icla.txt).

---



You can merge this pull request into a Git repository by running:

$ git pull https://github.com/peihe/incubator-beam fix-pr-1359

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-beam/pull/1611.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1611


commit a42055ec58c0f458a1939f4682d0c77bac9127bb
Author: Pei He 
Date:   2016-12-14T02:29:17Z

[BEAM-1153] GcsUtil: use non-batch API for single file size requests.




> GcsUtil needs to set timeout and retry explicitly in BatchRequest.
> --
>
> Key: BEAM-1153
> URL: https://issues.apache.org/jira/browse/BEAM-1153
> Project: Beam
>  Issue Type: Bug
>  Components: sdk-java-core
>Reporter: Pei He
>Assignee: Pei He
>Priority: Blocker
>
> Google Cloud auto generated Json library doesn't set HttpRequestInitializer 
> for batch requests. (But, it doesn't for other non-batch requests.)
> Possible fix is: https://github.com/apache/incubator-beam/pull/1608
> In additional, we can partially rollback 
> https://github.com/apache/incubator-beam/pull/1359 to keep using non-batch 
> API for fileSize() for single files. This will make sure existing code will 
> keep work as the same way.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[GitHub] incubator-beam pull request #1611: [BEAM-1153] GcsUtil: use non-batch API fo...

[peihe]

GitHub user peihe opened a pull request:

https://github.com/apache/incubator-beam/pull/1611

[BEAM-1153] GcsUtil: use non-batch API for single file size requests.

Be sure to do all of the following to help us incorporate your contribution
quickly and easily:

 - [ ] Make sure the PR title is formatted like:
   `[BEAM-] Description of pull request`
 - [ ] Make sure tests pass via `mvn clean verify`. (Even better, enable
   Travis-CI on your fork and ensure the whole test matrix passes).
 - [ ] Replace `` in the title with the actual Jira issue
   number, if there is one.
 - [ ] If this contribution is large, please file an Apache
   [Individual Contributor License 
Agreement](https://www.apache.org/licenses/icla.txt).

---



You can merge this pull request into a Git repository by running:

$ git pull https://github.com/peihe/incubator-beam fix-pr-1359

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-beam/pull/1611.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1611


commit a42055ec58c0f458a1939f4682d0c77bac9127bb
Author: Pei He 
Date:   2016-12-14T02:29:17Z

[BEAM-1153] GcsUtil: use non-batch API for single file size requests.




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[jira] [Updated] (BEAM-1153) GcsUtil needs to set timeout and retry explicitly in BatchRequest.

[Pei He (JIRA)]

 [ 
https://issues.apache.org/jira/browse/BEAM-1153?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Pei He updated BEAM-1153:
-
Priority: Blocker  (was: Major)

> GcsUtil needs to set timeout and retry explicitly in BatchRequest.
> --
>
> Key: BEAM-1153
> URL: https://issues.apache.org/jira/browse/BEAM-1153
> Project: Beam
>  Issue Type: Bug
>  Components: sdk-java-core
>Reporter: Pei He
>Assignee: Pei He
>Priority: Blocker
>
> Google Cloud auto generated Json library doesn't set HttpRequestInitializer 
> for batch requests. (But, it doesn't for other non-batch requests.)
> Possible fix is: https://github.com/apache/incubator-beam/pull/1608
> In additional, we can partially rollback 
> https://github.com/apache/incubator-beam/pull/1359 to keep using non-batch 
> API for fileSize() for single files. This will make sure existing code will 
> keep work as the same way.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Updated] (BEAM-1153) GcsUtil needs to set timeout and retry explicitly in BatchRequest.

[Pei He (JIRA)]

 [ 
https://issues.apache.org/jira/browse/BEAM-1153?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Pei He updated BEAM-1153:
-
Component/s: sdk-java-core

> GcsUtil needs to set timeout and retry explicitly in BatchRequest.
> --
>
> Key: BEAM-1153
> URL: https://issues.apache.org/jira/browse/BEAM-1153
> Project: Beam
>  Issue Type: Bug
>  Components: sdk-java-core
>Reporter: Pei He
>Assignee: Pei He
>Priority: Blocker
>
> Google Cloud auto generated Json library doesn't set HttpRequestInitializer 
> for batch requests. (But, it doesn't for other non-batch requests.)
> Possible fix is: https://github.com/apache/incubator-beam/pull/1608
> In additional, we can partially rollback 
> https://github.com/apache/incubator-beam/pull/1359 to keep using non-batch 
> API for fileSize() for single files. This will make sure existing code will 
> keep work as the same way.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Created] (BEAM-1153) GcsUtil needs to set timeout and retry explicitly in BatchRequest.

[Pei He (JIRA)]

Pei He created BEAM-1153:


 Summary: GcsUtil needs to set timeout and retry explicitly in 
BatchRequest.
 Key: BEAM-1153
 URL: https://issues.apache.org/jira/browse/BEAM-1153
 Project: Beam
  Issue Type: Bug
Reporter: Pei He
Assignee: Pei He


Google Cloud auto generated Json library doesn't set HttpRequestInitializer for 
batch requests. (But, it doesn't for other non-batch requests.)

Possible fix is: https://github.com/apache/incubator-beam/pull/1608

In additional, we can partially rollback 
https://github.com/apache/incubator-beam/pull/1359 to keep using non-batch API 
for fileSize() for single files. This will make sure existing code will keep 
work as the same way.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[GitHub] incubator-beam pull request #1607: Cherry pick disabling of state & state te...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/incubator-beam/pull/1607


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[1/3] incubator-beam git commit: Revert "Allow stateful DoFn in DataflowRunner"

[kenn]

Repository: incubator-beam
Updated Branches:
  refs/heads/release-0.4.0-incubating b2780881a -> 10bb4767a


Revert "Allow stateful DoFn in DataflowRunner"

This reverts commit 42bb15d2df28b99b6788010450f41f2932095771.

The Dataflow service has introduced a bug that was masked by various
test disabling.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/c1af44fa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/c1af44fa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/c1af44fa

Branch: refs/heads/release-0.4.0-incubating
Commit: c1af44fa27633fd2a9592a13579415f6b974cfe6
Parents: f78d960
Author: Kenneth Knowles 
Authored: Tue Dec 13 16:36:42 2016 -0800
Committer: Kenneth Knowles 
Committed: Tue Dec 13 16:57:26 2016 -0800

--
 .../dataflow/DataflowPipelineTranslator.java| 22 +---
 1 file changed, 14 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c1af44fa/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
--
diff --git 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
index 8048df9..a56690c 100644
--- 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
+++ 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
@@ -77,7 +77,6 @@ import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.View;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.transforms.display.HasDisplayData;
-import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
 import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
 import org.apache.beam.sdk.transforms.windowing.Window;
@@ -956,6 +955,7 @@ public class DataflowPipelineTranslator {
   private  void translateMultiHelper(
   ParDo.BoundMulti transform,
   TranslationContext context) {
+rejectStatefulDoFn(transform.getNewFn());
 
 context.addStep(transform, "ParallelDo");
 translateInputs(context.getInput(transform), 
transform.getSideInputs(), context);
@@ -985,6 +985,7 @@ public class DataflowPipelineTranslator {
   private  void translateSingleHelper(
   ParDo.Bound transform,
   TranslationContext context) {
+rejectStatefulDoFn(transform.getNewFn());
 
 context.addStep(transform, "ParallelDo");
 translateInputs(context.getInput(transform), 
transform.getSideInputs(), context);
@@ -1032,6 +1033,18 @@ public class DataflowPipelineTranslator {
 registerTransformTranslator(Read.Bounded.class, new ReadTranslator());
   }
 
+  private static void rejectStatefulDoFn(DoFn doFn) {
+if (DoFnSignatures.getSignature(doFn.getClass()).isStateful()) {
+throw new UnsupportedOperationException(
+String.format(
+"Found %s annotations on %s, but %s cannot yet be used with state 
in the %s.",
+DoFn.StateId.class.getSimpleName(),
+doFn.getClass().getName(),
+DoFn.class.getSimpleName(),
+DataflowRunner.class.getSimpleName()));
+}
+  }
+
   private static void translateInputs(
   PCollection input,
   List sideInputs,
@@ -1063,9 +1076,6 @@ public class DataflowPipelineTranslator {
   TranslationContext context,
   long mainOutput,
   Map outputMap) {
-
-DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass());
-
 context.addInput(PropertyNames.USER_FN, fn.getClass().getName());
 context.addInput(
 PropertyNames.SERIALIZED_FN,
@@ -1073,10 +1083,6 @@ public class DataflowPipelineTranslator {
 serializeToByteArray(
 DoFnInfo.forFn(
 fn, windowingStrategy, sideInputs, inputCoder, mainOutput, 
outputMap;
-
-if (signature.isStateful()) {
-  context.addInput(PropertyNames.USES_KEYED_STATE, "true");
-}
   }
 
   private static BiMap translateOutputs(

[3/3] incubator-beam git commit: This closes #1607

[kenn]

This closes #1607


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/10bb4767
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/10bb4767
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/10bb4767

Branch: refs/heads/release-0.4.0-incubating
Commit: 10bb4767a1f989a1a75778828c07d9c72c450495
Parents: b278088 c1af44f
Author: Kenneth Knowles 
Authored: Tue Dec 13 17:08:42 2016 -0800
Committer: Kenneth Knowles 
Committed: Tue Dec 13 17:08:42 2016 -0800

--
 runners/google-cloud-dataflow-java/pom.xml  |  1 +
 .../dataflow/DataflowPipelineTranslator.java| 22 +---
 2 files changed, 15 insertions(+), 8 deletions(-)
--

[2/3] incubator-beam git commit: Re-exclude UsesStatefulParDo tests for Dataflow

[kenn]

Re-exclude UsesStatefulParDo tests for Dataflow


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/f78d9606
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/f78d9606
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/f78d9606

Branch: refs/heads/release-0.4.0-incubating
Commit: f78d96069b428356609f219357cbf0702ec56c26
Parents: b278088
Author: Kenneth Knowles 
Authored: Tue Dec 13 16:09:57 2016 -0800
Committer: Kenneth Knowles 
Committed: Tue Dec 13 16:57:26 2016 -0800

--
 runners/google-cloud-dataflow-java/pom.xml | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f78d9606/runners/google-cloud-dataflow-java/pom.xml
--
diff --git a/runners/google-cloud-dataflow-java/pom.xml 
b/runners/google-cloud-dataflow-java/pom.xml
index 1543c0e..93e4054 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -78,6 +78,7 @@
 runnable-on-service-tests
 
   
+org.apache.beam.sdk.testing.UsesStatefulParDo,
 org.apache.beam.sdk.testing.UsesTimersInParDo,
 org.apache.beam.sdk.testing.UsesSplittableParDo
   

Jenkins build is still unstable: beam_PostCommit_Java_RunnableOnService_Dataflow #1807

[Apache Jenkins Server]

See 

Jenkins build is still unstable: beam_PostCommit_Java_RunnableOnService_Dataflow #1807

[Apache Jenkins Server]

See 

Build failed in Jenkins: beam_PostCommit_Java_MavenInstall #2087

[Apache Jenkins Server]

See 


Changes:

[klk] Re-exclude UsesStatefulParDo tests for Dataflow

[klk] Revert "Allow stateful DoFn in DataflowRunner"

--
[...truncated 8491 lines...]
[INFO] 
[INFO] --- maven-source-plugin:2.4:jar-no-fork (attach-sources) @ 
beam-sdks-java-maven-archetypes-starter ---
[INFO] Building jar: 

[INFO] 
[INFO] --- maven-source-plugin:2.4:test-jar-no-fork (attach-test-sources) @ 
beam-sdks-java-maven-archetypes-starter ---
[INFO] Building jar: 

[INFO] 
[INFO] --- maven-dependency-plugin:2.10:analyze-only (default) @ 
beam-sdks-java-maven-archetypes-starter ---
[INFO] No dependency problems found
[INFO] 
[INFO] --- apache-rat-plugin:0.12:check (default) @ 
beam-sdks-java-maven-archetypes-starter ---
[INFO] Enabled default license matchers.
[INFO] Will parse SCM ignores for exclusions...
[INFO] Finished adding exclusions from SCM ignore files.
[INFO] 61 implicit excludes (use -debug for more details).
[INFO] Exclude: **/target/**/*
[INFO] Exclude: **/dependency-reduced-pom.xml
[INFO] Exclude: **/hs_err_pid*.log
[INFO] Exclude: .github/**/*
[INFO] Exclude: **/*.iml
[INFO] Exclude: **/.idea/**/*
[INFO] Exclude: **/package-list
[INFO] Exclude: **/user.avsc
[INFO] Exclude: **/test/resources/**/*.txt
[INFO] Exclude: **/test/**/.placeholder
[INFO] Exclude: .repository/**/*
[INFO] Exclude: **/.checkstyle
[INFO] Exclude: **/.classpath
[INFO] Exclude: **/.factorypath
[INFO] Exclude: **/.project
[INFO] Exclude: **/.settings/**/*
[INFO] 7 resources included (use -debug for more details)
[INFO] Rat check: Summary over all files. Unapproved: 0, unknown: 0, generated: 
0, approved: 7 licenses.
[INFO] 
[INFO] --- maven-install-plugin:2.5.2:install (default-install) @ 
beam-sdks-java-maven-archetypes-starter ---
[INFO] Installing 

 to 

[INFO] Installing 

 to 

[INFO] Installing 

 to 

[INFO] Installing 

 to 

[INFO] 
[INFO] --- maven-archetype-plugin:2.4:integration-test 
(default-integration-test) @ beam-sdks-java-maven-archetypes-starter ---
[INFO] Processing Archetype IT project: basic
[INFO] 

[INFO] Using following parameters for creating project from Archetype: 
beam-sdks-java-maven-archetypes-starter:0.5.0-incubating-SNAPSHOT
[INFO] 

[INFO] Parameter: groupId, Value: archetype.it
[INFO] Parameter: artifactId, Value: basic
[INFO] Parameter: version, Value: 0.1
[INFO] Parameter: package, Value: it.pkg
[INFO] Parameter: packageInPathFormat, Value: it/pkg
[INFO] Parameter: version, Value: 0.1
[INFO] Parameter: package, Value: it.pkg
[INFO] Parameter: groupId, Value: archetype.it
[INFO] Parameter: targetPlatform, Value: 1.7
[INFO] Parameter: artifactId, Value: basic
[INFO] project created from Archetype in dir: 

[GitHub] incubator-beam pull request #1610: starter: fix typo in pom.xml

[dhalperi]

GitHub user dhalperi opened a pull request:

https://github.com/apache/incubator-beam/pull/1610

starter: fix typo in pom.xml

Be sure to do all of the following to help us incorporate your contribution
quickly and easily:

 - [ ] Make sure the PR title is formatted like:
   `[BEAM-] Description of pull request`
 - [ ] Make sure tests pass via `mvn clean verify`. (Even better, enable
   Travis-CI on your fork and ensure the whole test matrix passes).
 - [ ] Replace `` in the title with the actual Jira issue
   number, if there is one.
 - [ ] If this contribution is large, please file an Apache
   [Individual Contributor License 
Agreement](https://www.apache.org/licenses/icla.txt).

---

Manual edit introduced in 
https://github.com/apache/incubator-beam/commit/25215889381f7da61766054af68c84ffed4c0c71

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/dhalperi/incubator-beam fix-starter

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-beam/pull/1610.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1610


commit 60c33dd5a40a80e2d782cddbbd6940b96f34d975
Author: Dan Halperin 
Date:   2016-12-14T01:27:07Z

starter: fix typo in pom.xml

Manual edit introduced in 
https://github.com/apache/incubator-beam/commit/25215889381f7da61766054af68c84ffed4c0c71




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[jira] [Commented] (BEAM-1151) BigQueryIO.Write has no way of handling failures

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-1151?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15746824#comment-15746824
 ] 

ASF GitHub Bot commented on BEAM-1151:
--

GitHub user reuvenlax opened a pull request:

https://github.com/apache/incubator-beam/pull/1609

BEAM-1151 Add failure handling to BigQueryIO.Write

Allow a pipeline to handle streaming insert failures when using 
BigQueryIO.Write in streaming mode. TableRow objects that fail to insert are 
returned to the user as a PCollection.

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/reuvenlax/incubator-beam bq_dead_letter

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-beam/pull/1609.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1609


commit 379af6129f91ffb1b1b3aa917f3e0f13cb412132
Author: Reuven Lax 
Date:   2016-11-17T18:57:41Z

Add dead-letter support to BigQueryIO

Add dead-letter support to BigQueryIO

Strip out non-testing changes.

commit 8b2f3010058674a5ed4359148397296b7aad31de
Author: Reuven Lax 
Date:   2016-11-19T23:09:10Z

Remove dead code.

commit 4cf5edf61d29466811cb3487561d02f51bbdfd5a
Author: Reuven Lax 
Date:   2016-11-21T20:30:29Z

Add testing coverage for BigQuery streaming writes.

commit fa33d7621907f9c7e37a769b530514eb44a4c10a
Author: Reuven Lax 
Date:   2016-11-21T21:52:01Z

Fix checkstyle errors and remove remaining tests out of 
BigQueryTableInserterTest.

commit a91ac51f3f127921e02c64c4f7c8e2c1c4828eaf
Author: Reuven Lax 
Date:   2016-11-21T22:32:03Z

Fix more checkstyle errors.

commit e31b593bd2f8d007efeeb14fd30145bc56fff01a
Author: Reuven Lax 
Date:   2016-11-26T12:55:28Z

Make sure to clear static cache of created tables between tests.

commit 2011433e201c983575208d1353f4221e31b44b0e
Author: Reuven Lax 
Date:   2016-12-09T20:01:10Z

Refactor tryCreateTable method. Removes unused create and write disposition 
code on this path.

commit d5b549c24870bcd6593dc6616360d6c9565381d2
Author: Reuven Lax 
Date:   2016-12-10T03:39:02Z

Fix test failure. Remove no-longer applicable test.

commit 98febf780981ec978fef9a5a9392a2e605e74326
Author: Reuven Lax 
Date:   2016-12-13T00:35:38Z

Apply review comments.

commit 9e8cadbff8ac7fe5f5fc3d4a6c07221316144ef4
Author: Reuven Lax 
Date:   2016-12-13T05:28:42Z

Fix failures

commit 1f840efe0a71469fde3db37fba910c296ce0d76c
Author: Reuven Lax 
Date:   2016-12-13T08:27:12Z

Fix broken test

commit 2a446a508822da167162a022448d3d5f347287f9
Author: Reuven Lax 
Date:   2016-12-13T21:52:44Z

Template code broke windowed BigQuery output - fix.
Fix invalid GCS access in BigQueryIOTest

commit 14da9adec51a43e6f02672677d640a88fd42894b
Author: Reuven Lax 
Date:   2016-11-17T18:57:41Z

Add dead-letter support to BigQueryIO

commit dfced01604d7dff5c46c99389f1f945ac5ab8995
Author: Reuven Lax 
Date:   2016-11-19T22:28:10Z

Add dead-letter support to BigQueryIO

commit 9e66dffabaa71c7e01f3cc46736df40b163e9e82
Author: Reuven Lax 
Date:   2016-11-26T12:53:55Z

Add some tests of dead letter policy, and default retry policies.

commit 3e3cfcf311c6db0ac97ae360c1b403d1a00cae2c
Author: Reuven Lax 
Date:   2016-12-01T08:29:58Z

Add testing.

commit 2af0de1c84ac48bfcf4ec17af8e9d80012ff90c2
Author: Reuven Lax 
Date:   2016-12-14T00:57:48Z

Fixes to test




> BigQueryIO.Write has no way of handling failures
> 
>
> Key: BEAM-1151
> URL: https://issues.apache.org/jira/browse/BEAM-1151
> Project: Beam
>  Issue Type: Bug
>  Components: sdk-java-core
>Affects Versions: 0.4.0-incubating
>Reporter: Reuven Lax
>Assignee: Reuven Lax
> Fix For: 0.3.0-incubating
>
>
> BigQueryIO.Write in streaming mode uses BigQuery's streaming insert API. 
> Inserts can fail for a variety of reasons, and the current connector provides 
> no way of 

[GitHub] incubator-beam pull request #1609: BEAM-1151 Add failure handling to BigQuer...

[reuvenlax]

GitHub user reuvenlax opened a pull request:

https://github.com/apache/incubator-beam/pull/1609

BEAM-1151 Add failure handling to BigQueryIO.Write

Allow a pipeline to handle streaming insert failures when using 
BigQueryIO.Write in streaming mode. TableRow objects that fail to insert are 
returned to the user as a PCollection.

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/reuvenlax/incubator-beam bq_dead_letter

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-beam/pull/1609.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1609


commit 379af6129f91ffb1b1b3aa917f3e0f13cb412132
Author: Reuven Lax 
Date:   2016-11-17T18:57:41Z

Add dead-letter support to BigQueryIO

Add dead-letter support to BigQueryIO

Strip out non-testing changes.

commit 8b2f3010058674a5ed4359148397296b7aad31de
Author: Reuven Lax 
Date:   2016-11-19T23:09:10Z

Remove dead code.

commit 4cf5edf61d29466811cb3487561d02f51bbdfd5a
Author: Reuven Lax 
Date:   2016-11-21T20:30:29Z

Add testing coverage for BigQuery streaming writes.

commit fa33d7621907f9c7e37a769b530514eb44a4c10a
Author: Reuven Lax 
Date:   2016-11-21T21:52:01Z

Fix checkstyle errors and remove remaining tests out of 
BigQueryTableInserterTest.

commit a91ac51f3f127921e02c64c4f7c8e2c1c4828eaf
Author: Reuven Lax 
Date:   2016-11-21T22:32:03Z

Fix more checkstyle errors.

commit e31b593bd2f8d007efeeb14fd30145bc56fff01a
Author: Reuven Lax 
Date:   2016-11-26T12:55:28Z

Make sure to clear static cache of created tables between tests.

commit 2011433e201c983575208d1353f4221e31b44b0e
Author: Reuven Lax 
Date:   2016-12-09T20:01:10Z

Refactor tryCreateTable method. Removes unused create and write disposition 
code on this path.

commit d5b549c24870bcd6593dc6616360d6c9565381d2
Author: Reuven Lax 
Date:   2016-12-10T03:39:02Z

Fix test failure. Remove no-longer applicable test.

commit 98febf780981ec978fef9a5a9392a2e605e74326
Author: Reuven Lax 
Date:   2016-12-13T00:35:38Z

Apply review comments.

commit 9e8cadbff8ac7fe5f5fc3d4a6c07221316144ef4
Author: Reuven Lax 
Date:   2016-12-13T05:28:42Z

Fix failures

commit 1f840efe0a71469fde3db37fba910c296ce0d76c
Author: Reuven Lax 
Date:   2016-12-13T08:27:12Z

Fix broken test

commit 2a446a508822da167162a022448d3d5f347287f9
Author: Reuven Lax 
Date:   2016-12-13T21:52:44Z

Template code broke windowed BigQuery output - fix.
Fix invalid GCS access in BigQueryIOTest

commit 14da9adec51a43e6f02672677d640a88fd42894b
Author: Reuven Lax 
Date:   2016-11-17T18:57:41Z

Add dead-letter support to BigQueryIO

commit dfced01604d7dff5c46c99389f1f945ac5ab8995
Author: Reuven Lax 
Date:   2016-11-19T22:28:10Z

Add dead-letter support to BigQueryIO

commit 9e66dffabaa71c7e01f3cc46736df40b163e9e82
Author: Reuven Lax 
Date:   2016-11-26T12:53:55Z

Add some tests of dead letter policy, and default retry policies.

commit 3e3cfcf311c6db0ac97ae360c1b403d1a00cae2c
Author: Reuven Lax 
Date:   2016-12-01T08:29:58Z

Add testing.

commit 2af0de1c84ac48bfcf4ec17af8e9d80012ff90c2
Author: Reuven Lax 
Date:   2016-12-14T00:57:48Z

Fixes to test




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[jira] [Created] (BEAM-1151) BigQueryIO.Write has no way of handling failures

[Reuven Lax (JIRA)]

Reuven Lax created BEAM-1151:


 Summary: BigQueryIO.Write has no way of handling failures
 Key: BEAM-1151
 URL: https://issues.apache.org/jira/browse/BEAM-1151
 Project: Beam
  Issue Type: Bug
  Components: sdk-java-core
Affects Versions: 0.4.0-incubating
Reporter: Reuven Lax
Assignee: Reuven Lax
 Fix For: 0.3.0-incubating


BigQueryIO.Write in streaming mode uses BigQuery's streaming insert API. 
Inserts can fail for a variety of reasons, and the current connector provides 
no way of managing those failures. This PR allows better failure handling.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[GitHub] incubator-beam pull request #1608: GcsUtil: set timeout and retry for BatchR...

[peihe]

GitHub user peihe opened a pull request:

https://github.com/apache/incubator-beam/pull/1608

GcsUtil: set timeout and retry for BatchRequest with HttpRequestInitializer.

Be sure to do all of the following to help us incorporate your contribution
quickly and easily:

 - [ ] Make sure the PR title is formatted like:
   `[BEAM-] Description of pull request`
 - [ ] Make sure tests pass via `mvn clean verify`. (Even better, enable
   Travis-CI on your fork and ensure the whole test matrix passes).
 - [ ] Replace `` in the title with the actual Jira issue
   number, if there is one.
 - [ ] If this contribution is large, please file an Apache
   [Individual Contributor License 
Agreement](https://www.apache.org/licenses/icla.txt).

---


You can merge this pull request into a Git repository by running:

$ git pull https://github.com/peihe/incubator-beam gsutil-timeout

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-beam/pull/1608.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1608


commit fecac4e5265373dd90d1b7cf4d11ea7d7f61a44a
Author: peihe 
Date:   2016-12-14T00:56:32Z

GcsUtil: set timeout and retry for BatchRequest with HttpRequestInitializer.




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-beam pull request #1607: Cherry pick disabling of state & state te...

[kennknowles]

GitHub user kennknowles opened a pull request:

https://github.com/apache/incubator-beam/pull/1607

Cherry pick disabling of state & state tests for Dataflow runner

Be sure to do all of the following to help us incorporate your contribution
quickly and easily:

 - [x] Make sure the PR title is formatted like:
   `[BEAM-] Description of pull request`
 - [ ] Make sure tests pass via `mvn clean verify`. (Even better, enable
   Travis-CI on your fork and ensure the whole test matrix passes).
 - [x] Replace `` in the title with the actual Jira issue
   number, if there is one.
 - [x] If this contribution is large, please file an Apache
   [Individual Contributor License 
Agreement](https://www.apache.org/licenses/icla.txt).

---


You can merge this pull request into a Git repository by running:

$ git pull https://github.com/kennknowles/incubator-beam cherry-pick-state

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-beam/pull/1607.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1607


commit f78d96069b428356609f219357cbf0702ec56c26
Author: Kenneth Knowles 
Date:   2016-12-14T00:09:57Z

Re-exclude UsesStatefulParDo tests for Dataflow

commit c1af44fa27633fd2a9592a13579415f6b974cfe6
Author: Kenneth Knowles 
Date:   2016-12-14T00:36:42Z

Revert "Allow stateful DoFn in DataflowRunner"

This reverts commit 42bb15d2df28b99b6788010450f41f2932095771.

The Dataflow service has introduced a bug that was masked by various
test disabling.




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[2/2] incubator-beam git commit: This closes #1606

[kenn]

This closes #1606


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/b742a2c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/b742a2c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/b742a2c0

Branch: refs/heads/master
Commit: b742a2c09611b03356cafc1a2560d2c0fe177b40
Parents: d99eec1 df97fe4
Author: Kenneth Knowles 
Authored: Tue Dec 13 16:51:39 2016 -0800
Committer: Kenneth Knowles 
Committed: Tue Dec 13 16:51:39 2016 -0800

--
 .../dataflow/DataflowPipelineTranslator.java| 22 +---
 1 file changed, 14 insertions(+), 8 deletions(-)
--

[GitHub] incubator-beam pull request #1606: Revert "Allow stateful DoFn in DataflowRu...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/incubator-beam/pull/1606


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[1/2] incubator-beam git commit: Revert "Allow stateful DoFn in DataflowRunner"

[kenn]

Repository: incubator-beam
Updated Branches:
  refs/heads/master d99eec15d -> b742a2c09


Revert "Allow stateful DoFn in DataflowRunner"

This reverts commit 42bb15d2df28b99b6788010450f41f2932095771.

The Dataflow service has introduced a bug that was masked by various
test disabling.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/df97fe48
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/df97fe48
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/df97fe48

Branch: refs/heads/master
Commit: df97fe4836a669b11ddeb37ef467bbfee1d803ea
Parents: d99eec1
Author: Kenneth Knowles 
Authored: Tue Dec 13 16:36:42 2016 -0800
Committer: Kenneth Knowles 
Committed: Tue Dec 13 16:37:52 2016 -0800

--
 .../dataflow/DataflowPipelineTranslator.java| 22 +---
 1 file changed, 14 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/df97fe48/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
--
diff --git 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
index 8048df9..a56690c 100644
--- 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
+++ 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
@@ -77,7 +77,6 @@ import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.View;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.transforms.display.HasDisplayData;
-import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
 import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
 import org.apache.beam.sdk.transforms.windowing.Window;
@@ -956,6 +955,7 @@ public class DataflowPipelineTranslator {
   private  void translateMultiHelper(
   ParDo.BoundMulti transform,
   TranslationContext context) {
+rejectStatefulDoFn(transform.getNewFn());
 
 context.addStep(transform, "ParallelDo");
 translateInputs(context.getInput(transform), 
transform.getSideInputs(), context);
@@ -985,6 +985,7 @@ public class DataflowPipelineTranslator {
   private  void translateSingleHelper(
   ParDo.Bound transform,
   TranslationContext context) {
+rejectStatefulDoFn(transform.getNewFn());
 
 context.addStep(transform, "ParallelDo");
 translateInputs(context.getInput(transform), 
transform.getSideInputs(), context);
@@ -1032,6 +1033,18 @@ public class DataflowPipelineTranslator {
 registerTransformTranslator(Read.Bounded.class, new ReadTranslator());
   }
 
+  private static void rejectStatefulDoFn(DoFn doFn) {
+if (DoFnSignatures.getSignature(doFn.getClass()).isStateful()) {
+throw new UnsupportedOperationException(
+String.format(
+"Found %s annotations on %s, but %s cannot yet be used with state 
in the %s.",
+DoFn.StateId.class.getSimpleName(),
+doFn.getClass().getName(),
+DoFn.class.getSimpleName(),
+DataflowRunner.class.getSimpleName()));
+}
+  }
+
   private static void translateInputs(
   PCollection input,
   List sideInputs,
@@ -1063,9 +1076,6 @@ public class DataflowPipelineTranslator {
   TranslationContext context,
   long mainOutput,
   Map outputMap) {
-
-DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass());
-
 context.addInput(PropertyNames.USER_FN, fn.getClass().getName());
 context.addInput(
 PropertyNames.SERIALIZED_FN,
@@ -1073,10 +1083,6 @@ public class DataflowPipelineTranslator {
 serializeToByteArray(
 DoFnInfo.forFn(
 fn, windowingStrategy, sideInputs, inputCoder, mainOutput, 
outputMap;
-
-if (signature.isStateful()) {
-  context.addInput(PropertyNames.USES_KEYED_STATE, "true");
-}
   }
 
   private static BiMap translateOutputs(

Build failed in Jenkins: beam_PostCommit_Java_MavenInstall #2086

[Apache Jenkins Server]

See 


Changes:

[davor] Update version from 0.5.0-SNAPSHOT to 0.5.0-incubating-SNAPSHOT

[klk] [BEAM-1149] Explode windows when fn uses side inputs

[dhalperi] Some minor changes and fixes for sorter module

--
[...truncated 8421 lines...]
[INFO] 
[INFO] --- maven-source-plugin:2.4:jar-no-fork (attach-sources) @ 
beam-sdks-java-maven-archetypes-starter ---
[INFO] Building jar: 

[INFO] 
[INFO] --- maven-source-plugin:2.4:test-jar-no-fork (attach-test-sources) @ 
beam-sdks-java-maven-archetypes-starter ---
[INFO] Building jar: 

[INFO] 
[INFO] --- maven-dependency-plugin:2.10:analyze-only (default) @ 
beam-sdks-java-maven-archetypes-starter ---
[INFO] No dependency problems found
[INFO] 
[INFO] --- apache-rat-plugin:0.12:check (default) @ 
beam-sdks-java-maven-archetypes-starter ---
[INFO] Enabled default license matchers.
[INFO] Will parse SCM ignores for exclusions...
[INFO] Finished adding exclusions from SCM ignore files.
[INFO] 61 implicit excludes (use -debug for more details).
[INFO] Exclude: **/target/**/*
[INFO] Exclude: **/dependency-reduced-pom.xml
[INFO] Exclude: **/hs_err_pid*.log
[INFO] Exclude: .github/**/*
[INFO] Exclude: **/*.iml
[INFO] Exclude: **/.idea/**/*
[INFO] Exclude: **/package-list
[INFO] Exclude: **/user.avsc
[INFO] Exclude: **/test/resources/**/*.txt
[INFO] Exclude: **/test/**/.placeholder
[INFO] Exclude: .repository/**/*
[INFO] Exclude: **/.checkstyle
[INFO] Exclude: **/.classpath
[INFO] Exclude: **/.factorypath
[INFO] Exclude: **/.project
[INFO] Exclude: **/.settings/**/*
[INFO] 7 resources included (use -debug for more details)
[INFO] Rat check: Summary over all files. Unapproved: 0, unknown: 0, generated: 
0, approved: 7 licenses.
[INFO] 
[INFO] --- maven-install-plugin:2.5.2:install (default-install) @ 
beam-sdks-java-maven-archetypes-starter ---
[INFO] Installing 

 to 

[INFO] Installing 

 to 

[INFO] Installing 

 to 

[INFO] Installing 

 to 

[INFO] 
[INFO] --- maven-archetype-plugin:2.4:integration-test 
(default-integration-test) @ beam-sdks-java-maven-archetypes-starter ---
[INFO] Processing Archetype IT project: basic
[INFO] 

[INFO] Using following parameters for creating project from Archetype: 
beam-sdks-java-maven-archetypes-starter:0.5.0-incubating-SNAPSHOT
[INFO] 

[INFO] Parameter: groupId, Value: archetype.it
[INFO] Parameter: artifactId, Value: basic
[INFO] Parameter: version, Value: 0.1
[INFO] Parameter: package, Value: it.pkg
[INFO] Parameter: packageInPathFormat, Value: it/pkg
[INFO] Parameter: version, Value: 0.1
[INFO] Parameter: package, Value: it.pkg
[INFO] Parameter: groupId, Value: archetype.it
[INFO] Parameter: targetPlatform, Value: 1.7
[INFO] Parameter: artifactId, Value: basic
[INFO] project created from Archetype 

[GitHub] incubator-beam pull request #1606: Revert "Allow stateful DoFn in DataflowRu...

[kennknowles]

GitHub user kennknowles opened a pull request:

https://github.com/apache/incubator-beam/pull/1606

Revert "Allow stateful DoFn in DataflowRunner"

Be sure to do all of the following to help us incorporate your contribution
quickly and easily:

 - [x] Make sure the PR title is formatted like:
   `[BEAM-] Description of pull request`
 - [x] Make sure tests pass via `mvn clean verify`. (Even better, enable
   Travis-CI on your fork and ensure the whole test matrix passes).
 - [x] Replace `` in the title with the actual Jira issue
   number, if there is one.
 - [x] If this contribution is large, please file an Apache
   [Individual Contributor License 
Agreement](https://www.apache.org/licenses/icla.txt).

---

This reverts commit 42bb15d2df28b99b6788010450f41f2932095771.

The Dataflow service has introduced a bug that was masked by various test 
disabling.

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/kennknowles/incubator-beam 
no-state-in-Dataflow

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-beam/pull/1606.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1606


commit df97fe4836a669b11ddeb37ef467bbfee1d803ea
Author: Kenneth Knowles 
Date:   2016-12-14T00:36:42Z

Revert "Allow stateful DoFn in DataflowRunner"

This reverts commit 42bb15d2df28b99b6788010450f41f2932095771.

The Dataflow service has introduced a bug that was masked by various
test disabling.




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-beam pull request #1605: Re-exclude UsesStatefulParDo tests for Da...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/incubator-beam/pull/1605


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[2/2] incubator-beam git commit: This closes #1605

[kenn]

This closes #1605


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/d99eec15
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/d99eec15
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/d99eec15

Branch: refs/heads/master
Commit: d99eec15d17f40b453d81d21e23ff36c33017b24
Parents: a5f2df2 82249ab
Author: Kenneth Knowles 
Authored: Tue Dec 13 16:11:55 2016 -0800
Committer: Kenneth Knowles 
Committed: Tue Dec 13 16:11:55 2016 -0800

--
 runners/google-cloud-dataflow-java/pom.xml | 1 +
 1 file changed, 1 insertion(+)
--

[1/2] incubator-beam git commit: Re-exclude UsesStatefulParDo tests for Dataflow

[kenn]

Repository: incubator-beam
Updated Branches:
  refs/heads/master a5f2df2bd -> d99eec15d


Re-exclude UsesStatefulParDo tests for Dataflow


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/82249abe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/82249abe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/82249abe

Branch: refs/heads/master
Commit: 82249abe6f4e3f37ac46f2eef07208b7ac97756a
Parents: a5f2df2
Author: Kenneth Knowles 
Authored: Tue Dec 13 16:09:57 2016 -0800
Committer: Kenneth Knowles 
Committed: Tue Dec 13 16:09:57 2016 -0800

--
 runners/google-cloud-dataflow-java/pom.xml | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/82249abe/runners/google-cloud-dataflow-java/pom.xml
--
diff --git a/runners/google-cloud-dataflow-java/pom.xml 
b/runners/google-cloud-dataflow-java/pom.xml
index 4c2378c..77187d6 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -78,6 +78,7 @@
 runnable-on-service-tests
 
   
+org.apache.beam.sdk.testing.UsesStatefulParDo,
 org.apache.beam.sdk.testing.UsesTimersInParDo,
 org.apache.beam.sdk.testing.UsesSplittableParDo
   

[GitHub] incubator-beam pull request #1605: Re-exclude UsesStatefulParDo tests for Da...

[kennknowles]

GitHub user kennknowles opened a pull request:

https://github.com/apache/incubator-beam/pull/1605

Re-exclude UsesStatefulParDo tests for Dataflow

Be sure to do all of the following to help us incorporate your contribution
quickly and easily:

 - [x] Make sure the PR title is formatted like:
   `[BEAM-] Description of pull request`
 - [x] Make sure tests pass via `mvn clean verify`. (Even better, enable
   Travis-CI on your fork and ensure the whole test matrix passes).
 - [x] Replace `` in the title with the actual Jira issue
   number, if there is one.
 - [x] If this contribution is large, please file an Apache
   [Individual Contributor License 
Agreement](https://www.apache.org/licenses/icla.txt).

---

R: @dhalperi 

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/kennknowles/incubator-beam sickbay-state

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-beam/pull/1605.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1605


commit 82249abe6f4e3f37ac46f2eef07208b7ac97756a
Author: Kenneth Knowles 
Date:   2016-12-14T00:09:57Z

Re-exclude UsesStatefulParDo tests for Dataflow




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-beam pull request #1603: Revert "This closes #1541": Move PerKeyCo...

[kennknowles]

Github user kennknowles closed the pull request at:

https://github.com/apache/incubator-beam/pull/1603


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[jira] [Commented] (BEAM-840) Add Java SDK extension to support non-distributed sorting

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-840?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15746667#comment-15746667
 ] 

ASF GitHub Bot commented on BEAM-840:
-

Github user asfgit closed the pull request at:

https://github.com/apache/incubator-beam/pull/1327


> Add Java SDK extension to support non-distributed sorting
> -
>
> Key: BEAM-840
> URL: https://issues.apache.org/jira/browse/BEAM-840
> Project: Beam
>  Issue Type: New Feature
>  Components: sdk-java-extensions
>Reporter: Mitch Shanklin
>Assignee: Mitch Shanklin
>Priority: Minor
> Fix For: 0.4.0-incubating
>
>
> Add an extension that provides a PTransform which performs 
> local(non-distributed) sorting. It will sort in memory until the buffer is 
> full, then flush to disk and use external sorting.
> 
> Consumes a PCollection of KVs from primary key to iterable of secondary key 
> and value KVs and sorts the iterables. Would probably be called after a 
> GroupByKey. Uses coders to convert secondary keys and values into byte arrays 
> and does a lexicographical comparison on the secondary keys.
> Uses Hadoop as an external sorting library.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[2/2] incubator-beam git commit: Closes #1327

[dhalperi]

Closes #1327


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a5f2df2b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a5f2df2b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a5f2df2b

Branch: refs/heads/master
Commit: a5f2df2bdc05a3e2a3bc32a7a0663c6fdfde8c33
Parents: 0bdf7fc 5a04492
Author: Dan Halperin 
Authored: Tue Dec 13 15:42:11 2016 -0800
Committer: Dan Halperin 
Committed: Tue Dec 13 15:42:11 2016 -0800

--
 sdks/java/extensions/sorter/README.md   |  2 +-
 sdks/java/extensions/sorter/pom.xml |  8 ++
 .../sorter/BufferedExternalSorter.java  |  6 -
 .../sdk/extensions/sorter/ExternalSorter.java   | 15 ++-
 .../sdk/extensions/sorter/InMemorySorter.java   | 26 ++--
 .../sorter/BufferedExternalSorterTest.java  | 16 
 .../extensions/sorter/ExternalSorterTest.java   | 16 
 .../extensions/sorter/InMemorySorterTest.java   |  8 ++
 8 files changed, 75 insertions(+), 22 deletions(-)
--

[1/2] incubator-beam git commit: Some minor changes and fixes for sorter module

[dhalperi]

Repository: incubator-beam
Updated Branches:
  refs/heads/master 0bdf7fc04 -> a5f2df2bd


Some minor changes and fixes for sorter module

Includes:

* Limit max memory for ExternalSorter and BufferedExternalSorter to 2047 MB to 
prevent int overflow within Hadoop's sorting library
* Fix int overflow for large memory values in InMemorySorter
* Add note about estimated disk use to README.MD
* Fix to make Hadoop's sorting library put all temp files under the specified 
directory
* Have Hadoop clean up the temp directory on exit
* Stop shading hadoop dependencies. Some context:
** The existing shading is broken (modules that depend on this one cannot use 
it successfully).
** Hadoop's use of reflection in several instances makes shading the dependency 
"in a good way" nearly impossible. It requires a couple of rather brittle 
hacks, and, for clients that depend on certain conflicting versions of hadoop 
these hacks can mean it doesn't meet its intended goal of preventing conflicts 
anyway.
** From what I can tell, there's no good way to shade this to make it 
universally usable, so leaving it unshaded seems like a reasonable default.
** Without shading Hadoop, this module can be successfully used from Beam's 
wordcount example (which actually does have pre-existing hadoop dependencies 
already).


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/5a04492e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/5a04492e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/5a04492e

Branch: refs/heads/master
Commit: 5a04492e5b7c5d5b4cb2da0f7a80ed8f0c2f2eb4
Parents: 0bdf7fc
Author: Mitch Shanklin 
Authored: Wed Nov 9 14:09:49 2016 -0800
Committer: Dan Halperin 
Committed: Tue Dec 13 15:42:10 2016 -0800

--
 sdks/java/extensions/sorter/README.md   |  2 +-
 sdks/java/extensions/sorter/pom.xml |  8 ++
 .../sorter/BufferedExternalSorter.java  |  6 -
 .../sdk/extensions/sorter/ExternalSorter.java   | 15 ++-
 .../sdk/extensions/sorter/InMemorySorter.java   | 26 ++--
 .../sorter/BufferedExternalSorterTest.java  | 16 
 .../extensions/sorter/ExternalSorterTest.java   | 16 
 .../extensions/sorter/InMemorySorterTest.java   |  8 ++
 8 files changed, 75 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5a04492e/sdks/java/extensions/sorter/README.md
--
diff --git a/sdks/java/extensions/sorter/README.md 
b/sdks/java/extensions/sorter/README.md
index 18bd0d2..6ff3dbe 100644
--- a/sdks/java/extensions/sorter/README.md
+++ b/sdks/java/extensions/sorter/README.md
@@ -22,7 +22,7 @@ This module provides the SortValues transform, which takes a 
`PCollection>` is sorted on a single worker using local memory 
and disk. This means that `SortValues` may be a performance and/or scalability 
bottleneck when used in different pipelines. For example, users are discouraged 
from using `SortValues` on a `PCollection` of a single element to globally sort 
a large `PCollection`.
+* Each `Iterable>` is sorted on a single worker using local memory 
and disk. This means that `SortValues` may be a performance and/or scalability 
bottleneck when used in different pipelines. For example, users are discouraged 
from using `SortValues` on a `PCollection` of a single element to globally sort 
a large `PCollection`. A (rough) estimate of the number of bytes of disk space 
utilized if sorting spills to disk is `numRecords * 
(numSecondaryKeyBytesPerRecord + numValueBytesPerRecord + 16) * 3`.
 
 ##Options
 * The user can customize the temporary location used if sorting requires 
spilling to disk and the maximum amount of memory to use by creating a custom 
instance of `BufferedExternalSorter.Options` to pass into `SortValues.create`.

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5a04492e/sdks/java/extensions/sorter/pom.xml
--
diff --git a/sdks/java/extensions/sorter/pom.xml 
b/sdks/java/extensions/sorter/pom.xml
index a99a793..c8dfd52 100644
--- a/sdks/java/extensions/sorter/pom.xml
+++ b/sdks/java/extensions/sorter/pom.xml
@@ -69,8 +69,6 @@
   true
   
 
-  
org.apache.hadoop:hadoop-mapreduce-client-core
-  

[GitHub] incubator-beam pull request #1327: [BEAM-840] Some minor changes and fixes f...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/incubator-beam/pull/1327


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[jira] [Resolved] (BEAM-1149) Side input access fails in direct runner (possibly others too) when input element in multiple windows

[Eugene Kirpichov (JIRA)]

 [ 
https://issues.apache.org/jira/browse/BEAM-1149?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Eugene Kirpichov resolved BEAM-1149.

   Resolution: Fixed
Fix Version/s: 0.4.0-incubating

> Side input access fails in direct runner (possibly others too) when input 
> element in multiple windows
> -
>
> Key: BEAM-1149
> URL: https://issues.apache.org/jira/browse/BEAM-1149
> Project: Beam
>  Issue Type: Bug
>Reporter: Eugene Kirpichov
>Assignee: Eugene Kirpichov
>Priority: Blocker
> Fix For: 0.4.0-incubating
>
>
> {code:java}
>   private static class FnWithSideInputs extends DoFn {
> private final PCollectionView view;
> private FnWithSideInputs(PCollectionView view) {
>   this.view = view;
> }
> @ProcessElement
> public void processElement(ProcessContext c) {
>   c.output(c.element() + ":" + c.sideInput(view));
> }
>   }
>   @Test
>   public void testSideInputsWithMultipleWindows() {
> Pipeline p = TestPipeline.create();
> MutableDateTime mutableNow = Instant.now().toMutableDateTime();
> mutableNow.setMillisOfSecond(0);
> Instant now = mutableNow.toInstant();
> SlidingWindows windowFn =
> 
> SlidingWindows.of(Duration.standardSeconds(5)).every(Duration.standardSeconds(1));
> PCollectionView view = 
> p.apply(Create.of(1)).apply(View.asSingleton());
> PCollection res =
> p.apply(Create.timestamped(TimestampedValue.of("a", now)))
> .apply(Window.into(windowFn))
> .apply(ParDo.of(new FnWithSideInputs(view)).withSideInputs(view));
> PAssert.that(res).containsInAnyOrder("a:1");
> p.run();
>   }
> {code}
> This fails with the following exception:
> {code}
> org.apache.beam.sdk.Pipeline$PipelineExecutionException: 
> java.lang.IllegalStateException: sideInput called when main input element is 
> in multiple windows
>   at 
> org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:343)
>   at org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:1)
>   at org.apache.beam.sdk.Pipeline.run(Pipeline.java:176)
>   at org.apache.beam.sdk.testing.TestPipeline.run(TestPipeline.java:112)
>   at 
> Caused by: java.lang.IllegalStateException: sideInput called when main input 
> element is in multiple windows
>   at 
> org.apache.beam.runners.core.SimpleDoFnRunner$DoFnProcessContext.sideInput(SimpleDoFnRunner.java:514)
>   at 
> org.apache.beam.sdk.transforms.ParDoTest$FnWithSideInputs.processElement(ParDoTest.java:738)
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Commented] (BEAM-1149) Side input access fails in direct runner (possibly others too) when input element in multiple windows

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-1149?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15746646#comment-15746646
 ] 

ASF GitHub Bot commented on BEAM-1149:
--

Github user kennknowles closed the pull request at:

https://github.com/apache/incubator-beam/pull/1602


> Side input access fails in direct runner (possibly others too) when input 
> element in multiple windows
> -
>
> Key: BEAM-1149
> URL: https://issues.apache.org/jira/browse/BEAM-1149
> Project: Beam
>  Issue Type: Bug
>Reporter: Eugene Kirpichov
>Assignee: Eugene Kirpichov
>Priority: Blocker
>
> {code:java}
>   private static class FnWithSideInputs extends DoFn {
> private final PCollectionView view;
> private FnWithSideInputs(PCollectionView view) {
>   this.view = view;
> }
> @ProcessElement
> public void processElement(ProcessContext c) {
>   c.output(c.element() + ":" + c.sideInput(view));
> }
>   }
>   @Test
>   public void testSideInputsWithMultipleWindows() {
> Pipeline p = TestPipeline.create();
> MutableDateTime mutableNow = Instant.now().toMutableDateTime();
> mutableNow.setMillisOfSecond(0);
> Instant now = mutableNow.toInstant();
> SlidingWindows windowFn =
> 
> SlidingWindows.of(Duration.standardSeconds(5)).every(Duration.standardSeconds(1));
> PCollectionView view = 
> p.apply(Create.of(1)).apply(View.asSingleton());
> PCollection res =
> p.apply(Create.timestamped(TimestampedValue.of("a", now)))
> .apply(Window.into(windowFn))
> .apply(ParDo.of(new FnWithSideInputs(view)).withSideInputs(view));
> PAssert.that(res).containsInAnyOrder("a:1");
> p.run();
>   }
> {code}
> This fails with the following exception:
> {code}
> org.apache.beam.sdk.Pipeline$PipelineExecutionException: 
> java.lang.IllegalStateException: sideInput called when main input element is 
> in multiple windows
>   at 
> org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:343)
>   at org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:1)
>   at org.apache.beam.sdk.Pipeline.run(Pipeline.java:176)
>   at org.apache.beam.sdk.testing.TestPipeline.run(TestPipeline.java:112)
>   at 
> Caused by: java.lang.IllegalStateException: sideInput called when main input 
> element is in multiple windows
>   at 
> org.apache.beam.runners.core.SimpleDoFnRunner$DoFnProcessContext.sideInput(SimpleDoFnRunner.java:514)
>   at 
> org.apache.beam.sdk.transforms.ParDoTest$FnWithSideInputs.processElement(ParDoTest.java:738)
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Commented] (BEAM-1149) Side input access fails in direct runner (possibly others too) when input element in multiple windows

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-1149?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15746647#comment-15746647
 ] 

ASF GitHub Bot commented on BEAM-1149:
--

Github user asfgit closed the pull request at:

https://github.com/apache/incubator-beam/pull/1602


> Side input access fails in direct runner (possibly others too) when input 
> element in multiple windows
> -
>
> Key: BEAM-1149
> URL: https://issues.apache.org/jira/browse/BEAM-1149
> Project: Beam
>  Issue Type: Bug
>Reporter: Eugene Kirpichov
>Assignee: Eugene Kirpichov
>Priority: Blocker
>
> {code:java}
>   private static class FnWithSideInputs extends DoFn {
> private final PCollectionView view;
> private FnWithSideInputs(PCollectionView view) {
>   this.view = view;
> }
> @ProcessElement
> public void processElement(ProcessContext c) {
>   c.output(c.element() + ":" + c.sideInput(view));
> }
>   }
>   @Test
>   public void testSideInputsWithMultipleWindows() {
> Pipeline p = TestPipeline.create();
> MutableDateTime mutableNow = Instant.now().toMutableDateTime();
> mutableNow.setMillisOfSecond(0);
> Instant now = mutableNow.toInstant();
> SlidingWindows windowFn =
> 
> SlidingWindows.of(Duration.standardSeconds(5)).every(Duration.standardSeconds(1));
> PCollectionView view = 
> p.apply(Create.of(1)).apply(View.asSingleton());
> PCollection res =
> p.apply(Create.timestamped(TimestampedValue.of("a", now)))
> .apply(Window.into(windowFn))
> .apply(ParDo.of(new FnWithSideInputs(view)).withSideInputs(view));
> PAssert.that(res).containsInAnyOrder("a:1");
> p.run();
>   }
> {code}
> This fails with the following exception:
> {code}
> org.apache.beam.sdk.Pipeline$PipelineExecutionException: 
> java.lang.IllegalStateException: sideInput called when main input element is 
> in multiple windows
>   at 
> org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:343)
>   at org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:1)
>   at org.apache.beam.sdk.Pipeline.run(Pipeline.java:176)
>   at org.apache.beam.sdk.testing.TestPipeline.run(TestPipeline.java:112)
>   at 
> Caused by: java.lang.IllegalStateException: sideInput called when main input 
> element is in multiple windows
>   at 
> org.apache.beam.runners.core.SimpleDoFnRunner$DoFnProcessContext.sideInput(SimpleDoFnRunner.java:514)
>   at 
> org.apache.beam.sdk.transforms.ParDoTest$FnWithSideInputs.processElement(ParDoTest.java:738)
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[GitHub] incubator-beam pull request #1602: [BEAM-1149] Cherry pick side inputs fix t...

[kennknowles]

Github user kennknowles closed the pull request at:

https://github.com/apache/incubator-beam/pull/1602


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[1/2] incubator-beam git commit: [BEAM-1149] Explode windows when fn uses side inputs

[kenn]

Repository: incubator-beam
Updated Branches:
  refs/heads/release-0.4.0-incubating 9f3b06359 -> b2780881a


[BEAM-1149] Explode windows when fn uses side inputs


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/8d89bfce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/8d89bfce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/8d89bfce

Branch: refs/heads/release-0.4.0-incubating
Commit: 8d89bfce251d27c7fc6ff522a2cfeb676ec2c633
Parents: 9f3b063
Author: Eugene Kirpichov 
Authored: Tue Dec 13 14:35:33 2016 -0800
Committer: Kenneth Knowles 
Committed: Tue Dec 13 15:17:18 2016 -0800

--
 .../core/PushbackSideInputDoFnRunner.java   | 23 +++---
 .../core/PushbackSideInputDoFnRunnerTest.java   | 16 +++
 .../apache/beam/sdk/transforms/ParDoTest.java   | 45 
 3 files changed, 60 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/8d89bfce/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java
--
diff --git 
a/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java
 
b/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java
index 460154d..0bb9153 100644
--- 
a/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java
+++ 
b/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java
@@ -71,32 +71,23 @@ public class PushbackSideInputDoFnRunner 
implements DoFnRunner<
*/
   public Iterable 
processElementInReadyWindows(WindowedValue elem) {
 if (views.isEmpty()) {
+  // When there are no side inputs, we can preserve the compressed 
representation.
   processElement(elem);
   return Collections.emptyList();
 }
-ImmutableList.Builder readyWindowsBuilder = 
ImmutableList.builder();
-ImmutableList.Builder pushedBackWindowsBuilder = 
ImmutableList.builder();
+ImmutableList.Builder pushedBack = 
ImmutableList.builder();
 for (WindowedValue windowElem : elem.explodeWindows()) {
   BoundedWindow mainInputWindow = 
Iterables.getOnlyElement(windowElem.getWindows());
   if (isReady(mainInputWindow)) {
-readyWindowsBuilder.add(mainInputWindow);
+// When there are any side inputs, we have to process the element in 
each window
+// individually, to disambiguate access to per-window side inputs.
+processElement(windowElem);
   } else {
 notReadyWindows.add(mainInputWindow);
-pushedBackWindowsBuilder.add(mainInputWindow);
+pushedBack.add(windowElem);
   }
 }
-ImmutableList readyWindows = readyWindowsBuilder.build();
-ImmutableList pushedBackWindows = 
pushedBackWindowsBuilder.build();
-if (!readyWindows.isEmpty()) {
-  processElement(
-  WindowedValue.of(
-  elem.getValue(), elem.getTimestamp(), readyWindows, 
elem.getPane()));
-}
-return pushedBackWindows.isEmpty()
-? ImmutableList.of()
-: ImmutableList.of(
-WindowedValue.of(
-elem.getValue(), elem.getTimestamp(), pushedBackWindows, 
elem.getPane()));
+return pushedBack.build();
   }
 
   private boolean isReady(BoundedWindow mainInputWindow) {

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/8d89bfce/runners/core-java/src/test/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunnerTest.java
--
diff --git 
a/runners/core-java/src/test/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunnerTest.java
 
b/runners/core-java/src/test/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunnerTest.java
index f8f4604..176ab26 100644
--- 
a/runners/core-java/src/test/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunnerTest.java
+++ 
b/runners/core-java/src/test/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunnerTest.java
@@ -17,9 +17,9 @@
  */
 package org.apache.beam.runners.core;
 
-import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertThat;
 import static org.mockito.Mockito.when;
@@ -130,7 +130,7 @@ public class PushbackSideInputDoFnRunnerTest {
 PaneInfo.ON_TIME_AND_ONLY_FIRING);
 Iterable multiWindowPushback =
 

[GitHub] incubator-beam pull request #1602: [BEAM-1149] Cherry pick side inputs fix t...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/incubator-beam/pull/1602


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[2/2] incubator-beam git commit: This closes #1602

[kenn]

This closes #1602


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/b2780881
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/b2780881
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/b2780881

Branch: refs/heads/release-0.4.0-incubating
Commit: b2780881a6d735420a2d29c3270b781ea8a3b74f
Parents: 9f3b063 8d89bfc
Author: Kenneth Knowles 
Authored: Tue Dec 13 15:47:41 2016 -0800
Committer: Kenneth Knowles 
Committed: Tue Dec 13 15:47:41 2016 -0800

--
 .../core/PushbackSideInputDoFnRunner.java   | 23 +++---
 .../core/PushbackSideInputDoFnRunnerTest.java   | 16 +++
 .../apache/beam/sdk/transforms/ParDoTest.java   | 45 
 3 files changed, 60 insertions(+), 24 deletions(-)
--

[GitHub] incubator-beam pull request #1604: Python Precommit Maven Install

[markflyhigh]

GitHub user markflyhigh opened a pull request:

https://github.com/apache/incubator-beam/pull/1604

Python Precommit Maven Install

Be sure to do all of the following to help us incorporate your contribution
quickly and easily:

 - [ ] Make sure the PR title is formatted like:
   `[BEAM-] Description of pull request`
 - [ ] Make sure tests pass via `mvn clean verify`. (Even better, enable
   Travis-CI on your fork and ensure the whole test matrix passes).
 - [ ] Replace `` in the title with the actual Jira issue
   number, if there is one.
 - [ ] If this contribution is large, please file an Apache
   [Individual Contributor License 
Agreement](https://www.apache.org/licenses/icla.txt).

---

python project will be build in precommit if PR is created based on 
python-sdk branch.

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/markflyhigh/incubator-beam 
python-recommit-maven-install

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-beam/pull/1604.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1604


commit b1c9f05704391edd8a476b405022064ea211b388
Author: Mark Liu 
Date:   2016-12-13T22:38:59Z

Python Precommit Maven Install




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

Jenkins build became unstable: beam_PostCommit_Java_MavenInstall #2085

[Apache Jenkins Server]

See 

Jenkins build is still unstable: beam_PostCommit_Java_RunnableOnService_Dataflow #1806

[Apache Jenkins Server]

See 

[jira] [Commented] (BEAM-1149) Side input access fails in direct runner (possibly others too) when input element in multiple windows

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-1149?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15746557#comment-15746557
 ] 

ASF GitHub Bot commented on BEAM-1149:
--

Github user asfgit closed the pull request at:

https://github.com/apache/incubator-beam/pull/1601


> Side input access fails in direct runner (possibly others too) when input 
> element in multiple windows
> -
>
> Key: BEAM-1149
> URL: https://issues.apache.org/jira/browse/BEAM-1149
> Project: Beam
>  Issue Type: Bug
>Reporter: Eugene Kirpichov
>Assignee: Eugene Kirpichov
>Priority: Blocker
>
> {code:java}
>   private static class FnWithSideInputs extends DoFn {
> private final PCollectionView view;
> private FnWithSideInputs(PCollectionView view) {
>   this.view = view;
> }
> @ProcessElement
> public void processElement(ProcessContext c) {
>   c.output(c.element() + ":" + c.sideInput(view));
> }
>   }
>   @Test
>   public void testSideInputsWithMultipleWindows() {
> Pipeline p = TestPipeline.create();
> MutableDateTime mutableNow = Instant.now().toMutableDateTime();
> mutableNow.setMillisOfSecond(0);
> Instant now = mutableNow.toInstant();
> SlidingWindows windowFn =
> 
> SlidingWindows.of(Duration.standardSeconds(5)).every(Duration.standardSeconds(1));
> PCollectionView view = 
> p.apply(Create.of(1)).apply(View.asSingleton());
> PCollection res =
> p.apply(Create.timestamped(TimestampedValue.of("a", now)))
> .apply(Window.into(windowFn))
> .apply(ParDo.of(new FnWithSideInputs(view)).withSideInputs(view));
> PAssert.that(res).containsInAnyOrder("a:1");
> p.run();
>   }
> {code}
> This fails with the following exception:
> {code}
> org.apache.beam.sdk.Pipeline$PipelineExecutionException: 
> java.lang.IllegalStateException: sideInput called when main input element is 
> in multiple windows
>   at 
> org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:343)
>   at org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:1)
>   at org.apache.beam.sdk.Pipeline.run(Pipeline.java:176)
>   at org.apache.beam.sdk.testing.TestPipeline.run(TestPipeline.java:112)
>   at 
> Caused by: java.lang.IllegalStateException: sideInput called when main input 
> element is in multiple windows
>   at 
> org.apache.beam.runners.core.SimpleDoFnRunner$DoFnProcessContext.sideInput(SimpleDoFnRunner.java:514)
>   at 
> org.apache.beam.sdk.transforms.ParDoTest$FnWithSideInputs.processElement(ParDoTest.java:738)
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[1/2] incubator-beam git commit: This closes #1601

[kenn]

Repository: incubator-beam
Updated Branches:
  refs/heads/master dad5ba5a6 -> 0bdf7fc04


This closes #1601


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/0bdf7fc0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/0bdf7fc0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/0bdf7fc0

Branch: refs/heads/master
Commit: 0bdf7fc04546e6a6e88ba52110b4b79be5370a22
Parents: dad5ba5 a90c428
Author: Kenneth Knowles 
Authored: Tue Dec 13 15:06:46 2016 -0800
Committer: Kenneth Knowles 
Committed: Tue Dec 13 15:06:46 2016 -0800

--
 .../core/PushbackSideInputDoFnRunner.java   | 23 +++---
 .../core/PushbackSideInputDoFnRunnerTest.java   | 16 +++
 .../apache/beam/sdk/transforms/ParDoTest.java   | 45 
 3 files changed, 60 insertions(+), 24 deletions(-)
--

[2/2] incubator-beam git commit: [BEAM-1149] Explode windows when fn uses side inputs

[kenn]

[BEAM-1149] Explode windows when fn uses side inputs


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a90c4285
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a90c4285
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a90c4285

Branch: refs/heads/master
Commit: a90c4285053821d0015f56be52d81bd18994e405
Parents: dad5ba5
Author: Eugene Kirpichov 
Authored: Tue Dec 13 14:35:33 2016 -0800
Committer: Kenneth Knowles 
Committed: Tue Dec 13 15:06:46 2016 -0800

--
 .../core/PushbackSideInputDoFnRunner.java   | 23 +++---
 .../core/PushbackSideInputDoFnRunnerTest.java   | 16 +++
 .../apache/beam/sdk/transforms/ParDoTest.java   | 45 
 3 files changed, 60 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a90c4285/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java
--
diff --git 
a/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java
 
b/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java
index 460154d..0bb9153 100644
--- 
a/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java
+++ 
b/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java
@@ -71,32 +71,23 @@ public class PushbackSideInputDoFnRunner 
implements DoFnRunner<
*/
   public Iterable 
processElementInReadyWindows(WindowedValue elem) {
 if (views.isEmpty()) {
+  // When there are no side inputs, we can preserve the compressed 
representation.
   processElement(elem);
   return Collections.emptyList();
 }
-ImmutableList.Builder readyWindowsBuilder = 
ImmutableList.builder();
-ImmutableList.Builder pushedBackWindowsBuilder = 
ImmutableList.builder();
+ImmutableList.Builder pushedBack = 
ImmutableList.builder();
 for (WindowedValue windowElem : elem.explodeWindows()) {
   BoundedWindow mainInputWindow = 
Iterables.getOnlyElement(windowElem.getWindows());
   if (isReady(mainInputWindow)) {
-readyWindowsBuilder.add(mainInputWindow);
+// When there are any side inputs, we have to process the element in 
each window
+// individually, to disambiguate access to per-window side inputs.
+processElement(windowElem);
   } else {
 notReadyWindows.add(mainInputWindow);
-pushedBackWindowsBuilder.add(mainInputWindow);
+pushedBack.add(windowElem);
   }
 }
-ImmutableList readyWindows = readyWindowsBuilder.build();
-ImmutableList pushedBackWindows = 
pushedBackWindowsBuilder.build();
-if (!readyWindows.isEmpty()) {
-  processElement(
-  WindowedValue.of(
-  elem.getValue(), elem.getTimestamp(), readyWindows, 
elem.getPane()));
-}
-return pushedBackWindows.isEmpty()
-? ImmutableList.of()
-: ImmutableList.of(
-WindowedValue.of(
-elem.getValue(), elem.getTimestamp(), pushedBackWindows, 
elem.getPane()));
+return pushedBack.build();
   }
 
   private boolean isReady(BoundedWindow mainInputWindow) {

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a90c4285/runners/core-java/src/test/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunnerTest.java
--
diff --git 
a/runners/core-java/src/test/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunnerTest.java
 
b/runners/core-java/src/test/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunnerTest.java
index f8f4604..176ab26 100644
--- 
a/runners/core-java/src/test/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunnerTest.java
+++ 
b/runners/core-java/src/test/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunnerTest.java
@@ -17,9 +17,9 @@
  */
 package org.apache.beam.runners.core;
 
-import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertThat;
 import static org.mockito.Mockito.when;
@@ -130,7 +130,7 @@ public class PushbackSideInputDoFnRunnerTest {
 PaneInfo.ON_TIME_AND_ONLY_FIRING);
 Iterable multiWindowPushback =
 runner.processElementInReadyWindows(multiWindow);
-assertThat(multiWindowPushback, contains(multiWindow));
+assertThat(multiWindowPushback, 

[1/2] incubator-beam git commit: Update version from 0.5.0-SNAPSHOT to 0.5.0-incubating-SNAPSHOT

[davor]

Repository: incubator-beam
Updated Branches:
  refs/heads/master cf2c1d685 -> dad5ba5a6


Update version from 0.5.0-SNAPSHOT to 0.5.0-incubating-SNAPSHOT


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/25215889
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/25215889
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/25215889

Branch: refs/heads/master
Commit: 25215889381f7da61766054af68c84ffed4c0c71
Parents: cf2c1d6
Author: Davor Bonaci 
Authored: Tue Dec 13 14:27:55 2016 -0800
Committer: Davor Bonaci 
Committed: Tue Dec 13 14:27:55 2016 -0800

--
 examples/java/pom.xml   | 2 +-
 examples/java8/pom.xml  | 2 +-
 examples/pom.xml| 2 +-
 pom.xml | 2 +-
 runners/apex/pom.xml| 2 +-
 runners/core-java/pom.xml   | 2 +-
 runners/direct-java/pom.xml | 2 +-
 runners/flink/examples/pom.xml  | 2 +-
 runners/flink/pom.xml   | 2 +-
 runners/flink/runner/pom.xml| 2 +-
 runners/google-cloud-dataflow-java/pom.xml  | 2 +-
 runners/pom.xml | 2 +-
 runners/spark/pom.xml   | 2 +-
 sdks/java/build-tools/pom.xml   | 2 +-
 sdks/java/core/pom.xml  | 2 +-
 sdks/java/extensions/join-library/pom.xml   | 2 +-
 sdks/java/extensions/pom.xml| 2 +-
 sdks/java/extensions/sorter/pom.xml | 2 +-
 sdks/java/io/google-cloud-platform/pom.xml  | 2 +-
 sdks/java/io/hdfs/pom.xml   | 2 +-
 sdks/java/io/jdbc/pom.xml   | 2 +-
 sdks/java/io/jms/pom.xml| 2 +-
 sdks/java/io/kafka/pom.xml  | 2 +-
 sdks/java/io/kinesis/pom.xml| 2 +-
 sdks/java/io/mongodb/pom.xml| 2 +-
 sdks/java/io/pom.xml| 2 +-
 sdks/java/java8tests/pom.xml| 2 +-
 sdks/java/maven-archetypes/examples-java8/pom.xml   | 2 +-
 .../src/main/resources/archetype-resources/pom.xml  | 2 +-
 sdks/java/maven-archetypes/examples/pom.xml | 2 +-
 .../examples/src/main/resources/archetype-resources/pom.xml | 2 +-
 sdks/java/maven-archetypes/pom.xml  | 2 +-
 sdks/java/maven-archetypes/starter/pom.xml  | 2 +-
 .../starter/src/main/resources/archetype-resources/pom.xml  | 5 ++---
 .../starter/src/test/resources/projects/basic/reference/pom.xml | 2 +-
 sdks/java/pom.xml   | 2 +-
 sdks/pom.xml| 2 +-
 37 files changed, 38 insertions(+), 39 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/25215889/examples/java/pom.xml
--
diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index dc86532..f1e0fe1 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.beam
 beam-examples-parent
-0.5.0-SNAPSHOT
+0.5.0-incubating-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/25215889/examples/java8/pom.xml
--
diff --git a/examples/java8/pom.xml b/examples/java8/pom.xml
index 370d79f..1abf6fd 100644
--- a/examples/java8/pom.xml
+++ b/examples/java8/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.beam
 beam-examples-parent
-0.5.0-SNAPSHOT
+0.5.0-incubating-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/25215889/examples/pom.xml
--
diff --git a/examples/pom.xml b/examples/pom.xml
index adfbaa9..3e0fbdd 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.beam
 beam-parent
-0.5.0-SNAPSHOT
+0.5.0-incubating-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/25215889/pom.xml

[GitHub] incubator-beam pull request #1600: Update version from 0.5.0-SNAPSHOT to 0.5...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/incubator-beam/pull/1600


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[2/2] incubator-beam git commit: This closes #1600

[davor]

This closes #1600


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/dad5ba5a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/dad5ba5a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/dad5ba5a

Branch: refs/heads/master
Commit: dad5ba5a66b971c272b25b8da368df8593196edb
Parents: cf2c1d6 2521588
Author: Davor Bonaci 
Authored: Tue Dec 13 14:44:59 2016 -0800
Committer: Davor Bonaci 
Committed: Tue Dec 13 14:44:59 2016 -0800

--
 examples/java/pom.xml   | 2 +-
 examples/java8/pom.xml  | 2 +-
 examples/pom.xml| 2 +-
 pom.xml | 2 +-
 runners/apex/pom.xml| 2 +-
 runners/core-java/pom.xml   | 2 +-
 runners/direct-java/pom.xml | 2 +-
 runners/flink/examples/pom.xml  | 2 +-
 runners/flink/pom.xml   | 2 +-
 runners/flink/runner/pom.xml| 2 +-
 runners/google-cloud-dataflow-java/pom.xml  | 2 +-
 runners/pom.xml | 2 +-
 runners/spark/pom.xml   | 2 +-
 sdks/java/build-tools/pom.xml   | 2 +-
 sdks/java/core/pom.xml  | 2 +-
 sdks/java/extensions/join-library/pom.xml   | 2 +-
 sdks/java/extensions/pom.xml| 2 +-
 sdks/java/extensions/sorter/pom.xml | 2 +-
 sdks/java/io/google-cloud-platform/pom.xml  | 2 +-
 sdks/java/io/hdfs/pom.xml   | 2 +-
 sdks/java/io/jdbc/pom.xml   | 2 +-
 sdks/java/io/jms/pom.xml| 2 +-
 sdks/java/io/kafka/pom.xml  | 2 +-
 sdks/java/io/kinesis/pom.xml| 2 +-
 sdks/java/io/mongodb/pom.xml| 2 +-
 sdks/java/io/pom.xml| 2 +-
 sdks/java/java8tests/pom.xml| 2 +-
 sdks/java/maven-archetypes/examples-java8/pom.xml   | 2 +-
 .../src/main/resources/archetype-resources/pom.xml  | 2 +-
 sdks/java/maven-archetypes/examples/pom.xml | 2 +-
 .../examples/src/main/resources/archetype-resources/pom.xml | 2 +-
 sdks/java/maven-archetypes/pom.xml  | 2 +-
 sdks/java/maven-archetypes/starter/pom.xml  | 2 +-
 .../starter/src/main/resources/archetype-resources/pom.xml  | 5 ++---
 .../starter/src/test/resources/projects/basic/reference/pom.xml | 2 +-
 sdks/java/pom.xml   | 2 +-
 sdks/pom.xml| 2 +-
 37 files changed, 38 insertions(+), 39 deletions(-)
--

[jira] [Commented] (BEAM-1149) Side input access fails in direct runner (possibly others too) when input element in multiple windows

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-1149?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15746497#comment-15746497
 ] 

ASF GitHub Bot commented on BEAM-1149:
--

GitHub user jkff opened a pull request:

https://github.com/apache/incubator-beam/pull/1601

[BEAM-1149] Explode windows when fn uses side inputs

This basically reverts 
https://github.com/apache/incubator-beam/commit/38f0b11cc9028cf347e3c96b6e6116e5a5a9972d
 and adds some comments and a test.

R: @kennknowles 

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/jkff/incubator-beam multi-window-side-inputs

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-beam/pull/1601.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1601


commit e2644e3a869c40bbb3614198dfed8a4b2ab35ab8
Author: Eugene Kirpichov 
Date:   2016-12-13T22:35:33Z

[BEAM-1149] Explode windows when fn uses side inputs




> Side input access fails in direct runner (possibly others too) when input 
> element in multiple windows
> -
>
> Key: BEAM-1149
> URL: https://issues.apache.org/jira/browse/BEAM-1149
> Project: Beam
>  Issue Type: Bug
>Reporter: Eugene Kirpichov
>Assignee: Eugene Kirpichov
>Priority: Blocker
>
> {code:java}
>   private static class FnWithSideInputs extends DoFn {
> private final PCollectionView view;
> private FnWithSideInputs(PCollectionView view) {
>   this.view = view;
> }
> @ProcessElement
> public void processElement(ProcessContext c) {
>   c.output(c.element() + ":" + c.sideInput(view));
> }
>   }
>   @Test
>   public void testSideInputsWithMultipleWindows() {
> Pipeline p = TestPipeline.create();
> MutableDateTime mutableNow = Instant.now().toMutableDateTime();
> mutableNow.setMillisOfSecond(0);
> Instant now = mutableNow.toInstant();
> SlidingWindows windowFn =
> 
> SlidingWindows.of(Duration.standardSeconds(5)).every(Duration.standardSeconds(1));
> PCollectionView view = 
> p.apply(Create.of(1)).apply(View.asSingleton());
> PCollection res =
> p.apply(Create.timestamped(TimestampedValue.of("a", now)))
> .apply(Window.into(windowFn))
> .apply(ParDo.of(new FnWithSideInputs(view)).withSideInputs(view));
> PAssert.that(res).containsInAnyOrder("a:1");
> p.run();
>   }
> {code}
> This fails with the following exception:
> {code}
> org.apache.beam.sdk.Pipeline$PipelineExecutionException: 
> java.lang.IllegalStateException: sideInput called when main input element is 
> in multiple windows
>   at 
> org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:343)
>   at org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:1)
>   at org.apache.beam.sdk.Pipeline.run(Pipeline.java:176)
>   at org.apache.beam.sdk.testing.TestPipeline.run(TestPipeline.java:112)
>   at 
> Caused by: java.lang.IllegalStateException: sideInput called when main input 
> element is in multiple windows
>   at 
> org.apache.beam.runners.core.SimpleDoFnRunner$DoFnProcessContext.sideInput(SimpleDoFnRunner.java:514)
>   at 
> org.apache.beam.sdk.transforms.ParDoTest$FnWithSideInputs.processElement(ParDoTest.java:738)
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[GitHub] incubator-beam pull request #1601: [BEAM-1149] Explode windows when fn uses ...

[jkff]

GitHub user jkff opened a pull request:

https://github.com/apache/incubator-beam/pull/1601

[BEAM-1149] Explode windows when fn uses side inputs

This basically reverts 
https://github.com/apache/incubator-beam/commit/38f0b11cc9028cf347e3c96b6e6116e5a5a9972d
 and adds some comments and a test.

R: @kennknowles 

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/jkff/incubator-beam multi-window-side-inputs

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-beam/pull/1601.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1601


commit e2644e3a869c40bbb3614198dfed8a4b2ab35ab8
Author: Eugene Kirpichov 
Date:   2016-12-13T22:35:33Z

[BEAM-1149] Explode windows when fn uses side inputs




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-beam pull request #1600: Update version from 0.5.0-SNAPSHOT to 0.5...

[davorbonaci]

GitHub user davorbonaci opened a pull request:

https://github.com/apache/incubator-beam/pull/1600

Update version from 0.5.0-SNAPSHOT to 0.5.0-incubating-SNAPSHOT

R: @dhalperi 

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/davorbonaci/beam incubating

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-beam/pull/1600.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1600


commit 25215889381f7da61766054af68c84ffed4c0c71
Author: Davor Bonaci 
Date:   2016-12-13T22:27:55Z

Update version from 0.5.0-SNAPSHOT to 0.5.0-incubating-SNAPSHOT




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[1/2] incubator-beam git commit: Update Maven Archetype versions after cutting the release branch

[davor]

Repository: incubator-beam
Updated Branches:
  refs/heads/master a52600b68 -> cf2c1d685


Update Maven Archetype versions after cutting the release branch


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/5d6a50c6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/5d6a50c6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/5d6a50c6

Branch: refs/heads/master
Commit: 5d6a50c63360d8e706c095b73b55734390615135
Parents: a52600b
Author: Davor Bonaci 
Authored: Tue Dec 13 13:43:58 2016 -0800
Committer: Davor Bonaci 
Committed: Tue Dec 13 14:20:51 2016 -0800

--
 .../src/main/resources/archetype-resources/pom.xml | 6 +++---
 .../examples/src/main/resources/archetype-resources/pom.xml| 2 +-
 .../starter/src/main/resources/archetype-resources/pom.xml | 2 +-
 .../src/test/resources/projects/basic/reference/pom.xml| 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5d6a50c6/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
--
diff --git 
a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
 
b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
index 693850a..a440424 100644
--- 
a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
+++ 
b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
@@ -27,7 +27,7 @@
   jar
 
   
-0.4.0-incubating-SNAPSHOT
+0.5.0-SNAPSHOT
   
 
   
@@ -177,14 +177,14 @@
 
   org.apache.beam
   beam-sdks-java-core
-  0.4.0-incubating-SNAPSHOT
+  ${beam.version}
 
 
 
 
   org.apache.beam
   beam-sdks-java-io-google-cloud-platform
-  0.4.0-incubating-SNAPSHOT
+  ${beam.version}
 
 
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5d6a50c6/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
--
diff --git 
a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
 
b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
index 9bbc646..b6eab15 100644
--- 
a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
+++ 
b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
@@ -27,7 +27,7 @@
   jar
 
   
-0.4.0-incubating-SNAPSHOT
+0.5.0-SNAPSHOT
   
 
   

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5d6a50c6/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
--
diff --git 
a/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
 
b/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
index efafeca..4a6e89d 100644
--- 
a/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
+++ 
b/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
@@ -25,7 +25,7 @@
   ${version}
 
   
-0.4.0-incubating-SNAPSHOT
+0.5.0-SNAPSHOT
   
 
   

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5d6a50c6/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
--
diff --git 
a/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
 
b/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
index a86bd11..97f9e72 100644
--- 
a/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
+++ 
b/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
@@ -25,7 +25,7 @@
   0.1
 
   
-0.4.0-incubating-SNAPSHOT
+0.5.0-SNAPSHOT
   
 
   

[GitHub] incubator-beam pull request #1599: Update Maven Archetype versions after cut...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/incubator-beam/pull/1599


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[2/2] incubator-beam git commit: This closes #1599

[davor]

This closes #1599


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/cf2c1d68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/cf2c1d68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/cf2c1d68

Branch: refs/heads/master
Commit: cf2c1d685035d7da00efb1a72cd06caa006cccd7
Parents: a52600b 5d6a50c
Author: Davor Bonaci 
Authored: Tue Dec 13 14:20:55 2016 -0800
Committer: Davor Bonaci 
Committed: Tue Dec 13 14:20:55 2016 -0800

--
 .../src/main/resources/archetype-resources/pom.xml | 6 +++---
 .../examples/src/main/resources/archetype-resources/pom.xml| 2 +-
 .../starter/src/main/resources/archetype-resources/pom.xml | 2 +-
 .../src/test/resources/projects/basic/reference/pom.xml| 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)
--

incubator-beam git commit: Update Maven Archetype version; fixup for Java8.

[davor]

Repository: incubator-beam
Updated Branches:
  refs/heads/release-0.4.0-incubating 2c1d4eb4d -> 9f3b06359


Update Maven Archetype version; fixup for Java8.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/9f3b0635
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/9f3b0635
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/9f3b0635

Branch: refs/heads/release-0.4.0-incubating
Commit: 9f3b0635971a45f8baf8fa68ba5ec2f8c47cec6b
Parents: 2c1d4eb
Author: Davor Bonaci 
Authored: Tue Dec 13 14:18:32 2016 -0800
Committer: Davor Bonaci 
Committed: Tue Dec 13 14:18:32 2016 -0800

--
 .../src/main/resources/archetype-resources/pom.xml   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9f3b0635/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
--
diff --git 
a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
 
b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
index 2771d5d..c9645d3 100644
--- 
a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
+++ 
b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
@@ -177,14 +177,14 @@
 
   org.apache.beam
   beam-sdks-java-core
-  0.4.0-incubating-SNAPSHOT
+  ${beam.version}
 
 
 
 
   org.apache.beam
   beam-sdks-java-io-google-cloud-platform
-  0.4.0-incubating-SNAPSHOT
+  ${beam.version}
 
 
 

[jira] [Updated] (BEAM-1149) Side input access fails in direct runner (possibly others too) when input element in multiple windows

[Kenneth Knowles (JIRA)]

 [ 
https://issues.apache.org/jira/browse/BEAM-1149?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Kenneth Knowles updated BEAM-1149:
--
Assignee: Eugene Kirpichov

> Side input access fails in direct runner (possibly others too) when input 
> element in multiple windows
> -
>
> Key: BEAM-1149
> URL: https://issues.apache.org/jira/browse/BEAM-1149
> Project: Beam
>  Issue Type: Bug
>Reporter: Eugene Kirpichov
>Assignee: Eugene Kirpichov
>Priority: Blocker
>
> {code:java}
>   private static class FnWithSideInputs extends DoFn {
> private final PCollectionView view;
> private FnWithSideInputs(PCollectionView view) {
>   this.view = view;
> }
> @ProcessElement
> public void processElement(ProcessContext c) {
>   c.output(c.element() + ":" + c.sideInput(view));
> }
>   }
>   @Test
>   public void testSideInputsWithMultipleWindows() {
> Pipeline p = TestPipeline.create();
> MutableDateTime mutableNow = Instant.now().toMutableDateTime();
> mutableNow.setMillisOfSecond(0);
> Instant now = mutableNow.toInstant();
> SlidingWindows windowFn =
> 
> SlidingWindows.of(Duration.standardSeconds(5)).every(Duration.standardSeconds(1));
> PCollectionView view = 
> p.apply(Create.of(1)).apply(View.asSingleton());
> PCollection res =
> p.apply(Create.timestamped(TimestampedValue.of("a", now)))
> .apply(Window.into(windowFn))
> .apply(ParDo.of(new FnWithSideInputs(view)).withSideInputs(view));
> PAssert.that(res).containsInAnyOrder("a:1");
> p.run();
>   }
> {code}
> This fails with the following exception:
> {code}
> org.apache.beam.sdk.Pipeline$PipelineExecutionException: 
> java.lang.IllegalStateException: sideInput called when main input element is 
> in multiple windows
>   at 
> org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:343)
>   at org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:1)
>   at org.apache.beam.sdk.Pipeline.run(Pipeline.java:176)
>   at org.apache.beam.sdk.testing.TestPipeline.run(TestPipeline.java:112)
>   at 
> Caused by: java.lang.IllegalStateException: sideInput called when main input 
> element is in multiple windows
>   at 
> org.apache.beam.runners.core.SimpleDoFnRunner$DoFnProcessContext.sideInput(SimpleDoFnRunner.java:514)
>   at 
> org.apache.beam.sdk.transforms.ParDoTest$FnWithSideInputs.processElement(ParDoTest.java:738)
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Commented] (BEAM-1149) Side input access fails in direct runner (possibly others too) when input element in multiple windows

[Eugene Kirpichov (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-1149?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15746424#comment-15746424
 ] 

Eugene Kirpichov commented on BEAM-1149:


Test works with 
https://github.com/apache/incubator-beam/commit/38f0b11cc9028cf347e3c96b6e6116e5a5a9972d
 reverted. Though that, in turn, might break other things. Investigating.

> Side input access fails in direct runner (possibly others too) when input 
> element in multiple windows
> -
>
> Key: BEAM-1149
> URL: https://issues.apache.org/jira/browse/BEAM-1149
> Project: Beam
>  Issue Type: Bug
>Reporter: Eugene Kirpichov
>Priority: Blocker
>
> {code:java}
>   private static class FnWithSideInputs extends DoFn {
> private final PCollectionView view;
> private FnWithSideInputs(PCollectionView view) {
>   this.view = view;
> }
> @ProcessElement
> public void processElement(ProcessContext c) {
>   c.output(c.element() + ":" + c.sideInput(view));
> }
>   }
>   @Test
>   public void testSideInputsWithMultipleWindows() {
> Pipeline p = TestPipeline.create();
> MutableDateTime mutableNow = Instant.now().toMutableDateTime();
> mutableNow.setMillisOfSecond(0);
> Instant now = mutableNow.toInstant();
> SlidingWindows windowFn =
> 
> SlidingWindows.of(Duration.standardSeconds(5)).every(Duration.standardSeconds(1));
> PCollectionView view = 
> p.apply(Create.of(1)).apply(View.asSingleton());
> PCollection res =
> p.apply(Create.timestamped(TimestampedValue.of("a", now)))
> .apply(Window.into(windowFn))
> .apply(ParDo.of(new FnWithSideInputs(view)).withSideInputs(view));
> PAssert.that(res).containsInAnyOrder("a:1");
> p.run();
>   }
> {code}
> This fails with the following exception:
> {code}
> org.apache.beam.sdk.Pipeline$PipelineExecutionException: 
> java.lang.IllegalStateException: sideInput called when main input element is 
> in multiple windows
>   at 
> org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:343)
>   at org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:1)
>   at org.apache.beam.sdk.Pipeline.run(Pipeline.java:176)
>   at org.apache.beam.sdk.testing.TestPipeline.run(TestPipeline.java:112)
>   at 
> Caused by: java.lang.IllegalStateException: sideInput called when main input 
> element is in multiple windows
>   at 
> org.apache.beam.runners.core.SimpleDoFnRunner$DoFnProcessContext.sideInput(SimpleDoFnRunner.java:514)
>   at 
> org.apache.beam.sdk.transforms.ParDoTest$FnWithSideInputs.processElement(ParDoTest.java:738)
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Closed] (BEAM-1150) Side inputs broken with sliding windows (or any multi-window WindowFn)

[Kenneth Knowles (JIRA)]

 [ 
https://issues.apache.org/jira/browse/BEAM-1150?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Kenneth Knowles closed BEAM-1150.
-

> Side inputs broken with sliding windows (or any multi-window WindowFn)
> --
>
> Key: BEAM-1150
> URL: https://issues.apache.org/jira/browse/BEAM-1150
> Project: Beam
>  Issue Type: Bug
>  Components: sdk-java-core
>Reporter: Kenneth Knowles
>Assignee: Eugene Kirpichov
>Priority: Blocker
> Fix For: Not applicable
>
>




--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Resolved] (BEAM-1150) Side inputs broken with sliding windows (or any multi-window WindowFn)

[Kenneth Knowles (JIRA)]

 [ 
https://issues.apache.org/jira/browse/BEAM-1150?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Kenneth Knowles resolved BEAM-1150.
---
   Resolution: Duplicate
Fix Version/s: Not applicable

> Side inputs broken with sliding windows (or any multi-window WindowFn)
> --
>
> Key: BEAM-1150
> URL: https://issues.apache.org/jira/browse/BEAM-1150
> Project: Beam
>  Issue Type: Bug
>  Components: sdk-java-core
>Reporter: Kenneth Knowles
>Assignee: Eugene Kirpichov
>Priority: Blocker
> Fix For: Not applicable
>
>




--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Created] (BEAM-1150) Side inputs broken with sliding windows (or any multi-window WindowFn)

[Kenneth Knowles (JIRA)]

Kenneth Knowles created BEAM-1150:
-

 Summary: Side inputs broken with sliding windows (or any 
multi-window WindowFn)
 Key: BEAM-1150
 URL: https://issues.apache.org/jira/browse/BEAM-1150
 Project: Beam
  Issue Type: Bug
  Components: sdk-java-core
Reporter: Kenneth Knowles
Assignee: Eugene Kirpichov
Priority: Blocker






--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[jira] [Created] (BEAM-1149) Side input access fails in direct runner (possibly others too) when input element in multiple windows

[Eugene Kirpichov (JIRA)]

Eugene Kirpichov created BEAM-1149:
--

 Summary: Side input access fails in direct runner (possibly others 
too) when input element in multiple windows
 Key: BEAM-1149
 URL: https://issues.apache.org/jira/browse/BEAM-1149
 Project: Beam
  Issue Type: Bug
Reporter: Eugene Kirpichov
Priority: Blocker


{code:java}
  private static class FnWithSideInputs extends DoFn {
private final PCollectionView view;

private FnWithSideInputs(PCollectionView view) {
  this.view = view;
}

@ProcessElement
public void processElement(ProcessContext c) {
  c.output(c.element() + ":" + c.sideInput(view));
}
  }

  @Test
  public void testSideInputsWithMultipleWindows() {
Pipeline p = TestPipeline.create();

MutableDateTime mutableNow = Instant.now().toMutableDateTime();
mutableNow.setMillisOfSecond(0);
Instant now = mutableNow.toInstant();

SlidingWindows windowFn =

SlidingWindows.of(Duration.standardSeconds(5)).every(Duration.standardSeconds(1));
PCollectionView view = 
p.apply(Create.of(1)).apply(View.asSingleton());
PCollection res =
p.apply(Create.timestamped(TimestampedValue.of("a", now)))
.apply(Window.into(windowFn))
.apply(ParDo.of(new FnWithSideInputs(view)).withSideInputs(view));

PAssert.that(res).containsInAnyOrder("a:1");

p.run();
  }
{code}

This fails with the following exception:
{code}

org.apache.beam.sdk.Pipeline$PipelineExecutionException: 
java.lang.IllegalStateException: sideInput called when main input element is in 
multiple windows

at 
org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:343)
at org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:1)
at org.apache.beam.sdk.Pipeline.run(Pipeline.java:176)
at org.apache.beam.sdk.testing.TestPipeline.run(TestPipeline.java:112)
at 
Caused by: java.lang.IllegalStateException: sideInput called when main input 
element is in multiple windows
at 
org.apache.beam.runners.core.SimpleDoFnRunner$DoFnProcessContext.sideInput(SimpleDoFnRunner.java:514)
at 
org.apache.beam.sdk.transforms.ParDoTest$FnWithSideInputs.processElement(ParDoTest.java:738)
{code}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

incubator-beam git commit: Update Maven Archetype versions to pin them to 0.4.0-incubating to be released

[davor]

Repository: incubator-beam
Updated Branches:
  refs/heads/release-0.4.0-incubating c8dd9989c -> 2c1d4eb4d


Update Maven Archetype versions to pin them to 0.4.0-incubating to be released


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/2c1d4eb4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/2c1d4eb4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/2c1d4eb4

Branch: refs/heads/release-0.4.0-incubating
Commit: 2c1d4eb4d878d1a8b8adf68b10eb8744cf524b5c
Parents: c8dd998
Author: Davor Bonaci 
Authored: Tue Dec 13 13:48:26 2016 -0800
Committer: Davor Bonaci 
Committed: Tue Dec 13 13:48:26 2016 -0800

--
 .../examples-java8/src/main/resources/archetype-resources/pom.xml  | 2 +-
 .../examples/src/main/resources/archetype-resources/pom.xml| 2 +-
 .../starter/src/main/resources/archetype-resources/pom.xml | 2 +-
 .../starter/src/test/resources/projects/basic/reference/pom.xml| 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2c1d4eb4/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
--
diff --git 
a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
 
b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
index 693850a..2771d5d 100644
--- 
a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
+++ 
b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
@@ -27,7 +27,7 @@
   jar
 
   
-0.4.0-incubating-SNAPSHOT
+0.4.0-incubating
   
 
   

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2c1d4eb4/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
--
diff --git 
a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
 
b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
index 9bbc646..62a2de5 100644
--- 
a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
+++ 
b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
@@ -27,7 +27,7 @@
   jar
 
   
-0.4.0-incubating-SNAPSHOT
+0.4.0-incubating
   
 
   

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2c1d4eb4/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
--
diff --git 
a/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
 
b/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
index efafeca..08139b2 100644
--- 
a/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
+++ 
b/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
@@ -25,7 +25,7 @@
   ${version}
 
   
-0.4.0-incubating-SNAPSHOT
+0.4.0-incubating
   
 
   

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2c1d4eb4/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
--
diff --git 
a/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
 
b/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
index a86bd11..2a668e8 100644
--- 
a/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
+++ 
b/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
@@ -25,7 +25,7 @@
   0.1
 
   
-0.4.0-incubating-SNAPSHOT
+0.4.0-incubating
   
 
   

[GitHub] incubator-beam pull request #1599: Update Maven Archetype versions after cut...

[davorbonaci]

GitHub user davorbonaci opened a pull request:

https://github.com/apache/incubator-beam/pull/1599

Update Maven Archetype versions after cutting the release branch

R: @dhalperi 

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/davorbonaci/beam arc-master

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-beam/pull/1599.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1599


commit 861d1c04cc48adee3d05c8b4e65f6f64da8fb487
Author: Davor Bonaci 
Date:   2016-12-13T21:43:58Z

Update Maven Archetype versions after cutting the release branch




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[jira] [Commented] (BEAM-498) Make DoFnWithContext the new DoFn

[ASF GitHub Bot (JIRA)]

[ 
https://issues.apache.org/jira/browse/BEAM-498?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15746345#comment-15746345
 ] 

ASF GitHub Bot commented on BEAM-498:
-

Github user asfgit closed the pull request at:

https://github.com/apache/incubator-beam/pull/1541


> Make DoFnWithContext the new DoFn
> -
>
> Key: BEAM-498
> URL: https://issues.apache.org/jira/browse/BEAM-498
> Project: Beam
>  Issue Type: New Feature
>  Components: sdk-java-core
>Reporter: Kenneth Knowles
>Assignee: Kenneth Knowles
>  Labels: backward-incompatible
>




--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

[2/3] incubator-beam git commit: Update Dataflow worker to beam-master-20161212

[kenn]

Update Dataflow worker to beam-master-20161212


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/d323d5d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/d323d5d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/d323d5d9

Branch: refs/heads/master
Commit: d323d5d9f0396450ac78cfc747a7bc80695cbdf7
Parents: 596f255
Author: Kenneth Knowles 
Authored: Mon Dec 12 13:48:24 2016 -0800
Committer: Kenneth Knowles 
Committed: Tue Dec 13 13:37:26 2016 -0800

--
 .../org/apache/beam/runners/dataflow/dataflow.properties | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d323d5d9/runners/google-cloud-dataflow-java/src/main/resources/org/apache/beam/runners/dataflow/dataflow.properties
--
diff --git 
a/runners/google-cloud-dataflow-java/src/main/resources/org/apache/beam/runners/dataflow/dataflow.properties
 
b/runners/google-cloud-dataflow-java/src/main/resources/org/apache/beam/runners/dataflow/dataflow.properties
index 1eae8cb..d43881d 100644
--- 
a/runners/google-cloud-dataflow-java/src/main/resources/org/apache/beam/runners/dataflow/dataflow.properties
+++ 
b/runners/google-cloud-dataflow-java/src/main/resources/org/apache/beam/runners/dataflow/dataflow.properties
@@ -18,6 +18,6 @@
 
 environment.major.version=6
 
-worker.image.batch=dataflow.gcr.io/v1beta3/beam-java-batch:beam-master-20161205
+worker.image.batch=dataflow.gcr.io/v1beta3/beam-java-batch:beam-master-20161212
 
-worker.image.streaming=dataflow.gcr.io/v1beta3/beam-java-streaming:beam-master-20161205
+worker.image.streaming=dataflow.gcr.io/v1beta3/beam-java-streaming:beam-master-20161212