date:20170609

[2/2] beam git commit: Visit a Transform Hierarchy in Topological Order

2017-06-09 Thread tgroh

Visit a Transform Hierarchy in Topological Order

This reverts commit 6ad6433ec0c02aec8656e9e3b27f6e0f974f8ece.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/261e7df2
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/261e7df2
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/261e7df2

Branch: refs/heads/master
Commit: 261e7df2b860fe82d9f401e2621b020fe2020fea
Parents: d2c4093
Author: Thomas Groh 
Authored: Tue Jun 6 16:15:19 2017 -0700
Committer: Thomas Groh 
Committed: Fri Jun 9 14:44:42 2017 -0700

--
 .../spark/translation/StorageLevelTest.java |   4 +-
 .../beam/sdk/runners/TransformHierarchy.java|  79 +++-
 .../sdk/runners/TransformHierarchyTest.java | 197 +++
 3 files changed, 274 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/261e7df2/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
--
diff --git 
a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
 
b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
index 8f2e681..8bd6dae 100644
--- 
a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
+++ 
b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
@@ -58,12 +58,12 @@ public class StorageLevelTest {
 
   @Test
   public void test() throws Exception {
-PCollection pCollection = pipeline.apply(Create.of("foo"));
+PCollection pCollection = pipeline.apply("CreateFoo", 
Create.of("foo"));
 
 // by default, the Spark runner doesn't cache the RDD if it accessed only 
one time.
 // So, to "force" the caching of the RDD, we have to call the RDD at least 
two time.
 // That's why we are using Count fn on the PCollection.
-pCollection.apply(Count.globally());
+pCollection.apply("CountAll", Count.globally());
 
 PCollection output = pCollection.apply(new 
StorageLevelPTransform());
 

http://git-wip-us.apache.org/repos/asf/beam/blob/261e7df2/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
index ee1ce7b..5e048eb 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
@@ -208,7 +208,7 @@ public class TransformHierarchy {
   public Set visit(PipelineVisitor visitor) {
 finishSpecifying();
 Set visitedValues = new HashSet<>();
-root.visit(visitor, visitedValues);
+root.visit(visitor, visitedValues, new HashSet(), new 
HashSet());
 return visitedValues;
   }
 
@@ -503,10 +503,60 @@ public class TransformHierarchy {
 /**
  * Visit the transform node.
  *
+ * The visit proceeds in the following order:
+ *
+ * 
+ *   Visit all input {@link PValue PValues} returned by the flattened 
expansion of {@link
+ *   Node#getInputs()}.
+ *   If the node is a composite:
+ *   
+ * Enter the node via {@link 
PipelineVisitor#enterCompositeTransform(Node)}.
+ * If the result of {@link 
PipelineVisitor#enterCompositeTransform(Node)} was {@link
+ * CompositeBehavior#ENTER_TRANSFORM}, visit each child node 
of this {@link Node}.
+ * Leave the node via {@link 
PipelineVisitor#leaveCompositeTransform(Node)}.
+ *   
+ *   If the node is a primitive, visit it via {@link
+ *   PipelineVisitor#visitPrimitiveTransform(Node)}.
+ *   Visit each {@link PValue} that was output by this node.
+ * 
+ *
+ * Additionally, the following ordering restrictions are observed:
+ *
+ * 
+ *   A {@link Node} will be visited after its enclosing node has been 
entered and before its
+ *   enclosing node has been left
+ *   A {@link Node} will not be visited if any enclosing {@link Node} 
has returned {@link
+ *   CompositeBehavior#DO_NOT_ENTER_TRANSFORM} from the call to {@link
+ *   PipelineVisitor#enterCompositeTransform(Node)}.
+ *   A {@link PValue} will only be visited after the {@link Node} that 
originally produced
+ *   it has been visited.
+ * 
+ *
  * Provides an ordered visit of the input values, the primitive 
transform (or child nodes for
  * composite transforms), then the output values.
  */
-

[1/2] beam git commit: This closes #3329

2017-06-09 Thread tgroh

Repository: beam
Updated Branches:
  refs/heads/master d2c4093ed -> 33883ed88


This closes #3329


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/33883ed8
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/33883ed8
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/33883ed8

Branch: refs/heads/master
Commit: 33883ed88dfbd122c5a76d98833214044accc251
Parents: d2c4093 261e7df
Author: Thomas Groh 
Authored: Fri Jun 9 14:44:42 2017 -0700
Committer: Thomas Groh 
Committed: Fri Jun 9 14:44:42 2017 -0700

--
 .../spark/translation/StorageLevelTest.java |   4 +-
 .../beam/sdk/runners/TransformHierarchy.java|  79 +++-
 .../sdk/runners/TransformHierarchyTest.java | 197 +++
 3 files changed, 274 insertions(+), 6 deletions(-)
--

[GitHub] beam pull request #3329: Visit a Transform Hierarchy in Topological Order

2017-06-09 Thread asfgit

Github user asfgit closed the pull request at:

https://github.com/apache/beam/pull/3329


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[jira] [Commented] (BEAM-2371) Make Java DirectRunner demonstrate language-agnostic Runner API translation wrappers

2017-06-09 Thread ASF GitHub Bot (JIRA)


[ 
https://issues.apache.org/jira/browse/BEAM-2371?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16045043#comment-16045043
 ] 

ASF GitHub Bot commented on BEAM-2371:
--

GitHub user kennknowles opened a pull request:

https://github.com/apache/beam/pull/3338

[BEAM-2371] Port some DirectRunner overrides to SDK-agnostic APIs

Be sure to do all of the following to help us incorporate your contribution
quickly and easily:

 - [ ] Make sure the PR title is formatted like:
   `[BEAM-] Description of pull request`
 - [ ] Make sure tests pass via `mvn clean verify`.
 - [ ] Replace `` in the title with the actual Jira issue
   number, if there is one.
 - [ ] If this contribution is large, please file an Apache
   [Individual Contributor License 
Agreement](https://www.apache.org/licenses/icla.pdf).

---

@tgroh this is pulled off of #3330, the bits that I don't think are waiting 
for side inputs so I think it could go green right away. It should also be an 
easier review with fewer distractions.

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/kennknowles/beam DirectRunner-overrides-easy

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/beam/pull/3338.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #3338


commit e66724e69117961840ccc649a3d0ba723cc1c7dd
Author: Kenneth Knowles 
Date:   2017-05-26T21:26:55Z

Port ViewOverrideFactory to SDK-agnostic APIs

commit f77147a4642a7b3b7dfef0c00c752bdea07256fd
Author: Kenneth Knowles 
Date:   2017-05-26T21:27:23Z

Port DirectGroupByKey to SDK-agnostic APIs

commit eeffa9d69a9159b36733920cee9bc843d6b74307
Author: Kenneth Knowles 
Date:   2017-05-26T23:07:45Z

Port DirectRunner WriteFiles override to SDK-agnostic APIs

commit e35177b5f2ca9cbb6f7c08ee122b8d9fa15b6633
Author: Kenneth Knowles 
Date:   2017-06-08T20:39:32Z

Port DirectRunner TestStream override to SDK-agnostic APIs




> Make Java DirectRunner demonstrate language-agnostic Runner API translation 
> wrappers
> 
>
> Key: BEAM-2371
> URL: https://issues.apache.org/jira/browse/BEAM-2371
> Project: Beam
>  Issue Type: Improvement
>  Components: runner-direct
>Reporter: Kenneth Knowles
>Assignee: Kenneth Knowles
>
> This will complete the PoC for runners-core-construction-java and the Runner 
> API and show other runners the easy path to executing non-Java pipelines, 
> modulo Fn API.



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

[2/2] beam git commit: This closes #3337: Include Additional PTransform Inputs in Transform Nodes

2017-06-09 Thread kenn

This closes #3337: Include Additional PTransform Inputs in Transform Nodes

  Roll-forward Include Additional PTransform inputs in Transform Nodes


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b4c77167
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b4c77167
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b4c77167

Branch: refs/heads/master
Commit: b4c77167f20b8346d22ccff440489592e0de4b52
Parents: 33883ed 696f8b2
Author: Kenneth Knowles 
Authored: Fri Jun 9 15:04:02 2017 -0700
Committer: Kenneth Knowles 
Committed: Fri Jun 9 15:04:02 2017 -0700

--
 .../apex/translation/TranslationContext.java|   4 +-
 .../core/construction/TransformInputs.java  |  50 ++
 .../core/construction/TransformInputsTest.java  | 166 +++
 .../apache/beam/runners/direct/DirectGraph.java |  34 +++-
 .../beam/runners/direct/DirectGraphVisitor.java |  26 ++-
 .../direct/ExecutorServiceParallelExecutor.java |   2 +-
 .../runners/direct/ParDoEvaluatorFactory.java   |   9 +-
 ...littableProcessElementsEvaluatorFactory.java |   2 +
 .../direct/StatefulParDoEvaluatorFactory.java   |   1 +
 .../beam/runners/direct/WatermarkManager.java   |  14 +-
 .../runners/direct/DirectGraphVisitorTest.java  |  10 +-
 .../runners/direct/EvaluationContextTest.java   |   2 +-
 .../beam/runners/direct/ParDoEvaluatorTest.java |   6 +-
 .../flink/FlinkBatchTranslationContext.java |   3 +-
 .../flink/FlinkStreamingTranslationContext.java |   3 +-
 .../dataflow/DataflowPipelineTranslator.java|   5 +-
 .../spark/translation/EvaluationContext.java|   4 +-
 .../beam/sdk/runners/TransformHierarchy.java|  28 +++-
 18 files changed, 323 insertions(+), 46 deletions(-)
--

[1/2] beam git commit: Roll-forward Include Additional PTransform inputs in Transform Nodes

2017-06-09 Thread kenn

Repository: beam
Updated Branches:
  refs/heads/master 33883ed88 -> b4c77167f


Roll-forward Include Additional PTransform inputs in Transform Nodes

Update DirectGraph to have All and Non-Additional Inputs

This reverts commit 247f9bc1581984d026764b3d433cb594e700bc21


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/696f8b28
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/696f8b28
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/696f8b28

Branch: refs/heads/master
Commit: 696f8b28a3a17e7de81e2d46bb9774d57d6e265e
Parents: 33883ed
Author: Thomas Groh 
Authored: Tue Jun 6 17:00:09 2017 -0700
Committer: Thomas Groh 
Committed: Fri Jun 9 15:00:33 2017 -0700

--
 .../apex/translation/TranslationContext.java|   4 +-
 .../core/construction/TransformInputs.java  |  50 ++
 .../core/construction/TransformInputsTest.java  | 166 +++
 .../apache/beam/runners/direct/DirectGraph.java |  34 +++-
 .../beam/runners/direct/DirectGraphVisitor.java |  26 ++-
 .../direct/ExecutorServiceParallelExecutor.java |   2 +-
 .../runners/direct/ParDoEvaluatorFactory.java   |   9 +-
 ...littableProcessElementsEvaluatorFactory.java |   2 +
 .../direct/StatefulParDoEvaluatorFactory.java   |   1 +
 .../beam/runners/direct/WatermarkManager.java   |  14 +-
 .../runners/direct/DirectGraphVisitorTest.java  |  10 +-
 .../runners/direct/EvaluationContextTest.java   |   2 +-
 .../beam/runners/direct/ParDoEvaluatorTest.java |   6 +-
 .../flink/FlinkBatchTranslationContext.java |   3 +-
 .../flink/FlinkStreamingTranslationContext.java |   3 +-
 .../dataflow/DataflowPipelineTranslator.java|   5 +-
 .../spark/translation/EvaluationContext.java|   4 +-
 .../beam/sdk/runners/TransformHierarchy.java|  28 +++-
 18 files changed, 323 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
--
diff --git 
a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
 
b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
index aff3863..94d13e1 100644
--- 
a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
+++ 
b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
@@ -34,6 +34,7 @@ import 
org.apache.beam.runners.apex.translation.utils.ApexStateInternals;
 import 
org.apache.beam.runners.apex.translation.utils.ApexStateInternals.ApexStateBackend;
 import org.apache.beam.runners.apex.translation.utils.ApexStreamTuple;
 import org.apache.beam.runners.apex.translation.utils.CoderAdapterStreamCodec;
+import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
@@ -93,7 +94,8 @@ class TranslationContext {
   }
 
   public  InputT getInput() {
-return (InputT) 
Iterables.getOnlyElement(getCurrentTransform().getInputs().values());
+return (InputT)
+
Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform()));
   }
 
   public Map getOutputs() {

http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
--
diff --git 
a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
 
b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
new file mode 100644
index 000..2baf93a
--- /dev/null
+++ 
b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the

Jenkins build became unstable: beam_PostCommit_Java_ValidatesRunner_Spark #2324

2017-06-09 Thread Apache Jenkins Server

See

Build failed in Jenkins: beam_PostCommit_Java_MavenInstall_Windows #114

2017-06-09 Thread Apache Jenkins Server

See 


Changes:

[tgroh] Fix compile error occurs in some JDKs

[robertwb] Use beam.Map rather than beam.ParDo for PubSub encoding.

[tgroh] Visit a Transform Hierarchy in Topological Order

[tgroh] Roll-forward Include Additional PTransform inputs in Transform Nodes

--
[...truncated 2.64 MB...]
2017-06-10T00:34:31.278 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip/1.8/gossip-1.8.pom
2017-06-10T00:34:31.287 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip/1.8/gossip-1.8.pom
 (12 KB at 1250.7 KB/sec)
2017-06-10T00:34:31.291 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/sonatype/forge/forge-parent/9/forge-parent-9.pom
2017-06-10T00:34:31.305 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/sonatype/forge/forge-parent/9/forge-parent-9.pom
 (13 KB at 916.7 KB/sec)
2017-06-10T00:34:31.309 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip-core/1.8/gossip-core-1.8.pom
2017-06-10T00:34:31.317 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip-core/1.8/gossip-core-1.8.pom
 (3 KB at 271.6 KB/sec)
2017-06-10T00:34:31.320 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip-bootstrap/1.8/gossip-bootstrap-1.8.pom
2017-06-10T00:34:31.330 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip-bootstrap/1.8/gossip-bootstrap-1.8.pom
 (2 KB at 157.3 KB/sec)
2017-06-10T00:34:31.335 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/guava/guava/14.0.1/guava-14.0.1.pom
2017-06-10T00:34:31.342 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/guava/guava/14.0.1/guava-14.0.1.pom
 (6 KB at 750.0 KB/sec)
2017-06-10T00:34:31.346 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/guava/guava-parent/14.0.1/guava-parent-14.0.1.pom
2017-06-10T00:34:31.353 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/guava/guava-parent/14.0.1/guava-parent-14.0.1.pom
 (3 KB at 356.4 KB/sec)
2017-06-10T00:34:31.357 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-classworlds/2.4.2/plexus-classworlds-2.4.2.pom
2017-06-10T00:34:31.367 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-classworlds/2.4.2/plexus-classworlds-2.4.2.pom
 (4 KB at 342.7 KB/sec)
2017-06-10T00:34:31.372 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-interpolation/1.16/plexus-interpolation-1.16.pom
2017-06-10T00:34:31.379 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-interpolation/1.16/plexus-interpolation-1.16.pom
 (2 KB at 143.3 KB/sec)
2017-06-10T00:34:31.384 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/eclipse/aether/aether-api/0.9.0.M2/aether-api-0.9.0.M2.pom
2017-06-10T00:34:31.392 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/eclipse/aether/aether-api/0.9.0.M2/aether-api-0.9.0.M2.pom
 (2 KB at 212.0 KB/sec)
2017-06-10T00:34:31.397 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/gmaven/gmaven-adapter-api/2.0/gmaven-adapter-api-2.0.pom
2017-06-10T00:34:31.406 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/gmaven/gmaven-adapter-api/2.0/gmaven-adapter-api-2.0.pom
 (2 KB at 196.0 KB/sec)
2017-06-10T00:34:31.411 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/gmaven/gmaven-adapter-impl/2.0/gmaven-adapter-impl-2.0.pom
2017-06-10T00:34:31.421 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/gmaven/gmaven-adapter-impl/2.0/gmaven-adapter-impl-2.0.pom
 (3 KB at 265.4 KB/sec)
2017-06-10T00:34:31.427 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/groovy/groovy-all/2.1.5/groovy-all-2.1.5.pom
2017-06-10T00:34:31.444 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/groovy/groovy-all/2.1.5/groovy-all-2.1.5.pom
 (18 KB at 1036.2 KB/sec)
2017-06-10T00:34:31.449 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant/1.8.4/ant-1.8.4.pom
2017-06-10T00:34:31.458 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant/1.8.4/ant-1.8.4.pom (10 
KB at 1047.5 KB/sec)
2017-06-10T00:34:31.461 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant-parent/1.8.4/ant-parent-1.8.4.pom
2017-06-10T00:34:31.470 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant-parent/1.8.4/ant-parent-1.8.4.pom
 (5 KB at 494.9 KB/sec)
2017-06-10T00:34:31.475 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant-launcher/1.8.4/ant-launcher-1.8.4.pom
2017-06-10T00:34:31.484 [INFO] Downloaded:

Jenkins build is back to normal : beam_PostCommit_Java_JDK_Versions_Test » OpenJDK 7 (on Ubuntu only),beam #66

2017-06-09 Thread Apache Jenkins Server

See

Jenkins build is back to normal : beam_PostCommit_Python_Verify #2456

2017-06-09 Thread Apache Jenkins Server

See

Jenkins build is still unstable: beam_PostCommit_Java_MavenInstall #4074

2017-06-09 Thread Apache Jenkins Server

See

[GitHub] beam pull request #3339: Actually test the fn_api_runner.

2017-06-09 Thread robertwb

GitHub user robertwb opened a pull request:

https://github.com/apache/beam/pull/3339

Actually test the fn_api_runner.

The test suite was not being run due to a typo.
Fix breakage due to changes in the code in the meantime.

Be sure to do all of the following to help us incorporate your contribution
quickly and easily:

 - [ ] Make sure the PR title is formatted like:
   `[BEAM-] Description of pull request`
 - [ ] Make sure tests pass via `mvn clean verify`.
 - [ ] Replace `` in the title with the actual Jira issue
   number, if there is one.
 - [ ] If this contribution is large, please file an Apache
   [Individual Contributor License 
Agreement](https://www.apache.org/licenses/icla.pdf).

---


You can merge this pull request into a Git repository by running:

$ git pull https://github.com/robertwb/incubator-beam fn-api-runner-tests

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/beam/pull/3339.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #3339


commit 1db42923b81eab162eb6bf6a563505aff50cbbfa
Author: Robert Bradshaw 
Date:   2017-06-09T23:44:55Z

Actually test the fn_api_runner.

The test suite was not being run due to a typo.
Fix breakage due to changes in the code in the meantime.




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

Build failed in Jenkins: beam_PostCommit_Java_JDK_Versions_Test » JDK 1.7 (latest),beam #66

2017-06-09 Thread Apache Jenkins Server

See 


Changes:

[tgroh] Fix compile error occurs in some JDKs

[robertwb] Use beam.Map rather than beam.ParDo for PubSub encoding.

[tgroh] Visit a Transform Hierarchy in Topological Order

[tgroh] Roll-forward Include Additional PTransform inputs in Transform Nodes

--
[...truncated 1.28 MB...]
2017-06-10\T\00:31:32.525 [INFO] 2017-06-10\T\00:31:32.525 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/auth/google-auth-library-oauth2-http/0.4.0/google-auth-library-oauth2-http-0.4.0.pom
 (3 KB at 85.6 KB/sec)
2017-06-10\T\00:31:32.528 [INFO] 2017-06-10\T\00:31:32.528 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/api/grpc/proto-google-common-protos/0.1.11/proto-google-common-protos-0.1.11.pom
2017-06-10\T\00:31:32.555 [INFO] 2017-06-10\T\00:31:32.555 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/api/grpc/proto-google-common-protos/0.1.11/proto-google-common-protos-0.1.11.pom
 (2 KB at 58.8 KB/sec)
2017-06-10\T\00:31:32.558 [INFO] 2017-06-10\T\00:31:32.558 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/api/api-common/1.0.0-rc1/api-common-1.0.0-rc1.pom
2017-06-10\T\00:31:32.584 [INFO] 2017-06-10\T\00:31:32.584 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/api/api-common/1.0.0-rc1/api-common-1.0.0-rc1.pom
 (3 KB at 99.7 KB/sec)
2017-06-10\T\00:31:32.587 [INFO] 2017-06-10\T\00:31:32.587 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/joda-time/joda-time/2.8.2/joda-time-2.8.2.pom
2017-06-10\T\00:31:32.621 [INFO] 2017-06-10\T\00:31:32.621 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/joda-time/joda-time/2.8.2/joda-time-2.8.2.pom
 (29 KB at 825.0 KB/sec)
2017-06-10\T\00:31:32.642 [INFO] 2017-06-10\T\00:31:32.641 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/cloud/google-cloud-core/1.0.0-rc3/google-cloud-core-1.0.0-rc3.pom
2017-06-10\T\00:31:32.668 [INFO] 2017-06-10\T\00:31:32.668 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/cloud/google-cloud-core/1.0.0-rc3/google-cloud-core-1.0.0-rc3.pom
 (4 KB at 118.5 KB/sec)
2017-06-10\T\00:31:32.692 [INFO] 2017-06-10\T\00:31:32.692 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/api/grpc/grpc-google-common-protos/0.1.6/grpc-google-common-protos-0.1.6.pom
2017-06-10\T\00:31:32.719 [INFO] 2017-06-10\T\00:31:32.719 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/api/grpc/grpc-google-common-protos/0.1.6/grpc-google-common-protos-0.1.6.pom
 (2 KB at 67.7 KB/sec)
2017-06-10\T\00:31:32.721 [INFO] 2017-06-10\T\00:31:32.721 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/api/api-common/0.0.2/api-common-0.0.2.pom
2017-06-10\T\00:31:32.749 [INFO] 2017-06-10\T\00:31:32.749 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/api/api-common/0.0.2/api-common-0.0.2.pom
 (3 KB at 102.1 KB/sec)
2017-06-10\T\00:31:32.751 [INFO] 2017-06-10\T\00:31:32.751 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/inject/guice/4.0/guice-4.0.pom
2017-06-10\T\00:31:32.779 [INFO] 2017-06-10\T\00:31:32.779 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/inject/guice/4.0/guice-4.0.pom 
(11 KB at 370.8 KB/sec)
2017-06-10\T\00:31:32.781 [INFO] 2017-06-10\T\00:31:32.781 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/inject/guice-parent/4.0/guice-parent-4.0.pom
2017-06-10\T\00:31:32.812 [INFO] 2017-06-10\T\00:31:32.812 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/inject/guice-parent/4.0/guice-parent-4.0.pom
 (16 KB at 504.3 KB/sec)
2017-06-10\T\00:31:32.819 [INFO] 2017-06-10\T\00:31:32.819 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/guava/guava/16.0.1/guava-16.0.1.pom
2017-06-10\T\00:31:32.846 [INFO] 2017-06-10\T\00:31:32.846 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/guava/guava/16.0.1/guava-16.0.1.pom
 (6 KB at 220.7 KB/sec)
2017-06-10\T\00:31:32.848 [INFO] 2017-06-10\T\00:31:32.848 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/guava/guava-parent/16.0.1/guava-parent-16.0.1.pom
2017-06-10\T\00:31:32.875 [INFO] 2017-06-10\T\00:31:32.875 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/guava/guava-parent/16.0.1/guava-parent-16.0.1.pom
 (8 KB at 265.3 KB/sec)
2017-06-10\T\00:31:32.888 [INFO] 2017-06-10\T\00:31:32.888 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/auth/google-auth-library-oauth2-http/0.6.0/google-auth-library-oauth2-http-0.6.0.pom
2017-06-10\T\00:31:32.915 [INFO] 2017-06-10\T\00:31:32.915 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/auth/google-auth-library-oauth2-http/0.6.0/google-auth-library-oauth2-http-0.6.0.pom
 (3 KB at 79.1 KB/sec)

[GitHub] beam-site pull request #256: Add Hive (HCatalog) to the list of available I/...

2017-06-09 Thread iemejia

GitHub user iemejia opened a pull request:

https://github.com/apache/beam-site/pull/256

Add Hive (HCatalog) to the list of available I/O transforms



You can merge this pull request into a Git repository by running:

$ git pull https://github.com/iemejia/beam-site asf-site

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/beam-site/pull/256.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #256


commit 993d2c4ac6fe9c564d9c0e3166298c6f4edce099
Author: IsmaÃ«l MejÃa 
Date:   2017-06-10T00:22:22Z

Add Hive (HCatalog) to the list of available I/O transforms




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

Jenkins build is back to normal : beam_PostCommit_Java_JDK_Versions_Test » OpenJDK 8 (on Ubuntu only),beam #66

2017-06-09 Thread Apache Jenkins Server

See

[GitHub] beam pull request #3337: Include Additional PTransform Inputs in Transform N...

2017-06-09 Thread asfgit

Github user asfgit closed the pull request at:

https://github.com/apache/beam/pull/3337


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

Build failed in Jenkins: beam_PostCommit_Java_MavenInstall #4075

2017-06-09 Thread Apache Jenkins Server

See 


--
[...truncated 524.24 KB...]
[INFO] Retrying request to {s}->https://repo.maven.apache.org:443
[INFO] I/O exception (java.net.SocketException) caught when processing request 
to {s}->https://repo.maven.apache.org:443: Connection reset
[INFO] Retrying request to {s}->https://repo.maven.apache.org:443
[INFO] I/O exception (java.net.SocketException) caught when processing request 
to {s}->https://repo.maven.apache.org:443: Connection reset
[INFO] Retrying request to {s}->https://repo.maven.apache.org:443
2017-06-10T00:08:16.212 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/woodstox/woodstox-core-asl/4.4.1/woodstox-core-asl-4.4.1.pom
2017-06-10T00:08:16.238 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/woodstox/woodstox-core-asl/4.4.1/woodstox-core-asl-4.4.1.pom
 (2 KB at 67.8 KB/sec)
2017-06-10T00:08:16.240 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/cloud/dataflow/google-cloud-dataflow-java-proto-library-all/0.5.160304/google-cloud-dataflow-java-proto-library-all-0.5.160304.pom
2017-06-10T00:08:16.265 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/cloud/dataflow/google-cloud-dataflow-java-proto-library-all/0.5.160304/google-cloud-dataflow-java-proto-library-all-0.5.160304.pom
 (3 KB at 103.2 KB/sec)
2017-06-10T00:08:16.267 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/cloud/dataflow/google-cloud-dataflow-java-sdk-parent/1.4.0/google-cloud-dataflow-java-sdk-parent-1.4.0.pom
2017-06-10T00:08:16.302 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/cloud/dataflow/google-cloud-dataflow-java-sdk-parent/1.4.0/google-cloud-dataflow-java-sdk-parent-1.4.0.pom
 (12 KB at 337.8 KB/sec)
2017-06-10T00:08:16.303 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/google/5/google-5.pom
2017-06-10T00:08:16.327 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/google/5/google-5.pom (3 KB at 
100.0 KB/sec)
[JENKINS] Archiving disabled
2017-06-10T00:08:19.114 [INFO]  
   
2017-06-10T00:08:19.114 [INFO] 

2017-06-10T00:08:19.114 [INFO] Skipping Apache Beam :: Parent
2017-06-10T00:08:19.114 [INFO] This project has been banned from the build due 
to previous failures.
2017-06-10T00:08:19.114 [INFO] 

[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
[JENKINS] Archiving disabled
2017-06-10T00:08:33.640 [INFO] 

2017-06-10T00:08:33.640 [INFO] Reactor Summary:
2017-06-10T00:08:33.640 [INFO] 
2017-06-10T00:08:33.640 [INFO] Apache Beam :: Parent 
.. SUCCESS [ 32.179 s]
2017-06-10T00:08:33.640 [INFO] Apache Beam :: SDKs :: Java :: Build Tools 
. SUCCESS [  9.658 s]
2017-06-10T00:08:33.640 [INFO] Apache Beam :: SDKs 
 SUCCESS [  5.808 s]
2017-06-10T00:08:33.640 [INFO] Apache Beam :: SDKs :: Common 
.. SUCCESS [  1.705 s]
2017-06-10T00:08:33.640 [INFO] Apache Beam :: SDKs :: Common :: Runner API 
 SUCCESS [ 18.257 s]
2017-06-10T00:08:33.640 [INFO] Apache Beam :: SDKs :: Common :: Fn API 
 SUCCESS [ 17.316 s]
2017-06-10T00:08:33.640 [INFO] Apache Beam :: SDKs :: Java 
 SUCCESS [  1.830 s]
2017-06-10T00:08:33.640 [INFO] Apache Beam :: SDKs ::

[GitHub] beam pull request #3338: [BEAM-2371] Port some DirectRunner overrides to SDK...

2017-06-09 Thread kennknowles

GitHub user kennknowles opened a pull request:

https://github.com/apache/beam/pull/3338

[BEAM-2371] Port some DirectRunner overrides to SDK-agnostic APIs

Be sure to do all of the following to help us incorporate your contribution
quickly and easily:

 - [ ] Make sure the PR title is formatted like:
   `[BEAM-] Description of pull request`
 - [ ] Make sure tests pass via `mvn clean verify`.
 - [ ] Replace `` in the title with the actual Jira issue
   number, if there is one.
 - [ ] If this contribution is large, please file an Apache
   [Individual Contributor License 
Agreement](https://www.apache.org/licenses/icla.pdf).

---

@tgroh this is pulled off of #3330, the bits that I don't think are waiting 
for side inputs so I think it could go green right away. It should also be an 
easier review with fewer distractions.

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/kennknowles/beam DirectRunner-overrides-easy

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/beam/pull/3338.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #3338


commit e66724e69117961840ccc649a3d0ba723cc1c7dd
Author: Kenneth Knowles 
Date:   2017-05-26T21:26:55Z

Port ViewOverrideFactory to SDK-agnostic APIs

commit f77147a4642a7b3b7dfef0c00c752bdea07256fd
Author: Kenneth Knowles 
Date:   2017-05-26T21:27:23Z

Port DirectGroupByKey to SDK-agnostic APIs

commit eeffa9d69a9159b36733920cee9bc843d6b74307
Author: Kenneth Knowles 
Date:   2017-05-26T23:07:45Z

Port DirectRunner WriteFiles override to SDK-agnostic APIs

commit e35177b5f2ca9cbb6f7c08ee122b8d9fa15b6633
Author: Kenneth Knowles 
Date:   2017-06-08T20:39:32Z

Port DirectRunner TestStream override to SDK-agnostic APIs




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[1/3] beam-site git commit: Update IO with CassandraIO

2017-06-09 Thread iemejia

Repository: beam-site
Updated Branches:
  refs/heads/asf-site 5c993c612 -> ad4c5c011


Update IO with CassandraIO


Project: http://git-wip-us.apache.org/repos/asf/beam-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam-site/commit/df9f33da
Tree: http://git-wip-us.apache.org/repos/asf/beam-site/tree/df9f33da
Diff: http://git-wip-us.apache.org/repos/asf/beam-site/diff/df9f33da

Branch: refs/heads/asf-site
Commit: df9f33da6a083b154ec86a9daa8fd9e5a7561715
Parents: 5c993c6
Author: Jean-Baptiste OnofrÃ© 
Authored: Wed Jun 7 11:15:45 2017 +0200
Committer: IsmaÃ«l MejÃa 
Committed: Sat Jun 10 02:15:54 2017 +0200

--
 src/documentation/io/built-in.md | 5 +
 1 file changed, 1 insertion(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam-site/blob/df9f33da/src/documentation/io/built-in.md
--
diff --git a/src/documentation/io/built-in.md b/src/documentation/io/built-in.md
index 9dda7f4..7eb7c02 100644
--- a/src/documentation/io/built-in.md
+++ b/src/documentation/io/built-in.md
@@ -35,6 +35,7 @@ Consult the [Programming Guide I/O section]({{site.baseurl 
}}/documentation/prog
 https://github.com/apache/beam/blob/master/sdks/java/core/src/main/java/org/apache/beam/sdk/io;>Google
 Cloud PubSub
   
   
+https://github.com/apache/beam/tree/master/sdks/java/io/cassandra;>Apache 
Cassandra
 Apache 
Hadoop InputFormat
 https://github.com/apache/beam/tree/master/sdks/java/io/hbase;>Apache 
HBase
 https://github.com/apache/beam/tree/master/sdks/java/io/mongodb;>MongoDB
@@ -73,10 +74,6 @@ This table contains I/O transforms that are currently 
planned or in-progress. St
 https://issues.apache.org/jira/browse/BEAM-1237;>BEAM-1237
   
   
-Apache CassandraJava
-https://issues.apache.org/jira/browse/BEAM-245;>BEAM-245
-  
-  
 Apache DistributedLogJava
 https://issues.apache.org/jira/browse/BEAM-607;>BEAM-607

[3/3] beam-site git commit: This closes #255

2017-06-09 Thread iemejia

This closes #255


Project: http://git-wip-us.apache.org/repos/asf/beam-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam-site/commit/ad4c5c01
Tree: http://git-wip-us.apache.org/repos/asf/beam-site/tree/ad4c5c01
Diff: http://git-wip-us.apache.org/repos/asf/beam-site/diff/ad4c5c01

Branch: refs/heads/asf-site
Commit: ad4c5c0110aecbc89a140f748a437360885010fa
Parents: 5c993c6 110da3a
Author: IsmaÃ«l MejÃa 
Authored: Sat Jun 10 02:17:28 2017 +0200
Committer: IsmaÃ«l MejÃa 
Committed: Sat Jun 10 02:17:28 2017 +0200

--
 content/documentation/io/built-in/index.html | 5 +
 src/documentation/io/built-in.md | 5 +
 2 files changed, 2 insertions(+), 8 deletions(-)
--

[GitHub] beam-site pull request #255: Update IO with CassandraIO

2017-06-09 Thread asfgit

Github user asfgit closed the pull request at:

https://github.com/apache/beam-site/pull/255


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[2/3] beam-site git commit: Regenerate website

2017-06-09 Thread iemejia

Regenerate website


Project: http://git-wip-us.apache.org/repos/asf/beam-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam-site/commit/110da3a9
Tree: http://git-wip-us.apache.org/repos/asf/beam-site/tree/110da3a9
Diff: http://git-wip-us.apache.org/repos/asf/beam-site/diff/110da3a9

Branch: refs/heads/asf-site
Commit: 110da3a9926792abfe9e36fb4341192d32cb449c
Parents: df9f33d
Author: IsmaÃ«l MejÃa 
Authored: Sat Jun 10 02:17:28 2017 +0200
Committer: IsmaÃ«l MejÃa 
Committed: Sat Jun 10 02:17:28 2017 +0200

--
 content/documentation/io/built-in/index.html | 5 +
 1 file changed, 1 insertion(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam-site/blob/110da3a9/content/documentation/io/built-in/index.html
--
diff --git a/content/documentation/io/built-in/index.html 
b/content/documentation/io/built-in/index.html
index bb6b642..5860f77 100644
--- a/content/documentation/io/built-in/index.html
+++ b/content/documentation/io/built-in/index.html
@@ -167,6 +167,7 @@
 https://github.com/apache/beam/blob/master/sdks/java/core/src/main/java/org/apache/beam/sdk/io;>Google
 Cloud PubSub
   
   
+https://github.com/apache/beam/tree/master/sdks/java/io/cassandra;>Apache 
Cassandra
 Apache Hadoop 
InputFormat
 https://github.com/apache/beam/tree/master/sdks/java/io/hbase;>Apache 
HBase
 https://github.com/apache/beam/tree/master/sdks/java/io/mongodb;>MongoDB
@@ -205,10 +206,6 @@
 https://issues.apache.org/jira/browse/BEAM-1237;>BEAM-1237
   
   
-Apache CassandraJava
-https://issues.apache.org/jira/browse/BEAM-245;>BEAM-245
-  
-  
 Apache DistributedLogJava
 https://issues.apache.org/jira/browse/BEAM-607;>BEAM-607

Jenkins build is still unstable: beam_PostCommit_Java_ValidatesRunner_Spark #2325

2017-06-09 Thread Apache Jenkins Server

See

[jira] [Created] (BEAM-2429) Conflicting filesystems with used of HadoopFileSystem

2017-06-09 Thread JIRA

François Wagner created BEAM-2429:
-

 Summary: Conflicting filesystems with used of HadoopFileSystem
 Key: BEAM-2429
 URL: https://issues.apache.org/jira/browse/BEAM-2429
 Project: Beam
  Issue Type: Bug
  Components: sdk-java-extensions
Affects Versions: 2.0.0
Reporter: François Wagner
Assignee: Davor Bonaci


I'm facing issue when trying to use HadoopFileSystem in my pipeline. It looks 
like HadoopFileSystem is registring itself under the `file` schema 
(https://github.com/apache/beam/pull/2777/files#diff-330bd0854dcab6037ef0e52c05d68eb2L79),
 hence the following Exception is thrown when trying to register 
HadoopFileSystem.

java.lang.IllegalStateException: Scheme: [file] has conflicting filesystems: 
[org.apache.beam.sdk.io.LocalFileSystem, 
org.apache.beam.sdk.io.hdfs.HadoopFileSystem]
at 
org.apache.beam.sdk.io.FileSystems.verifySchemesAreUnique(FileSystems.java:498)

What is the correct way to handle `hdfs` url out of the box with TextIO & 
AvroIO ?

String[] args = new String[]{
"--hdfsConfiguration=[{\"dfs.client.use.datanode.hostname\": 
\"true\"}]"};
HadoopFileSystemOptions options = PipelineOptionsFactory
.fromArgs(args)
.withValidation()
.as(HadoopFileSystemOptions.class);
Pipeline pipeline = Pipeline.create(options);
configuration.add(config);
options.setHdfsConfiguration(configuration);
Pipeline pipeline = Pipeline.create(options); 





--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Build failed in Jenkins: beam_PostCommit_Python_Verify #2452

2017-06-09 Thread Apache Jenkins Server

See 


--
[...truncated 503.47 KB...]
Successfully downloaded pyhamcrest mock setuptools six funcsigs pbr
Successfully downloaded pyhamcrest mock setuptools six funcsigs pbr
Successfully downloaded pyhamcrest mock setuptools six funcsigs pbr
test_read_from_text_metrics 
(apache_beam.transforms.ptransform_test.PTransformTest) ... ok
:135:
 UserWarning: Using fallback coder for typehint: Union[TaggedOutput, int].
  warnings.warn('Using fallback coder for typehint: %r.' % typehint)
DEPRECATION: pip install --download has been deprecated and will be removed in 
the future. Pip now has a download command that should be used instead.
Collecting pyhamcrest (from -r postcommit_requirements.txt (line 1))
:318:
 SNIMissingWarning: An HTTPS request has been made, but the SNI (Subject Name 
Indication) extension to TLS is not available on this platform. This may cause 
the server to present an incorrect TLS certificate, which can cause validation 
failures. You can upgrade to a newer version of Python to solve this. For more 
information, see 
https://urllib3.readthedocs.io/en/latest/security.html#snimissingwarning.
  SNIMissingWarning
:122:
 InsecurePlatformWarning: A true SSLContext object is not available. This 
prevents urllib3 from configuring SSL appropriately and may cause certain SSL 
connections to fail. You can upgrade to a newer version of Python to solve 
this. For more information, see 
https://urllib3.readthedocs.io/en/latest/security.html#insecureplatformwarning.
  InsecurePlatformWarning
  File was already downloaded 
/tmp/dataflow-requirements-cache/PyHamcrest-1.9.0.tar.gz
Collecting mock (from -r postcommit_requirements.txt (line 2))
  File was already downloaded /tmp/dataflow-requirements-cache/mock-2.0.0.tar.gz
Collecting setuptools (from pyhamcrest->-r postcommit_requirements.txt (line 1))
  File was already downloaded 
/tmp/dataflow-requirements-cache/setuptools-36.0.1.zip
Collecting six (from pyhamcrest->-r postcommit_requirements.txt (line 1))
  File was already downloaded /tmp/dataflow-requirements-cache/six-1.10.0.tar.gz
Collecting funcsigs>=1 (from mock->-r postcommit_requirements.txt (line 2))
  File was already downloaded 
/tmp/dataflow-requirements-cache/funcsigs-1.0.2.tar.gz
Collecting pbr>=0.11 (from mock->-r postcommit_requirements.txt (line 2))
  File was already downloaded /tmp/dataflow-requirements-cache/pbr-3.0.1.tar.gz
Successfully downloaded pyhamcrest mock setuptools six funcsigs pbr
test_par_do_with_multiple_outputs_and_using_return 
(apache_beam.transforms.ptransform_test.PTransformTest) ... ok
:135:
 UserWarning: Using fallback coder for typehint: List[Any].
  warnings.warn('Using fallback coder for typehint: %r.' % typehint)
:135:
 UserWarning: Using fallback coder for typehint: Union[].
  warnings.warn('Using fallback coder for typehint: %r.' % typehint)
DEPRECATION: pip install --download has been deprecated and will be removed in 
the future. Pip now has a download command that should be used instead.
Collecting pyhamcrest (from -r postcommit_requirements.txt (line 1))
  File was already downloaded 
/tmp/dataflow-requirements-cache/PyHamcrest-1.9.0.tar.gz
Collecting mock (from -r postcommit_requirements.txt (line 2))
  File was already downloaded /tmp/dataflow-requirements-cache/mock-2.0.0.tar.gz
Collecting setuptools (from pyhamcrest->-r postcommit_requirements.txt (line 1))
  File was already downloaded 
/tmp/dataflow-requirements-cache/setuptools-36.0.1.zip
Collecting six (from pyhamcrest->-r postcommit_requirements.txt (line 1))
  File was already downloaded /tmp/dataflow-requirements-cache/six-1.10.0.tar.gz
Collecting funcsigs>=1 (from mock->-r postcommit_requirements.txt (line 2))
  File was already downloaded 
/tmp/dataflow-requirements-cache/funcsigs-1.0.2.tar.gz
Collecting pbr>=0.11 (from mock->-r postcommit_requirements.txt (line 2))
  File was already downloaded /tmp/dataflow-requirements-cache/pbr-3.0.1.tar.gz
Successfully downloaded pyhamcrest mock setuptools six funcsigs pbr
test_multiple_empty_outputs 
(apache_beam.transforms.ptransform_test.PTransformTest) ... ok
:135:
 UserWarning: Using

Build failed in Jenkins: beam_PostCommit_Java_MavenInstall_Windows #111

2017-06-09 Thread Apache Jenkins Server

See 


Changes:

[jbonofre] [BEAM-2410] Remove TransportClient from ElasticSearchIO to decouple 
IO

--
[...truncated 2.64 MB...]
2017-06-09T06:34:18.594 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip/1.8/gossip-1.8.pom
 (12 KB at 804.0 KB/sec)
2017-06-09T06:34:18.598 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/sonatype/forge/forge-parent/9/forge-parent-9.pom
2017-06-09T06:34:18.610 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/sonatype/forge/forge-parent/9/forge-parent-9.pom
 (13 KB at 1069.5 KB/sec)
2017-06-09T06:34:18.614 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip-core/1.8/gossip-core-1.8.pom
2017-06-09T06:34:18.626 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip-core/1.8/gossip-core-1.8.pom
 (3 KB at 181.1 KB/sec)
2017-06-09T06:34:18.630 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip-bootstrap/1.8/gossip-bootstrap-1.8.pom
2017-06-09T06:34:18.640 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip-bootstrap/1.8/gossip-bootstrap-1.8.pom
 (2 KB at 157.3 KB/sec)
2017-06-09T06:34:18.645 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/guava/guava/14.0.1/guava-14.0.1.pom
2017-06-09T06:34:18.658 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/guava/guava/14.0.1/guava-14.0.1.pom
 (6 KB at 403.8 KB/sec)
2017-06-09T06:34:18.662 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/guava/guava-parent/14.0.1/guava-parent-14.0.1.pom
2017-06-09T06:34:18.671 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/guava/guava-parent/14.0.1/guava-parent-14.0.1.pom
 (3 KB at 277.2 KB/sec)
2017-06-09T06:34:18.675 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-classworlds/2.4.2/plexus-classworlds-2.4.2.pom
2017-06-09T06:34:18.685 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-classworlds/2.4.2/plexus-classworlds-2.4.2.pom
 (4 KB at 342.7 KB/sec)
2017-06-09T06:34:18.690 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-interpolation/1.16/plexus-interpolation-1.16.pom
2017-06-09T06:34:18.699 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-interpolation/1.16/plexus-interpolation-1.16.pom
 (2 KB at 111.4 KB/sec)
2017-06-09T06:34:18.704 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/eclipse/aether/aether-api/0.9.0.M2/aether-api-0.9.0.M2.pom
2017-06-09T06:34:18.711 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/eclipse/aether/aether-api/0.9.0.M2/aether-api-0.9.0.M2.pom
 (2 KB at 242.3 KB/sec)
2017-06-09T06:34:18.717 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/gmaven/gmaven-adapter-api/2.0/gmaven-adapter-api-2.0.pom
2017-06-09T06:34:18.726 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/gmaven/gmaven-adapter-api/2.0/gmaven-adapter-api-2.0.pom
 (2 KB at 196.0 KB/sec)
2017-06-09T06:34:18.733 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/gmaven/gmaven-adapter-impl/2.0/gmaven-adapter-impl-2.0.pom
2017-06-09T06:34:18.746 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/gmaven/gmaven-adapter-impl/2.0/gmaven-adapter-impl-2.0.pom
 (3 KB at 204.2 KB/sec)
2017-06-09T06:34:18.751 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/groovy/groovy-all/2.1.5/groovy-all-2.1.5.pom
2017-06-09T06:34:18.765 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/groovy/groovy-all/2.1.5/groovy-all-2.1.5.pom
 (18 KB at 1258.3 KB/sec)
2017-06-09T06:34:18.772 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant/1.8.4/ant-1.8.4.pom
2017-06-09T06:34:18.782 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant/1.8.4/ant-1.8.4.pom (10 
KB at 942.8 KB/sec)
2017-06-09T06:34:18.785 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant-parent/1.8.4/ant-parent-1.8.4.pom
2017-06-09T06:34:18.796 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant-parent/1.8.4/ant-parent-1.8.4.pom
 (5 KB at 404.9 KB/sec)
2017-06-09T06:34:18.800 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant-launcher/1.8.4/ant-launcher-1.8.4.pom
2017-06-09T06:34:18.810 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant-launcher/1.8.4/ant-launcher-1.8.4.pom
 (3 KB at 230.6 KB/sec)
2017-06-09T06:34:18.814 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/fusesource/jansi/jansi/1.11/jansi-1.11.pom
2017-06-09T06:34:18.827 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/fusesource/jansi/jansi/1.11/jansi-1.11.pom

Build failed in Jenkins: beam_PostCommit_Java_JDK_Versions_Test » JDK 1.7 (latest),beam #63

2017-06-09 Thread Apache Jenkins Server

See 


Changes:

[jbonofre] [BEAM-2410] Remove TransportClient from ElasticSearchIO to decouple 
IO

--
[...truncated 1.36 MB...]
2017-06-09\T\06:29:09.770 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/github/scopt/scopt_2.10/3.5.0/scopt_2.10-3.5.0.pom
 (2 KB at 65.4 KB/sec)
2017-06-09\T\06:29:09.772 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/twitter/chill_2.10/0.7.4/chill_2.10-0.7.4.pom
2017-06-09\T\06:29:09.797 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/twitter/chill_2.10/0.7.4/chill_2.10-0.7.4.pom
 (3 KB at 92.7 KB/sec)
2017-06-09\T\06:29:09.799 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.pom
2017-06-09\T\06:29:09.824 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.pom
 (2 KB at 79.8 KB/sec)
2017-06-09\T\06:29:09.826 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/twitter/chill-java/0.7.4/chill-java-0.7.4.pom
2017-06-09\T\06:29:09.861 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/twitter/chill-java/0.7.4/chill-java-0.7.4.pom
 (2 KB at 56.8 KB/sec)
2017-06-09\T\06:29:09.863 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-optimizer_2.10/1.3.0/flink-optimizer_2.10-1.3.0.pom
2017-06-09\T\06:29:09.888 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-optimizer_2.10/1.3.0/flink-optimizer_2.10-1.3.0.pom
 (5 KB at 168.7 KB/sec)
2017-06-09\T\06:29:09.895 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-streaming-java_2.10/1.3.0/flink-streaming-java_2.10-1.3.0.pom
2017-06-09\T\06:29:09.920 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-streaming-java_2.10/1.3.0/flink-streaming-java_2.10-1.3.0.pom
 (7 KB at 241.2 KB/sec)
2017-06-09\T\06:29:09.923 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/sling/org.apache.sling.commons.json/2.0.6/org.apache.sling.commons.json-2.0.6.pom
2017-06-09\T\06:29:09.948 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/sling/org.apache.sling.commons.json/2.0.6/org.apache.sling.commons.json-2.0.6.pom
 (4 KB at 126.6 KB/sec)
2017-06-09\T\06:29:09.949 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/sling/sling/10/sling-10.pom
2017-06-09\T\06:29:09.975 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/sling/sling/10/sling-10.pom (26 
KB at 964.1 KB/sec)
2017-06-09\T\06:29:09.977 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/apache/8/apache-8.pom
2017-06-09\T\06:29:10.002 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/apache/8/apache-8.pom (14 KB at 
549.0 KB/sec)
2017-06-09\T\06:29:10.015 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils_2.10/1.3.0/flink-test-utils_2.10-1.3.0.pom
2017-06-09\T\06:29:10.040 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils_2.10/1.3.0/flink-test-utils_2.10-1.3.0.pom
 (5 KB at 185.2 KB/sec)
2017-06-09\T\06:29:10.041 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-parent/1.3.0/flink-test-utils-parent-1.3.0.pom
2017-06-09\T\06:29:10.066 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-parent/1.3.0/flink-test-utils-parent-1.3.0.pom
 (2 KB at 56.3 KB/sec)
2017-06-09\T\06:29:10.069 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-junit/1.3.0/flink-test-utils-junit-1.3.0.pom
2017-06-09\T\06:29:10.095 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-junit/1.3.0/flink-test-utils-junit-1.3.0.pom
 (4 KB at 123.4 KB/sec)
2017-06-09\T\06:29:10.099 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/curator/curator-test/2.12.0/curator-test-2.12.0.pom
2017-06-09\T\06:29:10.126 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/curator/curator-test/2.12.0/curator-test-2.12.0.pom
 (5 KB at 150.0 KB/sec)
2017-06-09\T\06:29:10.127 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/curator/apache-curator/2.12.0/apache-curator-2.12.0.pom
2017-06-09\T\06:29:10.154 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/curator/apache-curator/2.12.0/apache-curator-2.12.0.pom
 (32 KB at 1169.3 KB/sec)
2017-06-09\T\06:29:10.157 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/zookeeper/zookeeper/3.4.8/zookeeper-3.4.8.pom
2017-06-09\T\06:29:10.182 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/zookeeper/zookeeper/3.4.8/zookeeper-3.4.8.pom
 (4 KB at 157.2 KB/sec)
2017-06-09\T\06:29:10.183 [INFO]

Build failed in Jenkins: beam_PostCommit_Java_JDK_Versions_Test » OpenJDK 7 (on Ubuntu only),beam #63

2017-06-09 Thread Apache Jenkins Server

See 


Changes:

[jbonofre] [BEAM-2410] Remove TransportClient from ElasticSearchIO to decouple 
IO

--
[...truncated 1.37 MB...]
2017-06-09\T\06:29:20.290 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/scala-lang/scala-library/2.10.3/scala-library-2.10.3.pom
 (2 KB at 73.9 KB/sec)
2017-06-09\T\06:29:20.292 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/github/scopt/scopt_2.10/3.5.0/scopt_2.10-3.5.0.pom
2017-06-09\T\06:29:20.319 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/github/scopt/scopt_2.10/3.5.0/scopt_2.10-3.5.0.pom
 (2 KB at 60.6 KB/sec)
2017-06-09\T\06:29:20.321 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/twitter/chill_2.10/0.7.4/chill_2.10-0.7.4.pom
2017-06-09\T\06:29:20.348 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/twitter/chill_2.10/0.7.4/chill_2.10-0.7.4.pom
 (3 KB at 85.8 KB/sec)
2017-06-09\T\06:29:20.350 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.pom
2017-06-09\T\06:29:20.377 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.pom
 (2 KB at 73.9 KB/sec)
2017-06-09\T\06:29:20.378 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/twitter/chill-java/0.7.4/chill-java-0.7.4.pom
2017-06-09\T\06:29:20.405 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/twitter/chill-java/0.7.4/chill-java-0.7.4.pom
 (2 KB at 73.6 KB/sec)
2017-06-09\T\06:29:20.406 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-optimizer_2.10/1.3.0/flink-optimizer_2.10-1.3.0.pom
2017-06-09\T\06:29:20.451 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-optimizer_2.10/1.3.0/flink-optimizer_2.10-1.3.0.pom
 (5 KB at 93.7 KB/sec)
2017-06-09\T\06:29:20.455 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-streaming-java_2.10/1.3.0/flink-streaming-java_2.10-1.3.0.pom
2017-06-09\T\06:29:20.482 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-streaming-java_2.10/1.3.0/flink-streaming-java_2.10-1.3.0.pom
 (7 KB at 223.3 KB/sec)
2017-06-09\T\06:29:20.486 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/sling/org.apache.sling.commons.json/2.0.6/org.apache.sling.commons.json-2.0.6.pom
2017-06-09\T\06:29:20.514 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/sling/org.apache.sling.commons.json/2.0.6/org.apache.sling.commons.json-2.0.6.pom
 (4 KB at 113.0 KB/sec)
2017-06-09\T\06:29:20.516 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/sling/sling/10/sling-10.pom
2017-06-09\T\06:29:20.546 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/sling/sling/10/sling-10.pom (26 
KB at 835.5 KB/sec)
2017-06-09\T\06:29:20.547 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/apache/8/apache-8.pom
2017-06-09\T\06:29:20.575 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/apache/8/apache-8.pom (14 KB at 
508.3 KB/sec)
2017-06-09\T\06:29:20.589 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils_2.10/1.3.0/flink-test-utils_2.10-1.3.0.pom
2017-06-09\T\06:29:20.616 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils_2.10/1.3.0/flink-test-utils_2.10-1.3.0.pom
 (5 KB at 171.5 KB/sec)
2017-06-09\T\06:29:20.618 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-parent/1.3.0/flink-test-utils-parent-1.3.0.pom
2017-06-09\T\06:29:20.645 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-parent/1.3.0/flink-test-utils-parent-1.3.0.pom
 (2 KB at 50.3 KB/sec)
2017-06-09\T\06:29:20.649 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-junit/1.3.0/flink-test-utils-junit-1.3.0.pom
2017-06-09\T\06:29:20.676 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-junit/1.3.0/flink-test-utils-junit-1.3.0.pom
 (4 KB at 118.9 KB/sec)
2017-06-09\T\06:29:20.682 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/curator/curator-test/2.12.0/curator-test-2.12.0.pom
2017-06-09\T\06:29:20.709 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/curator/curator-test/2.12.0/curator-test-2.12.0.pom
 (5 KB at 150.0 KB/sec)
2017-06-09\T\06:29:20.710 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/curator/apache-curator/2.12.0/apache-curator-2.12.0.pom
2017-06-09\T\06:29:20.738 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/curator/apache-curator/2.12.0/apache-curator-2.12.0.pom
 (32 KB at 1127.5 KB/sec)

[jira] [Updated] (BEAM-2429) Conflicting filesystems with used of HadoopFileSystem

2017-06-09 Thread JIRA


 [ 
https://issues.apache.org/jira/browse/BEAM-2429?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

François Wagner updated BEAM-2429:
--
Description: 
I'm facing issue when trying to use HadoopFileSystem in my pipeline. It looks 
like HadoopFileSystem is registring itself under the `file` schema 
(https://github.com/apache/beam/pull/2777/files#diff-330bd0854dcab6037ef0e52c05d68eb2L79),
 hence the following Exception is thrown when trying to register 
HadoopFileSystem.

java.lang.IllegalStateException: Scheme: [file] has conflicting filesystems: 
[org.apache.beam.sdk.io.LocalFileSystem, 
org.apache.beam.sdk.io.hdfs.HadoopFileSystem]
at 
org.apache.beam.sdk.io.FileSystems.verifySchemesAreUnique(FileSystems.java:498)

What is the correct way to handle `hdfs` url out of the box with TextIO & 
AvroIO ?
{code:java}
String[] args = new String[]{
"--hdfsConfiguration=[{\"dfs.client.use.datanode.hostname\": 
\"true\"}]"};
HadoopFileSystemOptions options = PipelineOptionsFactory
.fromArgs(args)
.withValidation()
.as(HadoopFileSystemOptions.class);
Pipeline pipeline = Pipeline.create(options); 
{code}


  was:
I'm facing issue when trying to use HadoopFileSystem in my pipeline. It looks 
like HadoopFileSystem is registring itself under the `file` schema 
(https://github.com/apache/beam/pull/2777/files#diff-330bd0854dcab6037ef0e52c05d68eb2L79),
 hence the following Exception is thrown when trying to register 
HadoopFileSystem.

java.lang.IllegalStateException: Scheme: [file] has conflicting filesystems: 
[org.apache.beam.sdk.io.LocalFileSystem, 
org.apache.beam.sdk.io.hdfs.HadoopFileSystem]
at 
org.apache.beam.sdk.io.FileSystems.verifySchemesAreUnique(FileSystems.java:498)

What is the correct way to handle `hdfs` url out of the box with TextIO & 
AvroIO ?
{code:java}
String[] args = new String[]{
"--hdfsConfiguration=[{\"dfs.client.use.datanode.hostname\": 
\"true\"}]"};
HadoopFileSystemOptions options = PipelineOptionsFactory
.fromArgs(args)
.withValidation()
.as(HadoopFileSystemOptions.class);
Pipeline pipeline = Pipeline.create(options);
configuration.add(config);
options.setHdfsConfiguration(configuration);
Pipeline pipeline = Pipeline.create(options); 
{code}



> Conflicting filesystems with used of HadoopFileSystem
> -
>
> Key: BEAM-2429
> URL: https://issues.apache.org/jira/browse/BEAM-2429
> Project: Beam
>  Issue Type: Bug
>  Components: sdk-java-extensions
>Affects Versions: 2.0.0
>Reporter: François Wagner
>Assignee: Davor Bonaci
>
> I'm facing issue when trying to use HadoopFileSystem in my pipeline. It looks 
> like HadoopFileSystem is registring itself under the `file` schema 
> (https://github.com/apache/beam/pull/2777/files#diff-330bd0854dcab6037ef0e52c05d68eb2L79),
>  hence the following Exception is thrown when trying to register 
> HadoopFileSystem.
> java.lang.IllegalStateException: Scheme: [file] has conflicting filesystems: 
> [org.apache.beam.sdk.io.LocalFileSystem, 
> org.apache.beam.sdk.io.hdfs.HadoopFileSystem]
>   at 
> org.apache.beam.sdk.io.FileSystems.verifySchemesAreUnique(FileSystems.java:498)
> What is the correct way to handle `hdfs` url out of the box with TextIO & 
> AvroIO ?
> {code:java}
> String[] args = new String[]{
> "--hdfsConfiguration=[{\"dfs.client.use.datanode.hostname\": 
> \"true\"}]"};
> HadoopFileSystemOptions options = PipelineOptionsFactory
> .fromArgs(args)
> .withValidation()
> .as(HadoopFileSystemOptions.class);
> Pipeline pipeline = Pipeline.create(options); 
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Build failed in Jenkins: beam_PostCommit_Java_JDK_Versions_Test » OpenJDK 8 (on Ubuntu only),beam #63

2017-06-09 Thread Apache Jenkins Server

See 


Changes:

[jbonofre] [BEAM-2410] Remove TransportClient from ElasticSearchIO to decouple 
IO

--
[...truncated 1.37 MB...]
2017-06-09\T\06:26:34.726 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/scala-lang/scala-library/2.10.3/scala-library-2.10.3.pom
 (2 KB at 79.8 KB/sec)
2017-06-09\T\06:26:34.728 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/github/scopt/scopt_2.10/3.5.0/scopt_2.10-3.5.0.pom
2017-06-09\T\06:26:34.753 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/github/scopt/scopt_2.10/3.5.0/scopt_2.10-3.5.0.pom
 (2 KB at 65.4 KB/sec)
2017-06-09\T\06:26:34.755 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/twitter/chill_2.10/0.7.4/chill_2.10-0.7.4.pom
2017-06-09\T\06:26:34.782 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/twitter/chill_2.10/0.7.4/chill_2.10-0.7.4.pom
 (3 KB at 85.8 KB/sec)
2017-06-09\T\06:26:34.783 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.pom
2017-06-09\T\06:26:34.809 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.pom
 (2 KB at 76.8 KB/sec)
2017-06-09\T\06:26:34.810 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/twitter/chill-java/0.7.4/chill-java-0.7.4.pom
2017-06-09\T\06:26:34.837 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/twitter/chill-java/0.7.4/chill-java-0.7.4.pom
 (2 KB at 73.6 KB/sec)
2017-06-09\T\06:26:34.839 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-optimizer_2.10/1.3.0/flink-optimizer_2.10-1.3.0.pom
2017-06-09\T\06:26:34.864 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-optimizer_2.10/1.3.0/flink-optimizer_2.10-1.3.0.pom
 (5 KB at 168.7 KB/sec)
2017-06-09\T\06:26:34.868 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-streaming-java_2.10/1.3.0/flink-streaming-java_2.10-1.3.0.pom
2017-06-09\T\06:26:34.893 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-streaming-java_2.10/1.3.0/flink-streaming-java_2.10-1.3.0.pom
 (7 KB at 241.2 KB/sec)
2017-06-09\T\06:26:34.897 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/sling/org.apache.sling.commons.json/2.0.6/org.apache.sling.commons.json-2.0.6.pom
2017-06-09\T\06:26:34.923 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/sling/org.apache.sling.commons.json/2.0.6/org.apache.sling.commons.json-2.0.6.pom
 (4 KB at 121.7 KB/sec)
2017-06-09\T\06:26:34.925 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/sling/sling/10/sling-10.pom
2017-06-09\T\06:26:34.952 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/sling/sling/10/sling-10.pom (26 
KB at 928.3 KB/sec)
2017-06-09\T\06:26:34.953 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/apache/8/apache-8.pom
2017-06-09\T\06:26:34.981 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/apache/8/apache-8.pom (14 KB at 
490.2 KB/sec)
2017-06-09\T\06:26:34.994 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils_2.10/1.3.0/flink-test-utils_2.10-1.3.0.pom
2017-06-09\T\06:26:35.023 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils_2.10/1.3.0/flink-test-utils_2.10-1.3.0.pom
 (5 KB at 159.7 KB/sec)
2017-06-09\T\06:26:35.024 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-parent/1.3.0/flink-test-utils-parent-1.3.0.pom
2017-06-09\T\06:26:35.050 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-parent/1.3.0/flink-test-utils-parent-1.3.0.pom
 (2 KB at 54.1 KB/sec)
2017-06-09\T\06:26:35.053 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-junit/1.3.0/flink-test-utils-junit-1.3.0.pom
2017-06-09\T\06:26:35.079 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-junit/1.3.0/flink-test-utils-junit-1.3.0.pom
 (4 KB at 123.4 KB/sec)
2017-06-09\T\06:26:35.084 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/curator/curator-test/2.12.0/curator-test-2.12.0.pom
2017-06-09\T\06:26:35.110 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/curator/curator-test/2.12.0/curator-test-2.12.0.pom
 (5 KB at 155.8 KB/sec)
2017-06-09\T\06:26:35.112 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/curator/apache-curator/2.12.0/apache-curator-2.12.0.pom
2017-06-09\T\06:26:35.141 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/curator/apache-curator/2.12.0/apache-curator-2.12.0.pom
 (32 KB at 1052.4 KB/sec)

[jira] [Updated] (BEAM-2429) Conflicting filesystems with used of HadoopFileSystem

2017-06-09 Thread JIRA


 [ 
https://issues.apache.org/jira/browse/BEAM-2429?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

François Wagner updated BEAM-2429:
--
Description: 
I'm facing issue when trying to use HadoopFileSystem in my pipeline. It looks 
like HadoopFileSystem is registring itself under the `file` schema 
(https://github.com/apache/beam/pull/2777/files#diff-330bd0854dcab6037ef0e52c05d68eb2L79),
 hence the following Exception is thrown when trying to register 
HadoopFileSystem.

java.lang.IllegalStateException: Scheme: [file] has conflicting filesystems: 
[org.apache.beam.sdk.io.LocalFileSystem, 
org.apache.beam.sdk.io.hdfs.HadoopFileSystem]
at 
org.apache.beam.sdk.io.FileSystems.verifySchemesAreUnique(FileSystems.java:498)

What is the correct way to handle `hdfs` url out of the box with TextIO & 
AvroIO ?
{code:java}
String[] args = new String[]{
"--hdfsConfiguration=[{\"dfs.client.use.datanode.hostname\": 
\"true\"}]"};
HadoopFileSystemOptions options = PipelineOptionsFactory
.fromArgs(args)
.withValidation()
.as(HadoopFileSystemOptions.class);
Pipeline pipeline = Pipeline.create(options);
configuration.add(config);
options.setHdfsConfiguration(configuration);
Pipeline pipeline = Pipeline.create(options); 
{code}


  was:
I'm facing issue when trying to use HadoopFileSystem in my pipeline. It looks 
like HadoopFileSystem is registring itself under the `file` schema 
(https://github.com/apache/beam/pull/2777/files#diff-330bd0854dcab6037ef0e52c05d68eb2L79),
 hence the following Exception is thrown when trying to register 
HadoopFileSystem.

java.lang.IllegalStateException: Scheme: [file] has conflicting filesystems: 
[org.apache.beam.sdk.io.LocalFileSystem, 
org.apache.beam.sdk.io.hdfs.HadoopFileSystem]
at 
org.apache.beam.sdk.io.FileSystems.verifySchemesAreUnique(FileSystems.java:498)

What is the correct way to handle `hdfs` url out of the box with TextIO & 
AvroIO ?

String[] args = new String[]{
"--hdfsConfiguration=[{\"dfs.client.use.datanode.hostname\": 
\"true\"}]"};
HadoopFileSystemOptions options = PipelineOptionsFactory
.fromArgs(args)
.withValidation()
.as(HadoopFileSystemOptions.class);
Pipeline pipeline = Pipeline.create(options);
configuration.add(config);
options.setHdfsConfiguration(configuration);
Pipeline pipeline = Pipeline.create(options); 




> Conflicting filesystems with used of HadoopFileSystem
> -
>
> Key: BEAM-2429
> URL: https://issues.apache.org/jira/browse/BEAM-2429
> Project: Beam
>  Issue Type: Bug
>  Components: sdk-java-extensions
>Affects Versions: 2.0.0
>Reporter: François Wagner
>Assignee: Davor Bonaci
>
> I'm facing issue when trying to use HadoopFileSystem in my pipeline. It looks 
> like HadoopFileSystem is registring itself under the `file` schema 
> (https://github.com/apache/beam/pull/2777/files#diff-330bd0854dcab6037ef0e52c05d68eb2L79),
>  hence the following Exception is thrown when trying to register 
> HadoopFileSystem.
> java.lang.IllegalStateException: Scheme: [file] has conflicting filesystems: 
> [org.apache.beam.sdk.io.LocalFileSystem, 
> org.apache.beam.sdk.io.hdfs.HadoopFileSystem]
>   at 
> org.apache.beam.sdk.io.FileSystems.verifySchemesAreUnique(FileSystems.java:498)
> What is the correct way to handle `hdfs` url out of the box with TextIO & 
> AvroIO ?
> {code:java}
> String[] args = new String[]{
> "--hdfsConfiguration=[{\"dfs.client.use.datanode.hostname\": 
> \"true\"}]"};
> HadoopFileSystemOptions options = PipelineOptionsFactory
> .fromArgs(args)
> .withValidation()
> .as(HadoopFileSystemOptions.class);
> Pipeline pipeline = Pipeline.create(options);
> configuration.add(config);
> options.setHdfsConfiguration(configuration);
> Pipeline pipeline = Pipeline.create(options); 
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

[GitHub] beam pull request #3336: Beam-2171 Power function

2017-06-09 Thread app-tarush

GitHub user app-tarush opened a pull request:

https://github.com/apache/beam/pull/3336

Beam-2171 Power function

Be sure to do all of the following to help us incorporate your contribution
quickly and easily:

 - [ ] Make sure the PR title is formatted like:
   `[BEAM-] Description of pull request`
 - [ ] Make sure tests pass via `mvn clean verify`.
 - [ ] Replace `` in the title with the actual Jira issue
   number, if there is one.
 - [ ] If this contribution is large, please file an Apache
   [Individual Contributor License 
Agreement](https://www.apache.org/licenses/icla.pdf).

---


You can merge this pull request into a Git repository by running:

$ git pull https://github.com/app-tarush/beam beam-2171-power-func

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/beam/pull/3336.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #3336


commit fec5f15d7e4e67eacd449a42b0576b7b549787b2
Author: tarushapptech 
Date:   2017-06-08T06:46:01Z

commiting changes for round function

commit 715883fc8eae7309fd916e34cd2770576b6925f0
Author: tarushapptech 
Date:   2017-06-09T09:23:52Z

PR review comments added and chenges done

commit 6701988e4a70f0666101f22eecc8191dcbebd7d6
Author: tarushapptech 
Date:   2017-06-09T16:02:39Z

commiting changes for power functions




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

Build failed in Jenkins: beam_PostCommit_Python_Verify #2453

2017-06-09 Thread Apache Jenkins Server

See 


--
[...truncated 503.12 KB...]
Successfully downloaded pyhamcrest mock setuptools six funcsigs pbr
Successfully downloaded pyhamcrest mock setuptools six funcsigs pbr
Successfully downloaded pyhamcrest mock setuptools six funcsigs pbr
test_read_from_text_metrics 
(apache_beam.transforms.ptransform_test.PTransformTest) ... ok
:135:
 UserWarning: Using fallback coder for typehint: Union[TaggedOutput, int].
  warnings.warn('Using fallback coder for typehint: %r.' % typehint)
DEPRECATION: pip install --download has been deprecated and will be removed in 
the future. Pip now has a download command that should be used instead.
Collecting pyhamcrest (from -r postcommit_requirements.txt (line 1))
  File was already downloaded 
/tmp/dataflow-requirements-cache/PyHamcrest-1.9.0.tar.gz
Collecting mock (from -r postcommit_requirements.txt (line 2))
  File was already downloaded /tmp/dataflow-requirements-cache/mock-2.0.0.tar.gz
Collecting setuptools (from pyhamcrest->-r postcommit_requirements.txt (line 1))
  File was already downloaded 
/tmp/dataflow-requirements-cache/setuptools-36.0.1.zip
Collecting six (from pyhamcrest->-r postcommit_requirements.txt (line 1))
  File was already downloaded /tmp/dataflow-requirements-cache/six-1.10.0.tar.gz
Collecting funcsigs>=1 (from mock->-r postcommit_requirements.txt (line 2))
  File was already downloaded 
/tmp/dataflow-requirements-cache/funcsigs-1.0.2.tar.gz
Collecting pbr>=0.11 (from mock->-r postcommit_requirements.txt (line 2))
  File was already downloaded /tmp/dataflow-requirements-cache/pbr-3.0.1.tar.gz
Successfully downloaded pyhamcrest mock setuptools six funcsigs pbr
test_multiple_empty_outputs 
(apache_beam.transforms.ptransform_test.PTransformTest) ... ok
:135:
 UserWarning: Using fallback coder for typehint: List[Any].
  warnings.warn('Using fallback coder for typehint: %r.' % typehint)
:135:
 UserWarning: Using fallback coder for typehint: Union[].
  warnings.warn('Using fallback coder for typehint: %r.' % typehint)
DEPRECATION: pip install --download has been deprecated and will be removed in 
the future. Pip now has a download command that should be used instead.
Collecting pyhamcrest (from -r postcommit_requirements.txt (line 1))
  File was already downloaded 
/tmp/dataflow-requirements-cache/PyHamcrest-1.9.0.tar.gz
test_par_do_with_multiple_outputs_and_using_yield 
(apache_beam.transforms.ptransform_test.PTransformTest) ... ok
:135:
 UserWarning: Using fallback coder for typehint: Union[Tuple[str, None], 
Tuple[str, int]].
  warnings.warn('Using fallback coder for typehint: %r.' % typehint)
:135:
 UserWarning: Using fallback coder for typehint: List[Any].
  warnings.warn('Using fallback coder for typehint: %r.' % typehint)
:135:
 UserWarning: Using fallback coder for typehint: Union[].
  warnings.warn('Using fallback coder for typehint: %r.' % typehint)
Collecting mock (from -r postcommit_requirements.txt (line 2))
  File was already downloaded /tmp/dataflow-requirements-cache/mock-2.0.0.tar.gz
Collecting setuptools (from pyhamcrest->-r postcommit_requirements.txt (line 1))
DEPRECATION: pip install --download has been deprecated and will be removed in 
the future. Pip now has a download command that should be used instead.
Collecting pyhamcrest (from -r postcommit_requirements.txt (line 1))
  File was already downloaded 
/tmp/dataflow-requirements-cache/PyHamcrest-1.9.0.tar.gz
  File was already downloaded 
/tmp/dataflow-requirements-cache/setuptools-36.0.1.zip
Collecting mock (from -r postcommit_requirements.txt (line 2))
  File was already downloaded /tmp/dataflow-requirements-cache/mock-2.0.0.tar.gz
Collecting six (from pyhamcrest->-r postcommit_requirements.txt (line 1))
  File was already downloaded /tmp/dataflow-requirements-cache/six-1.10.0.tar.gz
Collecting funcsigs>=1 (from mock->-r postcommit_requirements.txt (line 2))
  File was already downloaded 
/tmp/dataflow-requirements-cache/funcsigs-1.0.2.tar.gz
Collecting setuptools (from pyhamcrest->-r postcommit_requirements.txt (line 1))
Collecting pbr>=0.11 (from mock->-r postcommit_requirements.txt (line 2))
  File was already downloaded 
/tmp/dataflow-requirements-cache/setuptools-36.0.1.zip
  File was already downloaded

[GitHub] beam pull request #3175: Beam-2171 power function

2017-06-09 Thread app-tarush

Github user app-tarush closed the pull request at:

https://github.com/apache/beam/pull/3175


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[2/3] beam-site git commit: Add Beam Runner Guide

2017-06-09 Thread kenn

Add Beam Runner Guide


Project: http://git-wip-us.apache.org/repos/asf/beam-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam-site/commit/15175f79
Tree: http://git-wip-us.apache.org/repos/asf/beam-site/tree/15175f79
Diff: http://git-wip-us.apache.org/repos/asf/beam-site/diff/15175f79

Branch: refs/heads/asf-site
Commit: 15175f7924d0fb5bbc7222c99f53cd2d77e2ee65
Parents: 41904c0
Author: Kenneth Knowles 
Authored: Wed May 24 21:08:17 2017 -0700
Committer: Kenneth Knowles 
Committed: Fri Jun 9 09:42:41 2017 -0700

--
 src/_includes/header.html  |1 +
 src/contribute/runner-guide.md | 1104 +++
 2 files changed, 1105 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/beam-site/blob/15175f79/src/_includes/header.html
--
diff --git a/src/_includes/header.html b/src/_includes/header.html
index 69027d2..3bd4ced 100644
--- a/src/_includes/header.html
+++ b/src/_includes/header.html
@@ -74,6 +74,7 @@
 Testing 
Guide
 Release 
Guide
 PTransform Style Guide
+Runner 
Authoring Guide
 
 Technical References
 Design Principles

http://git-wip-us.apache.org/repos/asf/beam-site/blob/15175f79/src/contribute/runner-guide.md
--
diff --git a/src/contribute/runner-guide.md b/src/contribute/runner-guide.md
new file mode 100644
index 000..d638ed0
--- /dev/null
+++ b/src/contribute/runner-guide.md
@@ -0,0 +1,1104 @@
+---
+layout: default
+title: "Runner Authoring Guide"
+permalink: /contribute/runner-guide/
+---
+
+# Runner Authoring Guide
+
+This guide walks through how to implement a new runner. It is aimed at someone
+who has a data processing system and wants to use it to execute a Beam
+pipeline. The guide starts from the basics, to help you evaluate the work
+ahead. Then the sections become more and more detailed, to be a resource
+throughout the development of your runner.
+
+Topics covered:
+
+ * Basics of the Beam model, focused on what a runner author needs to know to
+   execute a pipeline
+ * Details of how to implement each of Beam's five primitives (of which two are
+   rather trivial and three are rather complex)
+ * Support code we provide for manipulating and executing pipelines
+ * How to test your runner
+ * How to make your runner play nice with SDKs
+ * The low-level Runner API protos used for cross-language pipelines
+
+* TOC
+{:toc}
+
+## Basics of the Beam model
+
+Suppose you have a data processing engine that can pretty easily process graphs
+of operations. You want to integrate it with the Beam ecosystem to get access
+to other languages, great event time processing, and a library of connectors.
+You need to know the core vocabulary:
+
+ * [_Pipeline_](#pipeline) - A pipeline is a graph of transformations that a 
user constructs
+   that defines the data processing they want to do.
+ * [_PCollection_](#pcollections) - Data being processed in a pipeline is part 
of a PCollection.
+ * [_PTransforms_](#ptransforms) - The operations executed within a pipeline. 
These are best
+   thought of as operations on PCollections.
+ * _SDK_ - A language-specific library for pipeline authors (we often call them
+   "users" even though we have many kinds of users) to build transforms,
+   construct their pipelines and submit them to a runner
+ * _Runner_ - You are going to write a piece of software called a runner that
+   takes a Beam pipeline and executes it using the capabilities of your data
+   processing engine.
+ 
+These concepts may be very similar to your processing engine's concepts. Since
+Beam's design is for cross-language operation and reusable libraries of
+transforms, there are some special features worth highlighting.
+
+### Pipeline
+
+A pipeline in Beam is a graph of PTransforms operating on PCollections. A
+pipeline is constructed by a user in their SDK of choice, and makes its way to
+your runner either via the SDK directly or via the Runner API's (forthcoming)
+RPC interfaces.
+
+### PTransforms
+
+In Beam, a PTransform can be one of the five primitives or it can be a
+composite transform encapsulating a subgraph. The primitives are:
+
+ * [_Read_](#implementing-the-read-primitive) - parallel connectors to external
+   systems
+ * [_ParDo_](#implementing-the-pardo-primitive) - per element processing
+ * [_GroupByKey_](#implementing-the-groupbykey-and-window-primitive) - 
+   aggregating elements per key and window
+ * [_Flatten_](#implementing-the-flatten-primitive) - union of PCollections
+ * [_Window_](#implementing-the-window-primitive) - set the windowing strategy
+   for a PCollection
+ 
+When implementing a runner, these are the operations you need

[GitHub] beam-site pull request #251: [BEAM-2360] Add Beam Runner Guide

2017-06-09 Thread asfgit

Github user asfgit closed the pull request at:

https://github.com/apache/beam-site/pull/251


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[1/3] beam-site git commit: Add support for "no-toggle" code snippets

2017-06-09 Thread kenn

Repository: beam-site
Updated Branches:
  refs/heads/asf-site 07a32b382 -> 93c7c5f11


Add support for "no-toggle" code snippets


Project: http://git-wip-us.apache.org/repos/asf/beam-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam-site/commit/41904c0d
Tree: http://git-wip-us.apache.org/repos/asf/beam-site/tree/41904c0d
Diff: http://git-wip-us.apache.org/repos/asf/beam-site/diff/41904c0d

Branch: refs/heads/asf-site
Commit: 41904c0de1679ac2f6b64eaeb5792e418c5046bd
Parents: 07a32b3
Author: Kenneth Knowles 
Authored: Wed May 24 19:20:49 2017 -0700
Committer: Kenneth Knowles 
Committed: Fri Jun 9 09:42:40 2017 -0700

--
 src/js/language-switch.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/beam-site/blob/41904c0d/src/js/language-switch.js
--
diff --git a/src/js/language-switch.js b/src/js/language-switch.js
index 0406b16..a402146 100644
--- a/src/js/language-switch.js
+++ b/src/js/language-switch.js
@@ -5,7 +5,7 @@ $(document).ready(function() {
 var prefix = id + "-";
 return {
 "id": id,
-"selector": "[class^=" + prefix + "]",
+"selector": "[class^=" + prefix + "]:not(.no-toggle)",
 "wrapper": prefix + "switcher", // Parent wrapper-class.
 "default": prefix + def, // Default type to display.
 "dbKey": id, // Local Storage Key

[3/3] beam-site git commit: This closes #251: [BEAM-2360] Add Beam Runner Guide

2017-06-09 Thread kenn

This closes #251: [BEAM-2360] Add Beam Runner Guide

  Add Beam Runner Guide
  Add support for "no-toggle" code snippets


Project: http://git-wip-us.apache.org/repos/asf/beam-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam-site/commit/93c7c5f1
Tree: http://git-wip-us.apache.org/repos/asf/beam-site/tree/93c7c5f1
Diff: http://git-wip-us.apache.org/repos/asf/beam-site/diff/93c7c5f1

Branch: refs/heads/asf-site
Commit: 93c7c5f1141813e9fb3a14e69ab1acfc6b22bf82
Parents: 07a32b3 15175f7
Author: Kenneth Knowles 
Authored: Fri Jun 9 09:51:22 2017 -0700
Committer: Kenneth Knowles 
Committed: Fri Jun 9 09:51:22 2017 -0700

--
 src/_includes/header.html  |1 +
 src/contribute/runner-guide.md | 1104 +++
 src/js/language-switch.js  |2 +-
 3 files changed, 1106 insertions(+), 1 deletion(-)
--

Jenkins build is still unstable: beam_PostCommit_Java_ValidatesRunner_Spark #2326

2017-06-09 Thread Apache Jenkins Server

See

[jira] [Commented] (BEAM-2371) Make Java DirectRunner demonstrate language-agnostic Runner API translation wrappers

2017-06-09 Thread ASF GitHub Bot (JIRA)


[ 
https://issues.apache.org/jira/browse/BEAM-2371?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16045351#comment-16045351
 ] 

ASF GitHub Bot commented on BEAM-2371:
--

Github user asfgit closed the pull request at:

https://github.com/apache/beam/pull/3338


> Make Java DirectRunner demonstrate language-agnostic Runner API translation 
> wrappers
> 
>
> Key: BEAM-2371
> URL: https://issues.apache.org/jira/browse/BEAM-2371
> Project: Beam
>  Issue Type: Improvement
>  Components: runner-direct
>Reporter: Kenneth Knowles
>Assignee: Kenneth Knowles
>
> This will complete the PoC for runners-core-construction-java and the Runner 
> API and show other runners the easy path to executing non-Java pipelines, 
> modulo Fn API.



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Jenkins build is back to normal : beam_PostCommit_Java_MavenInstall #4076

2017-06-09 Thread Apache Jenkins Server

See

[jira] [Created] (BEAM-2433) unify SQL/Sql words in class name

2017-06-09 Thread Xu Mingmin (JIRA)

Xu Mingmin created BEAM-2433:


 Summary: unify SQL/Sql words in class name
 Key: BEAM-2433
 URL: https://issues.apache.org/jira/browse/BEAM-2433
 Project: Beam
  Issue Type: Improvement
  Components: dsl-sql
Reporter: Xu Mingmin
Assignee: Xu Mingmin
Priority: Minor


Use {{Sql}} for class name



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

[jira] [Assigned] (BEAM-765) TriggerStateMachine does not need to be separate from OnceTriggerStateMachine

2017-06-09 Thread Kenneth Knowles (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-765?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Kenneth Knowles reassigned BEAM-765:


Assignee: Innocent

> TriggerStateMachine does not need to be separate from OnceTriggerStateMachine
> -
>
> Key: BEAM-765
> URL: https://issues.apache.org/jira/browse/BEAM-765
> Project: Beam
>  Issue Type: Improvement
>  Components: runner-core
>Reporter: Kenneth Knowles
>Assignee: Innocent
>Priority: Trivial
>
> In runners-core, the state machine implementation of triggers does not need 
> to have the fine-grained type-like enforcement of whether a trigger is a 
> {{OnceTrigger}} or not. It may simplify the code.



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

[jira] [Created] (BEAM-2432) Support PTransform overriding when input and output types are different

2017-06-09 Thread Chamikara Jayalath (JIRA)

Chamikara Jayalath created BEAM-2432:


 Summary: Support PTransform overriding when input and output types 
are different
 Key: BEAM-2432
 URL: https://issues.apache.org/jira/browse/BEAM-2432
 Project: Beam
  Issue Type: Improvement
  Components: sdk-py
Reporter: Chamikara Jayalath


https://github.com/apache/beam/pull/ adds ability to dynamically replace 
PTransforms but does not support replacements when input and/or output types 
are different. We have to address complexities due to type hints before 
supporting replacements with different types.



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Build failed in Jenkins: beam_PostCommit_Java_MavenInstall_Windows #112

2017-06-09 Thread Apache Jenkins Server

See 


--
[...truncated 2.64 MB...]
2017-06-09T12:37:57.024 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip/1.8/gossip-1.8.pom
 (12 KB at 375.2 KB/sec)
2017-06-09T12:37:57.028 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/sonatype/forge/forge-parent/9/forge-parent-9.pom
2017-06-09T12:37:57.058 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/sonatype/forge/forge-parent/9/forge-parent-9.pom
 (13 KB at 427.8 KB/sec)
2017-06-09T12:37:57.063 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip-core/1.8/gossip-core-1.8.pom
2017-06-09T12:37:57.095 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip-core/1.8/gossip-core-1.8.pom
 (3 KB at 67.9 KB/sec)
2017-06-09T12:37:57.100 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip-bootstrap/1.8/gossip-bootstrap-1.8.pom
2017-06-09T12:37:57.131 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip-bootstrap/1.8/gossip-bootstrap-1.8.pom
 (2 KB at 50.7 KB/sec)
2017-06-09T12:37:57.137 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/guava/guava/14.0.1/guava-14.0.1.pom
2017-06-09T12:37:57.172 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/guava/guava/14.0.1/guava-14.0.1.pom
 (6 KB at 150.0 KB/sec)
2017-06-09T12:37:57.176 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/guava/guava-parent/14.0.1/guava-parent-14.0.1.pom
2017-06-09T12:37:57.210 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/guava/guava-parent/14.0.1/guava-parent-14.0.1.pom
 (3 KB at 73.4 KB/sec)
2017-06-09T12:37:57.215 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-classworlds/2.4.2/plexus-classworlds-2.4.2.pom
2017-06-09T12:37:57.244 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-classworlds/2.4.2/plexus-classworlds-2.4.2.pom
 (4 KB at 118.2 KB/sec)
2017-06-09T12:37:57.250 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-interpolation/1.16/plexus-interpolation-1.16.pom
2017-06-09T12:37:57.280 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-interpolation/1.16/plexus-interpolation-1.16.pom
 (2 KB at 33.4 KB/sec)
2017-06-09T12:37:57.285 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/eclipse/aether/aether-api/0.9.0.M2/aether-api-0.9.0.M2.pom
2017-06-09T12:37:57.315 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/eclipse/aether/aether-api/0.9.0.M2/aether-api-0.9.0.M2.pom
 (2 KB at 56.5 KB/sec)
2017-06-09T12:37:57.321 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/gmaven/gmaven-adapter-api/2.0/gmaven-adapter-api-2.0.pom
2017-06-09T12:37:57.352 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/gmaven/gmaven-adapter-api/2.0/gmaven-adapter-api-2.0.pom
 (2 KB at 56.9 KB/sec)
2017-06-09T12:37:57.359 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/gmaven/gmaven-adapter-impl/2.0/gmaven-adapter-impl-2.0.pom
2017-06-09T12:37:57.389 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/gmaven/gmaven-adapter-impl/2.0/gmaven-adapter-impl-2.0.pom
 (3 KB at 88.5 KB/sec)
2017-06-09T12:37:57.395 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/groovy/groovy-all/2.1.5/groovy-all-2.1.5.pom
2017-06-09T12:37:57.427 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/groovy/groovy-all/2.1.5/groovy-all-2.1.5.pom
 (18 KB at 550.5 KB/sec)
2017-06-09T12:37:57.433 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant/1.8.4/ant-1.8.4.pom
2017-06-09T12:37:57.463 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant/1.8.4/ant-1.8.4.pom (10 
KB at 314.3 KB/sec)
2017-06-09T12:37:57.467 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant-parent/1.8.4/ant-parent-1.8.4.pom
2017-06-09T12:37:57.500 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant-parent/1.8.4/ant-parent-1.8.4.pom
 (5 KB at 139.2 KB/sec)
2017-06-09T12:37:57.504 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant-launcher/1.8.4/ant-launcher-1.8.4.pom
2017-06-09T12:37:57.534 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant-launcher/1.8.4/ant-launcher-1.8.4.pom
 (3 KB at 76.9 KB/sec)
2017-06-09T12:37:57.537 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/fusesource/jansi/jansi/1.11/jansi-1.11.pom
2017-06-09T12:37:57.570 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/fusesource/jansi/jansi/1.11/jansi-1.11.pom
 (4 KB at 106.6 KB/sec)
2017-06-09T12:37:57.574 [INFO] Downloading:

[2/5] beam git commit: Port DirectRunner WriteFiles override to SDK-agnostic APIs

2017-06-09 Thread kenn

Port DirectRunner WriteFiles override to SDK-agnostic APIs


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ed6bd18b
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ed6bd18b
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ed6bd18b

Branch: refs/heads/master
Commit: ed6bd18bffe8a51d5fc2a59ff9aaa731b196d58a
Parents: 02dbaef
Author: Kenneth Knowles 
Authored: Fri May 26 16:07:45 2017 -0700
Committer: Kenneth Knowles 
Committed: Fri Jun 9 19:56:52 2017 -0700

--
 .../core/construction/PTransformMatchers.java   | 17 ---
 .../direct/WriteWithShardingFactory.java| 30 ++--
 .../direct/WriteWithShardingFactoryTest.java| 26 +++--
 3 files changed, 52 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/ed6bd18b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
--
diff --git 
a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
 
b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
index c339891..0d27241 100644
--- 
a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
+++ 
b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
@@ -17,13 +17,14 @@
  */
 package org.apache.beam.runners.core.construction;
 
+import static 
org.apache.beam.runners.core.construction.PTransformTranslation.WRITE_FILES_TRANSFORM_URN;
+
 import com.google.common.base.MoreObjects;
 import java.io.IOException;
 import java.util.HashSet;
 import java.util.Set;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.io.WriteFiles;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformMatcher;
 import org.apache.beam.sdk.transforms.DoFn;
@@ -359,10 +360,18 @@ public class PTransformMatchers {
 return new PTransformMatcher() {
   @Override
   public boolean matches(AppliedPTransform application) {
-if (PTransformTranslation.WRITE_FILES_TRANSFORM_URN.equals(
+if (WRITE_FILES_TRANSFORM_URN.equals(
 
PTransformTranslation.urnForTransformOrNull(application.getTransform( {
-  WriteFiles write = (WriteFiles) application.getTransform();
-  return write.getSharding() == null && write.getNumShards() == null;
+  try {
+return WriteFilesTranslation.isRunnerDeterminedSharding(
+(AppliedPTransform) application);
+  } catch (IOException exc) {
+throw new RuntimeException(
+String.format(
+"Transform with URN %s failed to parse: %s",
+WRITE_FILES_TRANSFORM_URN, application.getTransform()),
+exc);
+  }
 }
 return false;
   }

http://git-wip-us.apache.org/repos/asf/beam/blob/ed6bd18b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java
--
diff --git 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java
 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java
index 65a5a19..d8734a1 100644
--- 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java
+++ 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java
@@ -21,11 +21,13 @@ package org.apache.beam.runners.direct;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Supplier;
 import com.google.common.base.Suppliers;
+import java.io.IOException;
 import java.io.Serializable;
 import java.util.Collections;
 import java.util.Map;
 import java.util.concurrent.ThreadLocalRandom;
 import org.apache.beam.runners.core.construction.PTransformReplacements;
+import org.apache.beam.runners.core.construction.WriteFilesTranslation;
 import org.apache.beam.sdk.io.WriteFiles;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
@@ -43,23 +45,33 @@ import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
 
 /**
- * A {@link PTransformOverrideFactory} that overrides {@link WriteFiles}
- * {@link PTransform PTransforms} with an unspecified number of shards with a 
write with a
- * specified number of

[1/5] beam git commit: Port DirectRunner TestStream override to SDK-agnostic APIs

2017-06-09 Thread kenn

Repository: beam
Updated Branches:
  refs/heads/master b4c77167f -> 1597f3ca6


Port DirectRunner TestStream override to SDK-agnostic APIs


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/eaaf45fa
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/eaaf45fa
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/eaaf45fa

Branch: refs/heads/master
Commit: eaaf45fa33d500a9f0fd0c2861aac4889ee5086c
Parents: ed6bd18
Author: Kenneth Knowles 
Authored: Thu Jun 8 13:39:32 2017 -0700
Committer: Kenneth Knowles 
Committed: Fri Jun 9 19:56:52 2017 -0700

--
 .../construction/TestStreamTranslation.java | 49 +++-
 .../direct/TestStreamEvaluatorFactory.java  | 20 ++--
 .../org/apache/beam/sdk/testing/TestStream.java | 12 +
 3 files changed, 75 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/eaaf45fa/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java
--
diff --git 
a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java
 
b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java
index 90e6304..515de57 100644
--- 
a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java
+++ 
b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java
@@ -18,6 +18,9 @@
 
 package org.apache.beam.runners.core.construction;
 
+import static com.google.common.base.Preconditions.checkArgument;
+import static 
org.apache.beam.runners.core.construction.PTransformTranslation.TEST_STREAM_TRANSFORM_URN;
+
 import com.google.auto.service.AutoService;
 import com.google.protobuf.Any;
 import com.google.protobuf.ByteString;
@@ -33,6 +36,8 @@ import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.TestStream;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.CoderUtils;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TimestampedValue;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
@@ -57,6 +62,48 @@ public class TestStreamTranslation {
 return builder.build();
   }
 
+  private static TestStream fromProto(
+  RunnerApi.TestStreamPayload testStreamPayload, RunnerApi.Components 
components)
+  throws IOException {
+
+Coder coder =
+(Coder)
+CoderTranslation.fromProto(
+components.getCodersOrThrow(testStreamPayload.getCoderId()), 
components);
+
+List events = new ArrayList<>();
+
+for (RunnerApi.TestStreamPayload.Event event : 
testStreamPayload.getEventsList()) {
+  events.add(fromProto(event, coder));
+}
+return TestStream.fromRawEvents(coder, events);
+  }
+
+  /**
+   * Converts an {@link AppliedPTransform}, which may be a rehydrated 
transform or an original
+   * {@link TestStream}, to a {@link TestStream}.
+   */
+  public static  TestStream getTestStream(
+  AppliedPTransform> application)
+  throws IOException {
+// For robustness, we don't take this shortcut:
+// if (application.getTransform() instanceof TestStream) {
+//   return application.getTransform()
+// }
+
+SdkComponents sdkComponents = SdkComponents.create();
+RunnerApi.PTransform transformProto = 
PTransformTranslation.toProto(application, sdkComponents);
+checkArgument(
+TEST_STREAM_TRANSFORM_URN.equals(transformProto.getSpec().getUrn()),
+"Attempt to get %s from a transform with wrong URN %s",
+TestStream.class.getSimpleName(),
+transformProto.getSpec().getUrn());
+RunnerApi.TestStreamPayload testStreamPayload =
+
transformProto.getSpec().getParameter().unpack(RunnerApi.TestStreamPayload.class);
+
+return (TestStream) fromProto(testStreamPayload, 
sdkComponents.toComponents());
+  }
+
   static  RunnerApi.TestStreamPayload.Event toProto(TestStream.Event 
event, Coder coder)
   throws IOException {
 switch (event.getType()) {
@@ -130,7 +177,7 @@ public class TestStreamTranslation {
   static class TestStreamTranslator implements 
TransformPayloadTranslator {
 @Override
 public String getUrn(TestStream transform) {
-  return PTransformTranslation.TEST_STREAM_TRANSFORM_URN;
+  return TEST_STREAM_TRANSFORM_URN;
 }
 
 @Override

[4/5] beam git commit: Port DirectGroupByKey to SDK-agnostic APIs

2017-06-09 Thread kenn

Port DirectGroupByKey to SDK-agnostic APIs


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/02dbaefd
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/02dbaefd
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/02dbaefd

Branch: refs/heads/master
Commit: 02dbaefd2bbad0f0ff0b87469d184137b220fae7
Parents: 8c5b57e
Author: Kenneth Knowles 
Authored: Fri May 26 14:27:23 2017 -0700
Committer: Kenneth Knowles 
Committed: Fri Jun 9 19:56:52 2017 -0700

--
 .../apache/beam/runners/direct/DirectGroupByKey.java  | 13 +++--
 .../direct/DirectGroupByKeyOverrideFactory.java   | 14 +++---
 2 files changed, 18 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/02dbaefd/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
--
diff --git 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
index 2fc0dd4..06b8e29 100644
--- 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
+++ 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
@@ -36,13 +36,17 @@ import org.apache.beam.sdk.values.WindowingStrategy;
 
 class DirectGroupByKey
 extends ForwardingPTransform>, PCollection>> {
-  private final GroupByKey original;
+  private final PTransform>, PCollection>> original;
 
   static final String DIRECT_GBKO_URN = 
"urn:beam:directrunner:transforms:gbko:v1";
   static final String DIRECT_GABW_URN = 
"urn:beam:directrunner:transforms:gabw:v1";
+  private final WindowingStrategy outputWindowingStrategy;
 
-  DirectGroupByKey(GroupByKey from) {
-this.original = from;
+  DirectGroupByKey(
+  PTransform>, PCollection>> 
original,
+  WindowingStrategy outputWindowingStrategy) {
+this.original = original;
+this.outputWindowingStrategy = outputWindowingStrategy;
   }
 
   @Override
@@ -57,9 +61,6 @@ class DirectGroupByKey
 // key/value input elements and the window merge operation of the
 // window function associated with the input PCollection.
 WindowingStrategy inputWindowingStrategy = 
input.getWindowingStrategy();
-// Update the windowing strategy as appropriate.
-WindowingStrategy outputWindowingStrategy =
-original.updateWindowingStrategy(inputWindowingStrategy);
 
 // By default, implement GroupByKey via a series of lower-level operations.
 return input

http://git-wip-us.apache.org/repos/asf/beam/blob/02dbaefd/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
--
diff --git 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
index c2eb5e7..9c2de3d 100644
--- 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
+++ 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
@@ -17,26 +17,34 @@
  */
 package org.apache.beam.runners.direct;
 
+import com.google.common.collect.Iterables;
 import org.apache.beam.runners.core.construction.PTransformReplacements;
 import 
org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
 import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 
 /** A {@link PTransformOverrideFactory} for {@link GroupByKey} PTransforms. */
 final class DirectGroupByKeyOverrideFactory
 extends SingleInputOutputOverrideFactory<
-PCollection>, PCollection>, GroupByKey> {
+PCollection>, PCollection>,
+PTransform>, PCollection>, PCollection>>
   getReplacementTransform(
   AppliedPTransform<
-  PCollection>, PCollection>, 
GroupByKey>
+  PCollection>, PCollection>,
+

[3/5] beam git commit: Port ViewOverrideFactory to SDK-agnostic APIs

2017-06-09 Thread kenn

Port ViewOverrideFactory to SDK-agnostic APIs


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/8c5b57ea
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/8c5b57ea
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/8c5b57ea

Branch: refs/heads/master
Commit: 8c5b57ea8445cd50a35c6dffb460dcf0f426e700
Parents: b4c7716
Author: Kenneth Knowles 
Authored: Fri May 26 14:26:55 2017 -0700
Committer: Kenneth Knowles 
Committed: Fri Jun 9 19:56:52 2017 -0700

--
 .../CreatePCollectionViewTranslation.java   |  4 +-
 .../runners/direct/ViewOverrideFactory.java | 48 
 .../direct/ViewEvaluatorFactoryTest.java|  3 +-
 .../runners/direct/ViewOverrideFactoryTest.java | 23 --
 .../beam/sdk/values/PCollectionViews.java   | 10 
 5 files changed, 62 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/8c5b57ea/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java
--
diff --git 
a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java
 
b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java
index aa24909..8fc99b9 100644
--- 
a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java
+++ 
b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java
@@ -56,8 +56,8 @@ public class CreatePCollectionViewTranslation {
   @Deprecated
   public static  PCollectionView getView(
   AppliedPTransform<
-  PCollection, PCollectionView,
-  PTransform>
+  PCollection, PCollection,
+  PTransform>
   application)
   throws IOException {
 

http://git-wip-us.apache.org/repos/asf/beam/blob/8c5b57ea/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
--
diff --git 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
index 06a7388..5dcf016 100644
--- 
a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
+++ 
b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
@@ -18,8 +18,9 @@
 
 package org.apache.beam.runners.direct;
 
+import java.io.IOException;
 import java.util.Map;
-import org.apache.beam.runners.core.construction.ForwardingPTransform;
+import 
org.apache.beam.runners.core.construction.CreatePCollectionViewTranslation;
 import org.apache.beam.runners.core.construction.PTransformReplacements;
 import 
org.apache.beam.runners.core.construction.PTransformTranslation.RawPTransform;
 import org.apache.beam.runners.core.construction.ReplacementOutputs;
@@ -43,16 +44,30 @@ import org.apache.beam.sdk.values.TupleTag;
  */
 class ViewOverrideFactory
 implements PTransformOverrideFactory<
-PCollection, PCollection, CreatePCollectionView> {
+PCollection, PCollection,
+PTransform> {
 
   @Override
   public PTransformReplacement 
getReplacementTransform(
   AppliedPTransform<
-  PCollection, PCollection, 
CreatePCollectionView>
+  PCollection, PCollection,
+  PTransform>
   transform) {
-return PTransformReplacement.of(
+
+PCollectionView view;
+try {
+  view = CreatePCollectionViewTranslation.getView(transform);
+} catch (IOException exc) {
+  throw new RuntimeException(
+  String.format(
+  "Could not extract %s from transform %s",
+  PCollectionView.class.getSimpleName(), transform),
+  exc);
+}
+
+  return PTransformReplacement.of(
 PTransformReplacements.getSingletonMainInput(transform),
-new GroupAndWriteView<>(transform.getTransform()));
+new GroupAndWriteView(view));
   }
 
   @Override
@@ -63,11 +78,11 @@ class ViewOverrideFactory
 
   /** The {@link DirectRunner} composite override for {@link 
CreatePCollectionView}. */
   static class GroupAndWriteView
-  extends ForwardingPTransform {
-private final

[GitHub] beam pull request #3338: [BEAM-2371] Port some DirectRunner overrides to SDK...

2017-06-09 Thread asfgit

Github user asfgit closed the pull request at:

https://github.com/apache/beam/pull/3338


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[5/5] beam git commit: This closes #3338: [BEAM-2371] Port some DirectRunner overrides to SDK-agnostic APIs

2017-06-09 Thread kenn

This closes #3338: [BEAM-2371] Port some DirectRunner overrides to SDK-agnostic 
APIs

  Port DirectRunner TestStream override to SDK-agnostic APIs
  Port DirectRunner WriteFiles override to SDK-agnostic APIs
  Port DirectGroupByKey to SDK-agnostic APIs
  Port ViewOverrideFactory to SDK-agnostic APIs


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/1597f3ca
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/1597f3ca
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/1597f3ca

Branch: refs/heads/master
Commit: 1597f3ca64558f0099237aeb618b144e132ddcc6
Parents: b4c7716 eaaf45f
Author: Kenneth Knowles 
Authored: Fri Jun 9 19:57:17 2017 -0700
Committer: Kenneth Knowles 
Committed: Fri Jun 9 19:57:17 2017 -0700

--
 .../CreatePCollectionViewTranslation.java   |  4 +-
 .../core/construction/PTransformMatchers.java   | 17 +--
 .../construction/TestStreamTranslation.java | 49 +++-
 .../beam/runners/direct/DirectGroupByKey.java   | 13 +++---
 .../direct/DirectGroupByKeyOverrideFactory.java | 14 --
 .../direct/TestStreamEvaluatorFactory.java  | 20 ++--
 .../runners/direct/ViewOverrideFactory.java | 48 +++
 .../direct/WriteWithShardingFactory.java| 30 
 .../direct/ViewEvaluatorFactoryTest.java|  3 +-
 .../runners/direct/ViewOverrideFactoryTest.java | 23 +++--
 .../direct/WriteWithShardingFactoryTest.java| 26 +++
 .../org/apache/beam/sdk/testing/TestStream.java | 12 +
 .../beam/sdk/values/PCollectionViews.java   | 10 
 13 files changed, 207 insertions(+), 62 deletions(-)
--

[GitHub] beam pull request #3337: Include Additional PTransform Inputs in Transform N...

2017-06-09 Thread tgroh

GitHub user tgroh opened a pull request:

https://github.com/apache/beam/pull/3337

Include Additional PTransform Inputs in Transform Nodes

Be sure to do all of the following to help us incorporate your contribution
quickly and easily:

 - [ ] Make sure the PR title is formatted like:
   `[BEAM-] Description of pull request`
 - [ ] Make sure tests pass via `mvn clean verify`.
 - [ ] Replace `` in the title with the actual Jira issue
   number, if there is one.
 - [ ] If this contribution is large, please file an Apache
   [Individual Contributor License 
Agreement](https://www.apache.org/licenses/icla.pdf).

---


You can merge this pull request into a Git repository by running:

$ git pull https://github.com/tgroh/beam all_inputs

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/beam/pull/3337.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #3337


commit 2d9f528de54fd648bc51ca07287a5b972c3caa5b
Author: Thomas Groh 
Date:   2017-06-06T23:15:19Z

Visit a Transform Hierarchy in Topological Order

This reverts commit 6ad6433ec0c02aec8656e9e3b27f6e0f974f8ece.

commit 2656abecfdef8dd6011ba1578db7d4b9741962c2
Author: Thomas Groh 
Date:   2017-06-07T00:00:09Z

Roll-forward Include Additional PTransform inputs in Transform Nodes

Update DirectGraph to have All and Non-Additional Inputs




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[jira] [Commented] (BEAM-2429) Conflicting filesystems with used of HadoopFileSystem

2017-06-09 Thread Davor Bonaci (JIRA)


[ 
https://issues.apache.org/jira/browse/BEAM-2429?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16044646#comment-16044646
 ] 

Davor Bonaci commented on BEAM-2429:


[~flaviocf], could you assist please?

> Conflicting filesystems with used of HadoopFileSystem
> -
>
> Key: BEAM-2429
> URL: https://issues.apache.org/jira/browse/BEAM-2429
> Project: Beam
>  Issue Type: Bug
>  Components: sdk-java-extensions
>Affects Versions: 2.0.0
>Reporter: François Wagner
>Assignee: Flavio Fiszman
>
> I'm facing issue when trying to use HadoopFileSystem in my pipeline. It looks 
> like HadoopFileSystem is registring itself under the `file` schema 
> (https://github.com/apache/beam/pull/2777/files#diff-330bd0854dcab6037ef0e52c05d68eb2L79),
>  hence the following Exception is thrown when trying to register 
> HadoopFileSystem.
> java.lang.IllegalStateException: Scheme: [file] has conflicting filesystems: 
> [org.apache.beam.sdk.io.LocalFileSystem, 
> org.apache.beam.sdk.io.hdfs.HadoopFileSystem]
>   at 
> org.apache.beam.sdk.io.FileSystems.verifySchemesAreUnique(FileSystems.java:498)
> What is the correct way to handle `hdfs` url out of the box with TextIO & 
> AvroIO ?
> {code:java}
> String[] args = new String[]{
> "--hdfsConfiguration=[{\"dfs.client.use.datanode.hostname\": 
> \"true\"}]"};
> HadoopFileSystemOptions options = PipelineOptionsFactory
> .fromArgs(args)
> .withValidation()
> .as(HadoopFileSystemOptions.class);
> Pipeline pipeline = Pipeline.create(options); 
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

[jira] [Assigned] (BEAM-2429) Conflicting filesystems with used of HadoopFileSystem

2017-06-09 Thread Davor Bonaci (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-2429?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Davor Bonaci reassigned BEAM-2429:
--

Assignee: Flavio Fiszman  (was: Davor Bonaci)

> Conflicting filesystems with used of HadoopFileSystem
> -
>
> Key: BEAM-2429
> URL: https://issues.apache.org/jira/browse/BEAM-2429
> Project: Beam
>  Issue Type: Bug
>  Components: sdk-java-extensions
>Affects Versions: 2.0.0
>Reporter: François Wagner
>Assignee: Flavio Fiszman
>
> I'm facing issue when trying to use HadoopFileSystem in my pipeline. It looks 
> like HadoopFileSystem is registring itself under the `file` schema 
> (https://github.com/apache/beam/pull/2777/files#diff-330bd0854dcab6037ef0e52c05d68eb2L79),
>  hence the following Exception is thrown when trying to register 
> HadoopFileSystem.
> java.lang.IllegalStateException: Scheme: [file] has conflicting filesystems: 
> [org.apache.beam.sdk.io.LocalFileSystem, 
> org.apache.beam.sdk.io.hdfs.HadoopFileSystem]
>   at 
> org.apache.beam.sdk.io.FileSystems.verifySchemesAreUnique(FileSystems.java:498)
> What is the correct way to handle `hdfs` url out of the box with TextIO & 
> AvroIO ?
> {code:java}
> String[] args = new String[]{
> "--hdfsConfiguration=[{\"dfs.client.use.datanode.hostname\": 
> \"true\"}]"};
> HadoopFileSystemOptions options = PipelineOptionsFactory
> .fromArgs(args)
> .withValidation()
> .as(HadoopFileSystemOptions.class);
> Pipeline pipeline = Pipeline.create(options); 
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

[jira] [Commented] (BEAM-2429) Conflicting filesystems with used of HadoopFileSystem

2017-06-09 Thread Flavio Fiszman (JIRA)


[ 
https://issues.apache.org/jira/browse/BEAM-2429?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16044757#comment-16044757
 ] 

Flavio Fiszman commented on BEAM-2429:
--

Hi François! In your hdfsConfiguration, you're specifying 
"use.datanode.hostname" for your use case. On top of that, you should also 
specify the field "fs.defaultFS" with hdfs:///, specifying the whole URI. You 
can just add that field to the map.

This is a related question for reference, and notice how fs.defaultFS is the 
new name for fs.default.name, if you see both of them appearing online. Let me 
know if you have any questions, thanks!

> Conflicting filesystems with used of HadoopFileSystem
> -
>
> Key: BEAM-2429
> URL: https://issues.apache.org/jira/browse/BEAM-2429
> Project: Beam
>  Issue Type: Bug
>  Components: sdk-java-extensions
>Affects Versions: 2.0.0
>Reporter: François Wagner
>Assignee: Flavio Fiszman
>
> I'm facing issue when trying to use HadoopFileSystem in my pipeline. It looks 
> like HadoopFileSystem is registring itself under the `file` schema 
> (https://github.com/apache/beam/pull/2777/files#diff-330bd0854dcab6037ef0e52c05d68eb2L79),
>  hence the following Exception is thrown when trying to register 
> HadoopFileSystem.
> java.lang.IllegalStateException: Scheme: [file] has conflicting filesystems: 
> [org.apache.beam.sdk.io.LocalFileSystem, 
> org.apache.beam.sdk.io.hdfs.HadoopFileSystem]
>   at 
> org.apache.beam.sdk.io.FileSystems.verifySchemesAreUnique(FileSystems.java:498)
> What is the correct way to handle `hdfs` url out of the box with TextIO & 
> AvroIO ?
> {code:java}
> String[] args = new String[]{
> "--hdfsConfiguration=[{\"dfs.client.use.datanode.hostname\": 
> \"true\"}]"};
> HadoopFileSystemOptions options = PipelineOptionsFactory
> .fromArgs(args)
> .withValidation()
> .as(HadoopFileSystemOptions.class);
> Pipeline pipeline = Pipeline.create(options); 
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Build failed in Jenkins: beam_PostCommit_Java_JDK_Versions_Test » JDK 1.7 (latest),beam #65

2017-06-09 Thread Apache Jenkins Server

See 


--
[...truncated 1.37 MB...]
2017-06-09\T\18:27:22.986 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/github/scopt/scopt_2.10/3.5.0/scopt_2.10-3.5.0.pom
 (2 KB at 62.9 KB/sec)
2017-06-09\T\18:27:22.988 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/twitter/chill_2.10/0.7.4/chill_2.10-0.7.4.pom
2017-06-09\T\18:27:23.014 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/twitter/chill_2.10/0.7.4/chill_2.10-0.7.4.pom
 (3 KB at 89.1 KB/sec)
2017-06-09\T\18:27:23.016 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.pom
2017-06-09\T\18:27:23.042 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.pom
 (2 KB at 76.8 KB/sec)
2017-06-09\T\18:27:23.044 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/twitter/chill-java/0.7.4/chill-java-0.7.4.pom
2017-06-09\T\18:27:23.069 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/twitter/chill-java/0.7.4/chill-java-0.7.4.pom
 (2 KB at 79.5 KB/sec)
2017-06-09\T\18:27:23.071 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-optimizer_2.10/1.3.0/flink-optimizer_2.10-1.3.0.pom
2017-06-09\T\18:27:23.098 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-optimizer_2.10/1.3.0/flink-optimizer_2.10-1.3.0.pom
 (5 KB at 162.2 KB/sec)
2017-06-09\T\18:27:23.103 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-streaming-java_2.10/1.3.0/flink-streaming-java_2.10-1.3.0.pom
2017-06-09\T\18:27:23.128 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-streaming-java_2.10/1.3.0/flink-streaming-java_2.10-1.3.0.pom
 (7 KB at 241.2 KB/sec)
2017-06-09\T\18:27:23.132 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/sling/org.apache.sling.commons.json/2.0.6/org.apache.sling.commons.json-2.0.6.pom
2017-06-09\T\18:27:23.158 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/sling/org.apache.sling.commons.json/2.0.6/org.apache.sling.commons.json-2.0.6.pom
 (4 KB at 121.7 KB/sec)
2017-06-09\T\18:27:23.160 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/sling/sling/10/sling-10.pom
2017-06-09\T\18:27:23.206 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/sling/sling/10/sling-10.pom (26 
KB at 544.9 KB/sec)
2017-06-09\T\18:27:23.207 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/apache/8/apache-8.pom
2017-06-09\T\18:27:23.233 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/apache/8/apache-8.pom (14 KB at 
527.9 KB/sec)
2017-06-09\T\18:27:23.248 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils_2.10/1.3.0/flink-test-utils_2.10-1.3.0.pom
2017-06-09\T\18:27:23.274 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils_2.10/1.3.0/flink-test-utils_2.10-1.3.0.pom
 (5 KB at 178.1 KB/sec)
2017-06-09\T\18:27:23.275 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-parent/1.3.0/flink-test-utils-parent-1.3.0.pom
2017-06-09\T\18:27:23.300 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-parent/1.3.0/flink-test-utils-parent-1.3.0.pom
 (2 KB at 56.3 KB/sec)
2017-06-09\T\18:27:23.305 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-junit/1.3.0/flink-test-utils-junit-1.3.0.pom
2017-06-09\T\18:27:23.331 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-junit/1.3.0/flink-test-utils-junit-1.3.0.pom
 (4 KB at 123.4 KB/sec)
2017-06-09\T\18:27:23.336 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/curator/curator-test/2.12.0/curator-test-2.12.0.pom
2017-06-09\T\18:27:23.362 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/curator/curator-test/2.12.0/curator-test-2.12.0.pom
 (5 KB at 155.8 KB/sec)
2017-06-09\T\18:27:23.363 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/curator/apache-curator/2.12.0/apache-curator-2.12.0.pom
2017-06-09\T\18:27:23.390 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/curator/apache-curator/2.12.0/apache-curator-2.12.0.pom
 (32 KB at 1169.3 KB/sec)
2017-06-09\T\18:27:23.393 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/zookeeper/zookeeper/3.4.8/zookeeper-3.4.8.pom
2017-06-09\T\18:27:23.420 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/zookeeper/zookeeper/3.4.8/zookeeper-3.4.8.pom
 (4 KB at 145.6 KB/sec)
2017-06-09\T\18:27:23.422 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/javassist/javassist/3.18.1-GA/javassist-3.18.1-GA.pom

[1/2] beam git commit: Fix compile error occurs in some JDKs

2017-06-09 Thread tgroh

Repository: beam
Updated Branches:
  refs/heads/master 1c6861f22 -> 97957ea0e


Fix compile error occurs in some JDKs


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b7ae7ecf
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b7ae7ecf
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b7ae7ecf

Branch: refs/heads/master
Commit: b7ae7ecffcd08b6a0ccc8296210d36b90306c171
Parents: 1c6861f
Author: Mark Liu 
Authored: Wed Jun 7 16:27:34 2017 -0700
Committer: Thomas Groh 
Committed: Fri Jun 9 10:54:12 2017 -0700

--
 .../runners/flink/FlinkStreamingTransformTranslators.java   | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/b7ae7ecf/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
--
diff --git 
a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
 
b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
index ef46b63..fef32de 100644
--- 
a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
+++ 
b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
@@ -363,8 +363,13 @@ class FlinkStreamingTransformTranslators {
   Map> tagsToOutputTags = 
Maps.newHashMap();
   for (Map.Entry entry : outputs.entrySet()) {
 if (!tagsToOutputTags.containsKey(entry.getKey())) {
-  tagsToOutputTags.put(entry.getKey(), new 
OutputTag<>(entry.getKey().getId(),
-  (TypeInformation) context.getTypeInfo((PCollection) 
entry.getValue(;
+  tagsToOutputTags.put(
+  entry.getKey(),
+  new OutputTag(
+  entry.getKey().getId(),
+  (TypeInformation) context.getTypeInfo((PCollection) 
entry.getValue())
+  )
+  );
 }
   }

[jira] [Created] (BEAM-2430) Java FnApiDoFnRunner to share across runners

2017-06-09 Thread Kenneth Knowles (JIRA)

Kenneth Knowles created BEAM-2430:
-

 Summary: Java FnApiDoFnRunner to share across runners
 Key: BEAM-2430
 URL: https://issues.apache.org/jira/browse/BEAM-2430
 Project: Beam
  Issue Type: New Feature
  Components: runner-core
Reporter: Kenneth Knowles


As the portability framework comes into focus, let's fill out the support code 
for making it easy to onboard a new runner.

There is some amount of using the Fn API that has to do only with the fact that 
a runner is implemented in Java, and is not specific to that runner. This 
should be part of the runners-core library, and designed so that a runner can 
set it up however it likes, and just pass elements without having to explicitly 
manage things like requests, responses, protos, and coders.




--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

[GitHub] beam pull request #3320: Fix Flink compile error occurs in some JDK versions

2017-06-09 Thread asfgit

Github user asfgit closed the pull request at:

https://github.com/apache/beam/pull/3320


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[jira] [Created] (BEAM-2431) Model Runner interactions in RPC layer for Runner API

2017-06-09 Thread Kenneth Knowles (JIRA)

Kenneth Knowles created BEAM-2431:
-

 Summary: Model Runner interactions in RPC layer for Runner API
 Key: BEAM-2431
 URL: https://issues.apache.org/jira/browse/BEAM-2431
 Project: Beam
  Issue Type: New Feature
  Components: beam-model-runner-api
Reporter: Kenneth Knowles
Assignee: Sourabh Bajaj


The "Runner API" today is actually just a definition of what constitutes a Beam 
pipeline. It needs to actually be a (very small) API.

This would allow e.g. a Java-based job launcher to respond to launch requests 
and state queries from a Python-based adapter.

The expected API would be something like a distillation of the APIs for 
PipelineRunner and PipelineResult (which is really "Job") via analyzing how 
these both look in Java and Python.



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Build failed in Jenkins: beam_PostCommit_Java_MavenInstall_Windows #113

2017-06-09 Thread Apache Jenkins Server

See 


--
[...truncated 2.64 MB...]
2017-06-09T18:35:36.463 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip/1.8/gossip-1.8.pom
 (12 KB at 1023.3 KB/sec)
2017-06-09T18:35:36.466 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/sonatype/forge/forge-parent/9/forge-parent-9.pom
2017-06-09T18:35:36.477 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/sonatype/forge/forge-parent/9/forge-parent-9.pom
 (13 KB at 1166.7 KB/sec)
2017-06-09T18:35:36.480 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip-core/1.8/gossip-core-1.8.pom
2017-06-09T18:35:36.489 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip-core/1.8/gossip-core-1.8.pom
 (3 KB at 241.4 KB/sec)
2017-06-09T18:35:36.492 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip-bootstrap/1.8/gossip-bootstrap-1.8.pom
2017-06-09T18:35:36.503 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/sonatype/gossip/gossip-bootstrap/1.8/gossip-bootstrap-1.8.pom
 (2 KB at 143.0 KB/sec)
2017-06-09T18:35:36.506 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/guava/guava/14.0.1/guava-14.0.1.pom
2017-06-09T18:35:36.515 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/guava/guava/14.0.1/guava-14.0.1.pom
 (6 KB at 583.3 KB/sec)
2017-06-09T18:35:36.518 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/google/guava/guava-parent/14.0.1/guava-parent-14.0.1.pom
2017-06-09T18:35:36.525 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/google/guava/guava-parent/14.0.1/guava-parent-14.0.1.pom
 (3 KB at 356.4 KB/sec)
2017-06-09T18:35:36.529 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-classworlds/2.4.2/plexus-classworlds-2.4.2.pom
2017-06-09T18:35:36.537 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-classworlds/2.4.2/plexus-classworlds-2.4.2.pom
 (4 KB at 428.3 KB/sec)
2017-06-09T18:35:36.542 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-interpolation/1.16/plexus-interpolation-1.16.pom
2017-06-09T18:35:36.550 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-interpolation/1.16/plexus-interpolation-1.16.pom
 (2 KB at 125.4 KB/sec)
2017-06-09T18:35:36.554 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/eclipse/aether/aether-api/0.9.0.M2/aether-api-0.9.0.M2.pom
2017-06-09T18:35:36.564 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/eclipse/aether/aether-api/0.9.0.M2/aether-api-0.9.0.M2.pom
 (2 KB at 169.6 KB/sec)
2017-06-09T18:35:36.569 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/gmaven/gmaven-adapter-api/2.0/gmaven-adapter-api-2.0.pom
2017-06-09T18:35:36.577 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/gmaven/gmaven-adapter-api/2.0/gmaven-adapter-api-2.0.pom
 (2 KB at 220.5 KB/sec)
2017-06-09T18:35:36.583 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/gmaven/gmaven-adapter-impl/2.0/gmaven-adapter-impl-2.0.pom
2017-06-09T18:35:36.592 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/gmaven/gmaven-adapter-impl/2.0/gmaven-adapter-impl-2.0.pom
 (3 KB at 294.9 KB/sec)
2017-06-09T18:35:36.598 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/codehaus/groovy/groovy-all/2.1.5/groovy-all-2.1.5.pom
2017-06-09T18:35:36.610 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/codehaus/groovy/groovy-all/2.1.5/groovy-all-2.1.5.pom
 (18 KB at 1468.0 KB/sec)
2017-06-09T18:35:36.615 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant/1.8.4/ant-1.8.4.pom
2017-06-09T18:35:36.624 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant/1.8.4/ant-1.8.4.pom (10 
KB at 1047.5 KB/sec)
2017-06-09T18:35:36.629 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant-parent/1.8.4/ant-parent-1.8.4.pom
2017-06-09T18:35:36.639 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant-parent/1.8.4/ant-parent-1.8.4.pom
 (5 KB at 445.4 KB/sec)
2017-06-09T18:35:36.644 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant-launcher/1.8.4/ant-launcher-1.8.4.pom
2017-06-09T18:35:36.654 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/ant/ant-launcher/1.8.4/ant-launcher-1.8.4.pom
 (3 KB at 230.6 KB/sec)
2017-06-09T18:35:36.658 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/fusesource/jansi/jansi/1.11/jansi-1.11.pom
2017-06-09T18:35:36.669 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/fusesource/jansi/jansi/1.11/jansi-1.11.pom
 (4 KB at 319.7 KB/sec)
2017-06-09T18:35:36.672 [INFO] Downloading:

[jira] [Resolved] (BEAM-2364) PCollectionView should not be a PValue, it should expand to its underlying PCollection

2017-06-09 Thread Thomas Groh (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-2364?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Thomas Groh resolved BEAM-2364.
---
   Resolution: Fixed
Fix Version/s: 2.1.0

The expansion of PCollectionView has changed to be a PCollection, and Transform 
Hierarchy now fully expands all values until it only has PCollections. There 
can be some cleanup work done, but (in order to not break any custom POutputs) 
this is effectively complete.

> PCollectionView should not be a PValue, it should expand to its underlying 
> PCollection
> --
>
> Key: BEAM-2364
> URL: https://issues.apache.org/jira/browse/BEAM-2364
> Project: Beam
>  Issue Type: Bug
>  Components: sdk-java-core
>Reporter: Kenneth Knowles
>Assignee: Thomas Groh
>  Labels: backward-incompatible
> Fix For: 2.1.0
>
>
> This bug has been present for a very long time.
> It is a change to {{@Internal}} details but is going to be 
> backwards-incompatible in a couple ways there, because backwards compatible 
> behavior is incorrect. But we need to be very careful with the surrounding 
> logic.
> The particular motivating need is that as long as a PCollectionView expands 
> to only itself, the outputs for surgery and serialization are not correct. 
> There may be a solution involving hardcoded logic, TBD.



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

[jira] [Assigned] (BEAM-2390) allow user to use .setTimePartitioning in BigQueryIO.write

2017-06-09 Thread Kenneth Knowles (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-2390?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Kenneth Knowles reassigned BEAM-2390:
-

Assignee: Eric Johston  (was: Kenneth Knowles)

> allow user to use .setTimePartitioning in BigQueryIO.write
> --
>
> Key: BEAM-2390
> URL: https://issues.apache.org/jira/browse/BEAM-2390
> Project: Beam
>  Issue Type: Improvement
>  Components: beam-model-runner-api
>Affects Versions: 2.0.0
>Reporter: Eric Johston
>Assignee: Eric Johston
>  Labels: easyfix, features, newbie
> Fix For: 2.0.0
>
>   Original Estimate: 2h
>  Remaining Estimate: 2h
>
> Currently when writing to a table with BigQueryIO sink, there is no way to 
> create a new table that is date partitioned. This would be very useful, since 
> currently the only way to do this is  by manually creating a table ahead of 
> time. We should be able to leverage the automatic table creation 
> functionality for date partitioned tables.
> The best way to do this would be to have a withTimePartitioning method in the 
> BigQueryIO class.



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

[jira] [Updated] (BEAM-2390) allow user to use .setTimePartitioning in BigQueryIO.write

2017-06-09 Thread Kenneth Knowles (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-2390?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Kenneth Knowles updated BEAM-2390:
--
Component/s: (was: beam-model-runner-api)
 sdk-java-gcp

> allow user to use .setTimePartitioning in BigQueryIO.write
> --
>
> Key: BEAM-2390
> URL: https://issues.apache.org/jira/browse/BEAM-2390
> Project: Beam
>  Issue Type: Improvement
>  Components: sdk-java-gcp
>Affects Versions: 2.0.0
>Reporter: Eric Johston
>Assignee: Eric Johston
>  Labels: easyfix, features, newbie
> Fix For: 2.1.0
>
>   Original Estimate: 2h
>  Remaining Estimate: 2h
>
> Currently when writing to a table with BigQueryIO sink, there is no way to 
> create a new table that is date partitioned. This would be very useful, since 
> currently the only way to do this is  by manually creating a table ahead of 
> time. We should be able to leverage the automatic table creation 
> functionality for date partitioned tables.
> The best way to do this would be to have a withTimePartitioning method in the 
> BigQueryIO class.



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Jenkins build is back to stable : beam_PostCommit_Java_ValidatesRunner_Spark #2321

2017-06-09 Thread Apache Jenkins Server

See

Jenkins build became unstable: beam_PostCommit_Java_MavenInstall #4070

2017-06-09 Thread Apache Jenkins Server

See

Jenkins build is back to normal : beam_PostCommit_Python_Verify #2454

2017-06-09 Thread Apache Jenkins Server

See

[jira] [Commented] (BEAM-2429) Conflicting filesystems with used of HadoopFileSystem

2017-06-09 Thread Flavio Fiszman (JIRA)


[ 
https://issues.apache.org/jira/browse/BEAM-2429?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16044762#comment-16044762
 ] 

Flavio Fiszman commented on BEAM-2429:
--

Forgot to attach the related question with some context, here it is: 
https://stackoverflow.com/questions/12391226/hadoop-hdfs-points-to-file-not-hdfs?rq=1

> Conflicting filesystems with used of HadoopFileSystem
> -
>
> Key: BEAM-2429
> URL: https://issues.apache.org/jira/browse/BEAM-2429
> Project: Beam
>  Issue Type: Bug
>  Components: sdk-java-extensions
>Affects Versions: 2.0.0
>Reporter: François Wagner
>Assignee: Flavio Fiszman
>
> I'm facing issue when trying to use HadoopFileSystem in my pipeline. It looks 
> like HadoopFileSystem is registring itself under the `file` schema 
> (https://github.com/apache/beam/pull/2777/files#diff-330bd0854dcab6037ef0e52c05d68eb2L79),
>  hence the following Exception is thrown when trying to register 
> HadoopFileSystem.
> java.lang.IllegalStateException: Scheme: [file] has conflicting filesystems: 
> [org.apache.beam.sdk.io.LocalFileSystem, 
> org.apache.beam.sdk.io.hdfs.HadoopFileSystem]
>   at 
> org.apache.beam.sdk.io.FileSystems.verifySchemesAreUnique(FileSystems.java:498)
> What is the correct way to handle `hdfs` url out of the box with TextIO & 
> AvroIO ?
> {code:java}
> String[] args = new String[]{
> "--hdfsConfiguration=[{\"dfs.client.use.datanode.hostname\": 
> \"true\"}]"};
> HadoopFileSystemOptions options = PipelineOptionsFactory
> .fromArgs(args)
> .withValidation()
> .as(HadoopFileSystemOptions.class);
> Pipeline pipeline = Pipeline.create(options); 
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Jenkins build became unstable: beam_PostCommit_Java_ValidatesRunner_Spark #2320

2017-06-09 Thread Apache Jenkins Server

See

[jira] [Resolved] (BEAM-2360) Add guide for writing a Beam runner

2017-06-09 Thread Kenneth Knowles (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-2360?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Kenneth Knowles resolved BEAM-2360.
---
   Resolution: Fixed
Fix Version/s: Not applicable

> Add guide for writing a Beam runner
> ---
>
> Key: BEAM-2360
> URL: https://issues.apache.org/jira/browse/BEAM-2360
> Project: Beam
>  Issue Type: Wish
>  Components: website
>Reporter: Kenneth Knowles
>Assignee: Kenneth Knowles
> Fix For: Not applicable
>
>
> We now have enough runners and enough interest in more that we could really 
> use a guide for authoring a Beam runner.



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

[1/3] beam-site git commit: Regenerate website

2017-06-09 Thread kenn

Repository: beam-site
Updated Branches:
  refs/heads/asf-site 93c7c5f11 -> 5c993c612


http://git-wip-us.apache.org/repos/asf/beam-site/blob/5c993c61/content/contribute/source-repository/index.html
--
diff --git a/content/contribute/source-repository/index.html 
b/content/contribute/source-repository/index.html
index d46338f..6195346 100644
--- a/content/contribute/source-repository/index.html
+++ b/content/contribute/source-repository/index.html
@@ -102,6 +102,7 @@
 Testing Guide
 Release Guide
 PTransform Style 
Guide
+Runner Authoring 
Guide
 
 Technical References
 Design 
Principles

http://git-wip-us.apache.org/repos/asf/beam-site/blob/5c993c61/content/contribute/team/index.html
--
diff --git a/content/contribute/team/index.html 
b/content/contribute/team/index.html
index 8ae9462..ade8a0c 100644
--- a/content/contribute/team/index.html
+++ b/content/contribute/team/index.html
@@ -102,6 +102,7 @@
 Testing Guide
 Release Guide
 PTransform Style 
Guide
+Runner Authoring 
Guide
 
 Technical References
 Design 
Principles

http://git-wip-us.apache.org/repos/asf/beam-site/blob/5c993c61/content/contribute/testing/index.html
--
diff --git a/content/contribute/testing/index.html 
b/content/contribute/testing/index.html
index cca76dd..8e7cec1 100644
--- a/content/contribute/testing/index.html
+++ b/content/contribute/testing/index.html
@@ -102,6 +102,7 @@
 Testing Guide
 Release Guide
 PTransform Style 
Guide
+Runner Authoring 
Guide
 
 Technical References
 Design 
Principles

http://git-wip-us.apache.org/repos/asf/beam-site/blob/5c993c61/content/contribute/work-in-progress/index.html
--
diff --git a/content/contribute/work-in-progress/index.html 
b/content/contribute/work-in-progress/index.html
index 598ae04..aa2bed6 100644
--- a/content/contribute/work-in-progress/index.html
+++ b/content/contribute/work-in-progress/index.html
@@ -102,6 +102,7 @@
 Testing Guide
 Release Guide
 PTransform Style 
Guide
+Runner Authoring 
Guide
 
 Technical References
 Design 
Principles

http://git-wip-us.apache.org/repos/asf/beam-site/blob/5c993c61/content/documentation/index.html
--
diff --git a/content/documentation/index.html b/content/documentation/index.html
index e6843a6..48a7f25 100644
--- a/content/documentation/index.html
+++ b/content/documentation/index.html
@@ -102,6 +102,7 @@
 Testing Guide
 Release Guide
 PTransform Style 
Guide
+Runner Authoring 
Guide
 
 Technical References
 Design 
Principles

http://git-wip-us.apache.org/repos/asf/beam-site/blob/5c993c61/content/documentation/io/authoring-java/index.html
--
diff --git a/content/documentation/io/authoring-java/index.html 
b/content/documentation/io/authoring-java/index.html
index 637300e..63f6291 100644
--- a/content/documentation/io/authoring-java/index.html
+++ b/content/documentation/io/authoring-java/index.html
@@ -102,6 +102,7 @@
 Testing Guide
 Release Guide
 PTransform Style 
Guide
+Runner Authoring 
Guide
 
 Technical References
 Design 
Principles

http://git-wip-us.apache.org/repos/asf/beam-site/blob/5c993c61/content/documentation/io/authoring-overview/index.html
--
diff --git a/content/documentation/io/authoring-overview/index.html 
b/content/documentation/io/authoring-overview/index.html
index a50e929..38bd5ca 100644
--- a/content/documentation/io/authoring-overview/index.html
+++ b/content/documentation/io/authoring-overview/index.html
@@ -102,6 +102,7 @@
 Testing Guide
 Release Guide
 PTransform Style 
Guide
+Runner Authoring 
Guide
 
 Technical References
 Design 
Principles

http://git-wip-us.apache.org/repos/asf/beam-site/blob/5c993c61/content/documentation/io/authoring-python/index.html
--
diff --git a/content/documentation/io/authoring-python/index.html 
b/content/documentation/io/authoring-python/index.html
index 3d3c553..3d83937 100644
--- a/content/documentation/io/authoring-python/index.html
+++

[2/3] beam-site git commit: Regenerate website

2017-06-09 Thread kenn

http://git-wip-us.apache.org/repos/asf/beam-site/blob/5c993c61/content/contribute/runner-guide/index.html
--
diff --git a/content/contribute/runner-guide/index.html 
b/content/contribute/runner-guide/index.html
new file mode 100644
index 000..2dc6917
--- /dev/null
+++ b/content/contribute/runner-guide/index.html
@@ -0,0 +1,1375 @@
+
+
+  
+  
+  
+  
+  Runner Authoring Guide
+  
+  https://fonts.googleapis.com/css?family=Roboto:100,300,400; 
rel="stylesheet">
+  
+  https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";>
+  
+  
+  https://beam.apache.org/contribute/runner-guide/; data-proofer-ignore>
+  
+  https://beam.apache.org/feed.xml;>
+  
+
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new 
Date();a=s.createElement(o),
+
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+
})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+ga('create', 'UA-73650088-1', 'auto');
+ga('send', 'pageview');
+  
+
+
+  
+
+
+  
+
+  
+
+  Toggle navigation
+  
+  
+  
+
+
+
+  
+
+  Get Started 
+  
+Beam Overview
+Quickstart - 
Java
+Quickstart - 
Python
+
+Example Walkthroughs
+WordCount
+Mobile 
Gaming
+
+Resources
+Downloads
+Support
+  
+
+
+  Documentation 
+  
+Using the Documentation
+
+Beam Concepts
+Programming 
Guide
+Additional 
Resources
+
+Pipeline Fundamentals
+Design Your 
Pipeline
+Create Your 
Pipeline
+Test 
Your Pipeline
+Pipeline I/O
+
+SDKs
+Java SDK
+Java SDK API Reference 
+
+Python SDK
+Python SDK API Reference 
+
+
+Runners
+Capability 
Matrix
+Direct Runner
+Apache Apex 
Runner
+Apache Flink 
Runner
+Apache Spark 
Runner
+Cloud Dataflow 
Runner
+  
+
+
+  Contribute 
+  
+Get Started Contributing
+
+Guides
+Contribution 
Guide
+Testing Guide
+Release Guide
+PTransform Style 
Guide
+Runner Authoring 
Guide
+
+Technical References
+Design 
Principles
+Ongoing 
Projects
+Source 
Repository
+
+Promotion
+Presentation 
Materials
+Logos and Design
+
+Maturity Model
+Team
+  
+
+
+Blog
+  
+  
+
+  https://www.apache.org/foundation/press/kit/feather_small.png; alt="Apache 
Logo" style="height:20px;">
+  
+http://www.apache.org/;>ASF Homepage
+http://www.apache.org/licenses/;>License
+http://www.apache.org/security/;>Security
+http://www.apache.org/foundation/thanks.html;>Thanks
+http://www.apache.org/foundation/sponsorship.html;>Sponsorship
+https://www.apache.org/foundation/policies/conduct;>Code of 
Conduct
+  
+
+  
+
+
+
+
+  Runner Authoring Guide
+
+This guide walks through how to implement a new runner. It is aimed at 
someone
+who has a data processing system and wants to use it to execute a Beam
+pipeline. The guide starts from the basics, to help you evaluate the work
+ahead. Then the sections become more and more detailed, to be a resource
+throughout the development of your runner.
+
+Topics covered:
+
+
+  Basics of the Beam model
+  Pipeline
+  PTransforms
+  PCollections
+  Bounded vs Unbounded
+  Timestamps
+  Watermarks
+  Windowed elements
+  Coder
+  Windowing Strategy
+
+  
+  User-Defined Functions 
(UDFs)
+  Runner
+
+  
+  Implementing the Beam 
Primitives
+  What if 
you havenât implemented some of these features?
+  Implementing the ParDo 
primitive
+  Bundles
+  The DoFn Lifecycle
+  DoFnRunner(s)
+  Side 
Inputs
+  State and Timers
+  Splittable DoFn
+
+  
+  Implementing 
the GroupByKey (and window) primitive
+  Group By Encoded Bytes
+  Window Merging
+  Implementing
 via GroupByKeyOnly + GroupAlsoByWindow
+  Dropping late data
+  Triggering
+  TimestampCombiner
+

[3/3] beam-site git commit: Regenerate website

2017-06-09 Thread kenn

Regenerate website


Project: http://git-wip-us.apache.org/repos/asf/beam-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam-site/commit/5c993c61
Tree: http://git-wip-us.apache.org/repos/asf/beam-site/tree/5c993c61
Diff: http://git-wip-us.apache.org/repos/asf/beam-site/diff/5c993c61

Branch: refs/heads/asf-site
Commit: 5c993c612fe47912d25c305e9ad15bbeee7c1ed1
Parents: 93c7c5f
Author: Kenneth Knowles 
Authored: Fri Jun 9 10:43:44 2017 -0700
Committer: Kenneth Knowles 
Committed: Fri Jun 9 10:43:44 2017 -0700

--
 .../2016/03/17/capability-matrix.html   |1 +
 .../2016/04/03/presentation-materials.html  |1 +
 .../sdk/2016/02/25/python-sdk-now-public.html   |1 +
 .../beam/release/2016/06/15/first-release.html  |1 +
 .../10/11/strata-hadoop-world-and-beam.html |1 +
 .../website/2016/02/22/beam-has-a-logo.html |1 +
 .../blog/2016/05/18/splitAtFraction-method.html |1 +
 .../05/27/where-is-my-pcollection-dot-map.html  |1 +
 .../06/13/flink-batch-runner-milestone.html |1 +
 content/blog/2016/08/03/six-months.html |1 +
 content/blog/2016/10/20/test-stream.html|1 +
 content/blog/2017/01/09/added-apex-runner.html  |1 +
 content/blog/2017/01/10/beam-graduates.html |1 +
 .../blog/2017/02/01/graduation-media-recap.html |1 +
 .../blog/2017/02/13/stateful-processing.html|1 +
 content/blog/2017/03/16/python-sdk-release.html |1 +
 .../2017/05/17/beam-first-stable-release.html   |1 +
 content/blog/index.html |1 +
 content/coming-soon.html|1 +
 .../contribute/contribution-guide/index.html|1 +
 content/contribute/design-principles/index.html |1 +
 content/contribute/index.html   |1 +
 content/contribute/logos/index.html |1 +
 content/contribute/maturity-model/index.html|1 +
 .../presentation-materials/index.html   |1 +
 .../ptransform-style-guide/index.html   |1 +
 content/contribute/release-guide/index.html |1 +
 content/contribute/runner-guide/index.html  | 1375 ++
 content/contribute/source-repository/index.html |1 +
 content/contribute/team/index.html  |1 +
 content/contribute/testing/index.html   |1 +
 content/contribute/work-in-progress/index.html  |1 +
 content/documentation/index.html|1 +
 .../documentation/io/authoring-java/index.html  |1 +
 .../io/authoring-overview/index.html|1 +
 .../io/authoring-python/index.html  |1 +
 .../documentation/io/built-in/hadoop/index.html |1 +
 content/documentation/io/built-in/index.html|1 +
 .../documentation/io/contributing/index.html|1 +
 content/documentation/io/io-toc/index.html  |1 +
 content/documentation/io/testing/index.html |1 +
 .../pipelines/create-your-pipeline/index.html   |1 +
 .../pipelines/design-your-pipeline/index.html   |1 +
 .../pipelines/test-your-pipeline/index.html |1 +
 .../documentation/programming-guide/index.html  |1 +
 content/documentation/resources/index.html  |1 +
 content/documentation/runners/apex/index.html   |1 +
 .../runners/capability-matrix/index.html|1 +
 .../documentation/runners/dataflow/index.html   |1 +
 content/documentation/runners/direct/index.html |1 +
 content/documentation/runners/flink/index.html  |1 +
 .../documentation/runners/gearpump/index.html   |1 +
 content/documentation/runners/spark/index.html  |1 +
 .../sdks/java-extensions/index.html |1 +
 content/documentation/sdks/java/index.html  |1 +
 .../sdks/python-custom-io/index.html|1 +
 .../python-pipeline-dependencies/index.html |1 +
 .../sdks/python-type-safety/index.html  |1 +
 content/documentation/sdks/python/index.html|1 +
 content/get-started/beam-overview/index.html|1 +
 content/get-started/downloads/index.html|1 +
 content/get-started/index.html  |1 +
 .../mobile-gaming-example/index.html|1 +
 content/get-started/quickstart-java/index.html  |1 +
 content/get-started/quickstart-py/index.html|1 +
 content/get-started/support/index.html  |1 +
 .../get-started/wordcount-example/index.html|1 +
 content/index.html  |1 +
 content/js/language-switch.js   |2 +-
 content/privacy_policy/index.html   |1 +
 70 files changed, 1444 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/beam-site/blob/5c993c61/content/beam/capability/2016/03/17/capability-matrix.html

Build failed in Jenkins: beam_PostCommit_Java_JDK_Versions_Test » OpenJDK 7 (on Ubuntu only),beam #65

2017-06-09 Thread Apache Jenkins Server

See 


--
[...truncated 1.37 MB...]
2017-06-09\T\18:35:58.023 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/scala-lang/scala-library/2.10.3/scala-library-2.10.3.pom
 (2 KB at 73.9 KB/sec)
2017-06-09\T\18:35:58.025 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/github/scopt/scopt_2.10/3.5.0/scopt_2.10-3.5.0.pom
2017-06-09\T\18:35:58.053 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/github/scopt/scopt_2.10/3.5.0/scopt_2.10-3.5.0.pom
 (2 KB at 58.4 KB/sec)
2017-06-09\T\18:35:58.055 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/twitter/chill_2.10/0.7.4/chill_2.10-0.7.4.pom
2017-06-09\T\18:35:58.082 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/twitter/chill_2.10/0.7.4/chill_2.10-0.7.4.pom
 (3 KB at 85.8 KB/sec)
2017-06-09\T\18:35:58.085 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.pom
2017-06-09\T\18:35:58.112 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.pom
 (2 KB at 73.9 KB/sec)
2017-06-09\T\18:35:58.114 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/com/twitter/chill-java/0.7.4/chill-java-0.7.4.pom
2017-06-09\T\18:35:58.141 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/com/twitter/chill-java/0.7.4/chill-java-0.7.4.pom
 (2 KB at 73.6 KB/sec)
2017-06-09\T\18:35:58.144 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-optimizer_2.10/1.3.0/flink-optimizer_2.10-1.3.0.pom
2017-06-09\T\18:35:58.171 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-optimizer_2.10/1.3.0/flink-optimizer_2.10-1.3.0.pom
 (5 KB at 156.2 KB/sec)
2017-06-09\T\18:35:58.177 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-streaming-java_2.10/1.3.0/flink-streaming-java_2.10-1.3.0.pom
2017-06-09\T\18:35:58.204 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-streaming-java_2.10/1.3.0/flink-streaming-java_2.10-1.3.0.pom
 (7 KB at 223.3 KB/sec)
2017-06-09\T\18:35:58.210 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/sling/org.apache.sling.commons.json/2.0.6/org.apache.sling.commons.json-2.0.6.pom
2017-06-09\T\18:35:58.238 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/sling/org.apache.sling.commons.json/2.0.6/org.apache.sling.commons.json-2.0.6.pom
 (4 KB at 113.0 KB/sec)
2017-06-09\T\18:35:58.240 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/sling/sling/10/sling-10.pom
2017-06-09\T\18:35:58.269 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/sling/sling/10/sling-10.pom (26 
KB at 864.3 KB/sec)
2017-06-09\T\18:35:58.271 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/apache/8/apache-8.pom
2017-06-09\T\18:35:58.302 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/apache/8/apache-8.pom (14 KB at 
442.7 KB/sec)
2017-06-09\T\18:35:58.322 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils_2.10/1.3.0/flink-test-utils_2.10-1.3.0.pom
2017-06-09\T\18:35:58.349 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils_2.10/1.3.0/flink-test-utils_2.10-1.3.0.pom
 (5 KB at 171.5 KB/sec)
2017-06-09\T\18:35:58.350 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-parent/1.3.0/flink-test-utils-parent-1.3.0.pom
2017-06-09\T\18:35:58.377 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-parent/1.3.0/flink-test-utils-parent-1.3.0.pom
 (2 KB at 52.1 KB/sec)
2017-06-09\T\18:35:58.384 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-junit/1.3.0/flink-test-utils-junit-1.3.0.pom
2017-06-09\T\18:35:58.412 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/flink/flink-test-utils-junit/1.3.0/flink-test-utils-junit-1.3.0.pom
 (4 KB at 114.6 KB/sec)
2017-06-09\T\18:35:58.419 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/curator/curator-test/2.12.0/curator-test-2.12.0.pom
2017-06-09\T\18:35:58.449 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/curator/curator-test/2.12.0/curator-test-2.12.0.pom
 (5 KB at 135.0 KB/sec)
2017-06-09\T\18:35:58.451 [INFO] Downloading: 
https://repo.maven.apache.org/maven2/org/apache/curator/apache-curator/2.12.0/apache-curator-2.12.0.pom
2017-06-09\T\18:35:58.490 [INFO] Downloaded: 
https://repo.maven.apache.org/maven2/org/apache/curator/apache-curator/2.12.0/apache-curator-2.12.0.pom
 (32 KB at 809.5 KB/sec)
2017-06-09\T\18:35:58.495 [INFO] Downloading:

[jira] [Assigned] (BEAM-2298) Java WordCount doesn't work in Window OS for glob expressions or file: prefixed paths

2017-06-09 Thread Thomas Groh (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-2298?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Thomas Groh reassigned BEAM-2298:
-

Assignee: (was: Thomas Groh)

> Java WordCount doesn't work in Window OS for glob expressions or file: 
> prefixed paths
> -
>
> Key: BEAM-2298
> URL: https://issues.apache.org/jira/browse/BEAM-2298
> Project: Beam
>  Issue Type: Bug
>  Components: sdk-java-core
>Reporter: Pei He
> Fix For: 2.1.0
>
>
> I am not able to build beam repo in Windows OS, so I copied the jar file from 
> my Mac.
> WordCount failed with the following cmd:
> java -cp beam-examples-java-2.0.0-jar-with-dependencies.jar
>  org.apache.beam.examples.WordCount --inputFile=input.txt --output=counts
> May 15, 2017 6:09:48 AM org.apache.beam.sdk.io.FileBasedSource 
> getEstimatedSizeB
> ytes
> INFO: Filepattern input.txt matched 1 files with total size 0
> May 15, 2017 6:09:48 AM org.apache.beam.sdk.io.FileBasedSource 
> expandFilePattern
> INFO: Matched 1 files for pattern input.txt
> May 15, 2017 6:09:48 AM org.apache.beam.sdk.io.FileBasedSource split
> INFO: Splitting filepattern input.txt into bundles of size 0 took 0 ms and 
> produ
> ced 1 files and 0 bundles
> May 15, 2017 6:09:48 AM org.apache.beam.sdk.io.WriteFiles$2 processElement
> INFO: Finalizing write operation 
> TextWriteOperation{tempDirectory=C:\Users\Pei\D
> esktop\.temp-beam-2017-05-135_13-09-48-1\, windowedWrites=false}.
> May 15, 2017 6:09:48 AM org.apache.beam.sdk.io.WriteFiles$2 processElement
> INFO: Creating 1 empty output shards in addition to 0 written for a total of 
> 1.
> Exception in thread "main" 
> org.apache.beam.sdk.Pipeline$PipelineExecutionExcepti
> on: java.lang.IllegalStateException: Unable to find registrar for c
> at 
> org.apache.beam.runners.direct.DirectRunner$DirectPipelineResult.wait
> UntilFinish(DirectRunner.java:322)
> at 
> org.apache.beam.runners.direct.DirectRunner$DirectPipelineResult.wait
> UntilFinish(DirectRunner.java:292)
> at 
> org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:200
> )
> at 
> org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:63)
> at org.apache.beam.sdk.Pipeline.run(Pipeline.java:295)
> at org.apache.beam.sdk.Pipeline.run(Pipeline.java:281)
> at org.apache.beam.examples.WordCount.main(WordCount.java:184)
> Caused by: java.lang.IllegalStateException: Unable to find registrar for c
> at 
> org.apache.beam.sdk.io.FileSystems.getFileSystemInternal(FileSystems.
> java:447)
> at org.apache.beam.sdk.io.FileSystems.match(FileSystems.java:111)
> at 
> org.apache.beam.sdk.io.FileSystems.matchResources(FileSystems.java:17
> 4)
> at 
> org.apache.beam.sdk.io.FileSystems.filterMissingFiles(FileSystems.jav
> a:367)
> at org.apache.beam.sdk.io.FileSystems.copy(FileSystems.java:251)
> at 
> org.apache.beam.sdk.io.FileBasedSink$WriteOperation.copyToOutputFiles
> (FileBasedSink.java:641)
> at 
> org.apache.beam.sdk.io.FileBasedSink$WriteOperation.finalize(FileBase
> dSink.java:529)
> at 
> org.apache.beam.sdk.io.WriteFiles$2.processElement(WriteFiles.java:59
> 2)



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Jenkins build is back to stable : beam_PostCommit_Java_MavenInstall #4071

2017-06-09 Thread Apache Jenkins Server

See

[GitHub] beam pull request #3332: Use beam.Map rather than beam.ParDo for PubSub enco...

2017-06-09 Thread asfgit

Github user asfgit closed the pull request at:

https://github.com/apache/beam/pull/3332


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[1/2] beam git commit: Closes #3332

2017-06-09 Thread robertwb

Repository: beam
Updated Branches:
  refs/heads/master 97957ea0e -> d2c4093ed


Closes #3332


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/d2c4093e
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/d2c4093e
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/d2c4093e

Branch: refs/heads/master
Commit: d2c4093ed2fd47e28b31898a17e28d0575aa9ff9
Parents: 97957ea fb61c54
Author: Robert Bradshaw 
Authored: Fri Jun 9 13:00:55 2017 -0700
Committer: Robert Bradshaw 
Committed: Fri Jun 9 13:00:55 2017 -0700

--
 sdks/python/apache_beam/io/gcp/pubsub.py  | 16 +++-
 sdks/python/apache_beam/io/gcp/pubsub_test.py | 10 --
 2 files changed, 3 insertions(+), 23 deletions(-)
--

[2/2] beam git commit: Use beam.Map rather than beam.ParDo for PubSub encoding.

2017-06-09 Thread robertwb

Use beam.Map rather than beam.ParDo for PubSub encoding.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/fb61c540
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/fb61c540
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/fb61c540

Branch: refs/heads/master
Commit: fb61c540bc15bafb959d7accb7c08f6a681f62ef
Parents: 97957ea
Author: Robert Bradshaw 
Authored: Thu Jun 8 15:01:53 2017 -0700
Committer: Robert Bradshaw 
Committed: Fri Jun 9 13:00:55 2017 -0700

--
 sdks/python/apache_beam/io/gcp/pubsub.py  | 16 +++-
 sdks/python/apache_beam/io/gcp/pubsub_test.py | 10 --
 2 files changed, 3 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/fb61c540/sdks/python/apache_beam/io/gcp/pubsub.py
--
diff --git a/sdks/python/apache_beam/io/gcp/pubsub.py 
b/sdks/python/apache_beam/io/gcp/pubsub.py
index 40326e1..6dc1528 100644
--- a/sdks/python/apache_beam/io/gcp/pubsub.py
+++ b/sdks/python/apache_beam/io/gcp/pubsub.py
@@ -29,7 +29,7 @@ from apache_beam.io.iobase import Read
 from apache_beam.io.iobase import Write
 from apache_beam.runners.dataflow.native_io import iobase as dataflow_io
 from apache_beam.transforms import PTransform
-from apache_beam.transforms import ParDo
+from apache_beam.transforms import Map
 from apache_beam.transforms.display import DisplayDataItem
 
 
@@ -71,7 +71,7 @@ class ReadStringsFromPubSub(PTransform):
   def expand(self, pvalue):
 pcoll = pvalue.pipeline | Read(self._source)
 pcoll.element_type = bytes
-pcoll = pcoll | 'decode string' >> ParDo(_decodeUtf8String)
+pcoll = pcoll | 'DecodeString' >> Map(lambda b: b.decode('utf-8'))
 pcoll.element_type = unicode
 return pcoll
 
@@ -89,7 +89,7 @@ class WriteStringsToPubSub(PTransform):
 self._sink = _PubSubPayloadSink(topic)
 
   def expand(self, pcoll):
-pcoll = pcoll | 'encode string' >> ParDo(_encodeUtf8String)
+pcoll = pcoll | 'EncodeString' >> Map(lambda s: s.encode('utf-8'))
 pcoll.element_type = bytes
 return pcoll | Write(self._sink)
 
@@ -162,16 +162,6 @@ class _PubSubPayloadSink(dataflow_io.NativeSink):
 'PubSubPayloadSink is not supported in local execution.')
 
 
-def _decodeUtf8String(encoded_value):
-  """Decodes a string in utf-8 format from bytes"""
-  return encoded_value.decode('utf-8')
-
-
-def _encodeUtf8String(value):
-  """Encodes a string in utf-8 format to bytes"""
-  return value.encode('utf-8')
-
-
 class PubSubSource(dataflow_io.NativeSource):
   """Deprecated: do not use.
 

http://git-wip-us.apache.org/repos/asf/beam/blob/fb61c540/sdks/python/apache_beam/io/gcp/pubsub_test.py
--
diff --git a/sdks/python/apache_beam/io/gcp/pubsub_test.py 
b/sdks/python/apache_beam/io/gcp/pubsub_test.py
index cf14e8c..5d3e985 100644
--- a/sdks/python/apache_beam/io/gcp/pubsub_test.py
+++ b/sdks/python/apache_beam/io/gcp/pubsub_test.py
@@ -22,8 +22,6 @@ import unittest
 
 import hamcrest as hc
 
-from apache_beam.io.gcp.pubsub import _decodeUtf8String
-from apache_beam.io.gcp.pubsub import _encodeUtf8String
 from apache_beam.io.gcp.pubsub import _PubSubPayloadSink
 from apache_beam.io.gcp.pubsub import _PubSubPayloadSource
 from apache_beam.io.gcp.pubsub import ReadStringsFromPubSub
@@ -120,14 +118,6 @@ class TestPubSubSink(unittest.TestCase):
 hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
 
 
-class TestEncodeDecodeUtf8String(unittest.TestCase):
-  def test_encode(self):
-self.assertEqual(b'test_data', _encodeUtf8String('test_data'))
-
-  def test_decode(self):
-self.assertEqual('test_data', _decodeUtf8String(b'test_data'))
-
-
 if __name__ == '__main__':
   logging.getLogger().setLevel(logging.INFO)
   unittest.main()

[jira] [Commented] (BEAM-2326) Verbose INFO logging with stateful DoFns and Dataflow

2017-06-09 Thread peay (JIRA)


[ 
https://issues.apache.org/jira/browse/BEAM-2326?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16044935#comment-16044935
 ] 

peay commented on BEAM-2326:


Thanks [~kenn]!

> Verbose INFO logging with stateful DoFns and Dataflow 
> --
>
> Key: BEAM-2326
> URL: https://issues.apache.org/jira/browse/BEAM-2326
> Project: Beam
>  Issue Type: Bug
>  Components: runner-dataflow
>Affects Versions: 0.6.0
>Reporter: peay
>Assignee: Kenneth Knowles
>  Labels: log
> Fix For: 2.0.0
>
>
> I am seeing a lot of INFO level logging:
> {code}
>  jsonPayload: {
>   logger: 
> "com.google.cloud.dataflow.worker.runners.worker.BatchModeUngroupingParDoFn" 
>   message: "Processing timers for key {} for stateful DoFn"
>  }
>  jsonPayload: {
>   message: "Processing key KV{one of my keys} for stateful DoFn"
>   logger: 
> "com.google.cloud.dataflow.worker.runners.worker.BatchModeUngroupingParDoFn"  
>   
> }
> {code}
> out of one of my stateful DoFn. There is one such group of logs for each key 
> I process, which leads to a very large amount of logs and possibly to a 
> significant slowdown.
> Also, not sure if the {{Processing timers}} log message is missing some 
> string interpolation or if the empty key is on purpose.
> At any rate, this seems more like something for {{DEBUG}} than {{INFO}} given 
> the large volume.



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Build failed in Jenkins: beam_PostCommit_Python_Verify #2455

2017-06-09 Thread Apache Jenkins Server

See 


Changes:

[robertwb] Use beam.Map rather than beam.ParDo for PubSub encoding.

--
[...truncated 39.75 KB...]
ERROR: invocation failed (exit code 1), logfile: 

ERROR: actionid: py27cython
msg: installpkg
cmdargs: 
[local('
 'install', 
'
env: {'JENKINS_HOME': '/x1/jenkins/jenkins-home', 'BUILD_CAUSE': 'SCMTRIGGER', 
'GIT_COMMIT': 'd2c4093ed2fd47e28b31898a17e28d0575aa9ff9', 'HUDSON_URL': 
'https://builds.apache.org/', 'BUILD_URL': 
'https://builds.apache.org/job/beam_PostCommit_Python_Verify/2455/', 
'GIT_PREVIOUS_COMMIT': '97957ea0ec1625b0e937a4f3c9ea2bee316d24d4', 'BUILD_TAG': 
'jenkins-beam_PostCommit_Python_Verify-2455', 'SSH_CLIENT': '140.211.11.14 
53170 22', 'JENKINS_URL': 'https://builds.apache.org/', 'LOGNAME': 'jenkins', 
'USER': 'jenkins', 'WORKSPACE': 
' 'HOME': 
'/home/jenkins', 'PATH': 
':/home/jenkins/tools/java/latest1.8/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games',
 'JOB_NAME': 'beam_PostCommit_Python_Verify', 'LANG': 'en_US.UTF-8', 
'RUN_DISPLAY_URL': 
'https://builds.apache.org/job/beam_PostCommit_Python_Verify/2455/display/redirect',
 'VIRTUAL_ENV': 
'
 'SHELL': '/bin/bash', 'GIT_PREVIOUS_SUCCESSFUL_COMMIT': 
'97957ea0ec1625b0e937a4f3c9ea2bee316d24d4', 'SHLVL': '3', 'NLSPATH': 
'/usr/dt/lib/nls/msg/%L/%N.cat', 'GIT_AUTHOR_EMAIL': 'bui...@apache.org', 
'HUDSON_HOME': '/x1/jenkins/jenkins-home', 'NODE_LABELS': 'beam beam3', 
'GIT_COMMITTER_EMAIL': 'bui...@apache.org', 'PYTHONHASHSEED': '1237540187', 
'JAVA_HOME': '/home/jenkins/tools/java/latest1.8', 'ROOT_BUILD_CAUSE': 
'SCMTRIGGER', 'BUILD_ID': '2455', 'BUILD_NUMBER': '2455', 'XDG_RUNTIME_DIR': 
'/run/user/9997', 'HUDSON_COOKIE': '59d39f58-c2ce-4f2c-8c76-046d6bd03a7b', 
'JOB_URL': 'https://builds.apache.org/job/beam_PostCommit_Python_Verify/', 
'TEST_HOST': 'beam3', 'GIT_BRANCH': 'origin/master', 'JOB_BASE_NAME': 
'beam_PostCommit_Python_Verify', 'GIT_AUTHOR_NAME': 'jenkins', 
'GIT_COMMITTER_NAME': 'jenkins', 'XDG_SESSION_ID': '48', '_': 
'/home/jenkins/.local/bin//tox', 'sha1': 'master', 'COVERALLS_REPO_TOKEN': 
'', 'JOB_DISPLAY_URL': 
'https://builds.apache.org/job/beam_PostCommit_Python_Verify/display/redirect', 
'BUILD_CAUSE_SCMTRIGGER': 'true', 'HUDSON_SERVER_COOKIE': 'f4ebd1e6b0d976e8', 
'EXECUTOR_NUMBER': '0', 'NODE_NAME': 'beam3', 'PWD': 
' 
'SPARK_LOCAL_IP': '127.0.0.1', 'JENKINS_SERVER_COOKIE': 'f4ebd1e6b0d976e8', 
'BUILD_DISPLAY_NAME': '#2455', 'ROOT_BUILD_CAUSE_SCMTRIGGER': 'true', 
'XFILESEARCHPATH': '/usr/dt/app-defaults/%L/Dt', 'MAIL': '/var/mail/jenkins', 
'SSH_CONNECTION': '140.211.11.14 53170 10.128.0.4 22', 
'RUN_CHANGES_DISPLAY_URL': 
'https://builds.apache.org/job/beam_PostCommit_Python_Verify/2455/display/redirect?page=changes',
 'GIT_URL': 'https://github.com/apache/beam.git'}

Processing ./target/.tox/dist/apache-beam-2.1.0.dev.zip
Collecting avro<2.0.0,>=1.8.1 (from apache-beam==2.1.0.dev0)
Collecting crcmod<2.0,>=1.7 (from apache-beam==2.1.0.dev0)
Collecting dill==0.2.6 (from apache-beam==2.1.0.dev0)
Requirement already satisfied: grpcio<2.0,>=1.0 in 
./target/.tox/py27cython/lib/python2.7/site-packages (from 
apache-beam==2.1.0.dev0)
Collecting httplib2<0.10,>=0.8 (from apache-beam==2.1.0.dev0)
Collecting mock<3.0.0,>=1.0.1 (from apache-beam==2.1.0.dev0)
  Using cached mock-2.0.0-py2.py3-none-any.whl
Collecting oauth2client<4.0.0,>=2.0.1 (from apache-beam==2.1.0.dev0)
Requirement already satisfied: protobuf<=3.3.0,>=3.2.0 in 
./target/.tox/py27cython/lib/python2.7/site-packages (from 
apache-beam==2.1.0.dev0)
Collecting pyyaml<4.0.0,>=3.12 (from apache-beam==2.1.0.dev0)
Requirement already satisfied: futures>=2.2.0 in 
./target/.tox/py27cython/lib/python2.7/site-packages (from 
grpcio<2.0,>=1.0->apache-beam==2.1.0.dev0)
Requirement already satisfied: enum34>=1.0.4 in 
./target/.tox/py27cython/lib/python2.7/site-packages (from 
grpcio<2.0,>=1.0->apache-beam==2.1.0.dev0)
Requirement already satisfied: six>=1.5.2 in 
./target/.tox/py27cython/lib/python2.7/site-packages (from 
grpcio<2.0,>=1.0->apache-beam==2.1.0.dev0)
Collecting pbr>=0.11 (from mock<3.0.0,>=1.0.1->apache-beam==2.1.0.dev0)
  Using cached pbr-3.0.1-py2.py3-none-any.whl
Collecting

[jira] [Assigned] (BEAM-2298) Java WordCount doesn't work in Window OS for glob expressions or file: prefixed paths

2017-06-09 Thread Flavio Fiszman (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-2298?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Flavio Fiszman reassigned BEAM-2298:


Assignee: Flavio Fiszman

> Java WordCount doesn't work in Window OS for glob expressions or file: 
> prefixed paths
> -
>
> Key: BEAM-2298
> URL: https://issues.apache.org/jira/browse/BEAM-2298
> Project: Beam
>  Issue Type: Bug
>  Components: sdk-java-core
>Reporter: Pei He
>Assignee: Flavio Fiszman
> Fix For: 2.1.0
>
>
> I am not able to build beam repo in Windows OS, so I copied the jar file from 
> my Mac.
> WordCount failed with the following cmd:
> java -cp beam-examples-java-2.0.0-jar-with-dependencies.jar
>  org.apache.beam.examples.WordCount --inputFile=input.txt --output=counts
> May 15, 2017 6:09:48 AM org.apache.beam.sdk.io.FileBasedSource 
> getEstimatedSizeB
> ytes
> INFO: Filepattern input.txt matched 1 files with total size 0
> May 15, 2017 6:09:48 AM org.apache.beam.sdk.io.FileBasedSource 
> expandFilePattern
> INFO: Matched 1 files for pattern input.txt
> May 15, 2017 6:09:48 AM org.apache.beam.sdk.io.FileBasedSource split
> INFO: Splitting filepattern input.txt into bundles of size 0 took 0 ms and 
> produ
> ced 1 files and 0 bundles
> May 15, 2017 6:09:48 AM org.apache.beam.sdk.io.WriteFiles$2 processElement
> INFO: Finalizing write operation 
> TextWriteOperation{tempDirectory=C:\Users\Pei\D
> esktop\.temp-beam-2017-05-135_13-09-48-1\, windowedWrites=false}.
> May 15, 2017 6:09:48 AM org.apache.beam.sdk.io.WriteFiles$2 processElement
> INFO: Creating 1 empty output shards in addition to 0 written for a total of 
> 1.
> Exception in thread "main" 
> org.apache.beam.sdk.Pipeline$PipelineExecutionExcepti
> on: java.lang.IllegalStateException: Unable to find registrar for c
> at 
> org.apache.beam.runners.direct.DirectRunner$DirectPipelineResult.wait
> UntilFinish(DirectRunner.java:322)
> at 
> org.apache.beam.runners.direct.DirectRunner$DirectPipelineResult.wait
> UntilFinish(DirectRunner.java:292)
> at 
> org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:200
> )
> at 
> org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:63)
> at org.apache.beam.sdk.Pipeline.run(Pipeline.java:295)
> at org.apache.beam.sdk.Pipeline.run(Pipeline.java:281)
> at org.apache.beam.examples.WordCount.main(WordCount.java:184)
> Caused by: java.lang.IllegalStateException: Unable to find registrar for c
> at 
> org.apache.beam.sdk.io.FileSystems.getFileSystemInternal(FileSystems.
> java:447)
> at org.apache.beam.sdk.io.FileSystems.match(FileSystems.java:111)
> at 
> org.apache.beam.sdk.io.FileSystems.matchResources(FileSystems.java:17
> 4)
> at 
> org.apache.beam.sdk.io.FileSystems.filterMissingFiles(FileSystems.jav
> a:367)
> at org.apache.beam.sdk.io.FileSystems.copy(FileSystems.java:251)
> at 
> org.apache.beam.sdk.io.FileBasedSink$WriteOperation.copyToOutputFiles
> (FileBasedSink.java:641)
> at 
> org.apache.beam.sdk.io.FileBasedSink$WriteOperation.finalize(FileBase
> dSink.java:529)
> at 
> org.apache.beam.sdk.io.WriteFiles$2.processElement(WriteFiles.java:59
> 2)



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

82 matches

Mail list logo