[beam] tag nightly-master updated (d93c591 -> 158e177)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to tag nightly-master in repository https://gitbox.apache.org/repos/asf/beam.git. *** WARNING: tag nightly-master was modified! *** from d93c591 (commit) to 158e177 (commit) from d93c591 Merge pull request #14667:[BEAM-12343] GroupByKeyTest for changing WindowFn from GlobalWindow after GBK add ce2aef9 [BEAM-12329] Drain S3 read InputStreams to avoid warnings add 96b3b03 Merge pull request #14794: [BEAM-12329] Drain S3 read InputStreams to avoid warnings add dd2f67b [BEAM-12320] Raise timeout and add logging in PubsubTableProviderIT. (#14785) add 67fbf95 [BEAM-12333] Changing TimerKey to include TimerFamilyId (#14802) add e296c00 [BEAM-12089] Clarify usage of --artifacts-dir for job servers. add 158e177 Merge pull request #14420 from ibzib/BEAM-12089 No new revisions were added by this update. Summary of changes: .../runners/jobsubmission/JobServerDriver.java | 6 +- .../samza/runtime/SamzaTimerInternalsFactory.java | 125 + .../runtime/SamzaTimerInternalsFactoryTest.java| 20 +++- .../provider/pubsub/PubsubTableProviderIT.java | 19 +++- .../io/aws/s3/S3ReadableSeekableByteChannel.java | 4 + .../io/aws2/s3/S3ReadableSeekableByteChannel.java | 2 + .../beam/sdk/io/gcp/pubsub/TestPubsubSignal.java | 3 + 7 files changed, 95 insertions(+), 84 deletions(-)
[beam] branch master updated: [BEAM-12089] Clarify usage of --artifacts-dir for job servers.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new e296c00 [BEAM-12089] Clarify usage of --artifacts-dir for job servers.
new 158e177 Merge pull request #14420 from ibzib/BEAM-12089
e296c00 is described below
commit e296c0009209f61504500a6cfcd10be512dc3925
Author: Kyle Weaver
AuthorDate: Fri Apr 2 16:08:11 2021 -0700
[BEAM-12089] Clarify usage of --artifacts-dir for job servers.
---
.../java/org/apache/beam/runners/jobsubmission/JobServerDriver.java | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git
a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/JobServerDriver.java
b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/JobServerDriver.java
index 499fc8c..7bc129e 100644
---
a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/JobServerDriver.java
+++
b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/JobServerDriver.java
@@ -90,7 +90,11 @@ public abstract class JobServerDriver implements Runnable {
usage = "The Java expansion service port. 0 to use a dynamic port.
(Default: 8097)")
private int expansionPort = 8097;
-@Option(name = "--artifacts-dir", usage = "The location to store staged
artifact files")
+@Option(
+name = "--artifacts-dir",
+usage =
+"The location to store staged artifact files. "
++ "If artifact staging is needed, this directory must be
accessible by the execution engine's workers.")
private String artifactStagingPath =
Paths.get(System.getProperty("java.io.tmpdir"),
"beam-artifact-staging").toString();
[beam] branch master updated: [BEAM-12333] Changing TimerKey to include TimerFamilyId (#14802)
This is an automated email from the ASF dual-hosted git repository.
xinyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 67fbf95 [BEAM-12333] Changing TimerKey to include TimerFamilyId
(#14802)
67fbf95 is described below
commit 67fbf9581cd2eb0850e0cdda4463bf0fdbe8bd2d
Author: ajo thomas
AuthorDate: Mon May 17 16:00:22 2021 -0700
[BEAM-12333] Changing TimerKey to include TimerFamilyId (#14802)
---
.../samza/runtime/SamzaTimerInternalsFactory.java | 125 +
.../runtime/SamzaTimerInternalsFactoryTest.java| 20 +++-
2 files changed, 67 insertions(+), 78 deletions(-)
diff --git
a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaTimerInternalsFactory.java
b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaTimerInternalsFactory.java
index ad426f2..2588223 100644
---
a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaTimerInternalsFactory.java
+++
b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaTimerInternalsFactory.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.runners.samza.runtime;
+import com.google.auto.value.AutoValue;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -445,7 +446,7 @@ public class SamzaTimerInternalsFactory implements
TimerInternalsFactory {
final Long timestamp = eventTimeTimerState.get(timerKey).read();
if (timestamp != null) {
-final KeyedTimerData keyedTimerDataInStore =
+final KeyedTimerData keyedTimerDataInStore =
TimerKey.toKeyedTimerData(timerKey, timestamp,
TimeDomain.EVENT_TIME, keyCoder);
timestampSortedEventTimeTimerState.remove(keyedTimerDataInStore);
}
@@ -553,7 +554,7 @@ public class SamzaTimerInternalsFactory implements
TimerInternalsFactory {
// inline the migration code
while (eventTimersIter.hasNext()) {
final Map.Entry, Long> entry = eventTimersIter.next();
-final KeyedTimerData keyedTimerData =
+final KeyedTimerData keyedTimerData =
TimerKey.toKeyedTimerData(
entry.getKey(), entry.getValue(), TimeDomain.EVENT_TIME,
keyCoder);
timestampSortedEventTimeTimerState.add(keyedTimerData);
@@ -569,99 +570,66 @@ public class SamzaTimerInternalsFactory implements
TimerInternalsFactory {
}
}
- private static class TimerKey {
-private final K key;
-private final StateNamespace stateNamespace;
-private final String timerId;
+ @AutoValue
+ abstract static class TimerKey {
+abstract K getKey();
+
+abstract StateNamespace getStateNamespace();
+
+abstract String getTimerId();
+
+abstract String getTimerFamilyId();
+
+static Builder builder() {
+ return new AutoValue_SamzaTimerInternalsFactory_TimerKey.Builder<>();
+}
static TimerKey of(KeyedTimerData keyedTimerData) {
final TimerInternals.TimerData timerData = keyedTimerData.getTimerData();
- return new TimerKey<>(
- keyedTimerData.getKey(), timerData.getNamespace(),
timerData.getTimerId());
+ return TimerKey.builder()
+ .setKey(keyedTimerData.getKey())
+ .setStateNamespace(timerData.getNamespace())
+ .setTimerId(timerData.getTimerId())
+ .setTimerFamilyId(timerData.getTimerFamilyId())
+ .build();
}
static KeyedTimerData toKeyedTimerData(
TimerKey timerKey, long timestamp, TimeDomain domain, Coder
keyCoder) {
byte[] keyBytes = null;
- if (keyCoder != null && timerKey.key != null) {
+ if (keyCoder != null && timerKey.getKey() != null) {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
try {
- keyCoder.encode(timerKey.key, baos);
+ keyCoder.encode(timerKey.getKey(), baos);
} catch (IOException e) {
- throw new RuntimeException("Could not encode key: " + timerKey.key,
e);
+ throw new RuntimeException("Could not encode key: " +
timerKey.getKey(), e);
}
keyBytes = baos.toByteArray();
}
- return new KeyedTimerData(
+ return new KeyedTimerData<>(
keyBytes,
- timerKey.key,
+ timerKey.getKey(),
TimerInternals.TimerData.of(
- timerKey.timerId,
- timerKey.stateNamespace,
+ timerKey.getTimerId(),
+ timerKey.getTimerFamilyId(),
+ timerKey.getStateNamespace(),
new Instant(timestamp),
new Instant(timestamp),
domain));
}
-private TimerKey(K key, StateNamespace stateNamespace, String timerId) {
- this.key = key;
- this.stateNamespace = stateNamespace;
- this.timerId = timerId;
-}
+
[beam] branch fix/overview-cross-language-pattern created (now ff744d6)
This is an automated email from the ASF dual-hosted git repository. mbae pushed a change to branch fix/overview-cross-language-pattern in repository https://gitbox.apache.org/repos/asf/beam.git. at ff744d6 Fix overview.md for cross-language pattern No new revisions were added by this update.
[beam] branch master updated (96b3b03 -> dd2f67b)
This is an automated email from the ASF dual-hosted git repository. bhulette pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from 96b3b03 Merge pull request #14794: [BEAM-12329] Drain S3 read InputStreams to avoid warnings add dd2f67b [BEAM-12320] Raise timeout and add logging in PubsubTableProviderIT. (#14785) No new revisions were added by this update. Summary of changes: .../meta/provider/pubsub/PubsubTableProviderIT.java | 19 ++- .../beam/sdk/io/gcp/pubsub/TestPubsubSignal.java | 3 +++ 2 files changed, 17 insertions(+), 5 deletions(-)
svn commit: r47768 - in /dev/beam/vendor/beam-vendor-bytebuddy-1_11_0: ./ 0.1/
Author: iemejia Date: Mon May 17 15:29:42 2021 New Revision: 47768 Log: Add vendor bytebuddy 1.11.0 Added: dev/beam/vendor/beam-vendor-bytebuddy-1_11_0/ dev/beam/vendor/beam-vendor-bytebuddy-1_11_0/0.1/ dev/beam/vendor/beam-vendor-bytebuddy-1_11_0/0.1/apache-beam-d93c591deb21237ddb656583d7ef7a4debba-source-release.zip (with props) dev/beam/vendor/beam-vendor-bytebuddy-1_11_0/0.1/apache-beam-d93c591deb21237ddb656583d7ef7a4debba-source-release.zip.asc dev/beam/vendor/beam-vendor-bytebuddy-1_11_0/0.1/apache-beam-d93c591deb21237ddb656583d7ef7a4debba-source-release.zip.sha512 Added: dev/beam/vendor/beam-vendor-bytebuddy-1_11_0/0.1/apache-beam-d93c591deb21237ddb656583d7ef7a4debba-source-release.zip == Binary file - no diff available. Propchange: dev/beam/vendor/beam-vendor-bytebuddy-1_11_0/0.1/apache-beam-d93c591deb21237ddb656583d7ef7a4debba-source-release.zip -- svn:mime-type = application/octet-stream Added: dev/beam/vendor/beam-vendor-bytebuddy-1_11_0/0.1/apache-beam-d93c591deb21237ddb656583d7ef7a4debba-source-release.zip.asc == --- dev/beam/vendor/beam-vendor-bytebuddy-1_11_0/0.1/apache-beam-d93c591deb21237ddb656583d7ef7a4debba-source-release.zip.asc (added) +++ dev/beam/vendor/beam-vendor-bytebuddy-1_11_0/0.1/apache-beam-d93c591deb21237ddb656583d7ef7a4debba-source-release.zip.asc Mon May 17 15:29:42 2021 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEENBVjFynhWzMFGttnCp2vZxO4Y0kFAmCiiIUACgkQCp2vZxO4 +Y0kxwg//Rk4VH2BFbhQg//kc8KGjmjyT22exx1M55ttHmDaYmRlGR8YYBoLg6nfK +uybzgQVHlAlMLqv3CuHyVX0YrA+iBc9AtvbwuIvHDhkwV5asyXbBhkUJ6nOzy6WY +wUWOA7TTpFEOZUJiVDfOLYICo7QEuStJSVaEFome1JSFyK18cIy7uAKQfYIC1n4S +Sl32RKYg4EXJACULO6sOnQq9D4sBeiqJ9IJcV/uGPwgN3ByJAk/GzsTnPsUrVA+B +8tsa0USllAhMB6jTr3fGIOMDHaJJnIMr+qi/0Wmuc1Lq07tSsUtK8jeTVrtYy6zr +FMQ6g2RKZq8ofQfPyPVLD7Glfb8COnJASgcGY6Iwlht9PRCmvUBaBuKN6Xz5pruS +QHbf+EsiWi4dU9VX/aWI8w7csKLh2mOiQf/8ARIqPdQTmGTsfmQyKfZQzjPgq+lg +Qw1j8lU/hMjCQmRU6RnDToJP70+THEK/PffiyycFsjSbkoGtu6CjROdjWVQQJALi +jtqxf0KxH+jaYmQ3DfwToj4CdeVoY1IpoqXv+Jt+qboWE+xiUJYpsTzDq2Agy8+d +kKTk/8dU/sxFNwwo2DgkryGw0sk+0PZfGAirBjVhn2CnAfsoaJvdWBqYnRCYLXYn +FrV5PKzsrTMe8SZK183KJudKpASGXj+t2dxRTSF3MMxk3HJJIlE= +=GL+y +-END PGP SIGNATURE- Added: dev/beam/vendor/beam-vendor-bytebuddy-1_11_0/0.1/apache-beam-d93c591deb21237ddb656583d7ef7a4debba-source-release.zip.sha512 == --- dev/beam/vendor/beam-vendor-bytebuddy-1_11_0/0.1/apache-beam-d93c591deb21237ddb656583d7ef7a4debba-source-release.zip.sha512 (added) +++ dev/beam/vendor/beam-vendor-bytebuddy-1_11_0/0.1/apache-beam-d93c591deb21237ddb656583d7ef7a4debba-source-release.zip.sha512 Mon May 17 15:29:42 2021 @@ -0,0 +1 @@ +e368451efee117d44d1286e87566e8a085395dad0c4c0051cd7e15b3cf87c56cfb8f63168efc5e1b44ccf408f90b1a3242915abdc780b41bb12630a454573697 apache-beam-d93c591deb21237ddb656583d7ef7a4debba-source-release.zip
[beam] branch master updated (d93c591 -> 96b3b03)
This is an automated email from the ASF dual-hosted git repository. aromanenko pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git. from d93c591 Merge pull request #14667:[BEAM-12343] GroupByKeyTest for changing WindowFn from GlobalWindow after GBK add ce2aef9 [BEAM-12329] Drain S3 read InputStreams to avoid warnings add 96b3b03 Merge pull request #14794: [BEAM-12329] Drain S3 read InputStreams to avoid warnings No new revisions were added by this update. Summary of changes: .../org/apache/beam/sdk/io/aws/s3/S3ReadableSeekableByteChannel.java | 4 .../org/apache/beam/sdk/io/aws2/s3/S3ReadableSeekableByteChannel.java | 2 ++ 2 files changed, 6 insertions(+)
