[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[ https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=140682&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-140682 ] ASF GitHub Bot logged work on BEAM-3702: Author: ASF GitHub Bot Created on: 03/Sep/18 18:13 Start Date: 03/Sep/18 18:13 Worklog Time Spent: 10m Work Description: stale[bot] commented on issue #4683: [BEAM-3702] adding fromJvm to create pipelineoptions from the system properties URL: https://github.com/apache/beam/pull/4683#issuecomment-418172030 This pull request has been closed due to lack of activity. If you think that is incorrect, or the pull request requires review, you can revive the PR at any time. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 140682) Time Spent: 11h 20m (was: 11h 10m) > Support system properties source for pipeline options > - > > Key: BEAM-3702 > URL: https://issues.apache.org/jira/browse/BEAM-3702 > Project: Beam > Issue Type: Task > Components: sdk-java-core >Reporter: Romain Manni-Bucau >Assignee: Romain Manni-Bucau >Priority: Major > Time Spent: 11h 20m > Remaining Estimate: 0h > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[
https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=140691&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-140691
]
ASF GitHub Bot logged work on BEAM-3702:
Author: ASF GitHub Bot
Created on: 03/Sep/18 18:14
Start Date: 03/Sep/18 18:14
Worklog Time Spent: 10m
Work Description: stale[bot] closed pull request #4683: [BEAM-3702]
adding fromJvm to create pipelineoptions from the system properties
URL: https://github.com/apache/beam/pull/4683
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
index 7630fb3e167..3e724c04b18 100644
---
a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
+++
b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
@@ -20,12 +20,16 @@
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Locale.ROOT;
+import static java.util.function.Function.identity;
+import static java.util.stream.Collectors.joining;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toMap;
+import static java.util.stream.Collectors.toSet;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.common.base.Predicate;
@@ -38,9 +42,12 @@
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
+import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
import com.google.common.collect.Ordering;
import com.google.common.collect.RowSortedTable;
import com.google.common.collect.Sets;
@@ -63,11 +70,13 @@
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
+import java.util.Hashtable;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
+import java.util.Properties;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.SortedMap;
@@ -75,9 +84,10 @@
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import java.util.stream.StreamSupport;
-import javax.annotation.Nonnull;
import org.apache.beam.sdk.PipelineRunner;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.options.Validation.Required;
@@ -174,6 +184,13 @@ public static Builder fromArgs(String... args) {
return new Builder().fromArgs(args);
}
+ /**
+ * @return a builder instance enabling you to build pipeline options.
+ */
+ public static Builder builder() {
+return new Builder();
+ }
+
/**
* After creation we will validate that {@code } conforms to all the
* validation criteria. See
@@ -187,20 +204,20 @@ public Builder withValidation() {
/** A fluent {@link PipelineOptions} builder. */
public static class Builder {
private final String defaultAppName;
-private final String[] args;
+private final Collection>>
options;
private final boolean validation;
private final boolean strictParsing;
private final boolean isCli;
// Do not allow direct instantiation
private Builder() {
- this(null, false, true, false);
+ this(new ArrayList<>(), false, true, false);
}
-private Builder(String[] args, boolean validation,
-boolean strictParsing, boolean isCli) {
+private Builder(Collection>> options,
+boolean validation, boolean strictParsing, boolean isCli) {
this.defaultAppName = findCallersClassName();
- this.args = args;
+ this.options = options;
this.validation = validation;
this.strictParsing = strictParsing;
this.isCli = isCli;
@@ -242,10 +259,36 @@ private Builder(String[] args, boolean validation,
* {@cod
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[ https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=138480&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-138480 ] ASF GitHub Bot logged work on BEAM-3702: Author: ASF GitHub Bot Created on: 27/Aug/18 17:21 Start Date: 27/Aug/18 17:21 Worklog Time Spent: 10m Work Description: stale[bot] commented on issue #4683: [BEAM-3702] adding fromJvm to create pipelineoptions from the system properties URL: https://github.com/apache/beam/pull/4683#issuecomment-416300452 This pull request has been marked as stale due to 60 days of inactivity. It will be closed in 1 week if no further activity occurs. If you think that’s incorrect or this pull request requires a review, please simply write any comment. If closed, you can revive the PR at any time and @mention a reviewer or discuss it on the d...@beam.apache.org list. Thank you for your contributions. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 138480) Time Spent: 11h 10m (was: 11h) > Support system properties source for pipeline options > - > > Key: BEAM-3702 > URL: https://issues.apache.org/jira/browse/BEAM-3702 > Project: Beam > Issue Type: Task > Components: sdk-java-core >Reporter: Romain Manni-Bucau >Assignee: Romain Manni-Bucau >Priority: Major > Time Spent: 11h 10m > Remaining Estimate: 0h > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[ https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=116881&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-116881 ] ASF GitHub Bot logged work on BEAM-3702: Author: ASF GitHub Bot Created on: 28/Jun/18 16:26 Start Date: 28/Jun/18 16:26 Worklog Time Spent: 10m Work Description: kennknowles commented on issue #4683: [BEAM-3702] adding fromJvm to create pipelineoptions from the system properties URL: https://github.com/apache/beam/pull/4683#issuecomment-401093844 We have turned on autoformatting of the codebase, which causes small conflicts across the board. You can probably safely rebase and just keep your changes. Like this: ``` $ git rebase ... see some conflicts $ git diff ... confirmed that the conflicts are just autoformatting ... so we can just keep our changes are do our own autoformat $ git checkout --theirs -- $ git add -u $ git rebase --continue $ ./gradlew spotlessJavaApply ``` Please ping me if you run into any difficulty. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 116881) Time Spent: 11h (was: 10h 50m) > Support system properties source for pipeline options > - > > Key: BEAM-3702 > URL: https://issues.apache.org/jira/browse/BEAM-3702 > Project: Beam > Issue Type: Task > Components: sdk-java-core >Reporter: Romain Manni-Bucau >Assignee: Romain Manni-Bucau >Priority: Major > Fix For: 2.6.0 > > Time Spent: 11h > Remaining Estimate: 0h > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[ https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=82979&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-82979 ] ASF GitHub Bot logged work on BEAM-3702: Author: ASF GitHub Bot Created on: 21/Mar/18 22:52 Start Date: 21/Mar/18 22:52 Worklog Time Spent: 10m Work Description: lukecwik commented on issue #4683: [BEAM-3702] adding fromJvm to create pipelineoptions from the system properties URL: https://github.com/apache/beam/pull/4683#issuecomment-375121943 Please fix the javadoc errors: ``` [ERROR] /home/jenkins/jenkins-slave/workspace/beam_PreCommit_Java_MavenInstall/src/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java:273: Redundant tag. [JavadocParagraph] Audit done. 2018-03-21T21:43:26.135 [INFO] There is 1 error reported by Checkstyle 8.7 with beam/checkstyle.xml ruleset. 2018-03-21T21:43:26.138 [ERROR] src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java:[273] (javadoc) JavadocParagraph: Redundant tag. ``` This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 82979) Time Spent: 10h 50m (was: 10h 40m) > Support system properties source for pipeline options > - > > Key: BEAM-3702 > URL: https://issues.apache.org/jira/browse/BEAM-3702 > Project: Beam > Issue Type: Task > Components: sdk-java-core >Reporter: Romain Manni-Bucau >Assignee: Romain Manni-Bucau >Priority: Major > Fix For: 2.5.0 > > Time Spent: 10h 50m > Remaining Estimate: 0h > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[ https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=82907&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-82907 ] ASF GitHub Bot logged work on BEAM-3702: Author: ASF GitHub Bot Created on: 21/Mar/18 20:47 Start Date: 21/Mar/18 20:47 Worklog Time Spent: 10m Work Description: rmannibucau commented on issue #4683: [BEAM-3702] adding fromJvm to create pipelineoptions from the system properties URL: https://github.com/apache/beam/pull/4683#issuecomment-375090210 fixed This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 82907) Time Spent: 10h 40m (was: 10.5h) > Support system properties source for pipeline options > - > > Key: BEAM-3702 > URL: https://issues.apache.org/jira/browse/BEAM-3702 > Project: Beam > Issue Type: Task > Components: sdk-java-core >Reporter: Romain Manni-Bucau >Assignee: Romain Manni-Bucau >Priority: Major > Fix For: 2.5.0 > > Time Spent: 10h 40m > Remaining Estimate: 0h > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[ https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=82486&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-82486 ] ASF GitHub Bot logged work on BEAM-3702: Author: ASF GitHub Bot Created on: 20/Mar/18 21:24 Start Date: 20/Mar/18 21:24 Worklog Time Spent: 10m Work Description: lukecwik commented on issue #4683: [BEAM-3702] adding fromJvm to create pipelineoptions from the system properties URL: https://github.com/apache/beam/pull/4683#issuecomment-374761801 Please fix up the javadoc errors: 2018-03-18T19:18:01.876 [ERROR] Failed to execute goal org.apache.maven.plugins:maven-javadoc-plugin:3.0.0-M1:jar (attach-javadocs) on project beam-sdks-java-core: MavenReportException: Error while generating Javadoc: 2018-03-18T19:18:01.876 [ERROR] Exit code: 1 - /home/jenkins/jenkins-slave/workspace/beam_PreCommit_Java_MavenInstall@2/src/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java:1924: error: unexpected end tag: 2018-03-18T19:18:01.876 [ERROR] * key/values. 2018-03-18T19:18:01.876 [ERROR] ^ 2018-03-18T19:18:01.876 [ERROR] /home/jenkins/jenkins-slave/workspace/beam_PreCommit_Java_MavenInstall@2/src/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java:276: error: unexpected end tag: 2018-03-18T19:18:01.876 [ERROR] * as a prefix in this case. 2018-03-18T19:18:01.876 [ERROR] ^ This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 82486) Time Spent: 10h 20m (was: 10h 10m) > Support system properties source for pipeline options > - > > Key: BEAM-3702 > URL: https://issues.apache.org/jira/browse/BEAM-3702 > Project: Beam > Issue Type: Task > Components: sdk-java-core >Reporter: Romain Manni-Bucau >Assignee: Romain Manni-Bucau >Priority: Major > Fix For: 2.5.0 > > Time Spent: 10h 20m > Remaining Estimate: 0h > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[ https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=82487&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-82487 ] ASF GitHub Bot logged work on BEAM-3702: Author: ASF GitHub Bot Created on: 20/Mar/18 21:24 Start Date: 20/Mar/18 21:24 Worklog Time Spent: 10m Work Description: lukecwik commented on issue #4683: [BEAM-3702] adding fromJvm to create pipelineoptions from the system properties URL: https://github.com/apache/beam/pull/4683#issuecomment-374761801 Please fix up the javadoc errors: ``` 2018-03-18T19:18:01.876 [ERROR] Failed to execute goal org.apache.maven.plugins:maven-javadoc-plugin:3.0.0-M1:jar (attach-javadocs) on project beam-sdks-java-core: MavenReportException: Error while generating Javadoc: 2018-03-18T19:18:01.876 [ERROR] Exit code: 1 - /home/jenkins/jenkins-slave/workspace/beam_PreCommit_Java_MavenInstall@2/src/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java:1924: error: unexpected end tag: 2018-03-18T19:18:01.876 [ERROR] * key/values. 2018-03-18T19:18:01.876 [ERROR] ^ 2018-03-18T19:18:01.876 [ERROR] /home/jenkins/jenkins-slave/workspace/beam_PreCommit_Java_MavenInstall@2/src/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java:276: error: unexpected end tag: 2018-03-18T19:18:01.876 [ERROR] * as a prefix in this case. 2018-03-18T19:18:01.876 [ERROR] ^ ``` This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 82487) Time Spent: 10.5h (was: 10h 20m) > Support system properties source for pipeline options > - > > Key: BEAM-3702 > URL: https://issues.apache.org/jira/browse/BEAM-3702 > Project: Beam > Issue Type: Task > Components: sdk-java-core >Reporter: Romain Manni-Bucau >Assignee: Romain Manni-Bucau >Priority: Major > Fix For: 2.5.0 > > Time Spent: 10.5h > Remaining Estimate: 0h > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[ https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=81683&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-81683 ] ASF GitHub Bot logged work on BEAM-3702: Author: ASF GitHub Bot Created on: 18/Mar/18 19:09 Start Date: 18/Mar/18 19:09 Worklog Time Spent: 10m Work Description: rmannibucau commented on issue #4683: [BEAM-3702] adding fromJvm to create pipelineoptions from the system properties URL: https://github.com/apache/beam/pull/4683#issuecomment-374034875 addressed the changes and merged with master. hope it is fine this time :) This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 81683) Time Spent: 10h 10m (was: 10h) > Support system properties source for pipeline options > - > > Key: BEAM-3702 > URL: https://issues.apache.org/jira/browse/BEAM-3702 > Project: Beam > Issue Type: Task > Components: sdk-java-core >Reporter: Romain Manni-Bucau >Assignee: Romain Manni-Bucau >Priority: Major > Fix For: 2.5.0 > > Time Spent: 10h 10m > Remaining Estimate: 0h > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[
https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=81286&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-81286
]
ASF GitHub Bot logged work on BEAM-3702:
Author: ASF GitHub Bot
Created on: 16/Mar/18 17:56
Start Date: 16/Mar/18 17:56
Worklog Time Spent: 10m
Work Description: lukecwik commented on a change in pull request #4683:
[BEAM-3702] adding fromJvm to create pipelineoptions from the system properties
URL: https://github.com/apache/beam/pull/4683#discussion_r175164646
##
File path:
sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
##
@@ -1792,4 +1804,82 @@ private synchronized void register(Class iface) {
return combinedCache.get(interfaces).getPropertyDescriptors();
}
}
+
+ /**
+ * @param prefix a prefix filter on the map keys.
+ * @param options the options to convert to a pipeline options.
+ * @return a pipeline options instance based on the specified configuration.
+ */
+ @Experimental
+ public static PipelineOptions fromMap(final String prefix, final Map options) {
Review comment:
Why is this important to have part of the API?
It seems like we should ask users to filter their map keys themselves.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 81286)
Time Spent: 8h 40m (was: 8.5h)
> Support system properties source for pipeline options
> -
>
> Key: BEAM-3702
> URL: https://issues.apache.org/jira/browse/BEAM-3702
> Project: Beam
> Issue Type: Task
> Components: sdk-java-core
>Reporter: Romain Manni-Bucau
>Assignee: Romain Manni-Bucau
>Priority: Major
> Fix For: 2.5.0
>
> Time Spent: 8h 40m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[
https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=81289&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-81289
]
ASF GitHub Bot logged work on BEAM-3702:
Author: ASF GitHub Bot
Created on: 16/Mar/18 17:56
Start Date: 16/Mar/18 17:56
Worklog Time Spent: 10m
Work Description: lukecwik commented on a change in pull request #4683:
[BEAM-3702] adding fromJvm to create pipelineoptions from the system properties
URL: https://github.com/apache/beam/pull/4683#discussion_r175169114
##
File path:
sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java
##
@@ -1797,6 +1799,44 @@ public void serialize(JacksonIncompatible
jacksonIncompatible, JsonGenerator jso
}
}
+ @Test
+ public void testPipelineOptionsFactoryFromProperties() {
+assertEquals("testAppName", PipelineOptionsFactory.fromProperties(new
Properties() {{
+ put("appName", "testAppName");
+}}).as(ApplicationNameOptions.class).getAppName());
+ }
+
+ @Test
+ public void testPipelineOptionsFactoryFromPropertiesWithPrefix() {
+assertEquals(
+ "testAppName",
+ PipelineOptionsFactory.fromProperties("prefix.", new Properties() {{
+put("prefix.appName", "testAppName");
+ }}).as(ApplicationNameOptions.class).getAppName());
+ }
+
+ @Test
+ public void testPipelineOptionsFactoryFromPropertiesWithLongPrefix() {
Review comment:
This test doesn't add value.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 81289)
Time Spent: 9h (was: 8h 50m)
> Support system properties source for pipeline options
> -
>
> Key: BEAM-3702
> URL: https://issues.apache.org/jira/browse/BEAM-3702
> Project: Beam
> Issue Type: Task
> Components: sdk-java-core
>Reporter: Romain Manni-Bucau
>Assignee: Romain Manni-Bucau
>Priority: Major
> Fix For: 2.5.0
>
> Time Spent: 9h
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[
https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=81298&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-81298
]
ASF GitHub Bot logged work on BEAM-3702:
Author: ASF GitHub Bot
Created on: 16/Mar/18 17:56
Start Date: 16/Mar/18 17:56
Worklog Time Spent: 10m
Work Description: lukecwik commented on a change in pull request #4683:
[BEAM-3702] adding fromJvm to create pipelineoptions from the system properties
URL: https://github.com/apache/beam/pull/4683#discussion_r175169354
##
File path:
sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java
##
@@ -1797,6 +1799,44 @@ public void serialize(JacksonIncompatible
jacksonIncompatible, JsonGenerator jso
}
}
+ @Test
+ public void testPipelineOptionsFactoryFromProperties() {
+assertEquals("testAppName", PipelineOptionsFactory.fromProperties(new
Properties() {{
+ put("appName", "testAppName");
+}}).as(ApplicationNameOptions.class).getAppName());
+ }
+
+ @Test
+ public void testPipelineOptionsFactoryFromPropertiesWithPrefix() {
+assertEquals(
+ "testAppName",
+ PipelineOptionsFactory.fromProperties("prefix.", new Properties() {{
+put("prefix.appName", "testAppName");
+ }}).as(ApplicationNameOptions.class).getAppName());
+ }
+
+ @Test
+ public void testPipelineOptionsFactoryFromPropertiesWithLongPrefix() {
+assertEquals(
+ "testAppName",
+ PipelineOptionsFactory.fromProperties("org.apache.beam.", new
Properties() {{
+put("org.apache.beam.appName", "testAppName");
+ }}).as(ApplicationNameOptions.class).getAppName());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testPipelineOptionsFactoryFromNullProperties() {
+assertNotNull(PipelineOptionsFactory.fromProperties(null));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testPipelineOptionsFactoryFromPropertiesAndNullPrefix() {
+assertEquals(
Review comment:
drop the assertEquals as it does nothing
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 81298)
Time Spent: 9h 50m (was: 9h 40m)
> Support system properties source for pipeline options
> -
>
> Key: BEAM-3702
> URL: https://issues.apache.org/jira/browse/BEAM-3702
> Project: Beam
> Issue Type: Task
> Components: sdk-java-core
>Reporter: Romain Manni-Bucau
>Assignee: Romain Manni-Bucau
>Priority: Major
> Fix For: 2.5.0
>
> Time Spent: 9h 50m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[
https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=81285&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-81285
]
ASF GitHub Bot logged work on BEAM-3702:
Author: ASF GitHub Bot
Created on: 16/Mar/18 17:56
Start Date: 16/Mar/18 17:56
Worklog Time Spent: 10m
Work Description: lukecwik commented on a change in pull request #4683:
[BEAM-3702] adding fromJvm to create pipelineoptions from the system properties
URL: https://github.com/apache/beam/pull/4683#discussion_r175161005
##
File path:
sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
##
@@ -245,7 +263,19 @@ private Builder(String[] args, boolean validation,
*/
public Builder fromArgs(String... args) {
checkNotNull(args, "Arguments should not be null.");
- return new Builder(args, validation, strictParsing, true);
+ options.add(current -> parseCommandLine(args, current.strictParsing));
+ return new Builder(options, validation, strictParsing, true);
+}
+
+/**
+ * A key/value set of options.
+ * @param configuration the options to convert to PipelineOptions.
Review comment:
Please fill out a comment stating how the parsing works similar to fromArgs()
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 81285)
Time Spent: 8.5h (was: 8h 20m)
> Support system properties source for pipeline options
> -
>
> Key: BEAM-3702
> URL: https://issues.apache.org/jira/browse/BEAM-3702
> Project: Beam
> Issue Type: Task
> Components: sdk-java-core
>Reporter: Romain Manni-Bucau
>Assignee: Romain Manni-Bucau
>Priority: Major
> Fix For: 2.5.0
>
> Time Spent: 8.5h
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[
https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=81295&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-81295
]
ASF GitHub Bot logged work on BEAM-3702:
Author: ASF GitHub Bot
Created on: 16/Mar/18 17:56
Start Date: 16/Mar/18 17:56
Worklog Time Spent: 10m
Work Description: lukecwik commented on a change in pull request #4683:
[BEAM-3702] adding fromJvm to create pipelineoptions from the system properties
URL: https://github.com/apache/beam/pull/4683#discussion_r175164842
##
File path:
sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
##
@@ -1792,4 +1804,82 @@ private synchronized void register(Class iface) {
return combinedCache.get(interfaces).getPropertyDescriptors();
}
}
+
+ /**
+ * @param prefix a prefix filter on the map keys.
+ * @param options the options to convert to a pipeline options.
+ * @return a pipeline options instance based on the specified configuration.
+ */
+ @Experimental
+ public static PipelineOptions fromMap(final String prefix, final Map options) {
+checkArgument(prefix != null, "prefix should not be null.");
+checkArgument(options != null, "options should not be null.");
+final Map filtered = options.entrySet()
+.stream()
+.filter(e -> e.getKey()
+ .startsWith(prefix))
+.collect(toMap(Map.Entry::getKey,
Map.Entry::getValue));
+return fromMap(filtered);
+ }
+
+ /**
+ * @param options the options to convert to a pipeline options.
+ * @return a pipeline options instance based on the specified configuration.
+ */
+ public static PipelineOptions fromMap(final Map options) {
+checkArgument(options != null, "options should not be null.");
+return new Builder().fromMap(options).create();
+ }
+
+ /**
+ * @param properties the properties to use to create the pipeline options
instance.
Review comment:
Please improve the javadoc to be similar to `fromArgs` explaining the
parsing.
It's also important to mention that you filter out non string key/value
pairs since many people may assume that might call toString on everything.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 81295)
Time Spent: 9h 40m (was: 9.5h)
> Support system properties source for pipeline options
> -
>
> Key: BEAM-3702
> URL: https://issues.apache.org/jira/browse/BEAM-3702
> Project: Beam
> Issue Type: Task
> Components: sdk-java-core
>Reporter: Romain Manni-Bucau
>Assignee: Romain Manni-Bucau
>Priority: Major
> Fix For: 2.5.0
>
> Time Spent: 9h 40m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[
https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=81288&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-81288
]
ASF GitHub Bot logged work on BEAM-3702:
Author: ASF GitHub Bot
Created on: 16/Mar/18 17:56
Start Date: 16/Mar/18 17:56
Worklog Time Spent: 10m
Work Description: lukecwik commented on a change in pull request #4683:
[BEAM-3702] adding fromJvm to create pipelineoptions from the system properties
URL: https://github.com/apache/beam/pull/4683#discussion_r175164744
##
File path:
sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
##
@@ -1792,4 +1804,82 @@ private synchronized void register(Class iface) {
return combinedCache.get(interfaces).getPropertyDescriptors();
}
}
+
+ /**
+ * @param prefix a prefix filter on the map keys.
+ * @param options the options to convert to a pipeline options.
+ * @return a pipeline options instance based on the specified configuration.
+ */
+ @Experimental
+ public static PipelineOptions fromMap(final String prefix, final Map options) {
+checkArgument(prefix != null, "prefix should not be null.");
+checkArgument(options != null, "options should not be null.");
+final Map filtered = options.entrySet()
+.stream()
+.filter(e -> e.getKey()
+ .startsWith(prefix))
+.collect(toMap(Map.Entry::getKey,
Map.Entry::getValue));
+return fromMap(filtered);
+ }
+
+ /**
+ * @param options the options to convert to a pipeline options.
Review comment:
Please improve the javadoc to be similar to `fromArgs` explaining the
parsing.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 81288)
Time Spent: 8h 50m (was: 8h 40m)
> Support system properties source for pipeline options
> -
>
> Key: BEAM-3702
> URL: https://issues.apache.org/jira/browse/BEAM-3702
> Project: Beam
> Issue Type: Task
> Components: sdk-java-core
>Reporter: Romain Manni-Bucau
>Assignee: Romain Manni-Bucau
>Priority: Major
> Fix For: 2.5.0
>
> Time Spent: 8h 50m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[
https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=81297&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-81297
]
ASF GitHub Bot logged work on BEAM-3702:
Author: ASF GitHub Bot
Created on: 16/Mar/18 17:56
Start Date: 16/Mar/18 17:56
Worklog Time Spent: 10m
Work Description: lukecwik commented on a change in pull request #4683:
[BEAM-3702] adding fromJvm to create pipelineoptions from the system properties
URL: https://github.com/apache/beam/pull/4683#discussion_r175169858
##
File path:
sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java
##
@@ -1797,6 +1799,44 @@ public void serialize(JacksonIncompatible
jacksonIncompatible, JsonGenerator jso
}
}
+ @Test
+ public void testPipelineOptionsFactoryFromProperties() {
+assertEquals("testAppName", PipelineOptionsFactory.fromProperties(new
Properties() {{
Review comment:
Add a string key with non string value to the properties and also a non
string key and string value and validate that they aren't used.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 81297)
Time Spent: 9h 50m (was: 9h 40m)
> Support system properties source for pipeline options
> -
>
> Key: BEAM-3702
> URL: https://issues.apache.org/jira/browse/BEAM-3702
> Project: Beam
> Issue Type: Task
> Components: sdk-java-core
>Reporter: Romain Manni-Bucau
>Assignee: Romain Manni-Bucau
>Priority: Major
> Fix For: 2.5.0
>
> Time Spent: 9h 50m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[
https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=81290&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-81290
]
ASF GitHub Bot logged work on BEAM-3702:
Author: ASF GitHub Bot
Created on: 16/Mar/18 17:56
Start Date: 16/Mar/18 17:56
Worklog Time Spent: 10m
Work Description: lukecwik commented on a change in pull request #4683:
[BEAM-3702] adding fromJvm to create pipelineoptions from the system properties
URL: https://github.com/apache/beam/pull/4683#discussion_r175168208
##
File path:
sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
##
@@ -1792,4 +1804,82 @@ private synchronized void register(Class iface) {
return combinedCache.get(interfaces).getPropertyDescriptors();
}
}
+
+ /**
+ * @param prefix a prefix filter on the map keys.
+ * @param options the options to convert to a pipeline options.
+ * @return a pipeline options instance based on the specified configuration.
+ */
+ @Experimental
+ public static PipelineOptions fromMap(final String prefix, final Map options) {
+checkArgument(prefix != null, "prefix should not be null.");
+checkArgument(options != null, "options should not be null.");
+final Map filtered = options.entrySet()
+.stream()
+.filter(e -> e.getKey()
+ .startsWith(prefix))
+.collect(toMap(Map.Entry::getKey,
Map.Entry::getValue));
+return fromMap(filtered);
+ }
+
+ /**
+ * @param options the options to convert to a pipeline options.
+ * @return a pipeline options instance based on the specified configuration.
+ */
+ public static PipelineOptions fromMap(final Map options) {
+checkArgument(options != null, "options should not be null.");
+return new Builder().fromMap(options).create();
+ }
+
+ /**
+ * @param properties the properties to use to create the pipeline options
instance.
+ * @return a pipeline options instance based on the specified properties.
+ */
+ @Experimental
+ public static PipelineOptions fromProperties(final Properties properties) {
+checkArgument(properties != null, "properties should not be null.");
+return fromMap(properties.stringPropertyNames()
+.stream()
+.collect(toMap(identity(), properties::getProperty)));
+ }
+
+ /**
+ * @param prefix the prefix filter applied on system properties keys.
+ * @param properties the properties to use to create the pipeline options
instance.
Review comment:
Please improve the javadoc to be similar to `fromArgs` explaining the
parsing.
It's also important to mention that you filter out non string key/value
pairs since many people may assume that might call toString on everything.
Please also mention how prefix is used.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 81290)
Time Spent: 9h 10m (was: 9h)
> Support system properties source for pipeline options
> -
>
> Key: BEAM-3702
> URL: https://issues.apache.org/jira/browse/BEAM-3702
> Project: Beam
> Issue Type: Task
> Components: sdk-java-core
>Reporter: Romain Manni-Bucau
>Assignee: Romain Manni-Bucau
>Priority: Major
> Fix For: 2.5.0
>
> Time Spent: 9h 10m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[
https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=81299&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-81299
]
ASF GitHub Bot logged work on BEAM-3702:
Author: ASF GitHub Bot
Created on: 16/Mar/18 17:56
Start Date: 16/Mar/18 17:56
Worklog Time Spent: 10m
Work Description: lukecwik commented on a change in pull request #4683:
[BEAM-3702] adding fromJvm to create pipelineoptions from the system properties
URL: https://github.com/apache/beam/pull/4683#discussion_r175168913
##
File path:
sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java
##
@@ -1797,6 +1799,44 @@ public void serialize(JacksonIncompatible
jacksonIncompatible, JsonGenerator jso
}
}
+ @Test
+ public void testPipelineOptionsFactoryFromProperties() {
+assertEquals("testAppName", PipelineOptionsFactory.fromProperties(new
Properties() {{
+ put("appName", "testAppName");
+}}).as(ApplicationNameOptions.class).getAppName());
+ }
+
+ @Test
+ public void testPipelineOptionsFactoryFromPropertiesWithPrefix() {
+assertEquals(
+ "testAppName",
+ PipelineOptionsFactory.fromProperties("prefix.", new Properties() {{
+put("prefix.appName", "testAppName");
Review comment:
Add a value which shouldn't be matched and test that it isn't set.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 81299)
Time Spent: 10h (was: 9h 50m)
> Support system properties source for pipeline options
> -
>
> Key: BEAM-3702
> URL: https://issues.apache.org/jira/browse/BEAM-3702
> Project: Beam
> Issue Type: Task
> Components: sdk-java-core
>Reporter: Romain Manni-Bucau
>Assignee: Romain Manni-Bucau
>Priority: Major
> Fix For: 2.5.0
>
> Time Spent: 10h
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[
https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=81291&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-81291
]
ASF GitHub Bot logged work on BEAM-3702:
Author: ASF GitHub Bot
Created on: 16/Mar/18 17:56
Start Date: 16/Mar/18 17:56
Worklog Time Spent: 10m
Work Description: lukecwik commented on a change in pull request #4683:
[BEAM-3702] adding fromJvm to create pipelineoptions from the system properties
URL: https://github.com/apache/beam/pull/4683#discussion_r175169413
##
File path:
sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java
##
@@ -1797,6 +1799,44 @@ public void serialize(JacksonIncompatible
jacksonIncompatible, JsonGenerator jso
}
}
+ @Test
+ public void testPipelineOptionsFactoryFromProperties() {
+assertEquals("testAppName", PipelineOptionsFactory.fromProperties(new
Properties() {{
+ put("appName", "testAppName");
+}}).as(ApplicationNameOptions.class).getAppName());
+ }
+
+ @Test
+ public void testPipelineOptionsFactoryFromPropertiesWithPrefix() {
+assertEquals(
+ "testAppName",
+ PipelineOptionsFactory.fromProperties("prefix.", new Properties() {{
+put("prefix.appName", "testAppName");
+ }}).as(ApplicationNameOptions.class).getAppName());
+ }
+
+ @Test
+ public void testPipelineOptionsFactoryFromPropertiesWithLongPrefix() {
+assertEquals(
+ "testAppName",
+ PipelineOptionsFactory.fromProperties("org.apache.beam.", new
Properties() {{
+put("org.apache.beam.appName", "testAppName");
+ }}).as(ApplicationNameOptions.class).getAppName());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testPipelineOptionsFactoryFromNullProperties() {
+assertNotNull(PipelineOptionsFactory.fromProperties(null));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testPipelineOptionsFactoryFromPropertiesAndNullPrefix() {
+assertEquals(
+ "testAppName",
+ PipelineOptionsFactory.fromProperties(null, new Properties() {{
+put("appName", "testAppName");
+ }}).as(ApplicationNameOptions.class).getAppName());
+ }
Review comment:
nit: whitespace
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 81291)
Time Spent: 9h 20m (was: 9h 10m)
> Support system properties source for pipeline options
> -
>
> Key: BEAM-3702
> URL: https://issues.apache.org/jira/browse/BEAM-3702
> Project: Beam
> Issue Type: Task
> Components: sdk-java-core
>Reporter: Romain Manni-Bucau
>Assignee: Romain Manni-Bucau
>Priority: Major
> Fix For: 2.5.0
>
> Time Spent: 9h 20m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[
https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=81293&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-81293
]
ASF GitHub Bot logged work on BEAM-3702:
Author: ASF GitHub Bot
Created on: 16/Mar/18 17:56
Start Date: 16/Mar/18 17:56
Worklog Time Spent: 10m
Work Description: lukecwik commented on a change in pull request #4683:
[BEAM-3702] adding fromJvm to create pipelineoptions from the system properties
URL: https://github.com/apache/beam/pull/4683#discussion_r175167532
##
File path:
sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
##
@@ -1792,4 +1804,82 @@ private synchronized void register(Class iface) {
return combinedCache.get(interfaces).getPropertyDescriptors();
}
}
+
+ /**
+ * @param prefix a prefix filter on the map keys.
+ * @param options the options to convert to a pipeline options.
+ * @return a pipeline options instance based on the specified configuration.
+ */
+ @Experimental
+ public static PipelineOptions fromMap(final String prefix, final Map options) {
+checkArgument(prefix != null, "prefix should not be null.");
+checkArgument(options != null, "options should not be null.");
+final Map filtered = options.entrySet()
+.stream()
+.filter(e -> e.getKey()
+ .startsWith(prefix))
+.collect(toMap(Map.Entry::getKey,
Map.Entry::getValue));
+return fromMap(filtered);
+ }
+
+ /**
+ * @param options the options to convert to a pipeline options.
+ * @return a pipeline options instance based on the specified configuration.
+ */
+ public static PipelineOptions fromMap(final Map options) {
+checkArgument(options != null, "options should not be null.");
+return new Builder().fromMap(options).create();
+ }
+
+ /**
+ * @param properties the properties to use to create the pipeline options
instance.
+ * @return a pipeline options instance based on the specified properties.
+ */
+ @Experimental
+ public static PipelineOptions fromProperties(final Properties properties) {
+checkArgument(properties != null, "properties should not be null.");
+return fromMap(properties.stringPropertyNames()
+.stream()
+.collect(toMap(identity(), properties::getProperty)));
+ }
+
+ /**
+ * @param prefix the prefix filter applied on system properties keys.
+ * @param properties the properties to use to create the pipeline options
instance.
+ * @return a pipeline options instance based on the specified properties
+ * filtered with the specified prefix.
+ */
+ @Experimental
+ public static PipelineOptions fromProperties(final String prefix, final
Properties properties) {
+checkArgument(prefix != null, "prefix should not be null.");
+checkArgument(properties != null, "properties should not be null.");
+final Properties instantiationProperties = properties.stringPropertyNames()
+ .stream()
+ .filter(k -> k.startsWith(prefix))
+ .collect(
+Properties::new,
+(p, k) -> p.setProperty(k.substring(prefix.length()),
properties.getProperty(k)),
+Hashtable::putAll);
+return fromProperties(instantiationProperties);
+ }
+
+ /**
+ * @param prefix the prefix filter applied on system properties keys.
+ * @return a pipeline options instance based on system properties key/values
Review comment:
Please improve/explain what is the contract for this method.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 81293)
> Support system properties source for pipeline options
> -
>
> Key: BEAM-3702
> URL: https://issues.apache.org/jira/browse/BEAM-3702
> Project: Beam
> Issue Type: Task
> Components: sdk-java-core
>Reporter: Romain Manni-Bucau
>Assignee: Romain Manni-Bucau
>Priority: Major
> Fix For: 2.5.0
>
> Time Spent: 9h 20m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[
https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=81287&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-81287
]
ASF GitHub Bot logged work on BEAM-3702:
Author: ASF GitHub Bot
Created on: 16/Mar/18 17:56
Start Date: 16/Mar/18 17:56
Worklog Time Spent: 10m
Work Description: lukecwik commented on a change in pull request #4683:
[BEAM-3702] adding fromJvm to create pipelineoptions from the system properties
URL: https://github.com/apache/beam/pull/4683#discussion_r175164167
##
File path:
sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
##
@@ -1182,7 +1194,7 @@ private static void
validateInheritedInterfacesExtendPipelineOptions(Class kl
private static class MultipleDefinitions {
private Method method;
-private SortedSet collidingMethods;
Review comment:
Its important that this remains being sorted for logging reasons.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 81287)
Time Spent: 8h 50m (was: 8h 40m)
> Support system properties source for pipeline options
> -
>
> Key: BEAM-3702
> URL: https://issues.apache.org/jira/browse/BEAM-3702
> Project: Beam
> Issue Type: Task
> Components: sdk-java-core
>Reporter: Romain Manni-Bucau
>Assignee: Romain Manni-Bucau
>Priority: Major
> Fix For: 2.5.0
>
> Time Spent: 8h 50m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[
https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=81296&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-81296
]
ASF GitHub Bot logged work on BEAM-3702:
Author: ASF GitHub Bot
Created on: 16/Mar/18 17:56
Start Date: 16/Mar/18 17:56
Worklog Time Spent: 10m
Work Description: lukecwik commented on a change in pull request #4683:
[BEAM-3702] adding fromJvm to create pipelineoptions from the system properties
URL: https://github.com/apache/beam/pull/4683#discussion_r175169560
##
File path:
sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java
##
@@ -1797,6 +1799,44 @@ public void serialize(JacksonIncompatible
jacksonIncompatible, JsonGenerator jso
}
}
+ @Test
+ public void testPipelineOptionsFactoryFromProperties() {
+assertEquals("testAppName", PipelineOptionsFactory.fromProperties(new
Properties() {{
+ put("appName", "testAppName");
+}}).as(ApplicationNameOptions.class).getAppName());
+ }
+
+ @Test
+ public void testPipelineOptionsFactoryFromPropertiesWithPrefix() {
+assertEquals(
+ "testAppName",
+ PipelineOptionsFactory.fromProperties("prefix.", new Properties() {{
+put("prefix.appName", "testAppName");
+ }}).as(ApplicationNameOptions.class).getAppName());
+ }
+
+ @Test
+ public void testPipelineOptionsFactoryFromPropertiesWithLongPrefix() {
+assertEquals(
+ "testAppName",
+ PipelineOptionsFactory.fromProperties("org.apache.beam.", new
Properties() {{
+put("org.apache.beam.appName", "testAppName");
+ }}).as(ApplicationNameOptions.class).getAppName());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testPipelineOptionsFactoryFromNullProperties() {
+assertNotNull(PipelineOptionsFactory.fromProperties(null));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testPipelineOptionsFactoryFromPropertiesAndNullPrefix() {
+assertEquals(
+ "testAppName",
+ PipelineOptionsFactory.fromProperties(null, new Properties() {{
+put("appName", "testAppName");
+ }}).as(ApplicationNameOptions.class).getAppName());
Review comment:
Please add tests for system properties
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 81296)
> Support system properties source for pipeline options
> -
>
> Key: BEAM-3702
> URL: https://issues.apache.org/jira/browse/BEAM-3702
> Project: Beam
> Issue Type: Task
> Components: sdk-java-core
>Reporter: Romain Manni-Bucau
>Assignee: Romain Manni-Bucau
>Priority: Major
> Fix For: 2.5.0
>
> Time Spent: 9h 40m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[
https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=81294&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-81294
]
ASF GitHub Bot logged work on BEAM-3702:
Author: ASF GitHub Bot
Created on: 16/Mar/18 17:56
Start Date: 16/Mar/18 17:56
Worklog Time Spent: 10m
Work Description: lukecwik commented on a change in pull request #4683:
[BEAM-3702] adding fromJvm to create pipelineoptions from the system properties
URL: https://github.com/apache/beam/pull/4683#discussion_r175168423
##
File path:
sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
##
@@ -1792,4 +1804,82 @@ private synchronized void register(Class iface) {
return combinedCache.get(interfaces).getPropertyDescriptors();
}
}
+
+ /**
+ * @param prefix a prefix filter on the map keys.
+ * @param options the options to convert to a pipeline options.
+ * @return a pipeline options instance based on the specified configuration.
+ */
+ @Experimental
+ public static PipelineOptions fromMap(final String prefix, final Map options) {
+checkArgument(prefix != null, "prefix should not be null.");
+checkArgument(options != null, "options should not be null.");
+final Map filtered = options.entrySet()
+.stream()
+.filter(e -> e.getKey()
+ .startsWith(prefix))
+.collect(toMap(Map.Entry::getKey,
Map.Entry::getValue));
+return fromMap(filtered);
+ }
+
+ /**
+ * @param options the options to convert to a pipeline options.
+ * @return a pipeline options instance based on the specified configuration.
+ */
+ public static PipelineOptions fromMap(final Map options) {
+checkArgument(options != null, "options should not be null.");
+return new Builder().fromMap(options).create();
+ }
+
+ /**
+ * @param properties the properties to use to create the pipeline options
instance.
+ * @return a pipeline options instance based on the specified properties.
+ */
+ @Experimental
+ public static PipelineOptions fromProperties(final Properties properties) {
+checkArgument(properties != null, "properties should not be null.");
+return fromMap(properties.stringPropertyNames()
+.stream()
+.collect(toMap(identity(), properties::getProperty)));
+ }
+
+ /**
+ * @param prefix the prefix filter applied on system properties keys.
+ * @param properties the properties to use to create the pipeline options
instance.
+ * @return a pipeline options instance based on the specified properties
+ * filtered with the specified prefix.
+ */
+ @Experimental
+ public static PipelineOptions fromProperties(final String prefix, final
Properties properties) {
+checkArgument(prefix != null, "prefix should not be null.");
+checkArgument(properties != null, "properties should not be null.");
+final Properties instantiationProperties = properties.stringPropertyNames()
+ .stream()
+ .filter(k -> k.startsWith(prefix))
+ .collect(
+Properties::new,
+(p, k) -> p.setProperty(k.substring(prefix.length()),
properties.getProperty(k)),
+Hashtable::putAll);
+return fromProperties(instantiationProperties);
+ }
+
+ /**
+ * @param prefix the prefix filter applied on system properties keys.
+ * @return a pipeline options instance based on system properties key/values
+ * and filtered with a custom prefix.
+ */
+ @Experimental
+ public static PipelineOptions fromSystemProperties(final String prefix) {
+checkArgument(prefix != null, "prefix should not be null.");
+return fromProperties(prefix, System.getProperties());
+ }
+
+ /**
+ * @return a pipeline options instance based on system properties
+ * filtered based on beam. prefix.
Review comment:
Please improve the contract of this method.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 81294)
Time Spent: 9.5h (was: 9h 20m)
> Support system properties source for pipeline options
> -
>
> Key: BEAM-3702
> URL: https://issues.apache.org/jira/browse/BEAM-3702
> Project: Beam
> Issue Type: Task
> Components: sdk-java-core
>Reporter: Romain Manni-Bucau
>Assignee: Romain Manni-Bucau
>Priority: Major
> Fi
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[
https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=81284&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-81284
]
ASF GitHub Bot logged work on BEAM-3702:
Author: ASF GitHub Bot
Created on: 16/Mar/18 17:56
Start Date: 16/Mar/18 17:56
Worklog Time Spent: 10m
Work Description: lukecwik commented on a change in pull request #4683:
[BEAM-3702] adding fromJvm to create pipelineoptions from the system properties
URL: https://github.com/apache/beam/pull/4683#discussion_r175162404
##
File path:
sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
##
@@ -245,7 +263,19 @@ private Builder(String[] args, boolean validation,
*/
public Builder fromArgs(String... args) {
checkNotNull(args, "Arguments should not be null.");
- return new Builder(args, validation, strictParsing, true);
+ options.add(current -> parseCommandLine(args, current.strictParsing));
Review comment:
It is important to point out that this is a subtle change in behavior.
Before fromArgs replaced the existing set of args and now we are accumulating
them. Please update the javadoc accordingly.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 81284)
Time Spent: 8h 20m (was: 8h 10m)
> Support system properties source for pipeline options
> -
>
> Key: BEAM-3702
> URL: https://issues.apache.org/jira/browse/BEAM-3702
> Project: Beam
> Issue Type: Task
> Components: sdk-java-core
>Reporter: Romain Manni-Bucau
>Assignee: Romain Manni-Bucau
>Priority: Major
> Fix For: 2.5.0
>
> Time Spent: 8h 20m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-3702) Support system properties source for pipeline options
[
https://issues.apache.org/jira/browse/BEAM-3702?focusedWorklogId=81292&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-81292
]
ASF GitHub Bot logged work on BEAM-3702:
Author: ASF GitHub Bot
Created on: 16/Mar/18 17:56
Start Date: 16/Mar/18 17:56
Worklog Time Spent: 10m
Work Description: lukecwik commented on a change in pull request #4683:
[BEAM-3702] adding fromJvm to create pipelineoptions from the system properties
URL: https://github.com/apache/beam/pull/4683#discussion_r175169264
##
File path:
sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java
##
@@ -1797,6 +1799,44 @@ public void serialize(JacksonIncompatible
jacksonIncompatible, JsonGenerator jso
}
}
+ @Test
+ public void testPipelineOptionsFactoryFromProperties() {
+assertEquals("testAppName", PipelineOptionsFactory.fromProperties(new
Properties() {{
+ put("appName", "testAppName");
+}}).as(ApplicationNameOptions.class).getAppName());
+ }
+
+ @Test
+ public void testPipelineOptionsFactoryFromPropertiesWithPrefix() {
+assertEquals(
+ "testAppName",
+ PipelineOptionsFactory.fromProperties("prefix.", new Properties() {{
+put("prefix.appName", "testAppName");
+ }}).as(ApplicationNameOptions.class).getAppName());
+ }
+
+ @Test
+ public void testPipelineOptionsFactoryFromPropertiesWithLongPrefix() {
+assertEquals(
+ "testAppName",
+ PipelineOptionsFactory.fromProperties("org.apache.beam.", new
Properties() {{
+put("org.apache.beam.appName", "testAppName");
+ }}).as(ApplicationNameOptions.class).getAppName());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testPipelineOptionsFactoryFromNullProperties() {
+assertNotNull(PipelineOptionsFactory.fromProperties(null));
Review comment:
Drop the assertNotNull, it doesn't do anything
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 81292)
Time Spent: 9h 20m (was: 9h 10m)
> Support system properties source for pipeline options
> -
>
> Key: BEAM-3702
> URL: https://issues.apache.org/jira/browse/BEAM-3702
> Project: Beam
> Issue Type: Task
> Components: sdk-java-core
>Reporter: Romain Manni-Bucau
>Assignee: Romain Manni-Bucau
>Priority: Major
> Fix For: 2.5.0
>
> Time Spent: 9h 20m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
