pnowojski commented on a change in pull request #10084: [FLINK-14382][yarn] Incorrect handling of FLINK_PLUGINS_DIR on Yarn URL: https://github.com/apache/flink/pull/10084#discussion_r346243587
########## File path: flink-java/src/main/java/org/apache/flink/api/java/utils/MultipleParameterTool.java ########## @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.java.utils; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.java.Utils; +import org.apache.flink.util.Preconditions; + +import org.apache.commons.lang3.math.NumberUtils; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * This class provides simple utility methods for reading and parsing program arguments from different sources. + * Multiple values parameter in args could be supported. For example, --multi multiValue1 --multi multiValue2. + * If {@link MultipleParameterTool} object is used for GlobalJobParameters, the last one of multiple values will be used. + * Navigate to {@link #toMap()} for more information. + */ +@PublicEvolving +public class MultipleParameterTool extends AbstractParameterTool { + private static final long serialVersionUID = 1L; + + // ------------------ Constructors ------------------------ + + /** + * Returns {@link MultipleParameterTool} for the given arguments. The arguments are keys followed by values. + * Keys have to start with '-' or '--' + * + * <p><strong>Example arguments:</strong> + * --key1 value1 --key2 value2 -key3 value3 + * --multi multiValue1 --multi multiValue2 + * + * @param args Input array arguments + * @return A {@link MultipleParameterTool} + */ + public static MultipleParameterTool fromArgs(String[] args) { + final Map<String, Collection<String>> map = new HashMap<>(args.length / 2); + + int i = 0; + while (i < args.length) { + final String key = Utils.getKeyFromArgs(args, i); + + i += 1; // try to find the value + + map.putIfAbsent(key, new ArrayList<>()); + if (i >= args.length) { + map.get(key).add(NO_VALUE_KEY); + } else if (NumberUtils.isNumber(args[i])) { + map.get(key).add(args[i]); + i += 1; + } else if (args[i].startsWith("--") || args[i].startsWith("-")) { + // the argument cannot be a negative number because we checked earlier + // -> the next argument is a parameter name + map.get(key).add(NO_VALUE_KEY); + } else { + map.get(key).add(args[i]); + i += 1; + } + } + + return fromMultiMap(map); + } + + /** + * Returns {@link MultipleParameterTool} for the given multi map. + * + * @param multiMap A map of arguments. Key is String and value is a Collection. + * @return A {@link MultipleParameterTool} + */ + public static MultipleParameterTool fromMultiMap(Map<String, Collection<String>> multiMap) { + Preconditions.checkNotNull(multiMap, "Unable to initialize from empty map"); + return new MultipleParameterTool(multiMap); + } + + // ------------------ ParameterUtil ------------------------ + protected final Map<String, Collection<String>> data; + + private MultipleParameterTool(Map<String, Collection<String>> data) { + this.data = Collections.unmodifiableMap(new HashMap<>(data)); + + this.defaultData = new ConcurrentHashMap<>(data.size()); + + this.unrequestedParameters = Collections.newSetFromMap(new ConcurrentHashMap<>(data.size())); + + unrequestedParameters.addAll(data.keySet()); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + MultipleParameterTool that = (MultipleParameterTool) o; + return Objects.equals(data, that.data) && + Objects.equals(defaultData, that.defaultData) && + Objects.equals(unrequestedParameters, that.unrequestedParameters); + } + + @Override + public int hashCode() { + return Objects.hash(data, defaultData, unrequestedParameters); + } + + // ------------------ Get data from the util ---------------- + + /** + * Returns number of parameters in {@link ParameterTool}. + */ + @Override + public int getNumberOfParameters() { + return data.size(); + } + + /** + * Returns the String value for the given key. The value should only have one item. + * Use {@link #getMultiParameter(String)} instead if want to get multiple values parameter. + * If the key does not exist it will return null. + */ + @Override + public String get(String key) { + addToDefaults(key, null); + unrequestedParameters.remove(key); + if (!data.containsKey(key)) { + return null; + } + Preconditions.checkState(data.get(key).size() == 1, + "Key %s should has only one value.", key); + return (String) data.get(key).toArray()[0]; + } + + /** + * Check if value is set. + */ + @Override + public boolean has(String value) { + addToDefaults(value, null); + unrequestedParameters.remove(value); + return data.containsKey(value); + } + + /** + * Returns the Collection of String values for the given key. + * If the key does not exist it will return null. + */ + public Collection<String> getMultiParameter(String key) { + addToDefaults(key, null); + unrequestedParameters.remove(key); + return data.getOrDefault(key, null); + } + + /** + * Returns the Collection of String values for the given key. + * If the key does not exist it will throw a {@link RuntimeException}. + */ + public Collection<String> getMultiParameterRequired(String key) { + addToDefaults(key, null); + Collection<String> value = getMultiParameter(key); + if (value == null) { + throw new RuntimeException("No data for required key '" + key + "'"); + } + return value; + } + + // ------------------------- Export to different targets ------------------------- + + /** + * Return MultiMap of all the parameters processed by {@link MultipleParameterTool}. + * + * @return MultiMap of the {@link MultipleParameterTool}. Key is String and Value is a Collection of String. + */ + public Map<String, Collection<String>> toMultiMap() { + return data; + } + + @Override + protected Object clone() throws CloneNotSupportedException { + return new MultipleParameterTool(this.data); + } + + // ------------------------- Interaction with other ParameterUtils ------------------------- + + /** + * Merges two {@link MultipleParameterTool}. + * + * @param other Other {@link MultipleParameterTool} object + * @return The Merged {@link MultipleParameterTool} + */ + public MultipleParameterTool mergeWith(MultipleParameterTool other) { + final Map<String, Collection<String>> resultData = new HashMap<>(data.size() + other.data.size()); + resultData.putAll(data); + resultData.putAll(other.data); Review comment: what if a key is present both in `this` and `other`? Should we join the collections? Frankly If you don't want to spend time fixing it, I would be fine with dropping this method as we do not need it for now. It always can be added in the future. Of course if you would prefer to keep it, I'm also fine with it. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
