[
https://issues.apache.org/jira/browse/STORM-1227?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15109154#comment-15109154
]
ASF GitHub Bot commented on STORM-1227:
---------------------------------------
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/1030#discussion_r50300594
--- Diff: storm-core/src/jvm/org/apache/storm/utils/ConfigUtils.java ---
@@ -0,0 +1,711 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.utils;
+
+import org.apache.storm.Config;
+import org.apache.storm.validation.ConfigValidation;
+import org.apache.storm.generated.StormTopology;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Collections;
+
+public class ConfigUtils {
+ private final static Logger LOG =
LoggerFactory.getLogger(ConfigUtils.class);
+ public final static String RESOURCES_SUBDIR = "resources";
+ public final static String NIMBUS_DO_NOT_REASSIGN =
"NIMBUS-DO-NOT-REASSIGN";
+ public static final String FILE_SEPARATOR = File.separator;
+ public final static String LOG_DIR;
+
+ static {
+ String dir;
+ Map conf;
+ if (System.getProperty("storm.log.dir") != null) {
+ dir = System.getProperty("storm.log.dir");
+ } else if ((conf = readStormConfig()).get("storm.log.dir") !=
null) {
+ dir = String.valueOf(conf.get("storm.log.dir"));
+ } else {
+ if (System.getProperty("storm.home") != null) {
+ dir = System.getProperty("storm.home") + FILE_SEPARATOR +
"logs";
+ } else {
+ dir = FILE_SEPARATOR + "logs";
+ }
+ }
+ try {
+ LOG_DIR = new File(dir).getCanonicalPath();
+ } catch (IOException ex) {
+ throw new IllegalArgumentException("Illegal storm.log.dir in
conf: " + dir);
+ }
+ }
+
+ public static String clojureConfigName(String name) {
+ return name.toUpperCase().replace("_", "-");
+ }
+
+ // ALL-CONFIGS is only used by executor.clj once, do we want to do it
here? TODO
+ public static List<Object> All_CONFIGS() {
+ List<Object> ret = new ArrayList<Object>();
+ Config config = new Config();
+ Class<?> ConfigClass = config.getClass();
+ Field[] fields = ConfigClass.getFields();
+ for (int i = 0; i < fields.length; i++) {
+ try {
+ Object obj = fields[i].get(null);
+ ret.add(obj);
+ } catch (IllegalArgumentException e) {
+ LOG.error(e.getMessage(), e);
+ } catch (IllegalAccessException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+ return ret;
+ }
+
+ public static String clusterMode(Map conf) {
+ String mode = (String) conf.get(Config.STORM_CLUSTER_MODE);
+ return mode;
+
+ }
+
+ public static boolean isLocalMode(Map conf) {
+ String mode = (String) conf.get(Config.STORM_CLUSTER_MODE);
+ if (mode != null) {
+ if ("local".equals(mode)) {
+ return true;
+ }
+ if ("distributed".equals(mode)) {
+ return false;
+ }
+ }
+ throw new IllegalArgumentException("Illegal cluster mode in conf:
" + mode);
+ }
+
+ public static int samplingRate(Map conf) {
+ double rate =
Utils.getDouble(conf.get(Config.TOPOLOGY_STATS_SAMPLE_RATE));
+ if (rate != 0) {
+ return (int) (1 / rate);
+ }
+ throw new IllegalArgumentException("Illegal
topology.stats.sample.rate in conf: " + rate);
+ }
+
+ // public static mkStatsSampler // depends on Utils.evenSampler()
TODO, this is sth we need to do after util
+ // public static readDefaultConfig // depends on
Utils.clojurifyStructure and Utils.readDefaultConfig // TODO
+ // validate-configs-with-schemas is just a wrapper of
ConfigValidation.validateFields(conf)
+
+ //For testing only
+ // for java
+ // try (SetMockedStormConfig mocked = new SetMockedStormConfig(conf)) {
+ // run test ...
+ // }
+ //
+ // for clojure
+ // (with-open [mock (SetMockedStormConfig. conf)]
+ // run test ...)
+ public static class SetMockedStormConfig implements Closeable {
+ public SetMockedStormConfig(Map conf) {
+ mockedStormConfig = conf;
+ }
+
+ @Override
+ public void close() {
+ mockedStormConfig = null;
+ }
+ }
+ private static Map mockedStormConfig = null;
+ public static Map readStormConfig() {
+ if (mockedStormConfig != null) return mockedStormConfig;
+ Map conf = Utils.readStormConfig();
+ ConfigValidation.validateFields(conf);
+ return conf;
+ }
+
+ public static Map readYamlConfig(String name, boolean mustExist) {
+ Map conf = Utils.findAndReadConfigFile(name, mustExist);
+ ConfigValidation.validateFields(conf);
+ return conf;
+ }
+
+ public static Map readYamlConfig(String name) {
+ return readYamlConfig(name, true);
+ }
+
+ public static String absoluteStormLocalDir(Map conf) {
+ String stormHome = System.getProperty("storm.home");
+ String localDir = (String) conf.get(Config.STORM_LOCAL_DIR);
+ if (localDir == null) {
+ return (stormHome + FILE_SEPARATOR + "storm-local");
+ } else {
+ if (new File(localDir).isAbsolute()) {
+ return localDir;
+ } else {
+ return (stormHome + FILE_SEPARATOR + localDir);
+ }
+ }
+
+ }
+
+ public static String absoluteHealthCheckDir(Map conf) {
+ String stormHome = System.getProperty("storm.home");
+ String healthCheckDir =
String.valueOf(conf.get(Config.STORM_HEALTH_CHECK_DIR));
+ if (healthCheckDir.equals("null")) {
+ return (stormHome + FILE_SEPARATOR + "healthchecks");
+ } else {
+ if (new File(healthCheckDir).isAbsolute()) {
+ return healthCheckDir;
+ } else {
+ return (stormHome + FILE_SEPARATOR + healthCheckDir);
+ }
+ }
+ }
+
+ public static String masterLocalDir(Map conf) throws IOException {
+ String ret = String.valueOf(conf.get(Config.STORM_LOCAL_DIR)) +
FILE_SEPARATOR + "nimbus";
+ try {
+ FileUtils.forceMkdir(new File(ret));
+ } catch (IOException e) {
+ LOG.error("Failed to create dir " + ret, e);
+ throw e;
+ }
+ return ret;
+ }
+
+
+ public static String masterStormJarKey(String topologyId) {
+ return (topologyId + "-stormjar.jar");
+ }
+
+ public static String masterStormCodeKey(String topologyId) {
+ return (topologyId + "-stormcode.ser");
+ }
+
+ public static String masterStormConfKey(String topologyId) {
+ return (topologyId + "-stormconf.ser");
+ }
+
+ public static String masterStormDistRoot(Map conf) throws IOException {
+ String ret = stormDistPath(masterLocalDir(conf));
+ FileUtils.forceMkdir(new File(ret));
+ return ret;
+ }
+
+ public static String masterStormDistRoot(Map conf, String stormId)
throws IOException {
+ return (masterStormDistRoot(conf) + FILE_SEPARATOR + stormId);
+ }
+
+ public static String stormDistPath(String stormRoot) {
+ String ret = "";
+ // we do this since to concat a null String will actually concat a
"null", which is not the expected: ""
+ if (stormRoot != null) {
+ ret = stormRoot;
+ }
+ return ret + FILE_SEPARATOR + "stormdist";
+ }
+
+ public static String stormTmpPath(String stormRoot) {
+ return stormRoot + FILE_SEPARATOR + "tmp";
+ }
+
+ /* Never get used TODO : delete it*/
+ public static String masterTmpDir(Map conf) throws IOException {
+ String ret = stormTmpPath(masterLocalDir(conf));
+ FileUtils.forceMkdir(new File(ret));
+ return ret;
+ }
+
+ public static Map readSupervisorStormConfGivenPath(Map conf, String
stormConfPath) throws IOException {
+ Map ret = new HashMap(conf);
+
ret.putAll(Utils.fromCompressedJsonConf(FileUtils.readFileToByteArray(new
File(stormConfPath))));
+ return ret;
+ }
+
+ /* Never get used TODO : may delete it*/
+ public static String masterStormMetaFilePath(String stormRoot) {
+ return (stormRoot + FILE_SEPARATOR +
"storm-code-distributor.meta");
+ }
+
+ public static String masterStormJarPath(String stormRoot) {
+ return (stormRoot + FILE_SEPARATOR + "stormjar.jar");
+ }
+
+ /* Never get used TODO : may delete it*/
+ public static String masterStormCodePath(String stormRoot) {
+ return (stormRoot + FILE_SEPARATOR + "stormcode.ser");
+ }
+
+ /* Never get used TODO : may delete it*/
+ public static String masterStormConfPath(String stormRoot) {
+ return (stormRoot + FILE_SEPARATOR + "stormconf.ser");
+ }
+
+ public static String masterInbox(Map conf) throws IOException {
+ String ret = masterLocalDir(conf) + FILE_SEPARATOR + "inbox";
+ try {
+ FileUtils.forceMkdir(new File(ret));
+ } catch (IOException e) {
+ LOG.error("Failed to create dir " + ret, e);
+ throw e;
+ }
+ return ret;
+ }
+
+ public static String masterInimbusDir(Map conf) throws IOException {
+ return (masterLocalDir(conf) + FILE_SEPARATOR + "inimbus");
+ }
+
+ //For testing only
+ // for java
+ // try (SetMockedSupervisorLocalDir mocked = new
SetMockedSupervisorLocalDir(dir)) {
+ // run test ...
+ // }
+ //
+ // for clojure
+ // (with-open [mock (SetMockedSupervisorLocalDir. dir)]
+ // run test ...)
+ public static class SetMockedSupervisorLocalDir implements Closeable {
+ public SetMockedSupervisorLocalDir(String dir) {
+ mockedSupervisorLocalDir = dir;
+ }
+ @Override
+ public void close() {
+ mockedSupervisorLocalDir = null;
+ }
+ }
+ private static String mockedSupervisorLocalDir = null;
+ public static String supervisorLocalDir(Map conf) throws IOException {
+ if (mockedSupervisorLocalDir != null) {
+ return null;
+ }
+ String ret = absoluteStormLocalDir(conf) + FILE_SEPARATOR +
"supervisor";
+ FileUtils.forceMkdir(new File(ret));
+ return ret;
+ }
+
+ public static String supervisorIsupervisorDir(Map conf) throws
IOException {
+ return ((supervisorLocalDir(conf) + FILE_SEPARATOR +
"isupervisor"));
+ }
+
+ //For testing only
+ // for java
+ // try (SetMockedSupervisorStormDistRoot mocked = new
SetMockedSupervisorStormDistRoot(conf)) {
+ // run test ...
+ // }
+ //
+ // for clojure
+ // (with-open [mock (SetMockedSupervisorStormDistRoot. conf)]
+ // run test ...)
+ public static class SetMockedSupervisorStormDistRoot implements
Closeable {
+ public SetMockedSupervisorStormDistRoot(Map conf) {
+ mockedSupervisorStormDistRoot = conf;
+ }
+ @Override
+ public void close() {
+ mockedSupervisorStormDistRoot = null;
+ }
+ }
+ private static Map mockedSupervisorStormDistRoot = null;
+ public static String supervisorStormDistRoot(Map conf) throws
IOException {
+ if (mockedSupervisorStormDistRoot != null) {
+ return null;
+ }
+ return stormDistPath(supervisorLocalDir(conf)); // TODO: no need
to forceMake here?, clj does not.
+ }
+
+ public static String supervisorStormDistRoot(Map conf, String stormId)
throws IOException {
+ if (mockedSupervisorStormDistRoot != null) {
+ return null;
+ }
+ return supervisorStormDistRoot(conf) + FILE_SEPARATOR + stormId;
// TODO: need to (url-encode storm-id)? Not.
+ }
+
+ public static String supervisorStormJarPath(String stormRoot) {
+ String ret = "";
+ // we do this since to concat a null String will actually concat a
"null", which is not the expected: ""
+ if (stormRoot != null) {
+ ret = stormRoot;
+ }
+ return (ret + FILE_SEPARATOR + "stormjar.jar");
+ }
+
+ /* Never get used TODO : may delete it*/
+ public static String supervisorStormMetaFilePath(String stormRoot) {
+ String ret = "";
+ if (stormRoot != null) {
+ ret = stormRoot;
+ }
+ return (ret + FILE_SEPARATOR + "storm-code-distributor.meta");
+ }
+
+ public static String supervisorStormCodePath(String stormRoot) {
+ String ret = "";
+ if (stormRoot != null) {
+ ret = stormRoot;
+ }
+ return (ret + FILE_SEPARATOR + "stormcode.ser");
+ }
+
+ public static String supervisorStormConfPath(String stormRoot) {
+ String ret = "";
+ if (stormRoot != null) {
+ ret = stormRoot;
+ }
+ return (ret + FILE_SEPARATOR + "stormconf.ser");
+ }
+
+ public static String supervisorTmpDir(Map conf) throws IOException {
+ String ret = supervisorLocalDir(conf) + FILE_SEPARATOR + "tmp";
+ FileUtils.forceMkdir(new File(ret));
+ return ret;
+ }
+
+ public static String supervisorStormResourcesPath(String stormRoot) {
+ String ret = "";
+ // we do this since to concat a null String will actually concat a
"null", which is not the expected: ""
+ if (stormRoot != null) {
+ ret = stormRoot;
+ }
+ return (ret + FILE_SEPARATOR + RESOURCES_SUBDIR);
+ }
+
+ //For testing only
+ // for java
+ // try (SetMockedSupervisorState mocked = new
SetMockedSupervisorState(conf)) {
+ // run test ...
+ // }
+ //
+ // for clojure
+ // (with-open [mock (SetMockedSupervisorState. conf)]
+ // run test ...)
+ public static class SetMockedSupervisorState implements Closeable {
+ public SetMockedSupervisorState(Map conf) {
+ mockedSupervisorState = conf;
+ }
+ @Override
+ public void close() {
+ mockedSupervisorState = null;
+ }
+ }
+ private static Map mockedSupervisorState = null;
+ public static LocalState supervisorState(Map conf) throws IOException {
+ if (mockedSupervisorState != null) {
+ return null;
+ }
+ return new LocalState((supervisorLocalDir(conf) + FILE_SEPARATOR +
"localstate"));
+ }
+
+ //For testing only
+ // for java
+ // try (SetMockedNimbusTopoHistoryState mocked = new
SetMockedNimbusTopoHistoryState(conf)) {
+ // run test ...
+ // }
+ //
+ // for clojure
+ // (with-open [mock (SetMockedNimbusTopoHistoryState. conf)]
+ // run test ...)
+ public static class SetMockedNimbusTopoHistoryState implements
Closeable {
+ public SetMockedNimbusTopoHistoryState(Map conf) {
+ mockedNimbusTopoHistoryState = conf;
+ }
+ @Override
+ public void close() {
+ mockedNimbusTopoHistoryState = null;
+ }
+ }
+ private static Map mockedNimbusTopoHistoryState = null;
+ public static LocalState nimbusTopoHistoryState(Map conf) throws
IOException {
+ if (mockedNimbusTopoHistoryState != null) {
+ return null;
+ }
+ return new LocalState((masterLocalDir(conf) + FILE_SEPARATOR +
"history"));
+ }
+
+ //For testing only
+ // for java
+ // try (SetMockedSupervisorStormConf mocked = new
SetMockedSupervisorStormConf(conf)) {
+ // run test ...
+ // }
+ //
+ // for clojure
+ // (with-open [mock (SetMockedSupervisorStormConf. conf)]
+ // run test ...)
+ public static class SetMockedSupervisorStormConf implements Closeable {
+ public SetMockedSupervisorStormConf(Map conf) {
+ mockedSupervisorStormConf = conf;
+ }
+
+ @Override
+ public void close() {
+ mockedSupervisorStormConf = null;
+ }
+ }
+ private static Map mockedSupervisorStormConf = null;
+ public static Map readSupervisorStormConf(Map conf, String stormId)
throws IOException {
+ if (mockedSupervisorStormConf != null) {
+ return mockedSupervisorStormConf;
+ }
+ String stormRoot = supervisorStormDistRoot(conf, stormId);
+ String confPath = supervisorStormConfPath(stormRoot);
+ return readSupervisorStormConfGivenPath(conf, confPath);
+ }
+
+ public static StormTopology readSupervisorTopology(Map conf, String
stormId) throws IOException {
+ String stormRoot = supervisorStormDistRoot(conf, stormId);
+ String topologyPath = supervisorStormCodePath(stormRoot);
+ return Utils.deserialize(FileUtils.readFileToByteArray(new
File(topologyPath)), StormTopology.class);
+ }
+
+ public static String workerUserRoot(Map conf) {
+ return (absoluteStormLocalDir(conf) + FILE_SEPARATOR +
"/workers-users");
+ }
+
+ /* Never get used TODO : may delete it*/
+ public static String workerUserFile(Map conf, String workerId) {
+ return (workerUserRoot(conf) + FILE_SEPARATOR + workerId);
+ }
+
+ public static String getWorkerUser(Map conf, String workerId) {
+ LOG.info("GET worker-user for {}", workerId);
+ File file = new File(workerUserFile(conf, workerId));
+
+ try (InputStream in = new FileInputStream(file);
+ Reader reader = new InputStreamReader(in);
+ BufferedReader br = new BufferedReader(reader);) {
+ StringBuilder sb = new StringBuilder();
+ int r;
+ while ((r = br.read()) != -1) {
+ char ch = (char)r;
+ sb.append(ch);
+ }
+ String ret = sb.toString().trim();
+ return ret;
+ } catch (IOException e) {
+ LOG.error("Failed to get worker user for " + workerId + ".");
+ return null;
+ }
+ }
+
+ public static String getIdFromBlobKey(String key) {
+ if (key == null) return null;
+ String ret = null;
+ if (key.endsWith("-stormjar.jar")) {
+ ret = key.substring(0, key.length() - 13);
+ } else if (key.endsWith("-stormcode.ser")) {
+ ret = key.substring(0, key.length() - 14);
+ } else if (key.endsWith("-stormconf.ser")) {
+ ret = key.substring(0, key.length() - 14);
+ }
+ return ret;
+ }
+
+ //For testing only
+ // for java
+ // try (SetMockedWorkerUserWSE mocked = new
SetMockedWorkerUserWSE(conf)) {
+ // run test ...
+ // }
+ //
+ // for clojure
+ // (with-open [mock (SetMockedWorkerUserWSE. conf)]
+ // run test ...)
+ public static class SetMockedWorkerUserWSE implements Closeable {
--- End diff --
This should be renamed to something like IgnoreMockedWorkerUserWSE
> port backtype.storm.config to java
> ----------------------------------
>
> Key: STORM-1227
> URL: https://issues.apache.org/jira/browse/STORM-1227
> Project: Apache Storm
> Issue Type: New Feature
> Components: storm-core
> Reporter: Robert Joseph Evans
> Assignee: Zhuo Liu
> Labels: java-migration, jstorm-merger
>
> port backtype.storm.config to java. There are some parts of this that for
> convince sake may need to stay in clojure, or have clojure equivalents.
> The code that turns Config.* into clojure constants needs to stay, and
> reading the config may need to stay in clojure until we can cleanup all of
> the places it is used to not need the migration.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
