Repository: asterixdb Updated Branches: refs/heads/master ffd6e4ac5 -> 146417247
Change IO directory for integration util to be a directory in target Change-Id: I3dfb74dd4228725fb624eb1d21f621b7855f3d37 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1789 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> BAD: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> Reviewed-by: Till Westmann <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/14641724 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/14641724 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/14641724 Branch: refs/heads/master Commit: 1464172470d9425a222e7a0f13d8d0f7179098bb Parents: ffd6e4a Author: Abdullah Alamoudi <[email protected]> Authored: Tue May 30 23:40:37 2017 -0700 Committer: abdullah alamoudi <[email protected]> Committed: Wed May 31 09:15:29 2017 -0700 ---------------------------------------------------------------------- .../common/AsterixHyracksIntegrationUtil.java | 20 ++++----- .../bootstrap/ApplicationConfigurator.java | 7 +-- .../asterix/test/runtime/LangExecutionUtil.java | 46 +++++++++----------- .../cluster_state_1/cluster_state_1.1.regexadm | 2 +- .../cluster_state_1_full.1.regexadm | 2 +- .../cluster_state_1_less.1.regexadm | 2 +- .../common/config/ClusterProperties.java | 4 +- .../common/controllers/ControllerConfig.java | 9 ++-- .../control/common/utils/ConfigurationUtil.java | 28 ++++++++++++ 9 files changed, 73 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java index 8d60ec6..2a0631c 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java @@ -61,7 +61,6 @@ public class AsterixHyracksIntegrationUtil { } } - protected static final String IO_DIR_KEY = "java.io.tmpdir"; public static final int DEFAULT_HYRACKS_CC_CLIENT_PORT = 1098; public static final int DEFAULT_HYRACKS_CC_CLUSTER_PORT = 1099; @@ -79,8 +78,7 @@ public class AsterixHyracksIntegrationUtil { final CCConfig ccConfig = createCCConfig(configManager); cc = new ClusterControllerService(ccConfig, ccApplication); - - nodeNames = ccConfig.getConfigManager().getNodeNames(); + nodeNames = ccConfig.getConfigManager().getNodeNames(); if (deleteOldInstanceData) { deleteTransactionLogs(); removeTestStorageFiles(); @@ -133,7 +131,7 @@ public class AsterixHyracksIntegrationUtil { ccConfig.setClusterListenPort(DEFAULT_HYRACKS_CC_CLUSTER_PORT); ccConfig.setResultTTL(120000L); ccConfig.setResultSweepThreshold(1000L); - configManager.set(ControllerConfig.Option.DEFAULT_DIR, joinPath(System.getProperty(IO_DIR_KEY), "asterixdb")); + configManager.set(ControllerConfig.Option.DEFAULT_DIR, joinPath(getDefaultStoragePath(), "asterixdb")); return ccConfig; } @@ -152,8 +150,7 @@ public class AsterixHyracksIntegrationUtil { ncConfig.setResultTTL(120000L); ncConfig.setResultSweepThreshold(1000L); ncConfig.setVirtualNC(true); - configManager.set(ControllerConfig.Option.DEFAULT_DIR, - joinPath(System.getProperty(IO_DIR_KEY), "asterixdb", ncName)); + configManager.set(ControllerConfig.Option.DEFAULT_DIR, joinPath(getDefaultStoragePath(), "asterixdb", ncName)); return ncConfig; } @@ -170,11 +167,10 @@ public class AsterixHyracksIntegrationUtil { if (nodeStores == null) { throw new IllegalStateException("Couldn't find stores for NC: " + ncConfig.getNodeId()); } - String tempDirPath = System.getProperty(IO_DIR_KEY); - LOGGER.info("Using the temp path: " + tempDirPath); + LOGGER.info("Using the path: " + getDefaultStoragePath()); for (int i = 0; i < nodeStores.length; i++) { // create IO devices based on stores - nodeStores[i] = joinPath(tempDirPath, ncConfig.getNodeId(), nodeStores[i]); + nodeStores[i] = joinPath(getDefaultStoragePath(), ncConfig.getNodeId(), nodeStores[i]); } ncConfig.getConfigManager().set(ncConfig.getNodeId(), NCConfig.Option.IODEVICES, nodeStores); return ncConfig; @@ -226,8 +222,12 @@ public class AsterixHyracksIntegrationUtil { hcc.waitForCompletion(jobId); } + protected String getDefaultStoragePath() { + return joinPath("target", "io", "dir"); + } + public void removeTestStorageFiles() { - File dir = new File(System.getProperty(IO_DIR_KEY)); + File dir = new File(getDefaultStoragePath()); for (String ncName : nodeNames) { File ncDir = new File(dir, ncName); FileUtils.deleteQuietly(ncDir); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationConfigurator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationConfigurator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationConfigurator.java index 9678e09..adb3c18 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationConfigurator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationConfigurator.java @@ -28,6 +28,7 @@ import org.apache.hyracks.api.config.IConfigManager; import org.apache.hyracks.control.common.controllers.CCConfig; import org.apache.hyracks.control.common.controllers.ControllerConfig; import org.apache.hyracks.control.common.controllers.NCConfig; +import org.apache.hyracks.control.common.utils.ConfigurationUtil; import org.apache.hyracks.util.file.FileUtil; class ApplicationConfigurator { @@ -37,12 +38,12 @@ class ApplicationConfigurator { static void registerConfigOptions(IConfigManager configManager) { AsterixProperties.registerConfigOptions(configManager); ControllerConfig.Option.DEFAULT_DIR - .setDefaultValue(FileUtil.joinPath(System.getProperty("java.io.tmpdir"), "asterixdb")); + .setDefaultValue(FileUtil.joinPath(System.getProperty(ConfigurationUtil.JAVA_IO_TMPDIR), "asterixdb")); NCConfig.Option.APP_CLASS.setDefaultValue(NCApplication.class.getName()); CCConfig.Option.APP_CLASS.setDefaultValue(CCApplication.class.getName()); try { - InputStream propertyStream = ApplicationConfigurator.class.getClassLoader() - .getResourceAsStream("git.properties"); + InputStream propertyStream = + ApplicationConfigurator.class.getClassLoader().getResourceAsStream("git.properties"); if (propertyStream != null) { Properties gitProperties = new Properties(); gitProperties.load(propertyStream); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java index 64877ea..ec8333a 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java @@ -32,12 +32,15 @@ import java.util.Collection; import java.util.List; import org.apache.asterix.app.external.TestLibrarian; +import org.apache.asterix.common.config.ClusterProperties; import org.apache.asterix.common.library.ILibraryManager; import org.apache.asterix.test.common.TestExecutor; import org.apache.asterix.testframework.context.TestCaseContext; import org.apache.commons.lang.SystemUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.hyracks.api.io.IODeviceHandle; import org.apache.hyracks.control.common.utils.ThreadDumpHelper; +import org.apache.hyracks.control.nc.NodeControllerService; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -48,8 +51,8 @@ import org.junit.runners.Parameterized; public class LangExecutionUtil { private static final String PATH_ACTUAL = "target" + File.separator + "rttest" + File.separator; - private static final String PATH_BASE = StringUtils.join(new String[] { "src", "test", "resources", "runtimets" }, - File.separator); + private static final String PATH_BASE = + StringUtils.join(new String[] { "src", "test", "resources", "runtimets" }, File.separator); private static final boolean cleanupOnStart = true; private static final boolean cleanupOnStop = true; @@ -137,31 +140,22 @@ public class LangExecutionUtil { // Checks whether data files are uniformly distributed among io devices. private static void checkStorageFiles() throws Exception { - String tempDirPath = System.getProperty("java.io.tmpdir"); - File dir = new File(tempDirPath); - File[] subDirs = dir.listFiles(); - List<File> ncStores = new ArrayList<>(); - // Finds nc stores. - for (File file : subDirs) { - if (file.getName().startsWith("asterix_nc")) { - ncStores.add(file); - } - } - // Checks that dataset files are uniformly distributed across each nc store. - for (File ncStore : ncStores) { - checkNcStore(ncStore); + NodeControllerService[] ncs = ExecutionTestUtil.integrationUtil.ncs; + // Checks that dataset files are uniformly distributed across each io device. + for (NodeControllerService nc : ncs) { + checkNcStore(nc); } } // For each NC, check whether data files are uniformly distributed among io devices. - private static void checkNcStore(File ncStore) throws Exception { - File[] ioDevices = ncStore.listFiles(); + private static void checkNcStore(NodeControllerService nc) throws Exception { + List<IODeviceHandle> ioDevices = nc.getIoManager().getIODevices(); int expectedPartitionNum = -1; - for (File ioDevice : ioDevices) { - File[] dataDirs = ioDevice.listFiles(); + for (IODeviceHandle ioDevice : ioDevices) { + File[] dataDirs = ioDevice.getMount().listFiles(); for (File dataDir : dataDirs) { String dirName = dataDir.getName(); - if (!dirName.equals("storage")) { + if (!dirName.equals(ClusterProperties.DEFAULT_STORAGE_DIR_NAME)) { // Skips non-storage directories. continue; } @@ -202,8 +196,8 @@ public class LangExecutionUtil { String threadDump = ThreadDumpHelper.takeDumpJSON(ManagementFactory.getThreadMXBean()); // Currently we only do sanity check for threads used in the execution engine. // Later we should check if there are leaked storage threads as well. - if (threadDump.contains("Operator") || threadDump.contains("SuperActivity") || threadDump - .contains("PipelinedPartition")) { + if (threadDump.contains("Operator") || threadDump.contains("SuperActivity") + || threadDump.contains("PipelinedPartition")) { System.out.print(threadDump); throw new AssertionError("There are leaked threads in the execution engine."); } @@ -219,8 +213,8 @@ public class LangExecutionUtil { String processId = processName.split("@")[0]; // Checks whether there are leaked run files from operators. - Process process = Runtime.getRuntime() - .exec(new String[] { "bash", "-c", "lsof -p " + processId + "|grep waf|wc -l" }); + Process process = + Runtime.getRuntime().exec(new String[] { "bash", "-c", "lsof -p " + processId + "|grep waf|wc -l" }); try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { int runFileCount = Integer.parseInt(reader.readLine().trim()); if (runFileCount != 0) { @@ -232,8 +226,8 @@ public class LangExecutionUtil { } private static void outputLeakedOpenFiles(String processId) throws IOException { - Process process = Runtime.getRuntime() - .exec(new String[] { "bash", "-c", "lsof -p " + processId + "|grep waf" }); + Process process = + Runtime.getRuntime().exec(new String[] { "bash", "-c", "lsof -p " + processId + "|grep waf" }); try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { String line; while ((line = reader.readLine()) != null) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm index 5ec13a4..e8f018e 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm @@ -11,7 +11,7 @@ "compiler\.joinmemory" : 262144, "compiler\.parallelism" : 0, "compiler\.sortmemory" : 327680, - "default\.dir" : "/.*/asterixdb", + "default\.dir" : "target/io/dir/asterixdb", "instance\.name" : "DEFAULT_INSTANCE", "log\.level" : "INFO", "max\.wait\.active\.cluster" : 60, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm index d845b2a..e099835 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm @@ -11,7 +11,7 @@ "compiler\.joinmemory" : 262144, "compiler\.parallelism" : -1, "compiler\.sortmemory" : 327680, - "default\.dir" : "/.*/asterixdb", + "default\.dir" : "target/io/dir/asterixdb", "instance\.name" : "DEFAULT_INSTANCE", "log\.level" : "WARNING", "max\.wait\.active\.cluster" : 60, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm index 1315220..b3fe5cc 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm @@ -11,7 +11,7 @@ "compiler\.joinmemory" : 262144, "compiler\.parallelism" : 3, "compiler\.sortmemory" : 327680, - "default\.dir" : "/.*/asterixdb", + "default\.dir" : "target/io/dir/asterixdb", "instance\.name" : "DEFAULT_INSTANCE", "log\.level" : "WARNING", "max\.wait\.active\.cluster" : 60, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ClusterProperties.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ClusterProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ClusterProperties.java index 1ba9471..0abb92f 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ClusterProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ClusterProperties.java @@ -37,8 +37,8 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; public class ClusterProperties { public static final ClusterProperties INSTANCE = new ClusterProperties(); - private static final String CLUSTER_CONFIGURATION_FILE = "cluster.xml"; - private static final String DEFAULT_STORAGE_DIR_NAME = "storage"; + public static final String CLUSTER_CONFIGURATION_FILE = "cluster.xml"; + public static final String DEFAULT_STORAGE_DIR_NAME = "storage"; private String nodeNamePrefix = StringUtils.EMPTY; private Cluster cluster; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java index a9b3f97..1745e2a 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java @@ -27,16 +27,19 @@ import org.apache.hyracks.api.config.IOptionType; import org.apache.hyracks.api.config.Section; import org.apache.hyracks.control.common.config.ConfigManager; import org.apache.hyracks.control.common.config.OptionTypes; +import org.apache.hyracks.control.common.utils.ConfigurationUtil; import org.apache.hyracks.util.file.FileUtil; public class ControllerConfig implements Serializable { + private static final long serialVersionUID = 1L; public enum Option implements IOption { CONFIG_FILE(OptionTypes.STRING, "Specify path to master configuration file", null), CONFIG_FILE_URL(OptionTypes.URL, "Specify URL to master configuration file", null), - DEFAULT_DIR(OptionTypes.STRING, "Directory where files are written to by default", - FileUtil.joinPath(System.getProperty("java.io.tmpdir"), "hyracks")), - ; + DEFAULT_DIR( + OptionTypes.STRING, + "Directory where files are written to by default", + FileUtil.joinPath(System.getProperty(ConfigurationUtil.JAVA_IO_TMPDIR), "hyracks")),; private final IOptionType type; private final String description; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/ConfigurationUtil.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/ConfigurationUtil.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/ConfigurationUtil.java new file mode 100644 index 0000000..d2a455f --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/ConfigurationUtil.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.control.common.utils; + +public class ConfigurationUtil { + public static final String JAVA_IO_TMPDIR = "java.io.tmpdir"; + + private ConfigurationUtil() { + + } + +}
