Michael Blow has submitted this change and it was merged. Change subject: Config Cleanup, Update Docs ......................................................................
Config Cleanup, Update Docs - added some missing descriptions - updated ncservice.md with updated parameter names / sections - minor cleanup - removed some unused properties - renamed feed properties to active Change-Id: I9edc9ced4c2b99db239a2075903f9d8b256c26e4 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1555 Tested-by: Jenkins <[email protected]> BAD: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java A asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationEntryPointHelper.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ActiveProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java D asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/FeedProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/IPropertiesProvider.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java M asterixdb/asterix-common/src/test/java/org/apache/asterix/test/memory/ConcurrentFramePoolUnitTest.java M asterixdb/asterix-doc/src/site/markdown/ncservice.md M asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/AppContextInfo.java M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigManager.java M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IOptionType.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/OptionTypes.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java M hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java 29 files changed, 530 insertions(+), 342 deletions(-) Approvals: Till Westmann: Looks good to me, approved Jenkins: Verified; No violations found Objections: Jenkins: Violations found diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java index eafe312..c2b44c9 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java @@ -136,7 +136,7 @@ } protected Predicate<IOption> getConfigSelector() { - return option -> option != ControllerConfig.Option.CONFIG_FILE + return option -> !option.hidden() && option != ControllerConfig.Option.CONFIG_FILE && option != ControllerConfig.Option.CONFIG_FILE_URL; } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java index 5eba31d..aacad8e 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java @@ -33,12 +33,12 @@ import org.apache.asterix.common.api.IDatasetLifecycleManager; import org.apache.asterix.common.api.ThreadExecutor; import org.apache.asterix.common.cluster.ClusterPartition; +import org.apache.asterix.common.config.ActiveProperties; import org.apache.asterix.common.config.AsterixExtension; import org.apache.asterix.common.config.BuildProperties; import org.apache.asterix.common.config.CompilerProperties; import org.apache.asterix.common.config.ExtensionProperties; import org.apache.asterix.common.config.ExternalProperties; -import org.apache.asterix.common.config.FeedProperties; import org.apache.asterix.common.config.MessagingProperties; import org.apache.asterix.common.config.MetadataProperties; import org.apache.asterix.common.config.NodeProperties; @@ -110,7 +110,7 @@ private MetadataProperties metadataProperties; private StorageProperties storageProperties; private TransactionProperties txnProperties; - private FeedProperties feedProperties; + private ActiveProperties activeProperties; private BuildProperties buildProperties; private ReplicationProperties replicationProperties; private MessagingProperties messagingProperties; @@ -148,7 +148,7 @@ metadataProperties = new MetadataProperties(propertiesAccessor); storageProperties = new StorageProperties(propertiesAccessor); txnProperties = new TransactionProperties(propertiesAccessor); - feedProperties = new FeedProperties(propertiesAccessor); + activeProperties = new ActiveProperties(propertiesAccessor); buildProperties = new BuildProperties(propertiesAccessor); replicationProperties = new ReplicationProperties(propertiesAccessor); messagingProperties = new MessagingProperties(propertiesAccessor); @@ -206,7 +206,7 @@ isShuttingdown = false; activeManager = new ActiveManager(threadExecutor, ncApplicationContext.getNodeId(), - feedProperties.getMemoryComponentGlobalBudget(), compilerProperties.getFrameSize()); + activeProperties.getMemoryComponentGlobalBudget(), compilerProperties.getFrameSize()); if (replicationProperties.isParticipant(ncApplicationContext.getNodeId())) { String nodeId = ncApplicationContext.getNodeId(); @@ -364,8 +364,8 @@ } @Override - public FeedProperties getFeedProperties() { - return feedProperties; + public ActiveProperties getActiveProperties() { + return activeProperties; } @Override diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationEntryPointHelper.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationEntryPointHelper.java new file mode 100644 index 0000000..8c3861a --- /dev/null +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationEntryPointHelper.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.hyracks.bootstrap; + +import org.apache.asterix.common.config.AsterixProperties; +import org.apache.hyracks.api.config.IConfigManager; +import org.apache.hyracks.control.common.controllers.CCConfig; +import org.apache.hyracks.control.common.controllers.ControllerConfig; +import org.apache.hyracks.control.common.controllers.NCConfig; +import org.apache.hyracks.util.file.FileUtil; + +class ApplicationEntryPointHelper { + private ApplicationEntryPointHelper() { + } + + static void registerConfigOptions(IConfigManager configManager) { + AsterixProperties.registerConfigOptions(configManager); + + ControllerConfig.defaultDir = FileUtil.joinPath(System.getProperty("java.io.tmpdir"), "asterixdb"); + NCConfig.defaultAppClass = NCApplicationEntryPoint.class.getName(); + CCConfig.defaultAppClass = CCApplicationEntryPoint.class.getName(); + } +} diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java index 54f2c06..b63bb31 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java @@ -52,7 +52,6 @@ import org.apache.asterix.app.replication.FaultToleranceStrategyFactory; import org.apache.asterix.common.api.AsterixThreadFactory; import org.apache.asterix.common.config.AsterixExtension; -import org.apache.asterix.common.config.AsterixProperties; import org.apache.asterix.common.config.ClusterProperties; import org.apache.asterix.common.config.ExternalProperties; import org.apache.asterix.common.config.MetadataProperties; @@ -83,7 +82,6 @@ import org.apache.hyracks.http.api.IServlet; import org.apache.hyracks.http.server.HttpServer; import org.apache.hyracks.http.server.WebManager; -import org.apache.hyracks.util.file.FileUtil; public class CCApplicationEntryPoint extends org.apache.hyracks.control.cc.CCApplicationEntryPoint { @@ -94,10 +92,6 @@ protected IStorageComponentProvider componentProvider; private IJobCapacityController jobCapacityController; protected WebManager webManager; - - public CCApplicationEntryPoint() { - CCConfig.defaultDir = FileUtil.joinPath(System.getProperty("java.io.tmpdir"), "asterixdb"); - } @Override public void start(ICCApplicationContext ccAppCtx, String[] args) throws Exception { @@ -237,7 +231,7 @@ protected HttpServer setupFeedServer(ExternalProperties externalProperties) throws Exception { HttpServer feedServer = new HttpServer(webManager.getBosses(), webManager.getWorkers(), - externalProperties.getFeedServerPort()); + externalProperties.getActiveServerPort()); feedServer.setAttribute(HYRACKS_CONNECTION_ATTR, getNewHyracksClientConnection()); feedServer.addServlet(new FeedServlet(feedServer.ctx(), new String[] { "/" })); return feedServer; @@ -313,7 +307,7 @@ @Override public void registerConfig(IConfigManager configManager) { super.registerConfig(configManager); - AsterixProperties.registerConfigOptions(configManager); + ApplicationEntryPointHelper.registerConfigOptions(configManager); } public static synchronized void setAsterixStateProxy(IAsterixStateProxy proxy) { diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java index 238e93c..07a9a61 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java @@ -30,7 +30,6 @@ import org.apache.asterix.common.api.AsterixThreadFactory; import org.apache.asterix.common.api.IAppRuntimeContext; import org.apache.asterix.common.config.AsterixExtension; -import org.apache.asterix.common.config.AsterixProperties; import org.apache.asterix.common.config.ClusterProperties; import org.apache.asterix.common.config.IPropertiesProvider; import org.apache.asterix.common.config.MessagingProperties; @@ -54,7 +53,6 @@ import org.apache.hyracks.api.messages.IMessageBroker; import org.apache.hyracks.control.common.controllers.NCConfig; import org.apache.hyracks.control.nc.NodeControllerService; -import org.apache.hyracks.util.file.FileUtil; public class NCApplicationEntryPoint extends org.apache.hyracks.control.nc.NCApplicationEntryPoint { private static final Logger LOGGER = Logger.getLogger(NCApplicationEntryPoint.class.getName()); @@ -65,15 +63,10 @@ private boolean stopInitiated = false; private SystemState systemState; - public NCApplicationEntryPoint() { - NCConfig.defaultDir = FileUtil.joinPath(System.getProperty("java.io.tmpdir"), "asterixdb"); - NCConfig.defaultAppClass = "org.apache.asterix.hyracks.bootstrap.NCApplicationEntryPoint"; - } - @Override public void registerConfigOptions(IConfigManager configManager) { super.registerConfigOptions(configManager); - AsterixProperties.registerConfigOptions(configManager); + ApplicationEntryPointHelper.registerConfigOptions(configManager); } @Override diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java index b96b7fe..9cd961a 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java @@ -24,102 +24,167 @@ import java.io.PrintStream; import java.util.ArrayList; import java.util.Comparator; +import java.util.EnumMap; import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Predicate; +import java.util.stream.Collectors; import org.apache.asterix.hyracks.bootstrap.CCApplicationEntryPoint; +import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.hyracks.api.config.IOption; import org.apache.hyracks.api.config.Section; import org.apache.hyracks.control.common.config.ConfigManager; +import org.apache.hyracks.control.common.controllers.ControllerConfig; import org.apache.hyracks.util.file.FileUtil; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @RunWith(JUnit4.class) public class ConfigUsageTest { + private ConfigManager configManager; + final EnumMap<Column, Integer> maxWidths = new EnumMap<>(Column.class); + + enum Column { + SECTION, + PARAMETER, + MEANING, + DEFAULT + } private static final String CSV_FILE = FileUtil.joinPath("target", "surefire-reports", "config-options.csv"); + @Before + public void setup() { + configManager = getConfigManager(); + } + @Test public void generateUsage() { - generateUsage("| ", " | ", " |", true, System.err); + final EnumMap<Column, Boolean> align = new EnumMap<>(Column.class); + align.put(Column.SECTION, true); + align.put(Column.PARAMETER, true); + System.err.println(); + generateUsage("| ", " | ", " |", align, System.err); } @Test public void generateUsageCSV() throws IOException { new File(CSV_FILE).getParentFile().mkdirs(); try (final PrintStream output = new PrintStream(new FileOutputStream(CSV_FILE))) { - generateUsage("\"", "\",\"", "\"", false, output); + generateUsage("\"", "\",\"", "\"", new EnumMap<>(Column.class), output); // TODO(mblow): add some validation (in addition to just ensuring no exceptions...) } } - public void generateUsage(String startDelim, String midDelim, String endDelim, boolean align, PrintStream output) { + protected ConfigManager getConfigManager() { ConfigManager configManager = new ConfigManager(); CCApplicationEntryPoint aep = new CCApplicationEntryPoint(); aep.registerConfig(configManager); + ControllerConfig.defaultDir = ControllerConfig.defaultDir.replace(System.getProperty("java.io.tmpdir"), + "${java.io.tmpdir}/"); + return configManager; + } + + protected Set<Section> getSections(ConfigManager configManager) { + TreeSet<Section> sections = new TreeSet<>(Comparator.comparing(Section::sectionName)); + sections.addAll(configManager.getSections()); + sections.remove(Section.LOCALNC); + return sections; + } + + protected Predicate<IOption> optionSelector() { + return o -> !o.hidden() && o != ControllerConfig.Option.CONFIG_FILE + && o != ControllerConfig.Option.CONFIG_FILE_URL; + } + + protected Set<IOption> getSectionOptions(ConfigManager configManager, Section section) { + return configManager.getOptions(section).stream().filter(optionSelector()).collect(Collectors.toSet()); + } + + public void generateUsage(String startDelim, String midDelim, String endDelim, EnumMap<Column, Boolean> align, + PrintStream output) { + ConfigManager configManager = getConfigManager(); StringBuilder buf = new StringBuilder(); - int maxSectionWidth = 0; - int maxNameWidth = 0; - int maxDescriptionWidth = 0; - int maxDefaultWidth = 0; - if (align) { - for (Section section : configManager.getSections()) { - maxSectionWidth = Math.max(maxSectionWidth, section.sectionName().length()); - for (IOption option : configManager.getOptions(section)) { - if (option.hidden()) { - continue; + + final Column[] columns = Column.values(); + for (Section section : getSections(configManager)) { + for (IOption option : getSectionOptions(configManager, section)) { + for (Column column : columns) { + if (align.computeIfAbsent(column, c -> false)) { + calculateMaxWidth(option, column); } - maxNameWidth = Math.max(maxNameWidth, option.ini().length()); - maxDescriptionWidth = Math.max(maxDescriptionWidth, - option.description() == null ? 0 : option.description().length()); - maxDefaultWidth = Math.max(maxDefaultWidth, configManager.defaultTextForUsage(option, IOption::ini) - .length()); } } } - maxDescriptionWidth = Math.min(80, maxDescriptionWidth); - for (Section section : configManager.getSections()) { - List<IOption> options = new ArrayList<>(configManager.getOptions(section)); + // output header + for (Column column : columns) { + buf.append(column.ordinal() == 0 ? startDelim : midDelim); + pad(buf, StringUtils.capitalize(column.name().toLowerCase()), calculateMaxWidth(column, column.name())); + } + buf.append(endDelim).append('\n'); + + StringBuilder sepLine = new StringBuilder(); + for (Column column : columns) { + sepLine.append(column.ordinal() == 0 ? startDelim : midDelim); + pad(sepLine, "", maxWidths.get(column), '-'); + } + sepLine.append(endDelim).append('\n'); + buf.append(sepLine.toString().replace(' ', '-')); + + for (Section section : getSections(configManager)) { + List<IOption> options = new ArrayList<>(getSectionOptions(configManager, section)); options.sort(Comparator.comparing(IOption::ini)); for (IOption option : options) { - if (option.hidden()) { - continue; + for (Column column : columns) { + buf.append(column.ordinal() == 0 ? startDelim : midDelim); + if (column == Column.SECTION) { + center(buf, extractValue(column, option), maxWidths.get(column)); + } else { + pad(buf, extractValue(column, option), maxWidths.get(column)); + } } - buf.append(startDelim); - center(buf, section.sectionName(), maxSectionWidth).append(midDelim); - pad(buf, option.ini(), maxNameWidth).append(midDelim); - String description = option.description() == null ? "" : option.description(); - String defaultText = configManager.defaultTextForUsage(option, IOption::ini); - boolean extra = false; - while (align && description.length() > maxDescriptionWidth) { - int cut = description.lastIndexOf(' ', maxDescriptionWidth); - pad(buf, description.substring(0, cut), maxDescriptionWidth).append(midDelim); - pad(buf, defaultText, maxDefaultWidth).append(endDelim).append('\n'); - defaultText = ""; - description = description.substring(cut + 1); - buf.append(startDelim); - pad(buf, "", maxSectionWidth).append(midDelim); - pad(buf, "", maxNameWidth).append(midDelim); - } - pad(buf, description, maxDescriptionWidth).append(midDelim); - pad(buf, defaultText, maxDefaultWidth).append(endDelim).append('\n'); - if (extra) { - buf.append(startDelim); - pad(buf, "", maxSectionWidth).append(midDelim); - pad(buf, "", maxNameWidth).append(midDelim); - pad(buf, "", maxDescriptionWidth).append(midDelim); - pad(buf, "", maxDefaultWidth).append(endDelim).append('\n'); - } + buf.append(endDelim).append('\n'); } } output.println(buf); + } + + protected int calculateMaxWidth(IOption option, Column column) { + final String string = extractValue(column, option); + return calculateMaxWidth(column, string); + } + + private int calculateMaxWidth(Column column, String string) { + final int maxWidth = Math.max(maxWidths.computeIfAbsent(column, c -> 0), string.length()); + maxWidths.put(column, maxWidth); + return maxWidth; + } + + private String extractValue(Column column, IOption option) { + switch (column) { + case SECTION: + return option.section().sectionName(); + case PARAMETER: + return option.ini(); + case MEANING: + return option.description() == null ? "N/A" : option.description(); + case DEFAULT: + return configManager.defaultTextForUsage(option, IOption::ini); + default: + throw new IllegalStateException(String.valueOf(column)); + } } private StringBuilder center(StringBuilder buf, String string, int width) { if (string == null) { string = ""; } + string = StringEscapeUtils.escapeHtml4(string); int pad = width - string.length(); int leftPad = pad / 2; for (int i = leftPad; i > 0; i--) { @@ -133,12 +198,17 @@ } private StringBuilder pad(StringBuilder buf, String string, int width) { + return pad(buf, string, width, ' '); + } + + private StringBuilder pad(StringBuilder buf, String string, int width, char padChar) { if (string == null) { string = ""; } + string = StringEscapeUtils.escapeHtml4(string); buf.append(string); for (int i = width - string.length(); i > 0; i--) { - buf.append(' '); + buf.append(padChar); } return buf; } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm index 03884bd..bb241ba 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm @@ -5,17 +5,12 @@ "threadDumpUri" : "http://127.0.0.1:19002/admin/cluster/cc/threaddump"; }, "config" : { + "active.memory.global.budget" : 67108864, "compiler.framesize" : 32768, "compiler.groupmemory" : 163840, "compiler.joinmemory" : 262144, "compiler.parallelism" : 0, - "compiler.pregelix.home" : "~/pregelix", "compiler.sortmemory" : 327680, - "feed.central.manager.port" : 4500, - "feed.max.threshold.period" : 5, - "feed.memory.available.wait.timeout" : 10, - "feed.memory.global.budget" : 67108864, - "feed.pending.work.threshold" : 50, "instance.name" : "DEFAULT_INSTANCE", "log.level" : "INFO", "max.wait.active.cluster" : 60, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm index 372ac00..3ee13c9 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm @@ -5,17 +5,12 @@ "threadDumpUri" : "http://127.0.0.1:19002/admin/cluster/cc/threaddump"; }, "config" : { + "active.memory.global.budget" : 67108864, "compiler.framesize" : 32768, "compiler.groupmemory" : 163840, "compiler.joinmemory" : 262144, "compiler.parallelism" : -1, - "compiler.pregelix.home" : "~/pregelix", "compiler.sortmemory" : 327680, - "feed.central.manager.port" : 4500, - "feed.max.threshold.period" : 5, - "feed.memory.available.wait.timeout" : 10, - "feed.memory.global.budget" : 67108864, - "feed.pending.work.threshold" : 50, "instance.name" : "DEFAULT_INSTANCE", "log.level" : "WARNING", "max.wait.active.cluster" : 60, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm index 9d1ba6e..64d2f94 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm @@ -5,17 +5,12 @@ "threadDumpUri" : "http://127.0.0.1:19002/admin/cluster/cc/threaddump"; }, "config" : { + "active.memory.global.budget" : 67108864, "compiler.framesize" : 32768, "compiler.groupmemory" : 163840, "compiler.joinmemory" : 262144, "compiler.parallelism" : 3, - "compiler.pregelix.home" : "~/pregelix", "compiler.sortmemory" : 327680, - "feed.central.manager.port" : 4500, - "feed.max.threshold.period" : 5, - "feed.memory.available.wait.timeout" : 10, - "feed.memory.global.budget" : 67108864, - "feed.pending.work.threshold" : 50, "instance.name" : "DEFAULT_INSTANCE", "log.level" : "WARNING", "max.wait.active.cluster" : 60, diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ActiveProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ActiveProperties.java new file mode 100644 index 0000000..8455a6f --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ActiveProperties.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.common.config; + +import static org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT; +import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE; + +import org.apache.hyracks.api.config.IOption; +import org.apache.hyracks.api.config.IOptionType; +import org.apache.hyracks.api.config.Section; +import org.apache.hyracks.util.StorageUtil; + +public class ActiveProperties extends AbstractProperties { + + public enum Option implements IOption { + ACTIVE_MEMORY_GLOBAL_BUDGET( + LONG_BYTE_UNIT, + StorageUtil.getLongSizeInBytes(64L, MEGABYTE), + "The memory budget (in bytes) for the active runtime"); + + private final IOptionType type; + private final Object defaultValue; + private final String description; + + Option(IOptionType type, Object defaultValue, String description) { + this.type = type; + this.defaultValue = defaultValue; + this.description = description; + } + + @Override + public Section section() { + return Section.COMMON; + } + + @Override + public String description() { + return description; + } + + @Override + public IOptionType type() { + return type; + } + + @Override + public Object defaultValue() { + return defaultValue; + } + } + + public ActiveProperties(PropertiesAccessor accessor) { + super(accessor); + } + + public long getMemoryComponentGlobalBudget() { + return accessor.getLong(Option.ACTIVE_MEMORY_GLOBAL_BUDGET); + } +} diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java index 5f3a1aa..690d326 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java @@ -39,14 +39,14 @@ CompilerProperties.Option.class, MetadataProperties.Option.class, ExternalProperties.Option.class, - FeedProperties.Option.class, + ActiveProperties.Option.class, MessagingProperties.Option.class, ReplicationProperties.Option.class, StorageProperties.Option.class, TransactionProperties.Option.class); // we need to process the old-style asterix config before we apply defaults! - configManager.addConfigurator(IConfigManager.APPLY_DEFAULTS_METRIC - 1, () -> { + configManager.addConfigurator(IConfigManager.ConfiguratorMetric.APPLY_DEFAULTS.metric() - 1, () -> { try { PropertiesAccessor.getInstance(configManager.getAppConfig()); } catch (AsterixException e) { diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java index f97c5a5..5ccff4f 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java @@ -33,19 +33,29 @@ public class CompilerProperties extends AbstractProperties { public enum Option implements IOption { - COMPILER_SORTMEMORY(LONG_BYTE_UNIT, StorageUtil.getLongSizeInBytes(32L, MEGABYTE)), - COMPILER_JOINMEMORY(LONG_BYTE_UNIT, StorageUtil.getLongSizeInBytes(32L, MEGABYTE)), - COMPILER_GROUPMEMORY(LONG_BYTE_UNIT, StorageUtil.getLongSizeInBytes(32L, MEGABYTE)), - COMPILER_FRAMESIZE(INTEGER_BYTE_UNIT, StorageUtil.getIntSizeInBytes(32, KILOBYTE)), - COMPILER_PARALLELISM(INTEGER, COMPILER_PARALLELISM_AS_STORAGE), - COMPILER_PREGELIX_HOME(STRING, "~/pregelix"); + COMPILER_SORTMEMORY(LONG_BYTE_UNIT, StorageUtil.getLongSizeInBytes(32L, MEGABYTE), + "The memory budget (in bytes) for a sort operator instance in a partition"), + COMPILER_JOINMEMORY(LONG_BYTE_UNIT, StorageUtil.getLongSizeInBytes(32L, MEGABYTE), + "The memory budget (in bytes) for a join operator instance in a partition"), + COMPILER_GROUPMEMORY(LONG_BYTE_UNIT, StorageUtil.getLongSizeInBytes(32L, MEGABYTE), + "The memory budget (in bytes) for a group by operator instance in a partition"), + COMPILER_FRAMESIZE(INTEGER_BYTE_UNIT, StorageUtil.getIntSizeInBytes(32, KILOBYTE), + "The page size (in bytes) for computation"), + COMPILER_PARALLELISM(INTEGER, COMPILER_PARALLELISM_AS_STORAGE, "The degree of parallelism for query " + + "execution. Zero means to use the storage parallelism as the query execution parallelism, while " + + "other integer values dictate the number of query execution parallel partitions. The system will " + + "fall back to use the number of all available CPU cores in the cluster as the degree of parallelism " + + "if the number set by a user is too large or too small"), + COMPILER_PREGELIX_HOME(STRING, "~/pregelix", "Pregelix installation root directory"); private final IOptionType type; private final Object defaultValue; + private final String description; - Option(IOptionType type, Object defaultValue) { + Option(IOptionType type, Object defaultValue, String description) { this.type = type; this.defaultValue = defaultValue; + this.description = description; } @Override @@ -55,7 +65,7 @@ @Override public String description() { - return ""; + return description; } @Override @@ -67,6 +77,11 @@ public Object defaultValue() { return defaultValue; } + + @Override + public boolean hidden() { + return this == COMPILER_PREGELIX_HOME; + } } public static final String COMPILER_SORTMEMORY_KEY = Option.COMPILER_SORTMEMORY.ini(); diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java index 2f85221..4ab1d97 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java @@ -27,22 +27,25 @@ public class ExternalProperties extends AbstractProperties { public enum Option implements IOption { - WEB_PORT(INTEGER, 19001), - WEB_QUERYINTERFACE_PORT(INTEGER, 19006), - API_PORT(INTEGER, 19002), - FEED_PORT(INTEGER, 19003), - LOG_LEVEL(LEVEL, java.util.logging.Level.WARNING), - MAX_WAIT_ACTIVE_CLUSTER(INTEGER, 60), - PLOT_ACTIVATE(BOOLEAN, false), - CC_JAVA_OPTS(STRING, "-Xmx1024m"), - NC_JAVA_OPTS(STRING, "-Xmx1024m"); + WEB_PORT(INTEGER, 19001, "The listen port of the legacy query interface"), + WEB_QUERYINTERFACE_PORT(INTEGER, 19006, "The listen port of the query web interface"), + API_PORT(INTEGER, 19002, "The listen port of the API server"), + ACTIVE_PORT(INTEGER, 19003, "The listen port of the active server"), + LOG_LEVEL(LEVEL, java.util.logging.Level.WARNING, "The logging level for master and slave processes"), + MAX_WAIT_ACTIVE_CLUSTER(INTEGER, 60, "The max pending time (in seconds) for cluster startup. After the " + + "threshold, if the cluster still is not up and running, it is considered unavailable"), + PLOT_ACTIVATE(BOOLEAN, false, null), + CC_JAVA_OPTS(STRING, "-Xmx1024m", "The JVM options passed to the cluster controller process by managix"), + NC_JAVA_OPTS(STRING, "-Xmx1024m", "The JVM options passed to the node controller process(es) by managix"); private final IOptionType type; private final Object defaultValue; + private final String description; - Option(IOptionType type, Object defaultValue) { + Option(IOptionType type, Object defaultValue, String description) { this.type = type; this.defaultValue = defaultValue; + this.description = description; } @Override @@ -51,7 +54,7 @@ case WEB_PORT: case WEB_QUERYINTERFACE_PORT: case API_PORT: - case FEED_PORT: + case ACTIVE_PORT: return Section.CC; case LOG_LEVEL: case MAX_WAIT_ACTIVE_CLUSTER: @@ -67,8 +70,7 @@ @Override public String description() { - // TODO(mblow): add descriptions - return null; + return description; } @Override @@ -98,8 +100,8 @@ return accessor.getInt(Option.API_PORT); } - public int getFeedServerPort() { - return accessor.getInt(Option.FEED_PORT); + public int getActiveServerPort() { + return accessor.getInt(Option.ACTIVE_PORT); } public java.util.logging.Level getLogLevel() { diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/FeedProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/FeedProperties.java deleted file mode 100644 index a96d746..0000000 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/FeedProperties.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.common.config; - -import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER; -import static org.apache.hyracks.control.common.config.OptionTypes.LONG; -import static org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT; -import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE; - -import org.apache.hyracks.api.config.IOption; -import org.apache.hyracks.api.config.IOptionType; -import org.apache.hyracks.api.config.Section; -import org.apache.hyracks.util.StorageUtil; - -public class FeedProperties extends AbstractProperties { - - public enum Option implements IOption { - FEED_PENDING_WORK_THRESHOLD(INTEGER, 50), - FEED_MEMORY_GLOBAL_BUDGET(LONG_BYTE_UNIT, StorageUtil.getLongSizeInBytes(64L, MEGABYTE)), - FEED_MEMORY_AVAILABLE_WAIT_TIMEOUT(LONG, 10L), - FEED_CENTRAL_MANAGER_PORT(INTEGER, 4500), - FEED_MAX_THRESHOLD_PERIOD(INTEGER, 5); - - private final IOptionType type; - private final Object defaultValue; - - Option(IOptionType type, Object defaultValue) { - this.type = type; - this.defaultValue = defaultValue; - } - - @Override - public Section section() { - return Section.COMMON; - } - - @Override - public String description() { - // TODO(mblow): add missing descriptions - switch (this) { - case FEED_CENTRAL_MANAGER_PORT: - return "port at which the Central Feed Manager listens for control messages from local Feed " + - "Managers"; - case FEED_MAX_THRESHOLD_PERIOD: - return "maximum length of input queue before triggering corrective action"; - default: - return null; - } - } - - @Override - public IOptionType type() { - return type; - } - - @Override - public Object defaultValue() { - return defaultValue; - } - } - - public FeedProperties(PropertiesAccessor accessor) { - super(accessor); - } - - public int getPendingWorkThreshold() { - return accessor.getInt(Option.FEED_PENDING_WORK_THRESHOLD); - } - - public long getMemoryComponentGlobalBudget() { - return accessor.getLong(Option.FEED_MEMORY_GLOBAL_BUDGET); - } - - public long getMemoryAvailableWaitTimeout() { - return accessor.getLong(Option.FEED_MEMORY_AVAILABLE_WAIT_TIMEOUT); - } - - public int getFeedCentralManagerPort() { - return accessor.getInt(Option.FEED_CENTRAL_MANAGER_PORT); - } - - public int getMaxSuccessiveThresholdPeriod() { - return accessor.getInt(Option.FEED_MAX_THRESHOLD_PERIOD); - } -} diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/IPropertiesProvider.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/IPropertiesProvider.java index 4637437..c1264a9 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/IPropertiesProvider.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/IPropertiesProvider.java @@ -29,7 +29,7 @@ ExternalProperties getExternalProperties(); - FeedProperties getFeedProperties(); + ActiveProperties getActiveProperties(); BuildProperties getBuildProperties(); diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java index 4175873..e3a5fb9 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.common.config; +import java.util.function.Supplier; + import org.apache.hyracks.api.config.IOption; import org.apache.hyracks.api.config.IOptionType; import org.apache.hyracks.api.config.Section; @@ -28,18 +30,32 @@ public class NodeProperties extends AbstractProperties { public enum Option implements IOption { - INITIAL_RUN(OptionTypes.BOOLEAN, false), - CORE_DUMP_DIR(OptionTypes.STRING, FileUtil.joinPath(NCConfig.defaultDir, "coredump")), - TXN_LOG_DIR(OptionTypes.STRING, FileUtil.joinPath(NCConfig.defaultDir, "txn-log")), - STORAGE_SUBDIR(OptionTypes.STRING, "storage"), + INITIAL_RUN(OptionTypes.BOOLEAN, false, "A flag indicating if it's the first time the NC is started"), + CORE_DUMP_DIR( + OptionTypes.STRING, + (Supplier<String>) () -> FileUtil.joinPath(NCConfig.defaultDir, "coredump"), + "The directory where node core dumps should be written"), + TXN_LOG_DIR( + OptionTypes.STRING, + (Supplier<String>) () -> FileUtil.joinPath(NCConfig.defaultDir, "txn-log"), + "The directory where transaction logs should be stored"), + STORAGE_SUBDIR(OptionTypes.STRING, "storage", "The subdirectory name under each iodevice used for storage"), ; private final IOptionType type; private final Object defaultValue; + private final String description; - <T> Option(IOptionType<T> type, T defaultValue) { + <T> Option(IOptionType<T> type, T defaultValue, String description) { this.type = type; this.defaultValue = defaultValue; + this.description = description; + } + + <T> Option(IOptionType<T> type, Supplier<T> defaultValue, String description) { + this.type = type; + this.defaultValue = defaultValue; + this.description = description; } @Override @@ -49,12 +65,7 @@ @Override public String description() { - switch (this) { - case INITIAL_RUN: - return "A flag indicating if it's the first time the NC is started"; - default: - return null; - } + return description; } @Override diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java index 73e8c4a..97ad6be 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java @@ -18,8 +18,11 @@ */ package org.apache.asterix.common.config; -import static org.apache.hyracks.control.common.config.OptionTypes.*; -import static org.apache.hyracks.util.StorageUtil.StorageUnit.*; +import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER; +import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER_BYTE_UNIT; +import static org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT; +import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE; +import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE; import java.util.Map; @@ -31,25 +34,31 @@ public class TransactionProperties extends AbstractProperties { public enum Option implements IOption { - TXN_LOG_BUFFER_NUMPAGES(INTEGER, 8), - TXN_LOG_BUFFER_PAGESIZE(INTEGER_BYTE_UNIT, StorageUtil.getIntSizeInBytes(128, KILOBYTE)), - TXN_LOG_PARTITIONSIZE(LONG_BYTE_UNIT, StorageUtil.getLongSizeInBytes(256L, MEGABYTE)), - TXN_LOG_CHECKPOINT_LSNTHRESHOLD(INTEGER_BYTE_UNIT, StorageUtil.getIntSizeInBytes(64, MEGABYTE)), - TXN_LOG_CHECKPOINT_POLLFREQUENCY(INTEGER, 120), - TXN_LOG_CHECKPOINT_HISTORY(INTEGER, 0), - TXN_LOCK_ESCALATIONTHRESHOLD(INTEGER, 1000), - TXN_LOCK_SHRINKTIMER(INTEGER, 5000), - TXN_LOCK_TIMEOUT_WAITTHRESHOLD(INTEGER, 60000), - TXN_LOCK_TIMEOUT_SWEEPTHRESHOLD(INTEGER, 10000), - TXN_COMMITPROFILER_REPORTINTERVAL(INTEGER, 5), - TXN_JOB_RECOVERY_MEMORYSIZE(LONG_BYTE_UNIT, StorageUtil.getLongSizeInBytes(64L, MEGABYTE)); + TXN_LOG_BUFFER_NUMPAGES(INTEGER, 8, "The number of pages in the transaction log tail"), + TXN_LOG_BUFFER_PAGESIZE(INTEGER_BYTE_UNIT, StorageUtil.getIntSizeInBytes(128, KILOBYTE), + "The page size (in bytes) for transaction log buffer"), + TXN_LOG_PARTITIONSIZE(LONG_BYTE_UNIT, StorageUtil.getLongSizeInBytes(256L, MEGABYTE), null), + TXN_LOG_CHECKPOINT_LSNTHRESHOLD(INTEGER_BYTE_UNIT, StorageUtil.getIntSizeInBytes(64, MEGABYTE), + "The checkpoint threshold (in terms of LSNs (log sequence numbers) that have been written to the " + + "transaction log, i.e., the length of the transaction log) for transaction logs"), + TXN_LOG_CHECKPOINT_POLLFREQUENCY(INTEGER, 120, null), + TXN_LOG_CHECKPOINT_HISTORY(INTEGER, 0, "The number of checkpoints to keep in the transaction log"), + TXN_LOCK_ESCALATIONTHRESHOLD(INTEGER, 1000, null), + TXN_LOCK_SHRINKTIMER(INTEGER, 5000, null), + TXN_LOCK_TIMEOUT_WAITTHRESHOLD(INTEGER, 60000, "Time out (in milliseconds) of waiting for a lock"), + TXN_LOCK_TIMEOUT_SWEEPTHRESHOLD(INTEGER, 10000, "Interval (in milliseconds) for checking lock timeout"), + TXN_COMMITPROFILER_REPORTINTERVAL(INTEGER, 5, null), + TXN_JOB_RECOVERY_MEMORYSIZE(LONG_BYTE_UNIT, StorageUtil.getLongSizeInBytes(64L, MEGABYTE), + "The memory budget (in bytes) used for recovery"); private final IOptionType type; private final Object defaultValue; + private final String description; - Option(IOptionType type, Object defaultValue) { + Option(IOptionType type, Object defaultValue, String description) { this.type = type; this.defaultValue = defaultValue; + this.description = description; } @Override @@ -59,8 +68,7 @@ @Override public String description() { - // TODO(mblow): add missing descriptions - return null; + return description; } @Override diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/memory/ConcurrentFramePoolUnitTest.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/memory/ConcurrentFramePoolUnitTest.java index 17e1c5a..9be5837 100644 --- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/memory/ConcurrentFramePoolUnitTest.java +++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/memory/ConcurrentFramePoolUnitTest.java @@ -24,7 +24,7 @@ import java.util.Random; import java.util.concurrent.LinkedBlockingDeque; -import org.apache.asterix.common.config.FeedProperties; +import org.apache.asterix.common.config.ActiveProperties; import org.apache.asterix.common.memory.ConcurrentFramePool; import org.apache.asterix.common.memory.FrameAction; import org.apache.hyracks.api.exceptions.HyracksDataException; @@ -58,7 +58,7 @@ @org.junit.Test public void testMemoryManager() { - FeedProperties afp = Mockito.mock(FeedProperties.class); + ActiveProperties afp = Mockito.mock(ActiveProperties.class); Mockito.when(afp.getMemoryComponentGlobalBudget()).thenReturn(FEED_MEM_BUDGET); ConcurrentFramePool fmm = new ConcurrentFramePool("TestNode", afp.getMemoryComponentGlobalBudget(), DEFAULT_FRAME_SIZE); @@ -73,7 +73,7 @@ @org.junit.Test public void testConcurrentMemoryManager() { try { - FeedProperties afp = Mockito.mock(FeedProperties.class); + ActiveProperties afp = Mockito.mock(ActiveProperties.class); Mockito.when(afp.getMemoryComponentGlobalBudget()).thenReturn(FEED_MEM_BUDGET); ConcurrentFramePool fmm = new ConcurrentFramePool("TestNode", afp.getMemoryComponentGlobalBudget(), DEFAULT_FRAME_SIZE); @@ -104,7 +104,7 @@ @org.junit.Test public void testVarSizeMemoryManager() { try { - FeedProperties afp = Mockito.mock(FeedProperties.class); + ActiveProperties afp = Mockito.mock(ActiveProperties.class); Mockito.when(afp.getMemoryComponentGlobalBudget()).thenReturn(FEED_MEM_BUDGET); ConcurrentFramePool fmm = new ConcurrentFramePool("TestNode", afp.getMemoryComponentGlobalBudget(), DEFAULT_FRAME_SIZE); @@ -139,7 +139,7 @@ @org.junit.Test public void testConcurrentVarSizeMemoryManager() { try { - FeedProperties afp = Mockito.mock(FeedProperties.class); + ActiveProperties afp = Mockito.mock(ActiveProperties.class); Mockito.when(afp.getMemoryComponentGlobalBudget()).thenReturn(FEED_MEM_BUDGET); ConcurrentFramePool fmm = new ConcurrentFramePool("TestNode", afp.getMemoryComponentGlobalBudget(), DEFAULT_FRAME_SIZE); @@ -178,7 +178,7 @@ @org.junit.Test public void testAcquireReleaseMemoryManager() throws HyracksDataException { - FeedProperties afp = Mockito.mock(FeedProperties.class); + ActiveProperties afp = Mockito.mock(ActiveProperties.class); Mockito.when(afp.getMemoryComponentGlobalBudget()).thenReturn(FEED_MEM_BUDGET); ConcurrentFramePool fmm = new ConcurrentFramePool("TestNode", afp.getMemoryComponentGlobalBudget(), DEFAULT_FRAME_SIZE); @@ -211,7 +211,7 @@ @org.junit.Test public void testConcurrentAcquireReleaseMemoryManager() { try { - FeedProperties afp = Mockito.mock(FeedProperties.class); + ActiveProperties afp = Mockito.mock(ActiveProperties.class); Mockito.when(afp.getMemoryComponentGlobalBudget()).thenReturn(FEED_MEM_BUDGET); ConcurrentFramePool fmm = new ConcurrentFramePool("TestNode", afp.getMemoryComponentGlobalBudget(), DEFAULT_FRAME_SIZE); @@ -242,7 +242,7 @@ @org.junit.Test public void testAcquireReleaseVarSizeMemoryManager() { try { - FeedProperties afp = Mockito.mock(FeedProperties.class); + ActiveProperties afp = Mockito.mock(ActiveProperties.class); Mockito.when(afp.getMemoryComponentGlobalBudget()).thenReturn(FEED_MEM_BUDGET); ConcurrentFramePool fmm = new ConcurrentFramePool("TestNode", afp.getMemoryComponentGlobalBudget(), DEFAULT_FRAME_SIZE); @@ -295,7 +295,7 @@ @org.junit.Test public void testConcurrentAcquireReleaseVarSizeMemoryManager() { try { - FeedProperties afp = Mockito.mock(FeedProperties.class); + ActiveProperties afp = Mockito.mock(ActiveProperties.class); Mockito.when(afp.getMemoryComponentGlobalBudget()).thenReturn(FEED_MEM_BUDGET); ConcurrentFramePool fmm = new ConcurrentFramePool("TestNode", afp.getMemoryComponentGlobalBudget(), DEFAULT_FRAME_SIZE); @@ -331,7 +331,7 @@ @org.junit.Test public void testFixedSizeSubscribtion() { try { - FeedProperties afp = Mockito.mock(FeedProperties.class); + ActiveProperties afp = Mockito.mock(ActiveProperties.class); Mockito.when(afp.getMemoryComponentGlobalBudget()).thenReturn(FEED_MEM_BUDGET); ConcurrentFramePool fmm = new ConcurrentFramePool("TestNode", afp.getMemoryComponentGlobalBudget(), DEFAULT_FRAME_SIZE); @@ -397,7 +397,7 @@ @org.junit.Test public void testgetWhileSubscribersExist() { try { - FeedProperties afp = Mockito.mock(FeedProperties.class); + ActiveProperties afp = Mockito.mock(ActiveProperties.class); Mockito.when(afp.getMemoryComponentGlobalBudget()).thenReturn(FEED_MEM_BUDGET); ConcurrentFramePool fmm = new ConcurrentFramePool("TestNode", afp.getMemoryComponentGlobalBudget(), DEFAULT_FRAME_SIZE); diff --git a/asterixdb/asterix-doc/src/site/markdown/ncservice.md b/asterixdb/asterix-doc/src/site/markdown/ncservice.md index 67b1970..243d908 100644 --- a/asterixdb/asterix-doc/src/site/markdown/ncservice.md +++ b/asterixdb/asterix-doc/src/site/markdown/ncservice.md @@ -208,14 +208,14 @@ `cc.conf`: [nc/red] - txnlogdir=/lv_scratch/asterix/red/txnlog - coredumpdir=/lv_scratch/asterix/red/coredump + txn.log.dir=/lv_scratch/asterix/red/txnlog + core.dump.dir=/lv_scratch/asterix/red/coredump iodevices=/lv_scratch/asterix/red address=cacofonix-2 [nc/blue] - txnlogdir=/lv_scratch/asterix/blue/txnlog - coredumpdir=/lv_scratch/asterix/blue/coredump + txn.log.dir=/lv_scratch/asterix/blue/txnlog + core.dump.dir=/lv_scratch/asterix/blue/coredump iodevices=/lv_scratch/asterix/blue address=cacofonix-3 @@ -225,7 +225,7 @@ command=asterixnc [cc] - cluster.address = cacofonix-1 + address = cacofonix-1 To deploy, first the `asterix-server` binary must be present on each machine. Any method to transfer the archive to each machine will work, but here `scp` will be used for simplicity's sake. @@ -255,57 +255,123 @@ The following parameters are for the master process, under the "[cc]" section. -| Parameter | Meaning | Default | -|----------|--------|-------| -| max.wait.active.cluster | The max pending time (in seconds) for cluster startup. After the threshold, if the cluster still is not up and running, it is considered unavailable. | 60 | -| address | The binding IP address for the AsterixDB instance | N/A | +| Section | Parameter | Meaning | Default | +|---------|-------------------------------------------|---|---| +| cc | active.port | The listen port of the active server | 19003 | +| cc | address | Default bind address for all services on this cluster controller | 127.0.0.1 | +| cc | api.port | The listen port of the API server | 19002 | +| cc | app.class | Application CC main class | org.apache.asterix.hyracks.bootstrap.CCApplicationEntryPoint | +| cc | client.listen.address | Sets the IP Address to listen for connections from clients | same as address | +| cc | client.listen.port | Sets the port to listen for connections from clients | 1098 | +| cc | cluster.listen.address | Sets the IP Address to listen for connections from NCs | same as address | +| cc | cluster.listen.port | Sets the port to listen for connections from node controllers | 1099 | +| cc | cluster.public.address | Address that NCs should use to contact this CC | same as cluster.listen.address | +| cc | cluster.public.port | Port that NCs should use to contact this CC | same as cluster.listen.port | +| cc | cluster.topology | Sets the XML file that defines the cluster topology | <undefined> | +| cc | console.listen.address | Sets the listen address for the Cluster Controller | same as address | +| cc | console.listen.port | Sets the http port for the Cluster Controller) | 16001 | +| cc | heartbeat.max.misses | Sets the maximum number of missed heartbeats before a node is marked as dead | 5 | +| cc | heartbeat.period | Sets the time duration between two heartbeats from each node controller in milliseconds | 10000 | +| cc | job.history.size | Limits the number of historical jobs remembered by the system to the specified value | 10 | +| cc | job.manager.class | Specify the implementation class name for the job manager | org.apache.hyracks.control.cc.job.JobManager | +| cc | job.queue.capacity | The maximum number of jobs to queue before rejecting new jobs | 4096 | +| cc | job.queue.class | Specify the implementation class name for the job queue | org.apache.hyracks.control.cc.scheduler.FIFOJobQueue | +| cc | profile.dump.period | Sets the time duration between two profile dumps from each node controller in milliseconds; 0 to disable | 0 | +| cc | result.sweep.threshold | The duration within which an instance of the result cleanup should be invoked in milliseconds | 60000 | +| cc | result.ttl | Limits the amount of time results for asynchronous jobs should be retained by the system in milliseconds | 86400000 | +| cc | root.dir | Sets the root folder used for file operations | ${java.io.tmpdir}/asterixdb/ClusterControllerService | +| cc | web.port | The listen port of the legacy query interface | 19001 | +| cc | web.queryinterface.port | The listen port of the query web interface | 19006 | The following parameters for slave processes, under "[nc]" sections. -| Parameter | Meaning | Default | -|----------|--------|-------| -| address | The binding IP address for the slave process | N/A | -| command | The command for the slave process | N/A (for AsterixDB, it should be "asterixnc") | -| core.dump.dir | The path for core dump | N/A | -| iodevices | Comma separated directory paths for both storage files and temporary files | N/A | -| jvm.args | The JVM arguments | -Xmx1536m | -| ncservice.port | The port on which the NCService for this NC is listening | 9090 | -| txn.log.dir | The directory for transaction logs | N/A | +| Section | Parameter | Meaning | Default | +|---------|-------------------------------------------|---|---| +| nc | address | Default IP Address to bind listeners on this NC. All services will bind on this address unless a service-specific listen address is supplied. | 127.0.0.1 | +| nc | app.class | Application NC Main Class | org.apache.asterix.hyracks.bootstrap.NCApplicationEntryPoint | +| nc | cluster.address | Cluster Controller address (required unless specified in config file) | <undefined> | +| nc | cluster.connect.retries | Number of attempts to contact CC before giving up | 5 | +| nc | cluster.listen.address | IP Address to bind cluster listener on this NC | same as address | +| nc | cluster.listen.port | IP port to bind cluster listener | 0 | +| nc | cluster.port | Cluster Controller port | 1099 | +| nc | cluster.public.address | Public IP Address to announce cluster listener | same as public.address | +| nc | cluster.public.port | Public IP port to announce cluster listener | same as cluster.listen.port | +| nc | command | Command NCService should invoke to start the NCDriver | hyracksnc | +| nc | core.dump.dir | The directory where node core dumps should be written | ${java.io.tmpdir}/asterixdb/coredump | +| nc | data.listen.address | IP Address to bind data listener | same as address | +| nc | data.listen.port | IP port to bind data listener | 0 | +| nc | data.public.address | Public IP Address to announce data listener | same as public.address | +| nc | data.public.port | Public IP port to announce data listener | same as data.listen.port | +| nc | iodevices | Comma separated list of IO Device mount points | ${java.io.tmpdir}/asterixdb/iodevice | +| nc | jvm.args | JVM args to pass to the NCDriver | <undefined> | +| nc | messaging.listen.address | IP Address to bind messaging listener | same as address | +| nc | messaging.listen.port | IP port to bind messaging listener | 0 | +| nc | messaging.public.address | Public IP Address to announce messaging listener | same as public.address | +| nc | messaging.public.port | Public IP port to announce messaging listener | same as messaging.listen.port | +| nc | ncservice.address | Address the CC should use to contact the NCService associated with this NC | same as public.address | +| nc | ncservice.pid | PID of the NCService which launched this NCDriver | -1 | +| nc | ncservice.port | Port the CC should use to contact the NCService associated with this NC | 9090 | +| nc | net.buffer.count | Number of network buffers per input/output channel | 1 | +| nc | net.thread.count | Number of threads to use for Network I/O | 1 | +| nc | public.address | Default public address that other processes should use to contact this NC. All services will advertise this address unless a service-specific public address is supplied. | same as address | +| nc | result.listen.address | IP Address to bind dataset result distribution listener | same as address | +| nc | result.listen.port | IP port to bind dataset result distribution listener | 0 | +| nc | result.manager.memory | Memory usable for result caching at this Node Controller in bytes | -1 (-1 B) | +| nc | result.public.address | Public IP Address to announce dataset result distribution listener | same as public.address | +| nc | result.public.port | Public IP port to announce dataset result distribution listener | same as result.listen.port | +| nc | result.sweep.threshold | The duration within which an instance of the result cleanup should be invoked in milliseconds | 60000 | +| nc | result.ttl | Limits the amount of time results for asynchronous jobs should be retained by the system in milliseconds | 86400000 | +| nc | storage.buffercache.maxopenfiles | The maximum number of open files in the buffer cache | 2147483647 | +| nc | storage.buffercache.pagesize | The page size in bytes for pages in the buffer cache | 131072 (128 kB) | +| nc | storage.buffercache.size | The size of memory allocated to the disk buffer cache. The value should be a multiple of the buffer cache page size. | 715915264 (682.75 MB) | +| nc | storage.lsm.bloomfilter.falsepositiverate | The maximum acceptable false positive rate for bloom filters associated with LSM indexes | 0.01 | +| nc | storage.memorycomponent.globalbudget | The size of memory allocated to the memory components. The value should be a multiple of the memory component page size | 715915264 (682.75 MB) | +| nc | storage.memorycomponent.numcomponents | The number of memory components to be used per lsm index | 2 | +| nc | storage.memorycomponent.numpages | The number of pages to allocate for a memory component. This budget is shared by all the memory components of the primary index and all its secondary indexes across all I/O devices on a node. Note: in-memory components usually has fill factor of 75% since the pages are 75% full and the remaining 25% is un-utilized | 1/16th of the storage.memorycomponent.globalbudget value | +| nc | storage.memorycomponent.pagesize | The page size in bytes for pages allocated to memory components | 131072 (128 kB) | +| nc | storage.metadata.memorycomponent.numpages | The number of pages to allocate for a metadata memory component | 1/64th of the storage.memorycomponent.globalbudget value or 256, whichever is larger | +| nc | storage.subdir | The subdirectory name under each iodevice used for storage | storage | +| nc | txn.log.dir | The directory where transaction logs should be stored | ${java.io.tmpdir}/asterixdb/txn-log | The following parameters are configured under the "[common]" section. -| Parameter | Meaning | Default | -|----------|--------|-------| -| instance.name | The name of the AsterixDB instance | "DEFAULT_INSTANCE" | -| log.level | The logging level for master and slave processes | "INFO" | -| compiler.framesize | The page size (in bytes) for computation | 32768 | -| compiler.groupmemory | The memory budget (in bytes) for a group by operator instance in a partition | 33554432 | -| compiler.joinmemory | The memory budget (in bytes) for a join operator instance in a partition | 33554432 | -| compiler.sortmemory | The memory budget (in bytes) for a sort operator instance in a partition | 33554432 | -| compiler.parallelism | The degree of parallelism for query execution. Zero means to use the storage parallelism as the query execution parallelism, while other integer values dictate the number of query execution parallel partitions. The system will fall back to use the number of all available CPU cores in the cluster as the degree of parallelism if the number set by a user is too large or too small. | 0 | -| metadata.callback.port | The port for metadata communication | 0 | -| metadata.listen.port | The metadata communication port on the metadata node. This parameter should only be present in the section of the metadata NC | 0 | -| metadata.registration.timeout.secs | The time out threshold (in seconds) for metadata node registration | 60 | -| storage.subdir | The directory for storage files | N/A | -| storage.buffercache.maxopenfiles | The maximum number of open files for the buffer cache. Note that this is the parameter for the AsterixDB and setting the operating system parameter is still required. | 2147483647 | -| storage.buffercache.pagesize | The page size (in bytes) for the disk buffer cache (for reads) | 131072 | -| storage.buffercache.size | The overall budget (in bytes) of the disk buffer cache (for reads) | 536870912 | -| storage.lsm.bloomfilter.falsepositiverate | The false positive rate for the bloom filter for each memory/disk components | 0.01 | -| storage.memorycomponent.globalbudget | The global budget (in bytes) for all memory components of all datasets and indexes (for writes) | 536870912 | -| storage.memorycomponent.numcomponents | The number of memory components per data partition per index | 2 | -| storage.memorycomponent.numpages | The number of pages for all memory components of a dataset, including those for secondary indexes | 256 | -| storage.memorycomponent.pagesize | The page size (in bytes) of memory components | 131072 | -| storage.metadata.memorycomponent.numpages | The number of pages for all memory components of a metadata dataset | 256 | -| txn.commitprofiler.reportinterval | The interval for reporting commit statistics | 5 | -| txn.job.recovery.memorysize | The memory budget (in bytes) used for recovery | 67108864 | -| txn.lock.timeout.sweepthreshold | Interval (in milliseconds) for checking lock timeout | 10000 | -| txn.lock.timeout.waitthreshold | Time out (in milliseconds) of waiting for a lock | 60000 | -| txn.log.buffer.numpages | The number of pages in the transaction log tail | 8 | -| txn.log.buffer.pagesize | The page size (in bytes) for transaction log buffer. | 131072 | -| txn.log.checkpoint.history | The number of checkpoints to keep in the transaction log | 0 | -| txn.log.checkpoint.lsnthreshold | The checkpoint threshold (in terms of LSNs (log sequence numbers) that have been written to the transaction log, i.e., the length of the transaction log) for transection logs | 67108864 | +| Section | Parameter | Meaning | Default | +|---------|-------------------------------------------|---|---| +| common | active.memory.global.budget | The memory budget (in bytes) for the active runtime | 67108864 (64 MB) | +| common | compiler.framesize | The page size (in bytes) for computation | 32768 (32 kB) | +| common | compiler.groupmemory | The memory budget (in bytes) for a group by operator instance in a partition | 33554432 (32 MB) | +| common | compiler.joinmemory | The memory budget (in bytes) for a join operator instance in a partition | 33554432 (32 MB) | +| common | compiler.parallelism | The degree of parallelism for query execution. Zero means to use the storage parallelism as the query execution parallelism, while other integer values dictate the number of query execution parallel partitions. The system will fall back to use the number of all available CPU cores in the cluster as the degree of parallelism if the number set by a user is too large or too small | 0 | +| common | compiler.sortmemory | The memory budget (in bytes) for a sort operator instance in a partition | 33554432 (32 MB) | +| common | instance.name | The name of this cluster instance | DEFAULT_INSTANCE | +| common | log.level | The logging level for master and slave processes | WARNING | +| common | max.wait.active.cluster | The max pending time (in seconds) for cluster startup. After the threshold, if the cluster still is not up and running, it is considered unavailable | 60 | +| common | messaging.frame.count | N/A | 512 | +| common | messaging.frame.size | N/A | 4096 (4 kB) | +| common | metadata.callback.port | IP port to bind metadata callback listener (0 = random port) | 0 | +| common | metadata.listen.port | IP port to bind metadata listener (0 = random port) | 0 | +| common | metadata.node | the node which should serve as the metadata node | <undefined> | +| common | metadata.registration.timeout.secs | how long in seconds to wait for the metadata node to register with the CC | 60 | +| common | plot.activate | N/A | false | +| common | replication.log.batchsize | N/A | 4096 (4 kB) | +| common | replication.log.buffer.numpages | N/A | 8 | +| common | replication.log.buffer.pagesize | N/A | 131072 (128 kB) | +| common | replication.max.remote.recovery.attempts | N/A | 5 | +| common | replication.timeout | N/A | 15 | +| common | txn.commitprofiler.reportinterval | N/A | 5 | +| common | txn.job.recovery.memorysize | The memory budget (in bytes) used for recovery | 67108864 (64 MB) | +| common | txn.lock.escalationthreshold | N/A | 1000 | +| common | txn.lock.shrinktimer | N/A | 5000 | +| common | txn.lock.timeout.sweepthreshold | Interval (in milliseconds) for checking lock timeout | 10000 | +| common | txn.lock.timeout.waitthreshold | Time out (in milliseconds) of waiting for a lock | 60000 | +| common | txn.log.buffer.numpages | The number of pages in the transaction log tail | 8 | +| common | txn.log.buffer.pagesize | The page size (in bytes) for transaction log buffer | 131072 (128 kB) | +| common | txn.log.checkpoint.history | The number of checkpoints to keep in the transaction log | 0 | +| common | txn.log.checkpoint.lsnthreshold | The checkpoint threshold (in terms of LSNs (log sequence numbers) that have been written to the transaction log, i.e., the length of the transaction log) for transaction logs | 67108864 (64 MB) | +| common | txn.log.checkpoint.pollfrequency | N/A | 120 | +| common | txn.log.partitionsize | N/A | 268435456 (256 MB) | # For the optional NCService process configuration file, the following parameters, under "[ncservice]" section. diff --git a/asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml b/asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml index f4bb47a..d6f0968 100644 --- a/asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml +++ b/asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml @@ -242,24 +242,12 @@ <description>The minimum log level to be displayed. (Default = INFO) </description> </property> + <property> - <name>feed.memory.global.budget</name> + <name>active.memory.global.budget</name> <value>1073741824</value> <description>Feed memory budget (1 GB = 1073741824 Bytes) </description> </property> - <property> - <name>feed.pending.work.threshold</name> - <value>1000</value> - <description>Feed pending work threshold - </description> - </property> - - <property> - <name>feed.max.threshold.period</name> - <value>100</value> - <description>Feed max threshold period - </description> - </property> </asterixConfiguration> diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/AppContextInfo.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/AppContextInfo.java index 9bcfea2..400bb5a 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/AppContextInfo.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/AppContextInfo.java @@ -23,11 +23,11 @@ import java.util.logging.Logger; import org.apache.asterix.common.cluster.IGlobalRecoveryManager; +import org.apache.asterix.common.config.ActiveProperties; import org.apache.asterix.common.config.BuildProperties; import org.apache.asterix.common.config.CompilerProperties; import org.apache.asterix.common.config.ExtensionProperties; import org.apache.asterix.common.config.ExternalProperties; -import org.apache.asterix.common.config.FeedProperties; import org.apache.asterix.common.config.IPropertiesProvider; import org.apache.asterix.common.config.MessagingProperties; import org.apache.asterix.common.config.MetadataProperties; @@ -64,7 +64,7 @@ private MetadataProperties metadataProperties; private StorageProperties storageProperties; private TransactionProperties txnProperties; - private FeedProperties feedProperties; + private ActiveProperties activeProperties; private BuildProperties buildProperties; private ReplicationProperties replicationProperties; private ExtensionProperties extensionProperties; @@ -100,7 +100,7 @@ INSTANCE.metadataProperties = new MetadataProperties(propertiesAccessor); INSTANCE.storageProperties = new StorageProperties(propertiesAccessor); INSTANCE.txnProperties = new TransactionProperties(propertiesAccessor); - INSTANCE.feedProperties = new FeedProperties(propertiesAccessor); + INSTANCE.activeProperties = new ActiveProperties(propertiesAccessor); INSTANCE.extensionProperties = new ExtensionProperties(propertiesAccessor); INSTANCE.replicationProperties = new ReplicationProperties(propertiesAccessor); INSTANCE.ftStrategy = ftStrategy; @@ -150,8 +150,8 @@ } @Override - public FeedProperties getFeedProperties() { - return feedProperties; + public ActiveProperties getActiveProperties() { + return activeProperties; } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigManager.java index fb1332b..3944820 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigManager.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigManager.java @@ -24,10 +24,23 @@ import org.kohsuke.args4j.OptionHandlerFilter; public interface IConfigManager { - int PARSE_INI_POINTERS_METRIC = 100; - int PARSE_INI_METRIC = 200; - int PARSE_COMMAND_LINE_METRIC = 300; - int APPLY_DEFAULTS_METRIC = 400; + + enum ConfiguratorMetric { + PARSE_INI_POINTERS(100), + PARSE_INI(200), + PARSE_COMMAND_LINE(300), + APPLY_DEFAULTS(400); + + private final int metric; + + ConfiguratorMetric(int metric) { + this.metric = metric; + } + + public int metric() { + return metric; + } + } void register(IOption... options); diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IOptionType.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IOptionType.java index 2a98fdc..1bd6097 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IOptionType.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IOptionType.java @@ -26,15 +26,24 @@ Class<T> targetType(); + /** + * @return the value in a format suitable for serialized JSON + */ default Object serializeToJSON(Object value) { return value; } + /** + * @return the value in a format suitable for serialized ini file + */ default String serializeToIni(Object value) { return String.valueOf(value); } - default String serializeToString(Object value) { + /** + * @return the value in human-readable form (e.g. for usage) + */ + default String serializeToHumanReadable(Object value) { return serializeToIni(value); } } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java index bfe759e..04cf704 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java @@ -92,15 +92,19 @@ for (Section section : Section.values()) { allSections.add(section.sectionName()); } - addConfigurator(PARSE_INI_POINTERS_METRIC, this::extractIniPointersFromCommandLine); - addConfigurator(PARSE_INI_METRIC, this::parseIni); - addConfigurator(PARSE_COMMAND_LINE_METRIC, this::processCommandLine); - addConfigurator(APPLY_DEFAULTS_METRIC, this::applyDefaults); + addConfigurator(ConfiguratorMetric.PARSE_INI_POINTERS, this::extractIniPointersFromCommandLine); + addConfigurator(ConfiguratorMetric.PARSE_INI, this::parseIni); + addConfigurator(ConfiguratorMetric.PARSE_COMMAND_LINE, this::processCommandLine); + addConfigurator(ConfiguratorMetric.APPLY_DEFAULTS, this::applyDefaults); } @Override public void addConfigurator(int metric, IConfigurator configurator) { configurators.computeIfAbsent(metric, metric1 -> new ArrayList<>()).add(configurator); + } + + private void addConfigurator(ConfiguratorMetric metric, IConfigurator configurator) { + addConfigurator(metric.metric(), configurator); } @Override @@ -340,10 +344,9 @@ new CompositeMap<>(nodeMap.getValue(), definedMap, new NoOpMapMutator()), option, nodeMap.getKey())); } - // also push the defaults to the shared map, if the CC requests NC properties, they should receive the - // defaults -- TODO (mblow): seems lame, should log warning on access + } else { + entry.getValue().values().forEach(option -> getOrDefault(configurationMap, option, null)); } - entry.getValue().values().forEach(option -> getOrDefault(configurationMap, option, null)); } } @@ -514,10 +517,11 @@ } else if (value instanceof Function) { // TODO(mblow): defer usage calculation to enable evaluation of function buf.append("<function>"); + } else if (value == null) { + buf.append("<undefined>"); } else { - buf.append(option.type().serializeToString(resolveDefault(option, appConfig))); + buf.append(option.type().serializeToHumanReadable(resolveDefault(option, appConfig))); } - // TODO(mblow): defer usage calculation to enable inclusion of evaluated actual default } return buf.toString(); } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/OptionTypes.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/OptionTypes.java index fc26b5e..02b9325 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/OptionTypes.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/OptionTypes.java @@ -41,6 +41,11 @@ public Class<Integer> targetType() { return Integer.class; } + + @Override + public String serializeToHumanReadable(Object value) { + return value + " (" + StorageUtil.toHumanReadableSize((int)value) + ")"; + } }; public static final IOptionType<Long> LONG_BYTE_UNIT = new IOptionType<Long>() { @@ -53,6 +58,11 @@ public Class<Long> targetType() { return Long.class; } + + @Override + public String serializeToHumanReadable(Object value) { + return value + " (" + StorageUtil.toHumanReadableSize((long)value) + ")"; + } }; public static final IOptionType<Integer> INTEGER = new IOptionType<Integer>() { diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java index 97438c7..51451ce 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java @@ -38,10 +38,10 @@ @SuppressWarnings("SameParameterValue") public class CCConfig extends ControllerConfig { - public static String defaultDir = System.getProperty("java.io.tmpdir"); + public static String defaultAppClass; public enum Option implements IOption { - APP_CLASS(STRING), + APP_CLASS(STRING, (Supplier<String>)() -> defaultAppClass), ADDRESS(STRING, InetAddress.getLoopbackAddress().getHostAddress()), CLUSTER_LISTEN_ADDRESS(STRING, ADDRESS), CLUSTER_LISTEN_PORT(INTEGER, 1099), diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java index 4aae6df..fdc0963 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java @@ -30,6 +30,8 @@ public class ControllerConfig implements Serializable { + public static String defaultDir = System.getProperty("java.io.tmpdir"); + public enum Option implements IOption { CONFIG_FILE(OptionTypes.STRING, "Specify path to master configuration file"), CONFIG_FILE_URL(OptionTypes.URL, "Specify URL to master configuration file"); diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java index a65719e..9cf4da3 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java @@ -40,8 +40,7 @@ public class NCConfig extends ControllerConfig { private static final long serialVersionUID = 3L; - public static String defaultDir = System.getProperty("java.io.tmpdir"); - public static String defaultAppClass = null; + public static String defaultAppClass; public enum Option implements IOption { ADDRESS(STRING, InetAddress.getLoopbackAddress().getHostAddress()), diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java index a9a529c..9001e1b 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java @@ -160,6 +160,12 @@ return bytes + " B"; } final int baseValue = (63 - Long.numberOfLeadingZeros(bytes)) / 10; - return String.format("%.2f %sB", (double) bytes / (1L << (baseValue * 10)), " kMGTPE".charAt(baseValue)); + final char bytePrefix = " kMGTPE" .charAt(baseValue); + final long divisor = 1L << (baseValue * 10); + if (bytes % divisor == 0) { + return String.format("%d %sB", bytes / divisor, bytePrefix); + } else { + return String.format("%.2f %sB", (double) bytes / divisor, bytePrefix); + } } } -- To view, visit https://asterix-gerrit.ics.uci.edu/1555 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I9edc9ced4c2b99db239a2075903f9d8b256c26e4 Gerrit-PatchSet: 16 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Michael Blow <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
