Michael Blow has uploaded a new change for review. https://asterix-gerrit.ics.uci.edu/1555
Change subject: Config Cleanup, Update Docs
......................................................................
Config Cleanup, Update Docs
- added some missing descriptions
- updated ncservice.md with updated parameter names / sections
- minor cleanup
Change-Id: I9edc9ced4c2b99db239a2075903f9d8b256c26e4
---
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
M asterixdb/asterix-doc/src/site/markdown/ncservice.md
M
hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigManager.java
M
hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java
8 files changed, 222 insertions(+), 95 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/55/1555/1
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java
index b96b7fe..36a3fa3 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java
@@ -42,6 +42,7 @@
@Test
public void generateUsage() {
+ System.err.println();
generateUsage("| ", " | ", " |", true, System.err);
}
@@ -78,7 +79,21 @@
}
}
}
- maxDescriptionWidth = Math.min(80, maxDescriptionWidth);
+ // output header
+ buf.append(startDelim);
+ pad(buf, "Section", maxSectionWidth).append(midDelim);
+ pad(buf, "Parameter", maxNameWidth).append(midDelim);
+ pad(buf, "Meaning", maxDescriptionWidth).append(midDelim);
+ pad(buf, "Default", maxDefaultWidth).append(endDelim).append('\n');
+
+ StringBuilder sepLine = new StringBuilder();
+ sepLine.append(startDelim);
+ pad(sepLine, "", maxSectionWidth, '-').append(midDelim);
+ pad(sepLine, "", maxNameWidth, '-').append(midDelim);
+ pad(sepLine, "", maxDescriptionWidth, '-').append(midDelim);
+ pad(sepLine, "", maxDefaultWidth, '-').append(endDelim).append('\n');
+ buf.append(sepLine.toString().replace(' ', '-'));
+
for (Section section : configManager.getSections()) {
List<IOption> options = new
ArrayList<>(configManager.getOptions(section));
options.sort(Comparator.comparing(IOption::ini));
@@ -89,7 +104,7 @@
buf.append(startDelim);
center(buf, section.sectionName(),
maxSectionWidth).append(midDelim);
pad(buf, option.ini(), maxNameWidth).append(midDelim);
- String description = option.description() == null ? "" :
option.description();
+ String description = option.description() == null ? "N/A" :
option.description();
String defaultText = configManager.defaultTextForUsage(option,
IOption::ini);
boolean extra = false;
while (align && description.length() > maxDescriptionWidth) {
@@ -133,12 +148,16 @@
}
private StringBuilder pad(StringBuilder buf, String string, int width) {
+ return pad(buf, string, width, ' ');
+ }
+
+ private StringBuilder pad(StringBuilder buf, String string, int width,
char padChar) {
if (string == null) {
string = "";
}
buf.append(string);
for (int i = width - string.length(); i > 0; i--) {
- buf.append(' ');
+ buf.append(padChar);
}
return buf;
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java
index 5f3a1aa..2e554a6 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java
@@ -46,7 +46,7 @@
TransactionProperties.Option.class);
// we need to process the old-style asterix config before we apply
defaults!
- configManager.addConfigurator(IConfigManager.APPLY_DEFAULTS_METRIC -
1, () -> {
+
configManager.addConfigurator(IConfigManager.ConfiguratorMetric.APPLY_DEFAULTS.metric()
- 1, () -> {
try {
PropertiesAccessor.getInstance(configManager.getAppConfig());
} catch (AsterixException e) {
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index f97c5a5..a9cfa40 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -33,19 +33,29 @@
public class CompilerProperties extends AbstractProperties {
public enum Option implements IOption {
- COMPILER_SORTMEMORY(LONG_BYTE_UNIT,
StorageUtil.getLongSizeInBytes(32L, MEGABYTE)),
- COMPILER_JOINMEMORY(LONG_BYTE_UNIT,
StorageUtil.getLongSizeInBytes(32L, MEGABYTE)),
- COMPILER_GROUPMEMORY(LONG_BYTE_UNIT,
StorageUtil.getLongSizeInBytes(32L, MEGABYTE)),
- COMPILER_FRAMESIZE(INTEGER_BYTE_UNIT,
StorageUtil.getIntSizeInBytes(32, KILOBYTE)),
- COMPILER_PARALLELISM(INTEGER, COMPILER_PARALLELISM_AS_STORAGE),
- COMPILER_PREGELIX_HOME(STRING, "~/pregelix");
+ COMPILER_SORTMEMORY(LONG_BYTE_UNIT,
StorageUtil.getLongSizeInBytes(32L, MEGABYTE),
+ "The memory budget (in bytes) for a sort operator instance in
a partition"),
+ COMPILER_JOINMEMORY(LONG_BYTE_UNIT,
StorageUtil.getLongSizeInBytes(32L, MEGABYTE),
+ "The memory budget (in bytes) for a join operator instance in
a partition"),
+ COMPILER_GROUPMEMORY(LONG_BYTE_UNIT,
StorageUtil.getLongSizeInBytes(32L, MEGABYTE),
+ "The memory budget (in bytes) for a group by operator instance
in a partition"),
+ COMPILER_FRAMESIZE(INTEGER_BYTE_UNIT,
StorageUtil.getIntSizeInBytes(32, KILOBYTE),
+ "The page size (in bytes) for computation"),
+ COMPILER_PARALLELISM(INTEGER, COMPILER_PARALLELISM_AS_STORAGE, "The
degree of parallelism for query " +
+ "execution. Zero means to use the storage parallelism as the
query execution parallelism, while " +
+ "other integer values dictate the number of query execution
parallel partitions. The system will " +
+ "fall back to use the number of all available CPU cores in the
cluster as the degree of parallelism " +
+ "if the number set by a user is too large or too small"),
+ COMPILER_PREGELIX_HOME(STRING, "~/pregelix", "Pregelix installation
root directory");
private final IOptionType type;
private final Object defaultValue;
+ private final String description;
- Option(IOptionType type, Object defaultValue) {
+ Option(IOptionType type, Object defaultValue, String description) {
this.type = type;
this.defaultValue = defaultValue;
+ this.description = description;
}
@Override
@@ -55,7 +65,7 @@
@Override
public String description() {
- return "";
+ return description;
}
@Override
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
index 2f85221..463e00d 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
@@ -27,22 +27,25 @@
public class ExternalProperties extends AbstractProperties {
public enum Option implements IOption {
- WEB_PORT(INTEGER, 19001),
- WEB_QUERYINTERFACE_PORT(INTEGER, 19006),
- API_PORT(INTEGER, 19002),
- FEED_PORT(INTEGER, 19003),
- LOG_LEVEL(LEVEL, java.util.logging.Level.WARNING),
- MAX_WAIT_ACTIVE_CLUSTER(INTEGER, 60),
- PLOT_ACTIVATE(BOOLEAN, false),
- CC_JAVA_OPTS(STRING, "-Xmx1024m"),
- NC_JAVA_OPTS(STRING, "-Xmx1024m");
+ WEB_PORT(INTEGER, 19001, "The listen port of the legacy query
interface"),
+ WEB_QUERYINTERFACE_PORT(INTEGER, 19006, "The listen port of the query
web interface"),
+ API_PORT(INTEGER, 19002, "The listen port of the API server"),
+ FEED_PORT(INTEGER, 19003, "The listen port of the feed server"),
+ LOG_LEVEL(LEVEL, java.util.logging.Level.WARNING, "The logging level
for master and slave processes"),
+ MAX_WAIT_ACTIVE_CLUSTER(INTEGER, 60, "The max pending time (in
seconds) for cluster startup. After the " +
+ "threshold, if the cluster still is not up and running, it is
considered unavailable"),
+ PLOT_ACTIVATE(BOOLEAN, false, null),
+ CC_JAVA_OPTS(STRING, "-Xmx1024m", "The JVM options passed to the
cluster controller process by managix"),
+ NC_JAVA_OPTS(STRING, "-Xmx1024m", "The JVM options passed to the node
controller process(es) by managix");
private final IOptionType type;
private final Object defaultValue;
+ private final String description;
- Option(IOptionType type, Object defaultValue) {
+ Option(IOptionType type, Object defaultValue, String description) {
this.type = type;
this.defaultValue = defaultValue;
+ this.description = description;
}
@Override
@@ -67,8 +70,7 @@
@Override
public String description() {
- // TODO(mblow): add descriptions
- return null;
+ return description;
}
@Override
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
index 73e8c4a..97ad6be 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
@@ -18,8 +18,11 @@
*/
package org.apache.asterix.common.config;
-import static org.apache.hyracks.control.common.config.OptionTypes.*;
-import static org.apache.hyracks.util.StorageUtil.StorageUnit.*;
+import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER;
+import static
org.apache.hyracks.control.common.config.OptionTypes.INTEGER_BYTE_UNIT;
+import static
org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT;
+import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
+import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE;
import java.util.Map;
@@ -31,25 +34,31 @@
public class TransactionProperties extends AbstractProperties {
public enum Option implements IOption {
- TXN_LOG_BUFFER_NUMPAGES(INTEGER, 8),
- TXN_LOG_BUFFER_PAGESIZE(INTEGER_BYTE_UNIT,
StorageUtil.getIntSizeInBytes(128, KILOBYTE)),
- TXN_LOG_PARTITIONSIZE(LONG_BYTE_UNIT,
StorageUtil.getLongSizeInBytes(256L, MEGABYTE)),
- TXN_LOG_CHECKPOINT_LSNTHRESHOLD(INTEGER_BYTE_UNIT,
StorageUtil.getIntSizeInBytes(64, MEGABYTE)),
- TXN_LOG_CHECKPOINT_POLLFREQUENCY(INTEGER, 120),
- TXN_LOG_CHECKPOINT_HISTORY(INTEGER, 0),
- TXN_LOCK_ESCALATIONTHRESHOLD(INTEGER, 1000),
- TXN_LOCK_SHRINKTIMER(INTEGER, 5000),
- TXN_LOCK_TIMEOUT_WAITTHRESHOLD(INTEGER, 60000),
- TXN_LOCK_TIMEOUT_SWEEPTHRESHOLD(INTEGER, 10000),
- TXN_COMMITPROFILER_REPORTINTERVAL(INTEGER, 5),
- TXN_JOB_RECOVERY_MEMORYSIZE(LONG_BYTE_UNIT,
StorageUtil.getLongSizeInBytes(64L, MEGABYTE));
+ TXN_LOG_BUFFER_NUMPAGES(INTEGER, 8, "The number of pages in the
transaction log tail"),
+ TXN_LOG_BUFFER_PAGESIZE(INTEGER_BYTE_UNIT,
StorageUtil.getIntSizeInBytes(128, KILOBYTE),
+ "The page size (in bytes) for transaction log buffer"),
+ TXN_LOG_PARTITIONSIZE(LONG_BYTE_UNIT,
StorageUtil.getLongSizeInBytes(256L, MEGABYTE), null),
+ TXN_LOG_CHECKPOINT_LSNTHRESHOLD(INTEGER_BYTE_UNIT,
StorageUtil.getIntSizeInBytes(64, MEGABYTE),
+ "The checkpoint threshold (in terms of LSNs (log sequence
numbers) that have been written to the " +
+ "transaction log, i.e., the length of the transaction
log) for transaction logs"),
+ TXN_LOG_CHECKPOINT_POLLFREQUENCY(INTEGER, 120, null),
+ TXN_LOG_CHECKPOINT_HISTORY(INTEGER, 0, "The number of checkpoints to
keep in the transaction log"),
+ TXN_LOCK_ESCALATIONTHRESHOLD(INTEGER, 1000, null),
+ TXN_LOCK_SHRINKTIMER(INTEGER, 5000, null),
+ TXN_LOCK_TIMEOUT_WAITTHRESHOLD(INTEGER, 60000, "Time out (in
milliseconds) of waiting for a lock"),
+ TXN_LOCK_TIMEOUT_SWEEPTHRESHOLD(INTEGER, 10000, "Interval (in
milliseconds) for checking lock timeout"),
+ TXN_COMMITPROFILER_REPORTINTERVAL(INTEGER, 5, null),
+ TXN_JOB_RECOVERY_MEMORYSIZE(LONG_BYTE_UNIT,
StorageUtil.getLongSizeInBytes(64L, MEGABYTE),
+ "The memory budget (in bytes) used for recovery");
private final IOptionType type;
private final Object defaultValue;
+ private final String description;
- Option(IOptionType type, Object defaultValue) {
+ Option(IOptionType type, Object defaultValue, String description) {
this.type = type;
this.defaultValue = defaultValue;
+ this.description = description;
}
@Override
@@ -59,8 +68,7 @@
@Override
public String description() {
- // TODO(mblow): add missing descriptions
- return null;
+ return description;
}
@Override
diff --git a/asterixdb/asterix-doc/src/site/markdown/ncservice.md
b/asterixdb/asterix-doc/src/site/markdown/ncservice.md
index 67b1970..51787b5 100644
--- a/asterixdb/asterix-doc/src/site/markdown/ncservice.md
+++ b/asterixdb/asterix-doc/src/site/markdown/ncservice.md
@@ -255,57 +255,129 @@
The following parameters are for the master process, under the "[cc]" section.
-| Parameter | Meaning | Default |
-|----------|--------|-------|
-| max.wait.active.cluster | The max pending time (in seconds) for cluster
startup. After the threshold, if the cluster still is not up and running, it is
considered unavailable. | 60 |
-| address | The binding IP address for the AsterixDB instance | N/A |
+| Section | Parameter | Meaning
| Default
|
+|-----------|-------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------|
+| cc | address | Default bind address
for all services on this cluster controller
| 127.0.0.1
|
+| cc | api.port | The listen port of
the API server
| 19002
|
+| cc | app.class | Application CC main
class
| null
|
+| cc | client.listen.address | Sets the IP Address
to listen for connections from clients
| same as address
|
+| cc | client.listen.port | Sets the port to
listen for connections from clients
| 1098
|
+| cc | cluster.listen.address | Sets the IP Address
to listen for connections from NCs
| same as address
|
+| cc | cluster.listen.port | Sets the port to
listen for connections from node controllers
| 1099
|
+| cc | cluster.public.address | Address that NCs
should use to contact this CC
| same as cluster.listen.address
|
+| cc | cluster.public.port | Port that NCs should
use to contact this CC
| same as cluster.listen.port
|
+| cc | cluster.topology | Sets the XML file
that defines the cluster topology
| null
|
+| cc | console.listen.address | Sets the listen
address for the Cluster Controller
| same as address
|
+| cc | console.listen.port | Sets the http port
for the Cluster Controller)
| 16001
|
+| cc | feed.port | The listen port of
the feed server
| 19003
|
+| cc | heartbeat.max.misses | Sets the maximum
number of missed heartbeats before a node is marked as dead
| 5
|
+| cc | heartbeat.period | Sets the time
duration between two heartbeats from each node controller in milliseconds
| 10000
|
+| cc | job.history.size | Limits the number of
historical jobs remembered by the system to the specified value
| 10
|
+| cc | job.manager.class | Specify the
implementation class name for the job manager
|
org.apache.hyracks.control.cc.job.JobManager
|
+| cc | job.queue.class | Specify the
implementation class name for the job queue
|
org.apache.hyracks.control.cc.scheduler.FIFOJobQueue
|
+| cc | profile.dump.period | Sets the time
duration between two profile dumps from each node controller in milliseconds; 0
to disable
| 0
|
+| cc | result.sweep.threshold | The duration within
which an instance of the result cleanup should be invoked in milliseconds
| 60000
|
+| cc | result.ttl | Limits the amount of
time results for asynchronous jobs should be retained by the system in
milliseconds
| 86400000
|
+| cc | root.dir | Sets the root folder
used for file operations
|
/var/folders/5x/qdtntlds0fgcgknzwf61khvh0000gn/T/asterixdb/ClusterControllerService
|
+| cc | web.port | The listen port of
the legacy query interface
| 19001
|
+| cc | web.queryinterface.port | The listen port of
the query web interface
| 19006
|
The following parameters for slave processes, under "[nc]" sections.
-| Parameter | Meaning | Default |
-|----------|--------|-------|
-| address | The binding IP address for the slave process | N/A |
-| command | The command for the slave process | N/A (for AsterixDB, it should
be "asterixnc") |
-| core.dump.dir | The path for core dump | N/A |
-| iodevices | Comma separated directory paths for both storage files and
temporary files | N/A |
-| jvm.args | The JVM arguments | -Xmx1536m |
-| ncservice.port | The port on which the NCService for this NC is listening |
9090 |
-| txn.log.dir | The directory for transaction logs | N/A |
+| Section | Parameter | Meaning
| Default
|
+|-----------|-------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------|
+| nc | address | Default IP Address
to bind listeners on this NC. All services will bind on this address unless a
service-specific listen address is supplied.
| 127.0.0.1
|
+| nc | app.class | Application NC Main
Class
| null
|
+| nc | cluster.address | Cluster Controller
address (required unless specified in config file)
| null
|
+| nc | cluster.connect.retries | Number of attempts
to contact CC before giving up
| 5
|
+| nc | cluster.listen.address | IP Address to bind
cluster listener on this NC
| same as address
|
+| nc | cluster.listen.port | IP port to bind
cluster listener
| 0
|
+| nc | cluster.port | Cluster Controller
port
| 1099
|
+| nc | cluster.public.address | Public IP Address to
announce cluster listener
| same as public.address
|
+| nc | cluster.public.port | Public IP port to
announce cluster listener
| same as cluster.listen.port
|
+| nc | command | Command NCService
should invoke to start the NCDriver
| hyracksnc
|
+| nc | core.dump.dir | N/A
|
/var/folders/5x/qdtntlds0fgcgknzwf61khvh0000gn/T/coredump
|
+| nc | data.listen.address | IP Address to bind
data listener
| same as address
|
+| nc | data.listen.port | IP port to bind data
listener
| 0
|
+| nc | data.public.address | Public IP Address to
announce data listener
| same as public.address
|
+| nc | data.public.port | Public IP port to
announce data listener
| same as data.listen.port
|
+| nc | iodevices | Comma separated list
of IO Device mount points
|
/var/folders/5x/qdtntlds0fgcgknzwf61khvh0000gn/T/iodevice
|
+| nc | jvm.args | JVM args to pass to
the NCDriver
| null
|
+| nc | messaging.listen.address | IP Address to bind
messaging listener
| same as address
|
+| nc | messaging.listen.port | IP port to bind
messaging listener
| 0
|
+| nc | messaging.public.address | Public IP Address to
announce messaging listener
| same as public.address
|
+| nc | messaging.public.port | Public IP port to
announce messaging listener
| same as messaging.listen.port
|
+| nc | ncservice.address | Address the CC
should use to contact the NCService associated with this NC
| same as public.address
|
+| nc | ncservice.pid | PID of the NCService
which launched this NCDriver
| -1
|
+| nc | ncservice.port | Port the CC should
use to contact the NCService associated with this NC
| 9090
|
+| nc | net.buffer.count | Number of network
buffers per input/output channel
| 1
|
+| nc | net.thread.count | Number of threads to
use for Network I/O
| 1
|
+| nc | public.address | Default public
address that other processes should use to contact this NC. All services will
advertise this address unless a service-specific public address is supplied.
| same as address
|
+| nc | result.listen.address | IP Address to bind
dataset result distribution listener
| same as address
|
+| nc | result.listen.port | IP port to bind
dataset result distribution listener
| 0
|
+| nc | result.manager.memory | Memory usable for
result caching at this Node Controller in bytes
| -1
|
+| nc | result.public.address | Public IP Address to
announce dataset result distribution listener
| same as public.address
|
+| nc | result.public.port | Public IP port to
announce dataset result distribution listener
| same as result.listen.port
|
+| nc | result.sweep.threshold | The duration within
which an instance of the result cleanup should be invoked in milliseconds
| 60000
|
+| nc | result.ttl | Limits the amount of
time results for asynchronous jobs should be retained by the system in
milliseconds
| 86400000
|
+| nc | storage.buffercache.maxopenfiles | The maximum number
of open files in the buffer cache
| 2147483647
|
+| nc | storage.buffercache.pagesize | The page size in
bytes for pages in the buffer cache
| 131072
|
+| nc | storage.buffercache.size | The size of memory
allocated to the disk buffer cache. The value should be a multiple of the
buffer cache page size.
| 715915264
|
+| nc | storage.lsm.bloomfilter.falsepositiverate | The maximum
acceptable false positive rate for bloom filters associated with LSM indexes
| 0.01
|
+| nc | storage.memorycomponent.globalbudget | The size of memory
allocated to the memory components. The value should be a multiple of the
memory component page size
| 715915264
|
+| nc | storage.memorycomponent.numcomponents | The number of memory
components to be used per lsm index
| 2
|
+| nc | storage.memorycomponent.numpages | The number of pages
to allocate for a memory component. This budget is shared by all the memory
components of the primary index and all its secondary indexes across all I/O
devices on a node. Note: in-memory components usually has fill factor of 75%
since the pages are 75% full and the remaining 25% is un-utilized
| 1/16th of the
storage.memorycomponent.globalbudget value |
+| nc | storage.memorycomponent.pagesize | The page size in
bytes for pages allocated to memory components
| 131072
|
+| nc | storage.metadata.memorycomponent.numpages | The number of pages
to allocate for a metadata memory component
| 1/64th of the
storage.memorycomponent.globalbudget value or 256, whichever is larger |
+| nc | storage.subdir | N/A
| storage
|
+| nc | txn.log.dir | N/A
|
/var/folders/5x/qdtntlds0fgcgknzwf61khvh0000gn/T/txn-log
|
The following parameters are configured under the "[common]" section.
-| Parameter | Meaning | Default |
-|----------|--------|-------|
-| instance.name | The name of the AsterixDB instance | "DEFAULT_INSTANCE" |
-| log.level | The logging level for master and slave processes | "INFO" |
-| compiler.framesize | The page size (in bytes) for computation | 32768 |
-| compiler.groupmemory | The memory budget (in bytes) for a group by operator
instance in a partition | 33554432 |
-| compiler.joinmemory | The memory budget (in bytes) for a join operator
instance in a partition | 33554432 |
-| compiler.sortmemory | The memory budget (in bytes) for a sort operator
instance in a partition | 33554432 |
-| compiler.parallelism | The degree of parallelism for query execution. Zero
means to use the storage parallelism as the query execution parallelism, while
other integer values dictate the number of query execution parallel partitions.
The system will fall back to use the number of all available CPU cores in the
cluster as the degree of parallelism if the number set by a user is too large
or too small. | 0 |
-| metadata.callback.port | The port for metadata communication | 0 |
-| metadata.listen.port | The metadata communication port on the metadata node.
This parameter should only be present in the section of the metadata NC | 0 |
-| metadata.registration.timeout.secs | The time out threshold (in seconds) for
metadata node registration | 60 |
-| storage.subdir | The directory for storage files | N/A |
-| storage.buffercache.maxopenfiles | The maximum number of open files for the
buffer cache. Note that this is the parameter for the AsterixDB and setting
the operating system parameter is still required. | 2147483647 |
-| storage.buffercache.pagesize | The page size (in bytes) for the disk buffer
cache (for reads) | 131072 |
-| storage.buffercache.size | The overall budget (in bytes) of the disk buffer
cache (for reads) | 536870912 |
-| storage.lsm.bloomfilter.falsepositiverate | The false positive rate for the
bloom filter for each memory/disk components | 0.01 |
-| storage.memorycomponent.globalbudget | The global budget (in bytes) for all
memory components of all datasets and indexes (for writes) | 536870912 |
-| storage.memorycomponent.numcomponents | The number of memory components per
data partition per index | 2 |
-| storage.memorycomponent.numpages | The number of pages for all memory
components of a dataset, including those for secondary indexes | 256 |
-| storage.memorycomponent.pagesize | The page size (in bytes) of memory
components | 131072 |
-| storage.metadata.memorycomponent.numpages | The number of pages for all
memory components of a metadata dataset | 256 |
-| txn.commitprofiler.reportinterval | The interval for reporting commit
statistics | 5 |
-| txn.job.recovery.memorysize | The memory budget (in bytes) used for
recovery | 67108864 |
-| txn.lock.timeout.sweepthreshold | Interval (in milliseconds) for checking
lock timeout | 10000 |
-| txn.lock.timeout.waitthreshold | Time out (in milliseconds) of waiting for a
lock | 60000 |
-| txn.log.buffer.numpages | The number of pages in the transaction log tail |
8 |
-| txn.log.buffer.pagesize | The page size (in bytes) for transaction log
buffer. | 131072 |
-| txn.log.checkpoint.history | The number of checkpoints to keep in the
transaction log | 0 |
-| txn.log.checkpoint.lsnthreshold | The checkpoint threshold (in terms of LSNs
(log sequence numbers) that have been written to the transaction log, i.e., the
length of the transaction log) for transection logs | 67108864 |
+| Section | Parameter | Meaning
| Default
|
+|-----------|-------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------|
+| common | compiler.framesize | The page size (in
bytes) for computation
| 32768
|
+| common | compiler.groupmemory | The memory budget
(in bytes) for a group by operator instance in a partition
| 33554432
|
+| common | compiler.joinmemory | The memory budget
(in bytes) for a join operator instance in a partition
| 33554432
|
+| common | compiler.parallelism | The degree of
parallelism for query execution. Zero means to use the storage parallelism as
the query execution parallelism, while other integer values dictate the number
of query execution parallel partitions. The system will fall back to use the
number of all available CPU cores in the cluster as the degree of parallelism
if the number set by a user is too large or too small | 0
|
+| common | compiler.pregelix.home | Pregelix
installation root directory
| ~/pregelix
|
+| common | compiler.sortmemory | The memory budget
(in bytes) for a sort operator instance in a partition
| 33554432
|
+| common | config.file | Specify path to
master configuration file
| null
|
+| common | config.file.url | Specify URL to
master configuration file
| null
|
+| common | feed.central.manager.port | port at which the
Central Feed Manager listens for control messages from local Feed Managers
| 4500
|
+| common | feed.max.threshold.period | maximum length of
input queue before triggering corrective action
| 5
|
+| common | feed.memory.available.wait.timeout | N/A
| 10
|
+| common | feed.memory.global.budget | N/A
| 67108864
|
+| common | feed.pending.work.threshold | N/A
| 50
|
+| common | instance.name | The name of this
cluster instance
| DEFAULT_INSTANCE
|
+| common | log.level | The logging level
for master and slave processes
| WARNING
|
+| common | max.wait.active.cluster | The max pending time
(in seconds) for cluster startup. After the threshold, if the cluster still is
not up and running, it is considered unavailable
| 60
|
+| common | messaging.frame.count | N/A
| 512
|
+| common | messaging.frame.size | N/A
| 4096
|
+| common | metadata.callback.port | IP port to bind
metadata callback listener (0 = random port)
| 0
|
+| common | metadata.listen.port | IP port to bind
metadata listener (0 = random port)
| 0
|
+| common | metadata.node | the node which
should serve as the metadata node
| null
|
+| common | metadata.registration.timeout.secs | how long in seconds
to wait for the metadata node to register with the CC
| 60
|
+| common | plot.activate | N/A
| false
|
+| common | replication.log.batchsize | N/A
| 4096
|
+| common | replication.log.buffer.numpages | N/A
| 8
|
+| common | replication.log.buffer.pagesize | N/A
| 131072
|
+| common | replication.max.remote.recovery.attempts | N/A
| 5
|
+| common | replication.timeout | N/A
| 15
|
+| common | txn.commitprofiler.reportinterval | N/A
| 5
|
+| common | txn.job.recovery.memorysize | The memory budget
(in bytes) used for recovery
| 67108864
|
+| common | txn.lock.escalationthreshold | N/A
| 1000
|
+| common | txn.lock.shrinktimer | N/A
| 5000
|
+| common | txn.lock.timeout.sweepthreshold | Interval (in
milliseconds) for checking lock timeout
| 10000
|
+| common | txn.lock.timeout.waitthreshold | Time out (in
milliseconds) of waiting for a lock
| 60000
|
+| common | txn.log.buffer.numpages | The number of pages
in the transaction log tail
| 8
|
+| common | txn.log.buffer.pagesize | The page size (in
bytes) for transaction log buffer
| 131072
|
+| common | txn.log.checkpoint.history | The number of
checkpoints to keep in the transaction log
| 0
|
+| common | txn.log.checkpoint.lsnthreshold | The checkpoint
threshold (in terms of LSNs (log sequence numbers) that have been written to
the transaction log, i.e., the length of the transaction log) for transaction
logs
| 67108864
|
+| common | txn.log.checkpoint.pollfrequency | N/A
| 120
|
+| common | txn.log.partitionsize | N/A
| 268435456
|
# For the optional NCService process configuration file, the following
parameters, under "[ncservice]" section.
diff --git
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigManager.java
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigManager.java
index fb1332b..3944820 100644
---
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigManager.java
@@ -24,10 +24,23 @@
import org.kohsuke.args4j.OptionHandlerFilter;
public interface IConfigManager {
- int PARSE_INI_POINTERS_METRIC = 100;
- int PARSE_INI_METRIC = 200;
- int PARSE_COMMAND_LINE_METRIC = 300;
- int APPLY_DEFAULTS_METRIC = 400;
+
+ enum ConfiguratorMetric {
+ PARSE_INI_POINTERS(100),
+ PARSE_INI(200),
+ PARSE_COMMAND_LINE(300),
+ APPLY_DEFAULTS(400);
+
+ private final int metric;
+
+ ConfiguratorMetric(int metric) {
+ this.metric = metric;
+ }
+
+ public int metric() {
+ return metric;
+ }
+ }
void register(IOption... options);
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java
index bfe759e..c895762 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java
@@ -92,15 +92,19 @@
for (Section section : Section.values()) {
allSections.add(section.sectionName());
}
- addConfigurator(PARSE_INI_POINTERS_METRIC,
this::extractIniPointersFromCommandLine);
- addConfigurator(PARSE_INI_METRIC, this::parseIni);
- addConfigurator(PARSE_COMMAND_LINE_METRIC, this::processCommandLine);
- addConfigurator(APPLY_DEFAULTS_METRIC, this::applyDefaults);
+ addConfigurator(ConfiguratorMetric.PARSE_INI_POINTERS,
this::extractIniPointersFromCommandLine);
+ addConfigurator(ConfiguratorMetric.PARSE_INI, this::parseIni);
+ addConfigurator(ConfiguratorMetric.PARSE_COMMAND_LINE,
this::processCommandLine);
+ addConfigurator(ConfiguratorMetric.APPLY_DEFAULTS,
this::applyDefaults);
}
@Override
public void addConfigurator(int metric, IConfigurator configurator) {
configurators.computeIfAbsent(metric, metric1 -> new
ArrayList<>()).add(configurator);
+ }
+
+ private void addConfigurator(ConfiguratorMetric metric, IConfigurator
configurator) {
+ addConfigurator(metric.metric(), configurator);
}
@Override
@@ -340,10 +344,9 @@
new CompositeMap<>(nodeMap.getValue(),
definedMap, new NoOpMapMutator()), option,
nodeMap.getKey()));
}
- // also push the defaults to the shared map, if the CC
requests NC properties, they should receive the
- // defaults -- TODO (mblow): seems lame, should log warning on
access
+ } else {
+ entry.getValue().values().forEach(option ->
getOrDefault(configurationMap, option, null));
}
- entry.getValue().values().forEach(option ->
getOrDefault(configurationMap, option, null));
}
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/1555
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I9edc9ced4c2b99db239a2075903f9d8b256c26e4
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <[email protected]>
