http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixRestartIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixRestartIT.java b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixRestartIT.java index 7cca5cc..ce4de7c 100644 --- a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixRestartIT.java +++ b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixRestartIT.java @@ -28,7 +28,7 @@ import org.apache.asterix.event.model.AsterixInstance.State; import org.apache.asterix.test.base.RetainLogsRule; import org.apache.asterix.test.common.TestExecutor; import org.apache.asterix.testframework.context.TestCaseContext; -import org.apache.commons.lang3.StringUtils; +import org.apache.hyracks.util.file.FileUtil; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -48,8 +48,7 @@ public class AsterixRestartIT { private static final String PATH_BASE = "src/test/resources/integrationts/restart/"; private static final String PATH_ACTUAL = "target" + File.separator + "ittest" + File.separator; private static final Logger LOGGER = Logger.getLogger(AsterixRestartIT.class.getName()); - private static String reportPath = new File( - StringUtils.join(new String[] { "target", "failsafe-reports" }, File.separator)).getAbsolutePath(); + private static String reportPath = new File(FileUtil.joinPath("target", "failsafe-reports")).getAbsolutePath(); private final TestExecutor testExecutor = new TestExecutor(); private TestCaseContext tcCtx;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ClusterExecutionIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ClusterExecutionIT.java b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ClusterExecutionIT.java index 9fb1e01..446e5e3 100644 --- a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ClusterExecutionIT.java +++ b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ClusterExecutionIT.java @@ -25,6 +25,7 @@ import org.apache.asterix.test.common.TestExecutor; import org.apache.asterix.test.runtime.HDFSCluster; import org.apache.asterix.testframework.context.TestCaseContext; import org.apache.commons.lang3.StringUtils; +import org.apache.hyracks.util.file.FileUtil; import org.codehaus.plexus.util.FileUtils; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -58,7 +59,7 @@ public class ClusterExecutionIT extends AbstractExecutionIT { AsterixClusterLifeCycleIT.setUp(); FileUtils.copyDirectoryStructure( - new File(StringUtils.join(new String[] { "..", "asterix-app", "data" }, File.separator)), + new File(FileUtil.joinPath("..", "asterix-app", "data")), new File(StringUtils.join( new String[] { "src", "test", "resources", "clusterts", "managix-working", "data" }, File.separator))); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ReplicationIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ReplicationIT.java b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ReplicationIT.java index bac93e4..44ce7f8 100644 --- a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ReplicationIT.java +++ b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ReplicationIT.java @@ -28,7 +28,7 @@ import org.apache.asterix.event.model.AsterixInstance.State; import org.apache.asterix.test.base.RetainLogsRule; import org.apache.asterix.test.common.TestExecutor; import org.apache.asterix.testframework.context.TestCaseContext; -import org.apache.commons.lang3.StringUtils; +import org.apache.hyracks.util.file.FileUtil; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; @@ -41,11 +41,12 @@ import org.junit.runners.Parameterized; @RunWith(Parameterized.class) public class ReplicationIT { - private static final String PATH_BASE = "src/test/resources/integrationts/replication/"; - private static final String PATH_ACTUAL = "target" + File.separator + "ittest" + File.separator; + private static final String PATH_BASE = FileUtil.joinPath("src", "test", "resources", "integrationts", + "replication"); + private static final String PATH_ACTUAL = FileUtil.joinPath("target", "ittest"); private static final Logger LOGGER = Logger.getLogger(ReplicationIT.class.getName()); private static String reportPath = new File( - StringUtils.join(new String[] { "target", "failsafe-reports" }, File.separator)).getAbsolutePath(); + FileUtil.joinPath("target", "failsafe-reports")).getAbsolutePath(); private final TestExecutor testExecutor = new TestExecutor(); private TestCaseContext tcCtx; @@ -67,9 +68,8 @@ public class ReplicationIT { pb = new ProcessBuilder(); env = pb.environment(); asterixInstallerPath = new File(System.getProperty("user.dir")); - scriptHomePath = asterixInstallerPath + File.separator + "src" + File.separator + "test" + File.separator - + "resources" + File.separator + "integrationts" + File.separator + "replication" + File.separator - + "scripts"; + scriptHomePath = FileUtil.joinPath(asterixInstallerPath.getPath(), "src", "test", "resources", + "integrationts", "replication", "scripts"); env.put("SCRIPT_HOME", scriptHomePath); } catch (Throwable th) { th.printStackTrace(); @@ -81,7 +81,7 @@ public class ReplicationIT { public void before() throws Exception { LOGGER.info("Creating new instance..."); AsterixInstallerIntegrationUtil.init(AsterixInstallerIntegrationUtil.LOCAL_CLUSTER_CHAINED_DECLUSTERING_REP_PATH); - LOGGER.info("Instacne created."); + LOGGER.info("Instance created."); AsterixInstallerIntegrationUtil.transformIntoRequiredState(State.ACTIVE); LOGGER.info("Instance is in ACTIVE state."); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/transaction/RecoveryIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/transaction/RecoveryIT.java b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/transaction/RecoveryIT.java index 804848e..a0612cc 100644 --- a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/transaction/RecoveryIT.java +++ b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/transaction/RecoveryIT.java @@ -106,7 +106,7 @@ public class RecoveryIT { HDFSCluster.getInstance().cleanup(); } - @Parameters + @Parameters(name = "RecoveryIT {index}: {0}") public static Collection<Object[]> tests() throws Exception { Collection<Object[]> testArgs = new ArrayList<Object[]>(); TestCaseContext.Builder b = new TestCaseContext.Builder(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml index 0064565..c1abade 100644 --- a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml +++ b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml @@ -97,7 +97,7 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> <configuration> - <usedDependencies> + <usedDependencies combine.children="append"> <usedDependency>org.apache.maven:maven-core</usedDependency> <usedDependency>org.apache.maven:maven-compat</usedDependency> </usedDependencies> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationChannel.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationChannel.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationChannel.java index 044707a..afc1dfd 100644 --- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationChannel.java +++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationChannel.java @@ -95,7 +95,7 @@ public class ReplicationChannel extends Thread implements IReplicationChannel { private final IReplicationManager replicationManager; private final ReplicationProperties replicationProperties; private final IAppRuntimeContextProvider appContextProvider; - private static final int INTIAL_BUFFER_SIZE = StorageUtil.getSizeInBytes(4, StorageUnit.KILOBYTE); + private static final int INTIAL_BUFFER_SIZE = StorageUtil.getIntSizeInBytes(4, StorageUnit.KILOBYTE); private final LinkedBlockingQueue<LSMComponentLSNSyncTask> lsmComponentRemoteLSN2LocalLSNMappingTaskQ; private final LinkedBlockingQueue<LogRecord> pendingNotificationRemoteLogsQ; private final Map<String, LSMComponentProperties> lsmComponentId2PropertiesMap; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java index 72cc7d1..447021c 100644 --- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java +++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java @@ -117,7 +117,7 @@ public class ReplicationManager implements IReplicationManager { private final AtomicBoolean replicationSuspended; private AtomicBoolean terminateJobsReplication; private AtomicBoolean jobsReplicationSuspended; - private static final int INITIAL_BUFFER_SIZE = StorageUtil.getSizeInBytes(4, StorageUnit.KILOBYTE); + private static final int INITIAL_BUFFER_SIZE = StorageUtil.getIntSizeInBytes(4, StorageUnit.KILOBYTE); private final Set<String> shuttingDownReplicaIds; //replication threads private ReplicationJobsProccessor replicationJobsProcessor; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-runtime/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/pom.xml b/asterixdb/asterix-runtime/pom.xml index 6458fb0..df8f840 100644 --- a/asterixdb/asterix-runtime/pom.xml +++ b/asterixdb/asterix-runtime/pom.xml @@ -126,10 +126,13 @@ <artifactId>hyracks-storage-common</artifactId> </dependency> <dependency> + <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-control-common</artifactId> + </dependency> + <dependency> <groupId>org.apache.asterix</groupId> <artifactId>asterix-transactions</artifactId> <version>${project.version}</version> - <scope>compile</scope> </dependency> <dependency> <groupId>org.ow2.asm</groupId> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/AppContextInfo.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/AppContextInfo.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/AppContextInfo.java index 6193931..9bcfea2 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/AppContextInfo.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/AppContextInfo.java @@ -18,8 +18,24 @@ */ package org.apache.asterix.runtime.utils; +import java.io.IOException; +import java.util.function.Supplier; +import java.util.logging.Logger; + import org.apache.asterix.common.cluster.IGlobalRecoveryManager; -import org.apache.asterix.common.config.*; +import org.apache.asterix.common.config.BuildProperties; +import org.apache.asterix.common.config.CompilerProperties; +import org.apache.asterix.common.config.ExtensionProperties; +import org.apache.asterix.common.config.ExternalProperties; +import org.apache.asterix.common.config.FeedProperties; +import org.apache.asterix.common.config.IPropertiesProvider; +import org.apache.asterix.common.config.MessagingProperties; +import org.apache.asterix.common.config.MetadataProperties; +import org.apache.asterix.common.config.NodeProperties; +import org.apache.asterix.common.config.PropertiesAccessor; +import org.apache.asterix.common.config.ReplicationProperties; +import org.apache.asterix.common.config.StorageProperties; +import org.apache.asterix.common.config.TransactionProperties; import org.apache.asterix.common.dataflow.IApplicationContextInfo; import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.library.ILibraryManager; @@ -31,10 +47,6 @@ import org.apache.hyracks.api.client.IHyracksClientConnection; import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider; import org.apache.hyracks.storage.common.IStorageManager; -import java.io.IOException; -import java.util.function.Supplier; -import java.util.logging.Logger; - /* * Acts as an holder class for IndexRegistryProvider, AsterixStorageManager * instances that are accessed from the NCs. In addition an instance of ICCApplicationContext @@ -57,6 +69,7 @@ public class AppContextInfo implements IApplicationContextInfo, IPropertiesProvi private ReplicationProperties replicationProperties; private ExtensionProperties extensionProperties; private MessagingProperties messagingProperties; + private NodeProperties nodeProperties; private Supplier<IMetadataBootstrap> metadataBootstrapSupplier; private IHyracksClientConnection hcc; private Object extensionManager; @@ -94,6 +107,7 @@ public class AppContextInfo implements IApplicationContextInfo, IPropertiesProvi INSTANCE.hcc = hcc; INSTANCE.buildProperties = new BuildProperties(propertiesAccessor); INSTANCE.messagingProperties = new MessagingProperties(propertiesAccessor); + INSTANCE.nodeProperties = new NodeProperties(propertiesAccessor); INSTANCE.metadataBootstrapSupplier = metadataBootstrapSupplier; INSTANCE.globalRecoveryManager = globalRecoveryManager; @@ -191,6 +205,11 @@ public class AppContextInfo implements IApplicationContextInfo, IPropertiesProvi return messagingProperties; } + @Override + public NodeProperties getNodeProperties() { + return nodeProperties; + } + public IResourceIdManager getResourceIdManager() { return resourceIdManager; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java index d975a98..49cbc54 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java @@ -37,12 +37,14 @@ import org.apache.asterix.common.replication.IFaultToleranceStrategy; import org.apache.asterix.event.schema.cluster.Cluster; import org.apache.asterix.event.schema.cluster.Node; import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; +import org.apache.hyracks.api.config.IOption; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.exceptions.HyracksException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.hyracks.control.common.controllers.NCConfig; /** * A holder class for properties related to the Asterix cluster. @@ -57,8 +59,7 @@ public class ClusterStateManager implements IClusterStateManager { private static final Logger LOGGER = Logger.getLogger(ClusterStateManager.class.getName()); public static final ClusterStateManager INSTANCE = new ClusterStateManager(); - private static final String IO_DEVICES = "iodevices"; - private final Map<String, Map<String, String>> activeNcConfiguration = new HashMap<>(); + private final Map<String, Map<IOption, Object>> activeNcConfiguration = new HashMap<>(); private final Cluster cluster; private ClusterState state = ClusterState.UNUSABLE; @@ -95,7 +96,7 @@ public class ClusterStateManager implements IClusterStateManager { ftStrategy.notifyNodeFailure(nodeId); } - public synchronized void addNCConfiguration(String nodeId, Map<String, String> configuration) + public synchronized void addNCConfiguration(String nodeId, Map<IOption, Object> configuration) throws HyracksException { if (LOGGER.isLoggable(Level.INFO)) { LOGGER.info("Registering configuration parameters for node id " + nodeId); @@ -177,7 +178,7 @@ public class ClusterStateManager implements IClusterStateManager { * @return a list of IO devices. */ public synchronized String[] getIODevices(String nodeId) { - Map<String, String> ncConfig = activeNcConfiguration.get(nodeId); + Map<IOption, Object> ncConfig = activeNcConfiguration.get(nodeId); if (ncConfig == null) { if (LOGGER.isLoggable(Level.WARNING)) { LOGGER.warning("Configuration parameters for nodeId " + nodeId @@ -185,7 +186,7 @@ public class ClusterStateManager implements IClusterStateManager { } return new String[0]; } - return ncConfig.get(IO_DEVICES).split(","); + return (String [])ncConfig.get(NCConfig.Option.IODEVICES); } @Override @@ -312,7 +313,7 @@ public class ClusterStateManager implements IClusterStateManager { } @Override - public Map<String, Map<String, String>> getActiveNcConfiguration() { + public Map<String, Map<IOption, Object>> getActiveNcConfiguration() { return Collections.unmodifiableMap(activeNcConfiguration); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-server/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml index 019ad04..eef2f35 100644 --- a/asterixdb/asterix-server/pom.xml +++ b/asterixdb/asterix-server/pom.xml @@ -239,7 +239,7 @@ <name>asterixcc</name> <mainClass>org.apache.hyracks.control.cc.CCDriver</mainClass> <commandLineArguments> - <commandLineArgument>-app-cc-main-class</commandLineArgument> + <commandLineArgument>-app-class</commandLineArgument> <commandLineArgument>org.apache.asterix.hyracks.bootstrap.CCApplicationEntryPoint</commandLineArgument> </commandLineArguments> </program> @@ -250,7 +250,7 @@ <name>asterixnc</name> <mainClass>org.apache.hyracks.control.nc.NCDriver</mainClass> <commandLineArguments> - <commandLineArgument>-app-nc-main-class</commandLineArgument> + <commandLineArgument>-app-class</commandLineArgument> <commandLineArgument>org.apache.asterix.hyracks.bootstrap.NCApplicationEntryPoint</commandLineArgument> </commandLineArguments> </program> @@ -270,7 +270,7 @@ <platform>booter-windows</platform> </platforms> <commandLineArguments> - <commandLineArgument>-app-cc-main-class</commandLineArgument> + <commandLineArgument>-app-class</commandLineArgument> <commandLineArgument>org.apache.asterix.hyracks.bootstrap.CCApplicationEntryPoint</commandLineArgument> </commandLineArguments> </daemon> @@ -281,7 +281,7 @@ <platform>booter-windows</platform> </platforms> <commandLineArguments> - <commandLineArgument>-app-nc-main-class</commandLineArgument> + <commandLineArgument>-app-class</commandLineArgument> <commandLineArgument>org.apache.asterix.hyracks.bootstrap.NCApplicationEntryPoint</commandLineArgument> </commandLineArguments> </daemon> @@ -407,10 +407,10 @@ <artifactId>maven-dependency-plugin</artifactId> <version>2.10</version> <configuration> - <ignoredDependencies> + <ignoredDependencies combine.children="append"> <ignoredDependency>org.apache.asterix:asterix-opt-bom:*</ignoredDependency> </ignoredDependencies> - <usedDependencies> + <usedDependencies combine.children="append"> <usedDependency>org.apache.asterix:asterix-app</usedDependency> <usedDependency>org.apache.asterix:asterix-client-helper</usedDependency> <usedDependency>org.apache.hadoop:hadoop-minicluster</usedDependency> @@ -548,5 +548,9 @@ <artifactId>hadoop-minicluster</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-util</artifactId> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-server/src/main/opt/ansible/conf/cc.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/conf/cc.conf b/asterixdb/asterix-server/src/main/opt/ansible/conf/cc.conf index ff95344..4887732 100644 --- a/asterixdb/asterix-server/src/main/opt/ansible/conf/cc.conf +++ b/asterixdb/asterix-server/src/main/opt/ansible/conf/cc.conf @@ -19,6 +19,6 @@ log.level=INFO [nc] -txnlogdir=txnlog +txn.log.dir=txnlog iodevices=iodevice command=asterixnc http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-server/src/main/opt/ansible/yaml/gen_conf.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/gen_conf.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/gen_conf.yml index f1be8a1..f1409ae 100644 --- a/asterixdb/asterix-server/src/main/opt/ansible/yaml/gen_conf.yml +++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/gen_conf.yml @@ -33,5 +33,5 @@ loop_var: node - name: Populate CC to the cluster configuration file "{{ ccconf }}" - shell: printf "\n[cc]\ncluster.address={{ groups['cc'][0] }}\n\n" >> "{{ ccconf }}" + shell: printf "\n[cc]\naddress={{ groups['cc'][0] }}\n\n" >> "{{ ccconf }}" http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml b/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml index e0abb12..802abcd 100644 --- a/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml +++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml @@ -72,7 +72,7 @@ shell: printf "\n[all:vars]\nansible_ssh_user={{ user }}\n" >> "{{ inventory }}" - name: Populate CC to the cluster configuration file "{{ ccconf }}" - shell: printf "[cc]\ncluster.address={{ cc.private_ip }}\n\n" > "{{ ccconf }}" + shell: printf "[cc]\naddress={{ cc.private_ip }}\n\n" > "{{ ccconf }}" - name: Populate NCs to the cluster configuration file shell: printf "[nc/1]\naddress={{ cc.private_ip }}\n\n" >> "{{ ccconf }}" @@ -93,7 +93,7 @@ loop_var: node - name: Populate the NC txnlogdir to the cluster configuration file {{ ccconf }} - shell: printf "[nc]\ntxnlogdir={{ home_dir }}/txnlog\n" >> "{{ ccconf }}" + shell: printf "[nc]\ntxn.log.dir={{ home_dir }}/txnlog\n" >> "{{ ccconf }}" - name: Populate NC iodevices to the cluster configuration file {{ ccconf }} shell: printf "iodevices={{ home_dir }}/iodevice\n" >> "{{ ccconf }}" http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf b/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf index 71b7514..d1f03bc 100644 --- a/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf +++ b/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf @@ -16,23 +16,23 @@ ; under the License. [nc/red] -txnlogdir=data/red/txnlog -coredumpdir=data/red/coredump +txn.log.dir=data/red/txnlog +core.dump.dir=data/red/coredump iodevices=data/red [nc/blue] -port=9091 -txnlogdir=data/blue/txnlog -coredumpdir=data/blue/coredump +ncservice.port=9091 +txn.log.dir=data/blue/txnlog +core.dump.dir=data/blue/coredump iodevices=data/blue [nc] -storagedir=storage +storage.subdir=storage address=127.0.0.1 command=${NC_COMMAND} [cc] -cluster.address = 127.0.0.1 +address = 127.0.0.1 -[app] +[common] log.level=INFO http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java index 7324a5c..566ebb8 100644 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java @@ -18,8 +18,6 @@ */ package org.apache.asterix.server.test; -import static org.apache.asterix.test.common.TestHelper.joinPath; - import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -35,6 +33,7 @@ import org.apache.asterix.test.common.TestHelper; import org.apache.asterix.testframework.context.TestCaseContext.OutputFormat; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; +import org.apache.hyracks.util.file.FileUtil; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -49,13 +48,13 @@ public class SampleLocalClusterIT { // Important paths and files for this test. // The "target" subdirectory of asterix-server. All outputs go here. - private static final String TARGET_DIR = joinPath(/*System.getProperty("basedir"),*/ "target"); + private static final String TARGET_DIR = FileUtil.joinPath("target"); // Directory where the NCs create and store all data, as configured by // src/test/resources/NCServiceExecutionIT/cc.conf. - private static final String OUTPUT_DIR = joinPath(TARGET_DIR, "sample-local-cluster"); + private static final String OUTPUT_DIR = FileUtil.joinPath(TARGET_DIR, "sample-local-cluster"); - private static final String LOCAL_SAMPLES_DIR = joinPath(OUTPUT_DIR, "opt", "local"); + private static final String LOCAL_SAMPLES_DIR = FileUtil.joinPath(OUTPUT_DIR, "opt", "local"); @Rule public TestRule watcher = new TestMethodTracer(); @@ -71,18 +70,18 @@ public class SampleLocalClusterIT { } outDir.mkdirs(); - String installerZip = joinPath(TARGET_DIR, - new File(TARGET_DIR).list((dir, name) -> name.matches("asterix-server.*-binary-assembly.zip"))[0]); + String[] pathElements = new String[]{TARGET_DIR, new File(TARGET_DIR).list((dir, name) -> name.matches("asterix-server.*-binary-assembly.zip"))[0]}; + String installerZip = FileUtil.joinPath(pathElements); TestHelper.unzip(installerZip, OUTPUT_DIR); } @Test public void test0_startCluster() throws Exception { - Process process = new ProcessBuilder(joinPath(LOCAL_SAMPLES_DIR, "bin/stop-sample-cluster.sh"), "-f") + Process process = new ProcessBuilder(FileUtil.joinPath(LOCAL_SAMPLES_DIR, "bin/stop-sample-cluster.sh"), "-f") .inheritIO().start(); Assert.assertEquals(0, process.waitFor()); - process = new ProcessBuilder(joinPath(LOCAL_SAMPLES_DIR, "bin/start-sample-cluster.sh")).inheritIO().start(); + process = new ProcessBuilder(FileUtil.joinPath(LOCAL_SAMPLES_DIR, "bin/start-sample-cluster.sh")).inheritIO().start(); Assert.assertEquals(0, process.waitFor()); } @@ -100,7 +99,7 @@ public class SampleLocalClusterIT { @Test public void test2_stopCluster() throws Exception { Process process = - new ProcessBuilder(joinPath(LOCAL_SAMPLES_DIR, "bin/stop-sample-cluster.sh")).inheritIO().start(); + new ProcessBuilder(FileUtil.joinPath(LOCAL_SAMPLES_DIR, "bin/stop-sample-cluster.sh")).inheritIO().start(); Assert.assertEquals(0, process.waitFor()); try { new URL("http://127.0.0.1:19002";).openConnection().connect(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf index 3f58362..cc87be4 100644 --- a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf +++ b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf @@ -16,27 +16,26 @@ ; under the License. [nc/asterix_nc1] -txnlogdir=../asterix-server/target/tmp/asterix_nc1/txnlog -coredumpdir=../asterix-server/target/tmp/asterix_nc1/coredump +txn.log.dir=../asterix-server/target/tmp/asterix_nc1/txnlog +core.dump.dir=../asterix-server/target/tmp/asterix_nc1/coredump iodevices=../asterix-server/target/tmp/asterix_nc1/iodevice1,../asterix-server/target/tmp/asterix_nc1/iodevice2 +#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006 [nc/asterix_nc2] -port=9091 -txnlogdir=../asterix-server/target/tmp/asterix_nc2/txnlog -coredumpdir=../asterix-server/target/tmp/asterix_nc2/coredump +ncservice.port=9091 +txn.log.dir=../asterix-server/target/tmp/asterix_nc2/txnlog +core.dump.dir=../asterix-server/target/tmp/asterix_nc2/coredump iodevices=../asterix-server/target/tmp/asterix_nc2/iodevice1,../asterix-server/target/tmp/asterix_nc2/iodevice2 +#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007 [nc] address=127.0.0.1 command=asterixnc app.class=org.apache.asterix.hyracks.bootstrap.NCApplicationEntryPoint jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory" -storagedir=test_storage +storage.subdir=test_storage +storage.memorycomponent.globalbudget = 1073741824 [cc] -cluster.address = 127.0.0.1 +address = 127.0.0.1 app.class=org.apache.asterix.hyracks.bootstrap.CCApplicationEntryPoint - -[app] -storage.memorycomponent.globalbudget = 1073741824 - http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-yarn/src/main/java/org/apache/asterix/aoya/AsterixApplicationMaster.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-yarn/src/main/java/org/apache/asterix/aoya/AsterixApplicationMaster.java b/asterixdb/asterix-yarn/src/main/java/org/apache/asterix/aoya/AsterixApplicationMaster.java index 56d3d07..b4aeeb1 100644 --- a/asterixdb/asterix-yarn/src/main/java/org/apache/asterix/aoya/AsterixApplicationMaster.java +++ b/asterixdb/asterix-yarn/src/main/java/org/apache/asterix/aoya/AsterixApplicationMaster.java @@ -1032,36 +1032,32 @@ public class AsterixApplicationMaster { //get our java opts vargs.add(ccJavaOpts); vargs.add(CC_CLASSNAME); - vargs.add("-app-cc-main-class org.apache.asterix.hyracks.bootstrap.CCApplicationEntryPoint"); - vargs.add("-cluster-net-ip-address " + cC.getClusterIp()); - vargs.add("-client-net-ip-address " + cC.getClientIp()); + vargs.add("-app-class org.apache.asterix.hyracks.bootstrap.CCApplicationEntryPoint"); + vargs.add("-address " + cC.getClusterIp()); + vargs.add("-client-listen-address " + cC.getClientIp()); //pass CC optional parameters if (clusterDesc.getHeartbeatPeriod() != null) { vargs.add("-heartbeat-period " + String.valueOf(clusterDesc.getHeartbeatPeriod().intValue())); } if (clusterDesc.getMaxHeartbeatLapsePeriods() != null) { - vargs.add("-max-heartbeat-lapse-periods " + vargs.add("-heartbeat-max-misses " + String.valueOf(clusterDesc.getMaxHeartbeatLapsePeriods().intValue())); } if (clusterDesc.getProfileDumpPeriod() != null) { vargs.add("-profile-dump-period " + String.valueOf(clusterDesc.getProfileDumpPeriod().intValue())); } - if (clusterDesc.getDefaultMaxJobAttempts() != null) { - vargs.add("-default-max-job-attempts " - + String.valueOf(clusterDesc.getDefaultMaxJobAttempts().intValue())); - } if (clusterDesc.getJobHistorySize() != null) { vargs.add("-job-history-size " + String.valueOf(clusterDesc.getJobHistorySize().intValue())); } if (clusterDesc.getResultTimeToLive() != null) { - vargs.add("-result-time-to-live " + String.valueOf(clusterDesc.getResultTimeToLive().intValue())); + vargs.add("-result-ttl " + String.valueOf(clusterDesc.getResultTimeToLive().intValue())); } if (clusterDesc.getResultSweepThreshold() != null) { vargs.add("-result-sweep-threshold " + String.valueOf(clusterDesc.getResultSweepThreshold().intValue())); } if (clusterDesc.getCcRoot() != null) { - vargs.add("-cc-root " + clusterDesc.getCcRoot()); + vargs.add("-root-dir " + clusterDesc.getCcRoot()); } ccStarted.set(true); @@ -1078,14 +1074,13 @@ public class AsterixApplicationMaster { } vargs.add(ncJavaOpts); vargs.add(NC_CLASSNAME); - vargs.add("-app-nc-main-class org.apache.asterix.hyracks.bootstrap.NCApplicationEntryPoint"); + vargs.add("-app-class org.apache.asterix.hyracks.bootstrap.NCApplicationEntryPoint"); vargs.add("-node-id " + local.getId()); - vargs.add("-cc-host " + cC.getClusterIp()); + vargs.add("-cluster-address " + cC.getClusterIp()); vargs.add("-iodevices " + iodevice); - vargs.add("-cluster-net-ip-address " + local.getClusterIp()); - vargs.add("-data-ip-address " + local.getClusterIp()); - vargs.add("-result-ip-address " + local.getClusterIp()); - vargs.add("--"); + vargs.add("-address " + local.getClusterIp()); + vargs.add("-data-listen-address " + local.getClusterIp()); + vargs.add("-result-listen-address " + local.getClusterIp()); if (initial) { vargs.add("-initial-run "); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/asterixdb/asterix-yarn/src/main/java/org/apache/asterix/aoya/AsterixYARNClient.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-yarn/src/main/java/org/apache/asterix/aoya/AsterixYARNClient.java b/asterixdb/asterix-yarn/src/main/java/org/apache/asterix/aoya/AsterixYARNClient.java index f9d10af..0ccaf07c 100644 --- a/asterixdb/asterix-yarn/src/main/java/org/apache/asterix/aoya/AsterixYARNClient.java +++ b/asterixdb/asterix-yarn/src/main/java/org/apache/asterix/aoya/AsterixYARNClient.java @@ -1362,12 +1362,12 @@ public class AsterixYARNClient { } configuration.setStore(stores); List<Coredump> coredump = new ArrayList<Coredump>(); - String coredumpDir = null; + String coredumpdir = null; List<TransactionLogDir> txnLogDirs = new ArrayList<TransactionLogDir>(); String txnLogDir = null; for (Node node : cluster.getNode()) { - coredumpDir = node.getLogDir() == null ? cluster.getLogDir() : node.getLogDir(); - coredump.add(new Coredump(node.getId(), coredumpDir + "coredump" + File.separator)); + coredumpdir = node.getLogDir() == null ? cluster.getLogDir() : node.getLogDir(); + coredump.add(new Coredump(node.getId(), coredumpdir + "coredump" + File.separator)); txnLogDir = node.getTxnLogDir() == null ? cluster.getTxnLogDir() : node.getTxnLogDir(); //node or cluster-wide txnLogDirs.add(new TransactionLogDir(node.getId(), txnLogDir + (txnLogDir.charAt(txnLogDir.length() - 1) == File.separatorChar ? File.separator : "") http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/algebricks/algebricks-tests/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-tests/pom.xml b/hyracks-fullstack/algebricks/algebricks-tests/pom.xml index 1d509cf..fe6472e 100644 --- a/hyracks-fullstack/algebricks/algebricks-tests/pom.xml +++ b/hyracks-fullstack/algebricks/algebricks-tests/pom.xml @@ -71,7 +71,7 @@ <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId> <configuration> - <excludes> + <excludes combine.children="append"> <exclude>data/**</exclude> <exclude>src/test/resources/results/scanMicroSortWrite.out</exclude> </excludes> @@ -169,6 +169,11 @@ <version>${project.version}</version> </dependency> <dependency> + <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-util</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java b/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java index 9880047..2971b72 100644 --- a/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java +++ b/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java @@ -18,6 +18,8 @@ */ package org.apache.hyracks.algebricks.tests.util; +import static org.apache.hyracks.util.file.FileUtil.joinPath; + import java.io.File; import java.util.EnumSet; @@ -47,43 +49,41 @@ public class AlgebricksHyracksIntegrationUtil { private static IHyracksClientConnection hcc; public static void init() throws Exception { - FileUtils.deleteQuietly(new File("target" + File.separator + "data")); - FileUtils.copyDirectory(new File("data"), new File("target" + File.separator + "data")); + FileUtils.deleteQuietly(new File(joinPath("target", "data"))); + FileUtils.copyDirectory(new File("data"), new File(joinPath("target", "data"))); CCConfig ccConfig = new CCConfig(); - ccConfig.clientNetIpAddress = "127.0.0.1"; - ccConfig.clientNetPort = TEST_HYRACKS_CC_CLIENT_NET_PORT; - ccConfig.clusterNetIpAddress = "127.0.0.1"; - ccConfig.clusterNetPort = TEST_HYRACKS_CC_CLUSTER_NET_PORT; + ccConfig.setClientListenAddress("127.0.0.1"); + ccConfig.setClientListenPort(TEST_HYRACKS_CC_CLIENT_NET_PORT); + ccConfig.setClusterListenAddress("127.0.0.1"); + ccConfig.setClusterListenPort(TEST_HYRACKS_CC_CLUSTER_NET_PORT); cc = new ClusterControllerService(ccConfig); cc.start(); - NCConfig ncConfig1 = new NCConfig(); - ncConfig1.ccHost = "localhost"; - ncConfig1.ccPort = TEST_HYRACKS_CC_CLUSTER_NET_PORT; - ncConfig1.clusterNetIPAddress = "127.0.0.1"; - ncConfig1.dataIPAddress = "127.0.0.1"; - ncConfig1.resultIPAddress = "127.0.0.1"; - ncConfig1.nodeId = NC1_ID; - ncConfig1.ioDevices = System.getProperty("user.dir") + File.separator + "target" + File.separator + "data" - + File.separator + "device0"; - FileUtils.forceMkdir(new File(ncConfig1.ioDevices)); + NCConfig ncConfig1 = new NCConfig(NC1_ID); + ncConfig1.setClusterAddress("localhost"); + ncConfig1.setClusterPort(TEST_HYRACKS_CC_CLUSTER_NET_PORT); + ncConfig1.setClusterListenAddress("127.0.0.1"); + ncConfig1.setDataListenAddress("127.0.0.1"); + ncConfig1.setResultListenAddress("127.0.0.1"); + ncConfig1.setIODevices(new String [] { joinPath(System.getProperty("user.dir"), "target", "data", + "device0") }); + FileUtils.forceMkdir(new File(ncConfig1.getIODevices()[0])); nc1 = new NodeControllerService(ncConfig1); nc1.start(); - NCConfig ncConfig2 = new NCConfig(); - ncConfig2.ccHost = "localhost"; - ncConfig2.ccPort = TEST_HYRACKS_CC_CLUSTER_NET_PORT; - ncConfig2.clusterNetIPAddress = "127.0.0.1"; - ncConfig2.dataIPAddress = "127.0.0.1"; - ncConfig2.resultIPAddress = "127.0.0.1"; - ncConfig2.nodeId = NC2_ID; - ncConfig1.ioDevices = System.getProperty("user.dir") + File.separator + "target" + File.separator + "data" - + File.separator + "device1"; - FileUtils.forceMkdir(new File(ncConfig1.ioDevices)); + NCConfig ncConfig2 = new NCConfig(NC2_ID); + ncConfig2.setClusterAddress("localhost"); + ncConfig2.setClusterPort(TEST_HYRACKS_CC_CLUSTER_NET_PORT); + ncConfig2.setClusterListenAddress("127.0.0.1"); + ncConfig2.setDataListenAddress("127.0.0.1"); + ncConfig2.setResultListenAddress("127.0.0.1"); + ncConfig2.setIODevices(new String [] { joinPath(System.getProperty("user.dir"), "target", "data", + "device1") }); + FileUtils.forceMkdir(new File(ncConfig1.getIODevices()[0])); nc2 = new NodeControllerService(ncConfig2); nc2.start(); - hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort); + hcc = new HyracksConnection(ccConfig.getClientListenAddress(), ccConfig.getClientListenPort()); } public static void deinit() throws Exception { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-api/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/pom.xml b/hyracks-fullstack/hyracks/hyracks-api/pom.xml index cde607f..ec60022 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-api/pom.xml @@ -94,5 +94,9 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>args4j</groupId> + <artifactId>args4j</artifactId> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IApplicationConfig.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IApplicationConfig.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IApplicationConfig.java deleted file mode 100644 index 278a9d7..0000000 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IApplicationConfig.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hyracks.api.application; - -import java.util.List; -import java.util.Map.Entry; -import java.util.Set; - -/** - * Accessor for the data contained in the global application configuration file. - */ -public interface IApplicationConfig { - String getString(String section, String key); - - String getString(String section, String key, String defaultValue); - - int getInt(String section, String key); - - int getInt(String section, String key, int defaultValue); - - long getLong(String section, String key); - - long getLong(String section, String key, long defaultValue); - - Set<String> getSections(); - - Set<String> getKeys(String section); - - String[] getStringArray(String section, String key); - - List<Set<Entry<String, String>>> getMultiSections(String section); -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IApplicationContext.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IApplicationContext.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IApplicationContext.java index 7b07174..c933d9d 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IApplicationContext.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IApplicationContext.java @@ -21,6 +21,7 @@ package org.apache.hyracks.api.application; import java.io.Serializable; import java.util.concurrent.ThreadFactory; +import org.apache.hyracks.api.config.IApplicationConfig; import org.apache.hyracks.api.job.IJobSerializerDeserializerContainer; import org.apache.hyracks.api.messages.IMessageBroker; import org.apache.hyracks.api.service.IControllerService; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/ICCApplicationEntryPoint.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/ICCApplicationEntryPoint.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/ICCApplicationEntryPoint.java index c11cc7a..4f6f450 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/ICCApplicationEntryPoint.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/ICCApplicationEntryPoint.java @@ -18,7 +18,9 @@ */ package org.apache.hyracks.api.application; +import org.apache.hyracks.api.config.IConfigManager; import org.apache.hyracks.api.job.resource.IJobCapacityController; +import org.kohsuke.args4j.OptionHandlerFilter; public interface ICCApplicationEntryPoint { void start(ICCApplicationContext ccAppCtx, String[] args) throws Exception; @@ -28,4 +30,10 @@ public interface ICCApplicationEntryPoint { void startupCompleted() throws Exception; IJobCapacityController getJobCapacityController(); + + void registerConfig(IConfigManager configManager); + + default OptionHandlerFilter getUsageFilter() { + return OptionHandlerFilter.PUBLIC; + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IClusterLifecycleListener.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IClusterLifecycleListener.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IClusterLifecycleListener.java index 191a4af..2c53e1c 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IClusterLifecycleListener.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IClusterLifecycleListener.java @@ -20,8 +20,8 @@ package org.apache.hyracks.api.application; import java.util.Collection; import java.util.Map; -import java.util.Set; +import org.apache.hyracks.api.config.IOption; import org.apache.hyracks.api.exceptions.HyracksException; /** @@ -40,9 +40,8 @@ public interface IClusterLifecycleListener { * @param nodeId * A unique identifier of a Node Controller * @param ncConfiguration - * A map containing the set of configuration parameters that were used to start the Node Controller */ - public void notifyNodeJoin(String nodeId, Map<String, String> ncConfiguration) throws HyracksException; + public void notifyNodeJoin(String nodeId, Map<IOption, Object> ncConfiguration) throws HyracksException; /** * @param deadNodeIds http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCApplicationEntryPoint.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCApplicationEntryPoint.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCApplicationEntryPoint.java index dea6e4b..a92cd4a 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCApplicationEntryPoint.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCApplicationEntryPoint.java @@ -18,7 +18,9 @@ */ package org.apache.hyracks.api.application; +import org.apache.hyracks.api.config.IConfigManager; import org.apache.hyracks.api.job.resource.NodeCapacity; +import org.kohsuke.args4j.OptionHandlerFilter; public interface INCApplicationEntryPoint { void start(INCApplicationContext ncAppCtx, String[] args) throws Exception; @@ -28,4 +30,10 @@ public interface INCApplicationEntryPoint { void stop() throws Exception; NodeCapacity getCapacity(); + + void registerConfigOptions(IConfigManager configManager); + + default OptionHandlerFilter getUsageFilter() { + return OptionHandlerFilter.PUBLIC; + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IApplicationConfig.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IApplicationConfig.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IApplicationConfig.java new file mode 100644 index 0000000..0335b80 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IApplicationConfig.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.api.config; + +import java.util.List; +import java.util.Set; +import java.util.function.Predicate; +import java.util.logging.Level; + +/** + * Accessor for the data contained in the global application configuration file. + */ +public interface IApplicationConfig { + String getString(String section, String key); + + int getInt(String section, String key); + + long getLong(String section, String key); + + Set<String> getSectionNames(); + + Set<String> getKeys(String section); + + Object getStatic(IOption option); + + List<String> getNCNames(); + + IOption lookupOption(String sectionName, String propertyName); + + Set<IOption> getOptions(); + + Set<IOption> getOptions(Section section); + + IApplicationConfig getNCEffectiveConfig(String nodeId); + + Set<Section> getSections(); + + Set<Section> getSections(Predicate<Section> predicate); + + default Object get(IOption option) { + return option.get(this); + } + + default long getLong(IOption option) { + return (long)get(option); + } + + default int getInt(IOption option) { + return (int)get(option); + } + + default String getString(IOption option) { + return (String)get(option); + } + + default boolean getBoolean(IOption option) { + return (boolean)get(option); + } + + default Level getLoggingLevel(IOption option) { + return (Level)get(option); + } + + default double getDouble(IOption option) { + return (double)get(option); + } + + default String [] getStringArray(IOption option) { + return (String [])get(option); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigManager.java new file mode 100644 index 0000000..fb1332b --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigManager.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.api.config; + +import java.util.Set; +import java.util.function.Predicate; + +import org.kohsuke.args4j.OptionHandlerFilter; + +public interface IConfigManager { + int PARSE_INI_POINTERS_METRIC = 100; + int PARSE_INI_METRIC = 200; + int PARSE_COMMAND_LINE_METRIC = 300; + int APPLY_DEFAULTS_METRIC = 400; + + void register(IOption... options); + + @SuppressWarnings("unchecked") + void register(Class<? extends IOption>... optionClasses); + + Set<Section> getSections(Predicate<Section> predicate); + + Set<Section> getSections(); + + Set<IOption> getOptions(Section section); + + IApplicationConfig getAppConfig(); + + void addConfigurator(int metric, IConfigurator configurator); + + void addIniParamOptions(IOption... options); + + void addCmdLineSections(Section... sections); + + void setUsageFilter(OptionHandlerFilter usageFilter); +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigurator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigurator.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigurator.java new file mode 100644 index 0000000..54f780d --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IConfigurator.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.api.config; + +import java.io.IOException; + +import org.kohsuke.args4j.CmdLineException; + +public interface IConfigurator { + void run() throws IOException, CmdLineException; +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IOption.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IOption.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IOption.java new file mode 100644 index 0000000..834d73c --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IOption.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.api.config; + +import java.util.function.Function; + +public interface IOption { + + String name(); + + Section section(); + + String description(); + + IOptionType type(); + + /** + * @return the unresolved default value of this option- + */ + Object defaultValue(); + + /** + * @return a string to describe the default value, or null if the default should be used + */ + default String usageDefaultOverride(IApplicationConfig accessor, Function<IOption, String> optionPrinter) { + return null; + } + + /** + * Implementations should override this default implementation if this property value is non-static and should be + * calculated on every call + * @return the current value of this property + */ + default Object get(IApplicationConfig appConfig) { + return appConfig.getStatic(this); + } + + /** + * @return a true value indicates this option should not be advertised (e.g. command-line usage, documentation) + */ + default boolean hidden() { return false; } + + default String cmdline() { + return "-" + name().toLowerCase().replace("_", "-"); + } + + default String ini() { + return name().toLowerCase().replace("_", "."); + } + + default String toIniString() { + return "[" + section().sectionName() + "] " + ini(); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IOptionType.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IOptionType.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IOptionType.java new file mode 100644 index 0000000..2a98fdc --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/IOptionType.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.api.config; + +public interface IOptionType<T> { + /** + * @throws IllegalArgumentException when the supplied string cannot be interpreted + */ + T parse(String s); + + Class<T> targetType(); + + default Object serializeToJSON(Object value) { + return value; + } + + default String serializeToIni(Object value) { + return String.valueOf(value); + } + + default String serializeToString(Object value) { + return serializeToIni(value); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/Section.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/Section.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/Section.java new file mode 100644 index 0000000..8b19d80 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/config/Section.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.api.config; + +public enum Section { + CC, + NC, + COMMON, + LOCALNC, + EXTENSION, + VIRTUAL; // virtual section indicates options which are not accessible from the cmd-line nor ini file + + public static Section parseSectionName(String name) { + for (Section section : values()) { + if (section.sectionName().equals(name)) { + return section; + } + } + return null; + } + + public String sectionName() { + return name().toLowerCase(); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/ICCContext.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/ICCContext.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/ICCContext.java index f9618cf..83e0482 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/ICCContext.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/ICCContext.java @@ -27,9 +27,9 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.topology.ClusterTopology; public interface ICCContext { - public ClusterControllerInfo getClusterControllerInfo(); + ClusterControllerInfo getClusterControllerInfo(); - public void getIPAddressNodeMap(Map<InetAddress, Set<String>> map) throws HyracksDataException; + void getIPAddressNodeMap(Map<InetAddress, Set<String>> map) throws HyracksDataException; - public ClusterTopology getClusterTopology(); + ClusterTopology getClusterTopology(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java index 552fbeb..34c58f8 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java @@ -22,7 +22,6 @@ import java.io.File; import java.io.Serializable; import java.util.ArrayList; import java.util.List; -import java.util.StringTokenizer; import org.apache.hyracks.api.exceptions.HyracksDataException; @@ -79,11 +78,10 @@ public class IODeviceHandle implements Serializable { * comma separated list of devices * @return */ - public static List<IODeviceHandle> getDevices(String ioDevices) { + public static List<IODeviceHandle> getDevices(String [] ioDevices) { List<IODeviceHandle> devices = new ArrayList<>(); - StringTokenizer tok = new StringTokenizer(ioDevices, ","); - while (tok.hasMoreElements()) { - String devPath = tok.nextToken().trim(); + for (String ioDevice : ioDevices) { + String devPath = ioDevice.trim(); devices.add(new IODeviceHandle(new File(devPath), ".")); } return devices; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-client/src/test/java/org/apache/hyracks/client/stats/HyracksUtils.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-client/src/test/java/org/apache/hyracks/client/stats/HyracksUtils.java b/hyracks-fullstack/hyracks/hyracks-client/src/test/java/org/apache/hyracks/client/stats/HyracksUtils.java index a41ddd9..54ae838 100644 --- a/hyracks-fullstack/hyracks/hyracks-client/src/test/java/org/apache/hyracks/client/stats/HyracksUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-client/src/test/java/org/apache/hyracks/client/stats/HyracksUtils.java @@ -50,37 +50,34 @@ public class HyracksUtils { public static void init() throws Exception { CCConfig ccConfig = new CCConfig(); - ccConfig.clientNetIpAddress = CC_HOST; - ccConfig.clusterNetIpAddress = CC_HOST; - ccConfig.clusterNetPort = TEST_HYRACKS_CC_PORT; - ccConfig.clientNetPort = TEST_HYRACKS_CC_CLIENT_PORT; - ccConfig.defaultMaxJobAttempts = 0; - ccConfig.jobHistorySize = 0; - ccConfig.profileDumpPeriod = -1; - ccConfig.heartbeatPeriod = 50; + ccConfig.setClientListenAddress(CC_HOST); + ccConfig.setClusterListenAddress(CC_HOST); + ccConfig.setClusterListenPort(TEST_HYRACKS_CC_PORT); + ccConfig.setClientListenPort(TEST_HYRACKS_CC_CLIENT_PORT); + ccConfig.setJobHistorySize(0); + ccConfig.setProfileDumpPeriod(-1); + ccConfig.setHeartbeatPeriod(50); // cluster controller cc = new ClusterControllerService(ccConfig); cc.start(); // two node controllers - NCConfig ncConfig1 = new NCConfig(); - ncConfig1.ccHost = "localhost"; - ncConfig1.clusterNetIPAddress = "localhost"; - ncConfig1.ccPort = TEST_HYRACKS_CC_PORT; - ncConfig1.dataIPAddress = "127.0.0.1"; - ncConfig1.resultIPAddress = "127.0.0.1"; - ncConfig1.nodeId = NC1_ID; + NCConfig ncConfig1 = new NCConfig(NC1_ID); + ncConfig1.setClusterAddress("localhost"); + ncConfig1.setClusterListenAddress("localhost"); + ncConfig1.setClusterPort(TEST_HYRACKS_CC_PORT); + ncConfig1.setDataListenAddress("127.0.0.1"); + ncConfig1.setResultListenAddress("127.0.0.1"); nc1 = new NodeControllerService(ncConfig1); nc1.start(); - NCConfig ncConfig2 = new NCConfig(); - ncConfig2.ccHost = "localhost"; - ncConfig2.clusterNetIPAddress = "localhost"; - ncConfig2.ccPort = TEST_HYRACKS_CC_PORT; - ncConfig2.dataIPAddress = "127.0.0.1"; - ncConfig2.resultIPAddress = "127.0.0.1"; - ncConfig2.nodeId = NC2_ID; + NCConfig ncConfig2 = new NCConfig(NC2_ID); + ncConfig2.setClusterAddress("localhost"); + ncConfig2.setClusterListenAddress("localhost"); + ncConfig2.setClusterPort(TEST_HYRACKS_CC_PORT); + ncConfig2.setDataListenAddress("127.0.0.1"); + ncConfig2.setResultListenAddress("127.0.0.1"); nc2 = new NodeControllerService(ncConfig2); nc2.start(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml index 9dc3c1a..ba086ad 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml @@ -63,7 +63,7 @@ <licenseFamilies combine.children="append"> <licenseFamily implementation="org.apache.rat.license.MITLicenseFamily"/> </licenseFamilies> - <excludes> + <excludes combine.children="append"> <!-- See hyracks-fullstack-license/src/main/licenses/templates/source_licenses.ftl --> <exclude>src/main/resources/static/javascript/flot/jquery.flot.resize.min.js</exclude> <exclude>src/main/resources/static/javascript/jsplumb/jquery.jsPlumb-1.3.5-all-min.js</exclude> @@ -104,7 +104,6 @@ <dependency> <groupId>args4j</groupId> <artifactId>args4j</artifactId> - <version>2.0.12</version> </dependency> <dependency> <groupId>org.apache.hyracks</groupId> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCApplicationEntryPoint.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCApplicationEntryPoint.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCApplicationEntryPoint.java new file mode 100644 index 0000000..07008df --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCApplicationEntryPoint.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.control.cc; + +import java.util.Arrays; + +import org.apache.hyracks.api.application.ICCApplicationContext; +import org.apache.hyracks.api.application.ICCApplicationEntryPoint; +import org.apache.hyracks.api.config.IConfigManager; +import org.apache.hyracks.api.config.Section; +import org.apache.hyracks.api.job.resource.DefaultJobCapacityController; +import org.apache.hyracks.api.job.resource.IJobCapacityController; +import org.apache.hyracks.control.common.controllers.CCConfig; +import org.apache.hyracks.control.common.controllers.ControllerConfig; +import org.apache.hyracks.control.common.controllers.NCConfig; + +public class CCApplicationEntryPoint implements ICCApplicationEntryPoint { + public static final ICCApplicationEntryPoint INSTANCE = new CCApplicationEntryPoint(); + + protected CCApplicationEntryPoint() { + } + + @Override + public void start(ICCApplicationContext ccAppCtx, String[] args) throws Exception { + if (args.length > 0) { + throw new IllegalArgumentException("Unrecognized argument(s): " + Arrays.toString(args)); + } + } + + @Override + public void stop() throws Exception { + // no-op + } + + @Override + public void startupCompleted() throws Exception { + // no-op + } + + @Override + public IJobCapacityController getJobCapacityController() { + return DefaultJobCapacityController.INSTANCE; + } + + @Override + public void registerConfig(IConfigManager configManager) { + configManager.addIniParamOptions(ControllerConfig.Option.CONFIG_FILE, ControllerConfig.Option.CONFIG_FILE_URL); + configManager.addCmdLineSections(Section.CC, Section.COMMON); + configManager.setUsageFilter(getUsageFilter()); + configManager.register(ControllerConfig.Option.class, CCConfig.Option.class, NCConfig.Option.class); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java index dff3107..754deac 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java @@ -18,32 +18,50 @@ */ package org.apache.hyracks.control.cc; -import org.kohsuke.args4j.CmdLineParser; +import static org.apache.hyracks.control.common.controllers.CCConfig.Option.APP_CLASS; +import java.io.IOException; +import java.util.Arrays; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.hyracks.api.application.ICCApplicationEntryPoint; +import org.apache.hyracks.control.common.config.ConfigManager; +import org.apache.hyracks.control.common.config.ConfigUtils; import org.apache.hyracks.control.common.controllers.CCConfig; +import org.kohsuke.args4j.CmdLineException; public class CCDriver { - public static void main(String args []) throws Exception { - try { - CCConfig ccConfig = new CCConfig(); - CmdLineParser cp = new CmdLineParser(ccConfig); - try { - cp.parseArgument(args); - } catch (Exception e) { - System.err.println(e.getMessage()); - cp.printUsage(System.err); - return; - } - ccConfig.loadConfigAndApplyDefaults(); + private static final Logger LOGGER = Logger.getLogger(CCDriver.class.getName()); + + private CCDriver() { + } - ClusterControllerService ccService = new ClusterControllerService(ccConfig); + public static void main(String[] args) throws Exception { + try { + final ConfigManager configManager = new ConfigManager(args); + ICCApplicationEntryPoint appEntryPoint = getAppEntryPoint(args); + appEntryPoint.registerConfig(configManager); + CCConfig ccConfig = new CCConfig(configManager); + ClusterControllerService ccService = new ClusterControllerService(ccConfig, appEntryPoint); ccService.start(); while (true) { Thread.sleep(100000); } + } catch (CmdLineException e) { + LOGGER.log(Level.FINE, "Exception parsing command line: " + Arrays.toString(args), e); + System.exit(2); } catch (Exception e) { - e.printStackTrace(); + LOGGER.log(Level.SEVERE, "Exiting NCDriver due to exception", e); System.exit(1); } } + + private static ICCApplicationEntryPoint getAppEntryPoint(String[] args) + throws ClassNotFoundException, InstantiationException, IllegalAccessException, IOException { + // determine app class so that we can use the correct implementation of the configuration... + String appClassName = ConfigUtils.getOptionValue(args, APP_CLASS); + return appClassName != null ? (ICCApplicationEntryPoint) (Class.forName(appClassName)).newInstance() + : CCApplicationEntryPoint.INSTANCE; + } }
