Repository: asterixdb Updated Branches: refs/heads/master a5bfa8b4b -> 4c7b5bfa2
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java index bf0ddb6..2ee9161 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java @@ -19,6 +19,7 @@ package org.apache.hyracks.control.nc; import java.io.File; +import java.io.IOException; import java.lang.management.GarbageCollectorMXBean; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; @@ -51,11 +52,11 @@ import org.apache.hyracks.api.exceptions.HyracksException; import org.apache.hyracks.api.io.IODeviceHandle; import org.apache.hyracks.api.job.ActivityClusterGraph; import org.apache.hyracks.api.job.JobId; -import org.apache.hyracks.api.job.resource.NodeCapacity; import org.apache.hyracks.api.lifecycle.ILifeCycleComponentManager; import org.apache.hyracks.api.lifecycle.LifeCycleComponentManager; import org.apache.hyracks.api.service.IControllerService; import org.apache.hyracks.control.common.base.IClusterController; +import org.apache.hyracks.control.common.config.ConfigManager; import org.apache.hyracks.control.common.context.ServerContext; import org.apache.hyracks.control.common.controllers.NCConfig; import org.apache.hyracks.control.common.controllers.NodeParameters; @@ -84,6 +85,7 @@ import org.apache.hyracks.ipc.api.IPCPerformanceCounters; import org.apache.hyracks.ipc.impl.IPCSystem; import org.apache.hyracks.net.protocols.muxdemux.FullFrameChannelInterfaceFactory; import org.apache.hyracks.net.protocols.muxdemux.MuxDemuxPerformanceCounters; +import org.kohsuke.args4j.CmdLineException; public class NodeControllerService implements IControllerService { private static final Logger LOGGER = Logger.getLogger(NodeControllerService.class.getName()); @@ -130,7 +132,7 @@ public class NodeControllerService implements IControllerService { private NCApplicationContext appCtx; - private INCApplicationEntryPoint ncAppEntryPoint; + private final INCApplicationEntryPoint ncAppEntryPoint; private final ILifeCycleComponentManager lccm; @@ -154,13 +156,25 @@ public class NodeControllerService implements IControllerService { private MessagingNetworkManager messagingNetManager; - public NodeControllerService(NCConfig ncConfig) throws Exception { - this.ncConfig = ncConfig; - id = ncConfig.nodeId; + private final ConfigManager configManager; - ioManager = new IOManager(IODeviceHandle.getDevices(ncConfig.ioDevices)); + public NodeControllerService(NCConfig config) throws Exception { + this(config, getApplicationEntryPoint(config)); + } + + public NodeControllerService(NCConfig config, INCApplicationEntryPoint aep) throws IOException, CmdLineException { + this.ncConfig = config; + this.configManager = ncConfig.getConfigManager(); + if (aep == null) { + throw new IllegalArgumentException("INCApplicationEntryPoint cannot be null"); + } + configManager.processConfig(); + this.ncAppEntryPoint = aep; + id = ncConfig.getNodeId(); + + ioManager = new IOManager(IODeviceHandle.getDevices(ncConfig.getIODevices())); if (id == null) { - throw new Exception("id not set"); + throw new HyracksException("id not set"); } lccm = new LifeCycleComponentManager(); @@ -224,14 +238,16 @@ public class NodeControllerService implements IControllerService { private void init() throws Exception { ioManager.setExecutor(executor); - datasetPartitionManager = new DatasetPartitionManager(this, executor, ncConfig.resultManagerMemory, - ncConfig.resultTTL, ncConfig.resultSweepThreshold); - datasetNetworkManager = new DatasetNetworkManager(ncConfig.resultIPAddress, ncConfig.resultPort, - datasetPartitionManager, ncConfig.nNetThreads, ncConfig.nNetBuffers, ncConfig.resultPublicIPAddress, - ncConfig.resultPublicPort, FullFrameChannelInterfaceFactory.INSTANCE); - if (ncConfig.messagingIPAddress != null && appCtx.getMessagingChannelInterfaceFactory() != null) { - messagingNetManager = new MessagingNetworkManager(this, ncConfig.messagingIPAddress, ncConfig.messagingPort, - ncConfig.nNetThreads, ncConfig.messagingPublicIPAddress, ncConfig.messagingPublicPort, + datasetPartitionManager = new DatasetPartitionManager(this, executor, ncConfig.getResultManagerMemory(), + ncConfig.getResultTTL(), ncConfig.getResultSweepThreshold()); + datasetNetworkManager = new DatasetNetworkManager(ncConfig.getResultListenAddress(), + ncConfig.getResultListenPort(), datasetPartitionManager, ncConfig.getNetThreadCount(), + ncConfig.getNetBufferCount(), ncConfig.getResultPublicAddress(), ncConfig.getResultPublicPort(), + FullFrameChannelInterfaceFactory.INSTANCE); + if (ncConfig.getMessagingListenAddress() != null && appCtx.getMessagingChannelInterfaceFactory() != null) { + messagingNetManager = new MessagingNetworkManager(this, ncConfig.getMessagingListenAddress(), + ncConfig.getMessagingListenPort(), ncConfig.getNetThreadCount(), + ncConfig.getMessagingPublicAddress(), ncConfig.getMessagingPublicPort(), appCtx.getMessagingChannelInterfaceFactory()); } } @@ -239,12 +255,13 @@ public class NodeControllerService implements IControllerService { @Override public void start() throws Exception { LOGGER.log(Level.INFO, "Starting NodeControllerService"); - ipc = new IPCSystem(new InetSocketAddress(ncConfig.clusterNetIPAddress, ncConfig.clusterNetPort), + ipc = new IPCSystem(new InetSocketAddress(ncConfig.getClusterListenAddress(), ncConfig.getClusterListenPort()), new NodeControllerIPCI(this), new CCNCFunctions.SerializerDeserializer()); ipc.start(); partitionManager = new PartitionManager(this); - netManager = new NetworkManager(ncConfig.dataIPAddress, ncConfig.dataPort, partitionManager, - ncConfig.nNetThreads, ncConfig.nNetBuffers, ncConfig.dataPublicIPAddress, ncConfig.dataPublicPort, + netManager = new NetworkManager(ncConfig.getDataListenAddress(), ncConfig.getDataListenPort(), partitionManager, + ncConfig.getNetThreadCount(), ncConfig.getNetBufferCount(), ncConfig.getDataPublicAddress(), + ncConfig.getDataPublicPort(), FullFrameChannelInterfaceFactory.INSTANCE); netManager.start(); @@ -255,8 +272,9 @@ public class NodeControllerService implements IControllerService { if (messagingNetManager != null) { messagingNetManager.start(); } - IIPCHandle ccIPCHandle = ipc.getHandle(new InetSocketAddress(ncConfig.ccHost, ncConfig.ccPort), - ncConfig.retries); + IIPCHandle ccIPCHandle = ipc.getHandle( + new InetSocketAddress(ncConfig.getClusterAddress(), ncConfig.getClusterPort()), + ncConfig.getClusterConnectRetries()); this.ccs = new ClusterControllerRemoteProxy(ccIPCHandle); HeartbeatSchema.GarbageCollectorInfo[] gcInfos = new HeartbeatSchema.GarbageCollectorInfo[gcMXBeans.size()]; for (int i = 0; i < gcInfos.length; ++i) { @@ -274,10 +292,7 @@ public class NodeControllerService implements IControllerService { runtimeMXBean.getVmName(), runtimeMXBean.getVmVersion(), runtimeMXBean.getVmVendor(), runtimeMXBean.getClassPath(), runtimeMXBean.getLibraryPath(), runtimeMXBean.getBootClassPath(), runtimeMXBean.getInputArguments(), runtimeMXBean.getSystemProperties(), hbSchema, meesagingPort, - ncAppEntryPoint == null - ? new NodeCapacity(Runtime.getRuntime().maxMemory(), allCores > 1 ? allCores - 1 : allCores) - : ncAppEntryPoint.getCapacity(), - PidHelper.getPid())); + ncAppEntryPoint.getCapacity(), PidHelper.getPid())); synchronized (this) { while (registrationPending) { @@ -307,21 +322,12 @@ public class NodeControllerService implements IControllerService { } LOGGER.log(Level.INFO, "Started NodeControllerService"); - if (ncAppEntryPoint != null) { - ncAppEntryPoint.notifyStartupComplete(); - } + ncAppEntryPoint.notifyStartupComplete(); } private void startApplication() throws Exception { appCtx = new NCApplicationContext(this, serverCtx, ioManager, id, memoryManager, lccm, ncConfig.getAppConfig()); - String className = ncConfig.appNCMainClass; - if (className != null) { - Class<?> c = Class.forName(className); - ncAppEntryPoint = (INCApplicationEntryPoint) c.newInstance(); - String[] args = ncConfig.appArgs == null ? new String[0] - : ncConfig.appArgs.toArray(new String[ncConfig.appArgs.size()]); - ncAppEntryPoint.start(appCtx, args); - } + ncAppEntryPoint.start(appCtx, ncConfig.getAppArgsArray()); executor = Executors.newCachedThreadPool(appCtx.getThreadFactory()); } @@ -341,10 +347,8 @@ public class NodeControllerService implements IControllerService { messagingNetManager.stop(); } workQueue.stop(); - if (ncAppEntryPoint != null) { - ncAppEntryPoint.stop(); - } - /** + ncAppEntryPoint.stop(); + /* * Stop heartbeat after NC has stopped to avoid false node failure detection * on CC if an NC takes a long time to stop. */ @@ -525,4 +529,14 @@ public class NodeControllerService implements IControllerService { public MessagingNetworkManager getMessagingNetworkManager() { return messagingNetManager; } + + private static INCApplicationEntryPoint getApplicationEntryPoint(NCConfig config) + throws ClassNotFoundException, IllegalAccessException, InstantiationException { + if (config.getAppClass() != null) { + Class<?> c = Class.forName(config.getAppClass()); + return (INCApplicationEntryPoint) c.newInstance(); + } else { + return NCApplicationEntryPoint.INSTANCE; + } + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCApplicationContext.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCApplicationContext.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCApplicationContext.java index f52099d..6d549c2 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCApplicationContext.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCApplicationContext.java @@ -22,10 +22,10 @@ import java.io.IOException; import java.io.OutputStream; import java.io.Serializable; -import org.apache.hyracks.api.application.IApplicationConfig; import org.apache.hyracks.api.application.INCApplicationContext; import org.apache.hyracks.api.application.IStateDumpHandler; import org.apache.hyracks.api.comm.IChannelInterfaceFactory; +import org.apache.hyracks.api.config.IApplicationConfig; import org.apache.hyracks.api.lifecycle.ILifeCycleComponentManager; import org.apache.hyracks.api.resources.memory.IMemoryManager; import org.apache.hyracks.api.service.IControllerService; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml index 8d9a93b..ae7f272 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml @@ -31,9 +31,6 @@ <dependency> <groupId>args4j</groupId> <artifactId>args4j</artifactId> - <version>2.0.12</version> - <type>jar</type> - <scope>compile</scope> </dependency> <dependency> <groupId>org.ini4j</groupId> @@ -42,6 +39,11 @@ </dependency> <dependency> <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-api</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hyracks</groupId> <artifactId>hyracks-control-common</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java index 9b00cc2..6b11ecc 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.ObjectInputStream; import java.io.StringReader; +import java.io.StringWriter; import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; import java.net.InetAddress; @@ -35,7 +36,9 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.lang3.SystemUtils; -import org.apache.hyracks.control.common.controllers.IniUtils; +import org.apache.hyracks.control.common.config.ConfigUtils; +import org.apache.hyracks.control.common.controllers.NCConfig; +import org.apache.hyracks.api.config.Section; import org.apache.hyracks.control.common.controllers.ServiceConstants; import org.apache.hyracks.control.common.controllers.ServiceConstants.ServiceCommand; import org.ini4j.Ini; @@ -85,7 +88,7 @@ public class NCService { // Find the command to run. For now, we allow overriding the name, but // still assume it's located in the bin/ directory of the deployment. // Even this is likely more configurability than we need. - String command = IniUtils.getString(ini, nodeSection, "command", "hyracksnc"); + String command = ConfigUtils.getString(ini, nodeSection, NCConfig.Option.COMMAND.ini(), "hyracksnc"); // app.home is specified by the Maven appassembler plugin. If it isn't set, // fall back to user's home dir. Again this is likely more flexibility // than we need. @@ -112,7 +115,7 @@ public class NCService { } private static void configEnvironment(Map<String, String> env) { - String jvmargs = IniUtils.getString(ini, nodeSection, "jvm.args", null); + String jvmargs = ConfigUtils.getString(ini, nodeSection, NCConfig.Option.JVM_ARGS.ini(), null); if (jvmargs != null) { LOGGER.info("Using JAVA_OPTS from conf file (jvm.args)"); } else { @@ -188,7 +191,13 @@ public class NCService { return retval == 0; } catch (Exception e) { if (LOGGER.isLoggable(Level.SEVERE)) { - LOGGER.log(Level.SEVERE, "Configuration from CC broken", e); + StringWriter sw = new StringWriter(); + try { + ini.store(sw); + LOGGER.log(Level.SEVERE, "Configuration from CC broken: \n" + sw.toString(), e); + } catch (IOException e1) { + LOGGER.log(Level.SEVERE, "Configuration from CC broken, failed to serialize", e1); + } } return false; } @@ -213,7 +222,7 @@ public class NCService { case START_NC: String iniString = ois.readUTF(); ini = new Ini(new StringReader(iniString)); - ncId = IniUtils.getString(ini, "localnc", "id", ""); + ncId = ConfigUtils.getString(ini, Section.LOCALNC, NCConfig.Option.NODE_ID, ""); nodeSection = "nc/" + ncId; return launchNCProcess(); case TERMINATE: @@ -272,7 +281,7 @@ public class NCService { try (ServerSocket listener = new ServerSocket(port, 5, addr)) { boolean launched = false; while (!launched) { - LOGGER.info("Waiting for connection from CC on " + addr + ":" + port); + LOGGER.info("Waiting for connection from CC on " + (addr == null ? "*" : addr) + ":" + port); try (Socket socket = listener.accept()) { // QQQ Because acceptConnection() doesn't return if the // service is started appropriately, the socket remains http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCServiceConfig.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCServiceConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCServiceConfig.java index 91c9f6d..10fa679 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCServiceConfig.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCServiceConfig.java @@ -18,7 +18,7 @@ */ package org.apache.hyracks.control.nc.service; -import org.apache.hyracks.control.common.controllers.IniUtils; +import org.apache.hyracks.control.common.config.ConfigUtils; import org.ini4j.Ini; import org.kohsuke.args4j.Option; @@ -58,10 +58,10 @@ public class NCServiceConfig { * It does not apply defaults or any logic. */ private void loadINIFile() throws IOException { - ini = IniUtils.loadINIFile(configFile); - address = IniUtils.getString(ini, "ncservice", "address", address); - port = IniUtils.getInt(ini, "ncservice", "port", port); - logdir = IniUtils.getString(ini, "ncservice", "logdir", logdir); + ini = ConfigUtils.loadINIFile(configFile); + address = ConfigUtils.getString(ini, "ncservice", "address", address); + port = ConfigUtils.getInt(ini, "ncservice", "port", port); + logdir = ConfigUtils.getString(ini, "ncservice", "logdir", logdir); } /** http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml b/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml index 6cfaaa5..e7ed599 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml @@ -47,7 +47,7 @@ <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId> <configuration> - <excludes> + <excludes combine.children="append"> <exclude>src/test/resources/data/beer.txt</exclude> </excludes> </configuration> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-dist/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dist/pom.xml b/hyracks-fullstack/hyracks/hyracks-dist/pom.xml index b88d292..7b6d7ec 100644 --- a/hyracks-fullstack/hyracks/hyracks-dist/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-dist/pom.xml @@ -88,7 +88,7 @@ <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId> <configuration> - <excludes> + <excludes combine.children="append"> <exclude>src/main/resources/conf/master</exclude> <exclude>src/main/resources/conf/slaves</exclude> </excludes> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-dist/src/main/resources/bin/startDebugNc.sh ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dist/src/main/resources/bin/startDebugNc.sh b/hyracks-fullstack/hyracks/hyracks-dist/src/main/resources/bin/startDebugNc.sh index 4d2fc51..9794b07 100755 --- a/hyracks-fullstack/hyracks/hyracks-dist/src/main/resources/bin/startDebugNc.sh +++ b/hyracks-fullstack/hyracks/hyracks-dist/src/main/resources/bin/startDebugNc.sh @@ -65,4 +65,4 @@ HYRACKS_HOME=`pwd` cd $NCTMP_DIR2 #Launch hyracks nc -$HYRACKS_HOME/hyracks-server/target/appassembler/bin/hyracksnc -cc-host $CCHOST -cc-port $CC_CLUSTERPORT -cluster-net-ip-address $IPADDR -data-ip-address $IPADDR -node-id $NODEID -iodevices "${IO_DIRS2}" &> $NCLOGS_DIR2/$NODEID.log & +$HYRACKS_HOME/hyracks-server/target/appassembler/bin/hyracksnc -cluster-address $CCHOST -cluster-port $CC_CLUSTERPORT -address $IPADDR -data-listen-address $IPADDR -node-id $NODEID -iodevices "${IO_DIRS2}" &> $NCLOGS_DIR2/$NODEID.log & http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-dist/src/main/resources/bin/startcc.sh ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dist/src/main/resources/bin/startcc.sh b/hyracks-fullstack/hyracks/hyracks-dist/src/main/resources/bin/startcc.sh index 5fa2b86..1bbbe10 100755 --- a/hyracks-fullstack/hyracks/hyracks-dist/src/main/resources/bin/startcc.sh +++ b/hyracks-fullstack/hyracks/hyracks-dist/src/main/resources/bin/startcc.sh @@ -42,8 +42,8 @@ export JAVA_OPTS=$CCJAVA_OPTS chmod -R 755 $HYRACKS_HOME if [ -f "conf/topology.xml" ]; then #Launch hyracks cc script with topology -$HYRACKS_HOME/hyracks-server/target/appassembler/bin/hyrackscc -client-net-ip-address $CCHOST -cluster-net-ip-address $CCHOST -client-net-port $CC_CLIENTPORT -cluster-net-port $CC_CLUSTERPORT -max-heartbeat-lapse-periods 999999 -default-max-job-attempts 0 -job-history-size 0 -cluster-topology "conf/topology.xml" &> $CCLOGS_DIR/cc.log & +$HYRACKS_HOME/hyracks-server/target/appassembler/bin/hyrackscc -client-listen-address $CCHOST -address $CCHOST -client-listen-port $CC_CLIENTPORT -cluster-listen-port $CC_CLUSTERPORT -heartbeat-max-misses 999999 -job-history-size 0 -cluster-topology "conf/topology.xml" &> $CCLOGS_DIR/cc.log & else #Launch hyracks cc script without toplogy -$HYRACKS_HOME/hyracks-server/target/appassembler/bin/hyrackscc -client-net-ip-address $CCHOST -cluster-net-ip-address $CCHOST -client-net-port $CC_CLIENTPORT -cluster-net-port $CC_CLUSTERPORT -max-heartbeat-lapse-periods 999999 -default-max-job-attempts 0 -job-history-size 0 &> $CCLOGS_DIR/cc.log & +$HYRACKS_HOME/hyracks-server/target/appassembler/bin/hyrackscc -client-listen-address $CCHOST -address $CCHOST -client-listen-port $CC_CLIENTPORT -cluster-listen-port $CC_CLUSTERPORT -heartbeat-max-misses 999999 -job-history-size 0 &> $CCLOGS_DIR/cc.log & fi http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-dist/src/main/resources/bin/startnc.sh ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dist/src/main/resources/bin/startnc.sh b/hyracks-fullstack/hyracks/hyracks-dist/src/main/resources/bin/startnc.sh index d09b9c1..62d671f 100755 --- a/hyracks-fullstack/hyracks/hyracks-dist/src/main/resources/bin/startnc.sh +++ b/hyracks-fullstack/hyracks/hyracks-dist/src/main/resources/bin/startnc.sh @@ -64,4 +64,4 @@ HYRACKS_HOME=`pwd` cd $NCTMP_DIR #Launch hyracks nc -$HYRACKS_HOME/hyracks-server/target/appassembler/bin/hyracksnc -cc-host $CCHOST -cc-port $CC_CLUSTERPORT -cluster-net-ip-address $IPADDR -data-ip-address $IPADDR -result-ip-address $IPADDR -node-id $NODEID -iodevices "${IO_DIRS}" &> $NCLOGS_DIR/$NODEID.log & +$HYRACKS_HOME/hyracks-server/target/appassembler/bin/hyracksnc -cluster-address $CCHOST -cluster-port $CC_CLUSTERPORT -address $IPADDR -data-listen-address $IPADDR -result-listen-address $IPADDR -node-id $NODEID -iodevices "${IO_DIRS}" &> $NCLOGS_DIR/$NODEID.log & http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/pom.xml index db62a85..2486fe8 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/pom.xml @@ -61,7 +61,6 @@ <dependency> <groupId>args4j</groupId> <artifactId>args4j</artifactId> - <version>2.0.12</version> </dependency> <dependency> <groupId>org.apache.hyracks</groupId> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/NCApplicationEntryPoint.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/NCApplicationEntryPoint.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/NCApplicationEntryPoint.java index 5180f23..eec28a2 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/NCApplicationEntryPoint.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/NCApplicationEntryPoint.java @@ -20,6 +20,7 @@ package org.apache.hyracks.examples.btree.helper; import org.apache.hyracks.api.application.INCApplicationContext; import org.apache.hyracks.api.application.INCApplicationEntryPoint; +import org.apache.hyracks.api.config.IConfigManager; import org.apache.hyracks.api.job.resource.NodeCapacity; public class NCApplicationEntryPoint implements INCApplicationEntryPoint { @@ -44,4 +45,10 @@ public class NCApplicationEntryPoint implements INCApplicationEntryPoint { public NodeCapacity getCapacity() { return new NodeCapacity(Runtime.getRuntime().maxMemory(), Runtime.getRuntime().availableProcessors() - 1); } + + @Override + public void registerConfigOptions(IConfigManager configManager) { + // no-op + } + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml index 630b984..b26f00f 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml @@ -41,7 +41,7 @@ <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId> <configuration> - <excludes> + <excludes combine.children="append"> <exclude>data/**</exclude> </excludes> </configuration> @@ -174,5 +174,10 @@ <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> </dependency> + <dependency> + <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-util</artifactId> + <version>${project.version}</version> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractIntegrationTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractIntegrationTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractIntegrationTest.java index a7677f8..82fd737 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractIntegrationTest.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractIntegrationTest.java @@ -18,6 +18,8 @@ */ package org.apache.hyracks.tests.integration; +import static org.apache.hyracks.util.file.FileUtil.joinPath; + import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; @@ -82,47 +84,43 @@ public abstract class AbstractIntegrationTest { @BeforeClass public static void init() throws Exception { CCConfig ccConfig = new CCConfig(); - ccConfig.clientNetIpAddress = "127.0.0.1"; - ccConfig.clientNetPort = 39000; - ccConfig.clusterNetIpAddress = "127.0.0.1"; - ccConfig.clusterNetPort = 39001; - ccConfig.profileDumpPeriod = 10000; + ccConfig.setClientListenAddress("127.0.0.1"); + ccConfig.setClientListenPort(39000); + ccConfig.setClusterListenAddress("127.0.0.1"); + ccConfig.setClusterListenPort(39001); + ccConfig.setProfileDumpPeriod(10000); FileUtils.deleteQuietly(new File("target" + File.separator + "data")); - FileUtils.copyDirectory(new File("data"), new File("target" + File.separator + "data")); + FileUtils.copyDirectory(new File("data"), new File(joinPath("target", "data"))); File outDir = new File("target" + File.separator + "ClusterController"); outDir.mkdirs(); File ccRoot = File.createTempFile(AbstractIntegrationTest.class.getName(), ".data", outDir); ccRoot.delete(); ccRoot.mkdir(); - ccConfig.ccRoot = ccRoot.getAbsolutePath(); + ccConfig.setRootDir(ccRoot.getAbsolutePath()); cc = new ClusterControllerService(ccConfig); cc.start(); - NCConfig ncConfig1 = new NCConfig(); - ncConfig1.ccHost = "localhost"; - ncConfig1.ccPort = 39001; - ncConfig1.clusterNetIPAddress = "127.0.0.1"; - ncConfig1.dataIPAddress = "127.0.0.1"; - ncConfig1.resultIPAddress = "127.0.0.1"; - ncConfig1.nodeId = NC1_ID; - ncConfig1.ioDevices = System.getProperty("user.dir") + File.separator + "target" + File.separator + "data" - + File.separator + "device0"; + NCConfig ncConfig1 = new NCConfig(NC1_ID); + ncConfig1.setClusterAddress("localhost"); + ncConfig1.setClusterPort(39001); + ncConfig1.setClusterListenAddress("127.0.0.1"); + ncConfig1.setDataListenAddress("127.0.0.1"); + ncConfig1.setResultListenAddress("127.0.0.1"); + ncConfig1.setIODevices(new String [] { joinPath(System.getProperty("user.dir"), "target", "data", "device0") }); nc1 = new NodeControllerService(ncConfig1); nc1.start(); - NCConfig ncConfig2 = new NCConfig(); - ncConfig2.ccHost = "localhost"; - ncConfig2.ccPort = 39001; - ncConfig2.clusterNetIPAddress = "127.0.0.1"; - ncConfig2.dataIPAddress = "127.0.0.1"; - ncConfig2.resultIPAddress = "127.0.0.1"; - ncConfig2.nodeId = NC2_ID; - ncConfig2.ioDevices = System.getProperty("user.dir") + File.separator + "target" + File.separator + "data" - + File.separator + "device1"; + NCConfig ncConfig2 = new NCConfig(NC2_ID); + ncConfig2.setClusterAddress("localhost"); + ncConfig2.setClusterPort(39001); + ncConfig2.setClusterListenAddress("127.0.0.1"); + ncConfig2.setDataListenAddress("127.0.0.1"); + ncConfig2.setResultListenAddress("127.0.0.1"); + ncConfig2.setIODevices(new String [] { joinPath(System.getProperty("user.dir"), "target", "data", "device1") }); nc2 = new NodeControllerService(ncConfig2); nc2.start(); - hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort); + hcc = new HyracksConnection(ccConfig.getClientListenAddress(), ccConfig.getClientListenPort()); if (LOGGER.isLoggable(Level.INFO)) { LOGGER.info("Starting CC in " + ccRoot.getAbsolutePath()); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java index 3d6ac00..f7959d8 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java @@ -26,9 +26,9 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import org.apache.commons.io.FileUtils; -import org.apache.hyracks.api.application.ICCApplicationContext; -import org.apache.hyracks.api.application.ICCApplicationEntryPoint; import org.apache.hyracks.api.client.HyracksConnection; import org.apache.hyracks.api.client.IHyracksClientConnection; import org.apache.hyracks.api.comm.IFrameTupleAccessor; @@ -42,6 +42,7 @@ import org.apache.hyracks.api.job.JobId; import org.apache.hyracks.api.job.JobSpecification; import org.apache.hyracks.api.job.resource.IJobCapacityController; import org.apache.hyracks.client.dataset.HyracksDataset; +import org.apache.hyracks.control.cc.CCApplicationEntryPoint; import org.apache.hyracks.control.cc.ClusterControllerService; import org.apache.hyracks.control.common.controllers.CCConfig; import org.apache.hyracks.control.common.controllers.NCConfig; @@ -54,9 +55,6 @@ import org.junit.BeforeClass; import org.junit.Rule; import org.junit.rules.TemporaryFolder; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; - public abstract class AbstractMultiNCIntegrationTest { private static final Logger LOGGER = Logger.getLogger(AbstractMultiNCIntegrationTest.class.getName()); @@ -82,18 +80,18 @@ public abstract class AbstractMultiNCIntegrationTest { @BeforeClass public static void init() throws Exception { CCConfig ccConfig = new CCConfig(); - ccConfig.clientNetIpAddress = "127.0.0.1"; - ccConfig.clientNetPort = 39000; - ccConfig.clusterNetIpAddress = "127.0.0.1"; - ccConfig.clusterNetPort = 39001; - ccConfig.profileDumpPeriod = 10000; + ccConfig.setClientListenAddress("127.0.0.1"); + ccConfig.setClientListenPort(39000); + ccConfig.setClusterListenAddress("127.0.0.1"); + ccConfig.setClusterListenPort(39001); + ccConfig.setProfileDumpPeriod(10000); File outDir = new File("target" + File.separator + "ClusterController"); outDir.mkdirs(); File ccRoot = File.createTempFile(AbstractMultiNCIntegrationTest.class.getName(), ".data", outDir); ccRoot.delete(); ccRoot.mkdir(); - ccConfig.ccRoot = ccRoot.getAbsolutePath(); - ccConfig.appCCMainClass = DummyApplicationEntryPoint.class.getName(); + ccConfig.setRootDir(ccRoot.getAbsolutePath()); + ccConfig.setAppClass(DummyApplicationEntryPoint.class.getName()); cc = new ClusterControllerService(ccConfig); cc.start(); @@ -102,19 +100,18 @@ public abstract class AbstractMultiNCIntegrationTest { File ioDev = new File("target" + File.separator + ASTERIX_IDS[i] + File.separator + "ioDevice"); FileUtils.forceMkdir(ioDev); FileUtils.copyDirectory(new File("data" + File.separator + "device0"), ioDev); - NCConfig ncConfig = new NCConfig(); - ncConfig.ccHost = "localhost"; - ncConfig.ccPort = 39001; - ncConfig.clusterNetIPAddress = "127.0.0.1"; - ncConfig.dataIPAddress = "127.0.0.1"; - ncConfig.resultIPAddress = "127.0.0.1"; - ncConfig.nodeId = ASTERIX_IDS[i]; - ncConfig.ioDevices = ioDev.getAbsolutePath(); + NCConfig ncConfig = new NCConfig(ASTERIX_IDS[i]); + ncConfig.setClusterAddress("localhost"); + ncConfig.setClusterPort(39001); + ncConfig.setClusterListenAddress("127.0.0.1"); + ncConfig.setDataListenAddress("127.0.0.1"); + ncConfig.setResultListenAddress("127.0.0.1"); + ncConfig.setIODevices(new String [] { ioDev.getAbsolutePath() }); asterixNCs[i] = new NodeControllerService(ncConfig); asterixNCs[i].start(); } - hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort); + hcc = new HyracksConnection(ccConfig.getClientListenAddress(), ccConfig.getClientListenPort()); if (LOGGER.isLoggable(Level.INFO)) { LOGGER.info("Starting CC in " + ccRoot.getAbsolutePath()); } @@ -219,22 +216,7 @@ public abstract class AbstractMultiNCIntegrationTest { return tempFile; } - public static class DummyApplicationEntryPoint implements ICCApplicationEntryPoint { - - @Override - public void start(ICCApplicationContext ccAppCtx, String[] args) throws Exception { - - } - - @Override - public void stop() throws Exception { - - } - - @Override - public void startupCompleted() throws Exception { - - } + public static class DummyApplicationEntryPoint extends CCApplicationEntryPoint { @Override public IJobCapacityController getJobCapacityController() { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/NodesAPIIntegrationTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/NodesAPIIntegrationTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/NodesAPIIntegrationTest.java index 6f05600..672d2c4 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/NodesAPIIntegrationTest.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/NodesAPIIntegrationTest.java @@ -48,7 +48,7 @@ public class NodesAPIIntegrationTest extends AbstractIntegrationTest { "net-signaling-bytes-written", "dataset-net-payload-bytes-read", "dataset-net-payload-bytes-written", "dataset-net-signaling-bytes-read", "dataset-net-signaling-bytes-written", "ipc-messages-sent", "ipc-message-bytes-sent", "ipc-messages-received", "ipc-message-bytes-received", "disk-reads", - "disk-writes", "ini" }; + "disk-writes", "config" }; public static final String ROOT_PATH = "/rest/nodes"; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/PredistributedJobsTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/PredistributedJobsTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/PredistributedJobsTest.java index 2509515..f911a75 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/PredistributedJobsTest.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/PredistributedJobsTest.java @@ -18,6 +18,7 @@ */ package org.apache.hyracks.tests.integration; +import static org.apache.hyracks.util.file.FileUtil.joinPath; import static org.mockito.Matchers.any; import static org.mockito.Mockito.verify; @@ -54,50 +55,46 @@ public class PredistributedJobsTest { @BeforeClass public static void init() throws Exception { CCConfig ccConfig = new CCConfig(); - ccConfig.clientNetIpAddress = "127.0.0.1"; - ccConfig.clientNetPort = 39000; - ccConfig.clusterNetIpAddress = "127.0.0.1"; - ccConfig.clusterNetPort = 39001; - ccConfig.profileDumpPeriod = 10000; - FileUtils.deleteQuietly(new File("target" + File.separator + "data")); - FileUtils.copyDirectory(new File("data"), new File("target" + File.separator + "data")); + ccConfig.setClientListenAddress("127.0.0.1"); + ccConfig.setClientListenPort(39000); + ccConfig.setClusterListenAddress("127.0.0.1"); + ccConfig.setClusterListenPort(39001); + ccConfig.setProfileDumpPeriod(10000); + FileUtils.deleteQuietly(new File(joinPath("target", "data"))); + FileUtils.copyDirectory(new File("data"), new File(joinPath("target", "data"))); File outDir = new File("target" + File.separator + "ClusterController"); outDir.mkdirs(); File ccRoot = File.createTempFile(AbstractIntegrationTest.class.getName(), ".data", outDir); ccRoot.delete(); ccRoot.mkdir(); - ccConfig.ccRoot = ccRoot.getAbsolutePath(); + ccConfig.setRootDir(ccRoot.getAbsolutePath()); ClusterControllerService ccBase = new ClusterControllerService(ccConfig); cc = Mockito.spy(ccBase); cc.start(); - NCConfig ncConfig1 = new NCConfig(); - ncConfig1.ccHost = "localhost"; - ncConfig1.ccPort = 39001; - ncConfig1.clusterNetIPAddress = "127.0.0.1"; - ncConfig1.dataIPAddress = "127.0.0.1"; - ncConfig1.resultIPAddress = "127.0.0.1"; - ncConfig1.nodeId = NC1_ID; - ncConfig1.ioDevices = System.getProperty("user.dir") + File.separator + "target" + File.separator + "data" - + File.separator + "device0"; + NCConfig ncConfig1 = new NCConfig(NC1_ID); + ncConfig1.setClusterAddress("localhost"); + ncConfig1.setClusterPort(39001); + ncConfig1.setClusterListenAddress("127.0.0.1"); + ncConfig1.setDataListenAddress("127.0.0.1"); + ncConfig1.setResultListenAddress("127.0.0.1"); + ncConfig1.setIODevices(new String [] { joinPath(System.getProperty("user.dir"), "target", "data", "device0") }); NodeControllerService nc1Base = new NodeControllerService(ncConfig1); nc1 = Mockito.spy(nc1Base); nc1.start(); - NCConfig ncConfig2 = new NCConfig(); - ncConfig2.ccHost = "localhost"; - ncConfig2.ccPort = 39001; - ncConfig2.clusterNetIPAddress = "127.0.0.1"; - ncConfig2.dataIPAddress = "127.0.0.1"; - ncConfig2.resultIPAddress = "127.0.0.1"; - ncConfig2.nodeId = NC2_ID; - ncConfig2.ioDevices = System.getProperty("user.dir") + File.separator + "target" + File.separator + "data" - + File.separator + "device1"; + NCConfig ncConfig2 = new NCConfig(NC2_ID); + ncConfig2.setClusterAddress("localhost"); + ncConfig2.setClusterPort(39001); + ncConfig2.setClusterListenAddress("127.0.0.1"); + ncConfig2.setDataListenAddress("127.0.0.1"); + ncConfig2.setResultListenAddress("127.0.0.1"); + ncConfig2.setIODevices(new String [] { joinPath(System.getProperty("user.dir"), "target", "data", "device1") }); NodeControllerService nc2Base = new NodeControllerService(ncConfig2); nc2 = Mockito.spy(nc2Base); nc2.start(); - hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort); + hcc = new HyracksConnection(ccConfig.getClientListenAddress(), ccConfig.getClientListenPort()); if (LOGGER.isLoggable(Level.INFO)) { LOGGER.info("Starting CC in " + ccRoot.getAbsolutePath()); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-examples/hyracks-shutdown-test/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-shutdown-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-shutdown-test/pom.xml index 0e9d24e..42a104d 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-shutdown-test/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-shutdown-test/pom.xml @@ -40,9 +40,10 @@ <artifactId>maven-dependency-plugin</artifactId> <version>2.10</version> <configuration> - <failOnWarning>true</failOnWarning> - <outputXML>true</outputXML> - <usedDependencies>org.apache.hyracks:hyracks-control-nc,org.apache.hyracks:hyracks-control-cc</usedDependencies> + <usedDependencies combine.children="append"> + <usedDependency>org.apache.hyracks:hyracks-control-nc</usedDependency> + <usedDependency>org.apache.hyracks:hyracks-control-cc</usedDependency> + </usedDependencies> </configuration> <executions> <execution> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-examples/text-example/textclient/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/text-example/textclient/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/text-example/textclient/pom.xml index e1155b7..9003724 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/text-example/textclient/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-examples/text-example/textclient/pom.xml @@ -51,7 +51,6 @@ <dependency> <groupId>args4j</groupId> <artifactId>args4j</artifactId> - <version>2.0.12</version> </dependency> <dependency> <groupId>org.apache.hyracks</groupId> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml index 394fa11..ed269ea 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml @@ -40,10 +40,11 @@ <artifactId>maven-dependency-plugin</artifactId> <version>2.10</version> <configuration> - <failOnWarning>true</failOnWarning> - <outputXML>true</outputXML> - <usedDependencies> - org.apache.hyracks:hyracks-control-nc,org.apache.hyracks:hyracks-control-cc,org.apache.hyracks:hyracks-dataflow-std,org.apache.hyracks:texthelper + <usedDependencies combine.children="append"> + <usedDependency>org.apache.hyracks:hyracks-control-nc</usedDependency> + <usedDependency>org.apache.hyracks:hyracks-control-cc</usedDependency> + <usedDependency>org.apache.hyracks:hyracks-dataflow-std</usedDependency> + <usedDependency>org.apache.hyracks:texthelper</usedDependency> </usedDependencies> </configuration> <executions> @@ -103,7 +104,7 @@ <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId> <configuration> - <excludes> + <excludes combine.children="append"> <exclude>data/file1.txt</exclude> <exclude>data/file2.txt</exclude> </excludes> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/pom.xml index 5bd7796..f8378c6 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-examples/tpch-example/tpchclient/pom.xml @@ -45,7 +45,6 @@ <dependency> <groupId>args4j</groupId> <artifactId>args4j</artifactId> - <version>2.0.12</version> </dependency> <dependency> <groupId>org.apache.hyracks</groupId> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml index b4abc4a..a8778d6 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml @@ -66,7 +66,7 @@ <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId> <configuration> - <excludes> + <excludes combine.children="append"> <exclude>src/test/resources/data/customer.tbl</exclude> <exclude>src/test/resources/expected/part-0</exclude> </excludes> @@ -406,5 +406,11 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-util</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java index 2307a43..bf7f2a0 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java @@ -19,8 +19,6 @@ package org.apache.hyracks.hdfs.dataflow; -import static org.apache.hyracks.test.support.TestUtils.joinPath; - import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; @@ -61,6 +59,7 @@ import org.apache.hyracks.hdfs.lib.TextTupleWriterFactory; import org.apache.hyracks.hdfs.scheduler.Scheduler; import org.apache.hyracks.hdfs.utils.HyracksUtils; import org.apache.hyracks.hdfs.utils.TestUtils; +import org.apache.hyracks.util.file.FileUtil; /** * Test the org.apache.hyracks.hdfs.dataflow package, @@ -69,19 +68,19 @@ import org.apache.hyracks.hdfs.utils.TestUtils; @SuppressWarnings({ "deprecation" }) public class DataflowTest extends TestCase { - protected static final String ACTUAL_RESULT_DIR = joinPath("target", "actual"); - private static final String TEST_RESOURCES = joinPath("src", "test", "resources"); - protected static final String EXPECTED_RESULT_PATH = joinPath(TEST_RESOURCES, "expected"); - private static final String PATH_TO_HADOOP_CONF = joinPath(TEST_RESOURCES, "hadoop", "conf"); - protected static final String BUILD_DIR = joinPath("target", "build"); + protected static final String ACTUAL_RESULT_DIR = FileUtil.joinPath("target", "actual"); + private static final String TEST_RESOURCES = FileUtil.joinPath("src", "test", "resources"); + protected static final String EXPECTED_RESULT_PATH = FileUtil.joinPath(TEST_RESOURCES, "expected"); + private static final String PATH_TO_HADOOP_CONF = FileUtil.joinPath(TEST_RESOURCES, "hadoop", "conf"); + protected static final String BUILD_DIR = FileUtil.joinPath("target", "build"); - private static final String DATA_PATH = joinPath(TEST_RESOURCES, "data", "customer.tbl"); + private static final String DATA_PATH = FileUtil.joinPath(TEST_RESOURCES, "data", "customer.tbl"); protected static final String HDFS_INPUT_PATH = "/customer/"; protected static final String HDFS_OUTPUT_PATH = "/customer_result/"; private static final String HADOOP_CONF_PATH = ACTUAL_RESULT_DIR + File.separator + "conf.xml"; - private static final String MINIDFS_BASEDIR = joinPath("target", "hdfs"); + private static final String MINIDFS_BASEDIR = FileUtil.joinPath("target", "hdfs"); private MiniDFSCluster dfsCluster; private JobConf conf = new JobConf(); @@ -121,8 +120,8 @@ public class DataflowTest extends TestCase { FileSystem lfs = FileSystem.getLocal(new Configuration()); lfs.delete(new Path(BUILD_DIR), true); - System.setProperty("hadoop.log.dir", joinPath("target", "logs")); - getConfiguration().set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, MINIDFS_BASEDIR); + System.setProperty("hadoop.log.dir", FileUtil.joinPath("target", "logs")); + getConfiguration().set("hdfs.minidfs.basedir", MINIDFS_BASEDIR); dfsCluster = getMiniDFSCluster(getConfiguration(), numberOfNC); FileSystem dfs = FileSystem.get(getConfiguration()); Path src = new Path(DATA_PATH); @@ -197,8 +196,8 @@ public class DataflowTest extends TestCase { Path actual = new Path(ACTUAL_RESULT_DIR); dfs.copyToLocalFile(result, actual); - TestUtils.compareWithResult(new File(joinPath(EXPECTED_RESULT_PATH, "part-0")), new File( - joinPath(ACTUAL_RESULT_DIR, "customer_result", "part-0"))); + TestUtils.compareWithResult(new File(FileUtil.joinPath(EXPECTED_RESULT_PATH, "part-0")), new File( + FileUtil.joinPath(ACTUAL_RESULT_DIR, "customer_result", "part-0"))); return true; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/HyracksUtils.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/HyracksUtils.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/HyracksUtils.java index cc32527..1fddc46 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/HyracksUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/HyracksUtils.java @@ -50,36 +50,33 @@ public class HyracksUtils { public static void init() throws Exception { CCConfig ccConfig = new CCConfig(); - ccConfig.clientNetIpAddress = CC_HOST; - ccConfig.clusterNetIpAddress = CC_HOST; - ccConfig.clusterNetPort = TEST_HYRACKS_CC_PORT; - ccConfig.clientNetPort = TEST_HYRACKS_CC_CLIENT_PORT; - ccConfig.defaultMaxJobAttempts = 0; - ccConfig.jobHistorySize = 0; - ccConfig.profileDumpPeriod = -1; + ccConfig.setClientListenAddress(CC_HOST); + ccConfig.setClusterListenAddress(CC_HOST); + ccConfig.setClusterListenPort(TEST_HYRACKS_CC_PORT); + ccConfig.setClientListenPort(TEST_HYRACKS_CC_CLIENT_PORT); + ccConfig.setJobHistorySize(0); + ccConfig.setProfileDumpPeriod(-1); // cluster controller cc = new ClusterControllerService(ccConfig); cc.start(); // two node controllers - NCConfig ncConfig1 = new NCConfig(); - ncConfig1.ccHost = "localhost"; - ncConfig1.clusterNetIPAddress = "localhost"; - ncConfig1.ccPort = TEST_HYRACKS_CC_PORT; - ncConfig1.dataIPAddress = "127.0.0.1"; - ncConfig1.resultIPAddress = "127.0.0.1"; - ncConfig1.nodeId = NC1_ID; + NCConfig ncConfig1 = new NCConfig(NC1_ID); + ncConfig1.setClusterAddress("localhost"); + ncConfig1.setClusterListenAddress("localhost"); + ncConfig1.setClusterPort(TEST_HYRACKS_CC_PORT); + ncConfig1.setDataListenAddress("127.0.0.1"); + ncConfig1.setResultListenAddress("127.0.0.1"); nc1 = new NodeControllerService(ncConfig1); nc1.start(); - NCConfig ncConfig2 = new NCConfig(); - ncConfig2.ccHost = "localhost"; - ncConfig2.clusterNetIPAddress = "localhost"; - ncConfig2.ccPort = TEST_HYRACKS_CC_PORT; - ncConfig2.dataIPAddress = "127.0.0.1"; - ncConfig2.resultIPAddress = "127.0.0.1"; - ncConfig2.nodeId = NC2_ID; + NCConfig ncConfig2 = new NCConfig(NC2_ID); + ncConfig2.setClusterAddress("localhost"); + ncConfig2.setClusterListenAddress("localhost"); + ncConfig2.setClusterPort(TEST_HYRACKS_CC_PORT); + ncConfig2.setDataListenAddress("127.0.0.1"); + ncConfig2.setResultListenAddress("127.0.0.1"); nc2 = new NodeControllerService(ncConfig2); nc2.start(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java index cb9deea..e4d9005 100644 --- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java +++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java @@ -38,7 +38,7 @@ public abstract class AbstractServlet implements IServlet { protected final ConcurrentMap<String, Object> ctx; private final int[] trims; - public AbstractServlet(ConcurrentMap<String, Object> ctx, String[] paths) { + public AbstractServlet(ConcurrentMap<String, Object> ctx, String... paths) { this.paths = paths; this.ctx = ctx; trims = new int[paths.length]; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/IPCConnectionManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/IPCConnectionManager.java b/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/IPCConnectionManager.java index 9ee135b..fe2bcae 100644 --- a/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/IPCConnectionManager.java +++ b/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/IPCConnectionManager.java @@ -64,14 +64,14 @@ public class IPCConnectionManager { IPCConnectionManager(IPCSystem system, InetSocketAddress socketAddress) throws IOException { this.system = system; - this.networkThread = new NetworkThread(); - this.networkThread.setPriority(Thread.MAX_PRIORITY); this.serverSocketChannel = ServerSocketChannel.open(); serverSocketChannel.socket().setReuseAddress(true); serverSocketChannel.configureBlocking(false); ServerSocket socket = serverSocketChannel.socket(); socket.bind(socketAddress); address = new InetSocketAddress(socket.getInetAddress(), socket.getLocalPort()); + networkThread = new NetworkThread(); + networkThread.setPriority(Thread.MAX_PRIORITY); ipcHandleMap = new HashMap<>(); pendingConnections = new ArrayList<>(); workingPendingConnections = new ArrayList<>(); @@ -175,7 +175,7 @@ public class IPCConnectionManager { private final Selector selector; public NetworkThread() { - super("IPC Network Listener Thread"); + super("IPC Network Listener Thread [" + address + "]"); setDaemon(true); try { selector = Selector.open(); @@ -323,7 +323,7 @@ public class IPCConnectionManager { failingLoops = 0; } catch (Exception e) { int sleepSecs = (int)Math.pow(2, Math.min(11, failingLoops++)); - LOGGER.log(Level.WARNING, "Exception processing message; sleeping " + sleepSecs + LOGGER.log(Level.SEVERE, "Exception processing message; sleeping " + sleepSecs + " seconds", e); try { Thread.sleep(TimeUnit.SECONDS.toMillis(sleepSecs)); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/org/apache/hyracks/maven/plugin/HyracksCCStartMojo.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/org/apache/hyracks/maven/plugin/HyracksCCStartMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/org/apache/hyracks/maven/plugin/HyracksCCStartMojo.java index a461064..5a9d9dd 100644 --- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/org/apache/hyracks/maven/plugin/HyracksCCStartMojo.java +++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/org/apache/hyracks/maven/plugin/HyracksCCStartMojo.java @@ -40,8 +40,8 @@ public class HyracksCCStartMojo extends AbstractHyracksServerMojo { if (port != 0) { cmdLineBuffer.append("-port ").append(port); } - cmdLineBuffer.append(" -client-net-ip-address 127.0.0.1"); - cmdLineBuffer.append(" -cluster-net-ip-address 127.0.0.1"); + cmdLineBuffer.append(" -client-listen-address 127.0.0.1"); + cmdLineBuffer.append(" -address 127.0.0.1"); String args = cmdLineBuffer.toString(); final Process proc = launch(new File(hyracksServerHome, makeScriptName(HYRACKS_CC_SCRIPT)), args, workingDir); HyracksServiceRegistry.INSTANCE.addServiceProcess(proc); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/org/apache/hyracks/maven/plugin/HyracksNCStartMojo.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/org/apache/hyracks/maven/plugin/HyracksNCStartMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/org/apache/hyracks/maven/plugin/HyracksNCStartMojo.java index 1f0e640..926e108 100644 --- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/org/apache/hyracks/maven/plugin/HyracksNCStartMojo.java +++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/org/apache/hyracks/maven/plugin/HyracksNCStartMojo.java @@ -55,13 +55,13 @@ public class HyracksNCStartMojo extends AbstractHyracksServerMojo { @Override public void execute() throws MojoExecutionException, MojoFailureException { StringBuilder cmdLineBuffer = new StringBuilder(); - cmdLineBuffer.append(" -cc-host ").append(ccHost); - cmdLineBuffer.append(" -data-ip-address ").append(dataIpAddress); + cmdLineBuffer.append(" -cluster-address ").append(ccHost); + cmdLineBuffer.append(" -data-listen-address ").append(dataIpAddress); cmdLineBuffer.append(" -node-id ").append(nodeId); - cmdLineBuffer.append(" -cluster-net-ip-address 127.0.0.1"); - cmdLineBuffer.append(" -result-ip-address 127.0.0.1"); + cmdLineBuffer.append(" -address 127.0.0.1"); + cmdLineBuffer.append(" -result-listen-address 127.0.0.1"); if (ccPort != 0) { - cmdLineBuffer.append(" -cc-port ").append(ccPort); + cmdLineBuffer.append(" -cluster-port ").append(ccPort); } String args = cmdLineBuffer.toString(); final Process proc = launch(new File(hyracksServerHome, makeScriptName(HYRACKS_NC_SCRIPT)), args, workingDir); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-server/docs/README ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-server/docs/README b/hyracks-fullstack/hyracks/hyracks-server/docs/README index 06bb1e1..44cdcd8 100644 --- a/hyracks-fullstack/hyracks/hyracks-server/docs/README +++ b/hyracks-fullstack/hyracks/hyracks-server/docs/README @@ -19,15 +19,15 @@ bin/hyrackscc -port <port> The node controller is started by running bin/hyracksnc. It requires at least the following two command line arguments. - -cc-host VAL : Cluster Controller host name - -data-ip-address VAL : IP Address to bind data listener + -cluster-address VAL : Cluster Controller host name + -data-listen-address VAL : IP Address to bind data listener If the cluster controller was directed to listen on a port other than the default, you will need to pass one more argument to hyracksnc. - -cc-port N : Cluster Controller port (default: 1099) + -cluster-port N : Cluster Controller port (default: 1099) -The data-ip-address is the interface on which the Node Controller must listen on -- in the event the machine is multi-homed it must listen on an IP that is reachable from -other Node Controllers. Make sure that the value passed to the data-ip-address is a valid IPv4 address (four octets separated by .). +The data-listen-address is the interface on which the Node Controller must listen on -- in the event the machine is multi-homed it must listen on an IP that is reachable from +other Node Controllers. Make sure that the value passed to the data-listen-address is a valid IPv4 address (four octets separated by .). 3. Running a job on Hyracks http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-server/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-server/pom.xml b/hyracks-fullstack/hyracks/hyracks-server/pom.xml index ded28ad..5e36b8a 100644 --- a/hyracks-fullstack/hyracks/hyracks-server/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-server/pom.xml @@ -47,9 +47,9 @@ <artifactId>maven-dependency-plugin</artifactId> <version>2.10</version> <configuration> - <failOnWarning>true</failOnWarning> - <outputXML>true</outputXML> - <usedDependencies>org.apache.hyracks:hyracks-control-nc</usedDependencies> + <usedDependencies combine.children="append"> + <usedDependency>org.apache.hyracks:hyracks-control-nc</usedDependency> + </usedDependencies> </configuration> <executions> <execution> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksCCProcess.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksCCProcess.java b/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksCCProcess.java index 4a70120..b2aa2d1 100644 --- a/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksCCProcess.java +++ b/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksCCProcess.java @@ -18,11 +18,11 @@ */ package org.apache.hyracks.server.process; -import org.apache.hyracks.control.cc.CCDriver; - import java.io.File; import java.util.List; +import org.apache.hyracks.control.cc.CCDriver; + public class HyracksCCProcess extends HyracksServerProcess { public HyracksCCProcess(File configFile, File logFile, File appHome, File workingDir) { @@ -41,5 +41,6 @@ public class HyracksCCProcess extends HyracksServerProcess { protected void addJvmArgs(List<String> cList) { // CC needs more than default memory cList.add("-Xmx1024m"); + //cList.add("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005"); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java index 2185826..fda099e 100644 --- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java +++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java @@ -34,8 +34,6 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import org.apache.hyracks.server.process.HyracksVirtualCluster; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf index 2339efb..69676f7 100644 --- a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf +++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf @@ -16,13 +16,15 @@ ; under the License. [nc/red] -address=127.0.0.1 +address = 127.0.0.1 +#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006 [nc/blue] -address=127.0.0.1 -port=9091 +address = 127.0.0.1 +ncservice.port = 9091 +#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007 [cc] -cluster.address = 127.0.0.1 -http.port = 12345 +address = 127.0.0.1 +console.listen.port = 12345 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-test-support/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/pom.xml b/hyracks-fullstack/hyracks/hyracks-test-support/pom.xml index 9e0535b..9a51b6c 100644 --- a/hyracks-fullstack/hyracks/hyracks-test-support/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-test-support/pom.xml @@ -105,9 +105,5 @@ <artifactId>hyracks-util</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCApplicationContext.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCApplicationContext.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCApplicationContext.java index 87739ea..81ee47b 100644 --- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCApplicationContext.java +++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCApplicationContext.java @@ -21,10 +21,10 @@ package org.apache.hyracks.test.support; import java.io.Serializable; import java.util.concurrent.ThreadFactory; -import org.apache.hyracks.api.application.IApplicationConfig; import org.apache.hyracks.api.application.INCApplicationContext; import org.apache.hyracks.api.application.IStateDumpHandler; import org.apache.hyracks.api.comm.IChannelInterfaceFactory; +import org.apache.hyracks.api.config.IApplicationConfig; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.api.job.IJobSerializerDeserializerContainer; import org.apache.hyracks.api.lifecycle.ILifeCycleComponentManager; @@ -32,7 +32,6 @@ import org.apache.hyracks.api.lifecycle.LifeCycleComponentManager; import org.apache.hyracks.api.messages.IMessageBroker; import org.apache.hyracks.api.resources.memory.IMemoryManager; import org.apache.hyracks.api.service.IControllerService; -import org.apache.hyracks.control.nc.io.IOManager; public class TestNCApplicationContext implements INCApplicationContext { private final ILifeCycleComponentManager lccm; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestUtils.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestUtils.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestUtils.java index 039cf7d..ab87f93 100644 --- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestUtils.java @@ -23,7 +23,6 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.Executors; -import org.apache.commons.lang3.StringUtils; import org.apache.hyracks.api.application.INCApplicationContext; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.dataflow.ActivityId; @@ -54,8 +53,4 @@ public class TestUtils { devices.add(new IODeviceHandle(new File(System.getProperty("java.io.tmpdir")), ".")); return new IOManager(devices, Executors.newCachedThreadPool()); } - - public static String joinPath(String... pathElements) { - return StringUtils.join(pathElements, File.separatorChar); - } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java index 31bce7a..a9a529c 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java @@ -66,7 +66,7 @@ public class StorageUtil { throw new AssertionError("This util class should not be initialized."); } - public static int getSizeInBytes(final int size, final StorageUnit unit) { + public static int getIntSizeInBytes(final int size, final StorageUnit unit) { double result = unit.toBytes(size); if (result > Integer.MAX_VALUE || result < Integer.MIN_VALUE) { throw new IllegalArgumentException("The given value:" + result + " is not within the integer range."); @@ -75,7 +75,7 @@ public class StorageUtil { } } - public static long getSizeInBytes(final long size, final StorageUnit unit) { + public static long getLongSizeInBytes(final long size, final StorageUnit unit) { double result = unit.toBytes(size); if (result > Long.MAX_VALUE || result < Long.MIN_VALUE) { throw new IllegalArgumentException("The given value:" + result + " is not within the long range."); @@ -85,8 +85,7 @@ public class StorageUtil { } /** - * Helper method to parse a byte unit string to its double value and unit - * (e.g., 10,345.8MB becomes Pair<10345.8, StorageUnit.MB>.) + * Helper method to parse a byte unit string to its double value in bytes * * @throws IllegalArgumentException */ http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java new file mode 100644 index 0000000..d44f1b4 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.util.file; + +import java.io.File; + +public class FileUtil { + private FileUtil() { + } + + public static String joinPath(String... elements) { + return String.join(File.separator, elements) + .replaceAll("([^:])(" + File.separator + ")+", "$1$2") + .replaceAll(File.separator + "$", ""); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c7b5bfa/hyracks-fullstack/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml index cbb83aa..3bf472d 100644 --- a/hyracks-fullstack/pom.xml +++ b/hyracks-fullstack/pom.xml @@ -140,6 +140,11 @@ <version>3.5</version> </dependency> <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-collections4</artifactId> + <version>4.1</version> + </dependency> + <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpcore</artifactId> <version>4.4.5</version> @@ -154,6 +159,11 @@ <artifactId>apache-rat-plugin</artifactId> <version>0.12</version> </dependency> + <dependency> + <groupId>args4j</groupId> + <artifactId>args4j</artifactId> + <version>2.33</version> + </dependency> </dependencies> </dependencyManagement> @@ -164,7 +174,7 @@ <artifactId>maven-jar-plugin</artifactId> <version>3.0.0</version> <configuration> - <excludes> + <excludes combine.children="append"> <exclude>**/DEPENDENCIES</exclude> </excludes> </configuration> @@ -176,6 +186,9 @@ <configuration> <failOnWarning>true</failOnWarning> <outputXML>true</outputXML> + <usedDependencies> + <usedDependency>org.slf4j:slf4j-simple</usedDependency> + </usedDependencies> </configuration> <executions> <execution> @@ -212,6 +225,10 @@ <licenseFamily implementation="org.apache.rat.license.Apache20LicenseFamily"/> </licenseFamilies> <excludeSubProjects>true</excludeSubProjects> + <excludes combine.children="append"> + <!-- TODO (mblow): ClusterControllerService should not get written outside of **/target/; remove once fixed --> + <exclude>**/ClusterControllerService/**</exclude> + </excludes> </configuration> </plugin> <plugin> @@ -232,7 +249,7 @@ <includes> <include>${global.test.includes},${test.includes}</include> </includes> - <excludes> + <excludes combine.children="append"> <exclude>${global.test.excludes},${test.excludes}</exclude> </excludes> </configuration> @@ -507,4 +524,14 @@ <module>algebricks</module> <module>hyracks-fullstack-license</module> </modules> + + <dependencies> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-simple</artifactId> + <version>1.7.22</version> + <scope>test</scope> + </dependency> + </dependencies> + </project>
