http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java index b933db8..c0c491a 100644 --- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java +++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java @@ -54,7 +54,6 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import org.apache.asterix.common.cluster.ClusterPartition; -import org.apache.asterix.common.config.ClusterProperties; import org.apache.asterix.common.config.ReplicationProperties; import org.apache.asterix.common.replication.IPartitionReplica; import org.apache.asterix.common.replication.IReplicaResourcesManager; @@ -64,6 +63,7 @@ import org.apache.asterix.common.replication.Replica; import org.apache.asterix.common.replication.Replica.ReplicaState; import org.apache.asterix.common.replication.ReplicaEvent; import org.apache.asterix.common.replication.ReplicationJob; +import org.apache.asterix.common.replication.ReplicationStrategyFactory; import org.apache.asterix.common.storage.DatasetResourceReference; import org.apache.asterix.common.storage.IIndexCheckpointManagerProvider; import org.apache.asterix.common.storage.ResourceReference; @@ -71,7 +71,6 @@ import org.apache.asterix.common.transactions.IAppRuntimeContextProvider; import org.apache.asterix.common.transactions.ILogManager; import org.apache.asterix.common.transactions.ILogRecord; import org.apache.asterix.common.transactions.LogType; -import org.apache.asterix.event.schema.cluster.Node; import org.apache.asterix.replication.functions.ReplicaFilesRequest; import org.apache.asterix.replication.functions.ReplicaIndexFlushRequest; import org.apache.asterix.replication.functions.ReplicationProtocol; @@ -83,11 +82,16 @@ import org.apache.asterix.replication.storage.LSMIndexFileProperties; import org.apache.asterix.replication.storage.ReplicaResourcesManager; import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository; import org.apache.hyracks.api.application.IClusterLifecycleListener.ClusterEventType; +import org.apache.hyracks.api.application.INCServiceContext; +import org.apache.hyracks.api.config.IApplicationConfig; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.replication.IReplicationJob; import org.apache.hyracks.api.replication.IReplicationJob.ReplicationExecutionType; import org.apache.hyracks.api.replication.IReplicationJob.ReplicationJobType; import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation; +import org.apache.hyracks.control.common.controllers.NCConfig; +import org.apache.hyracks.control.nc.NodeControllerService; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexReplicationJob; import org.apache.hyracks.util.StorageUtil; import org.apache.hyracks.util.StorageUtil.StorageUnit; @@ -137,26 +141,33 @@ public class ReplicationManager implements IReplicationManager { private Future<? extends Object> txnLogReplicatorTask; private SocketChannel[] logsRepSockets; private final ByteBuffer txnLogsBatchSizeBuffer = ByteBuffer.allocate(Integer.BYTES); - private final IReplicationStrategy replicationStrategy; + private IReplicationStrategy replicationStrategy; private final PersistentLocalResourceRepository localResourceRepo; + private NCConfig ncConfig; private final IIndexCheckpointManagerProvider indexCheckpointManagerProvider; //TODO this class needs to be refactored by moving its private classes to separate files //and possibly using MessageBroker to send/receive remote replicas events. public ReplicationManager(String nodeId, ReplicationProperties replicationProperties, IReplicaResourcesManager remoteResoucesManager, ILogManager logManager, - IAppRuntimeContextProvider asterixAppRuntimeContextProvider) { + IAppRuntimeContextProvider asterixAppRuntimeContextProvider, INCServiceContext ncServiceContext) { this.nodeId = nodeId; + this.ncConfig = ((NodeControllerService) ncServiceContext.getControllerService()).getConfiguration(); this.replicationProperties = replicationProperties; - replicationStrategy = replicationProperties.getReplicationStrategy(); + try { + replicationStrategy = ReplicationStrategyFactory.create(replicationProperties.getReplicationStrategy(), + replicationProperties, ncConfig.getConfigManager()); + } catch (HyracksDataException e) { + LOGGER.log(Level.WARNING, "Couldn't initialize replication strategy", e); + } this.replicaResourcesManager = (ReplicaResourcesManager) remoteResoucesManager; this.asterixAppRuntimeContextProvider = asterixAppRuntimeContextProvider; this.logManager = logManager; - this.indexCheckpointManagerProvider = - asterixAppRuntimeContextProvider.getAppContext().getIndexCheckpointManagerProvider(); localResourceRepo = (PersistentLocalResourceRepository) asterixAppRuntimeContextProvider.getLocalResourceRepository(); - this.hostIPAddressFirstOctet = replicationProperties.getReplicaIPAddress(nodeId).substring(0, 3); + this.hostIPAddressFirstOctet = ncConfig.getPublicAddress().substring(0, 3); + this.indexCheckpointManagerProvider = + asterixAppRuntimeContextProvider.getAppContext().getIndexCheckpointManagerProvider(); replicas = new HashMap<>(); replicationJobsQ = new LinkedBlockingQueue<>(); replicaEventsQ = new LinkedBlockingQueue<>(); @@ -169,7 +180,7 @@ public class ReplicationManager implements IReplicationManager { dataBuffer = ByteBuffer.allocate(INITIAL_BUFFER_SIZE); replicationMonitor = new ReplicasEventsMonitor(); //add list of replicas from configurations (To be read from another source e.g. Zookeeper) - Set<Replica> replicaNodes = replicationProperties.getReplicationStrategy().getRemoteReplicas(nodeId); + Set<Replica> replicaNodes = replicationStrategy.getRemoteReplicas(nodeId); //Used as async listeners from replicas replicationListenerThreads = Executors.newCachedThreadPool(); @@ -181,11 +192,11 @@ public class ReplicationManager implements IReplicationManager { for (Replica replica : replicaNodes) { replicas.put(replica.getId(), replica); //for each remote replica, get the list of replication clients - Set<String> nodeReplicationClients = replicationProperties.getRemotePrimaryReplicasIds(replica.getId()); + Set<Replica> nodeReplicationClients = replicationStrategy.getRemotePrimaryReplicas(replica.getId()); //get the partitions of each client List<Integer> clientPartitions = new ArrayList<>(); - for (String clientId : nodeReplicationClients) { - for (ClusterPartition clusterPartition : nodePartitions.get(clientId)) { + for (Replica client : nodeReplicationClients) { + for (ClusterPartition clusterPartition : nodePartitions.get(client.getId())) { clientPartitions.add(clusterPartition.getPartitionId()); } } @@ -313,8 +324,8 @@ public class ReplicationManager implements IReplicationManager { //send LSMComponent properties LSMComponentJob = (ILSMIndexReplicationJob) job; LSMComponentProperties lsmCompProp = new LSMComponentProperties(LSMComponentJob, nodeId); - requestBuffer = - ReplicationProtocol.writeLSMComponentPropertiesRequest(lsmCompProp, requestBuffer); + requestBuffer = ReplicationProtocol.writeLSMComponentPropertiesRequest(lsmCompProp, //NOSONAR + requestBuffer); sendRequest(replicasSockets, requestBuffer); } @@ -438,17 +449,17 @@ public class ReplicationManager implements IReplicationManager { @Override public boolean isReplicationEnabled() { - return replicationProperties.isParticipant(nodeId); + return replicationStrategy.isParticipant(nodeId); } @Override public synchronized void updateReplicaInfo(Replica replicaNode) { - Replica replica = replicas.get(replicaNode.getNode().getId()); + Replica replica = replicas.get(replicaNode.getId()); //should not update the info of an active replica if (replica.getState() == ReplicaState.ACTIVE) { return; } - replica.getNode().setClusterIp(replicaNode.getNode().getClusterIp()); + replica.setClusterIp(replicaNode.getClusterIp()); } /** @@ -616,10 +627,8 @@ public class ReplicationManager implements IReplicationManager { * @throws IOException */ private void sendShutdownNotifiction() throws IOException { - Node node = new Node(); - node.setId(nodeId); - node.setClusterIp(NetworkingUtil.getHostAddress(hostIPAddressFirstOctet)); - Replica replica = new Replica(node); + Replica replica = new Replica(nodeId, NetworkingUtil.getHostAddress(hostIPAddressFirstOctet), + ncConfig.getReplicationPublicPort()); ReplicaEvent event = new ReplicaEvent(replica, ClusterEventType.NODE_SHUTTING_DOWN); ByteBuffer buffer = ReplicationProtocol.writeReplicaEventRequest(event); Map<String, SocketChannel> replicaSockets = getActiveRemoteReplicasSockets(); @@ -678,7 +687,7 @@ public class ReplicationManager implements IReplicationManager { @Override public void initializeReplicasState() { for (Replica replica : replicas.values()) { - checkReplicaState(replica.getNode().getId(), false, false); + checkReplicaState(replica.getId(), false, false); } } @@ -696,7 +705,7 @@ public class ReplicationManager implements IReplicationManager { Replica replica = replicas.get(replicaId); ReplicaStateChecker connector = new ReplicaStateChecker(replica, replicationProperties.getReplicationTimeOut(), - this, replicationProperties, suspendReplication); + this, suspendReplication); Future<? extends Object> ft = asterixAppRuntimeContextProvider.getThreadExecutor().submit(connector); if (!async) { @@ -848,11 +857,11 @@ public class ReplicationManager implements IReplicationManager { * @throws IOException */ private SocketChannel getReplicaSocket(String replicaId) throws IOException { - Replica replica = replicationProperties.getReplicaById(replicaId); SocketChannel sc = SocketChannel.open(); sc.configureBlocking(true); - InetSocketAddress address = replica.getAddress(replicationProperties); - sc.connect(new InetSocketAddress(address.getHostString(), address.getPort())); + IApplicationConfig config = ncConfig.getConfigManager().getNodeEffectiveConfig(replicaId); + sc.connect(new InetSocketAddress(config.getString(NCConfig.Option.REPLICATION_LISTEN_ADDRESS), + config.getInt(NCConfig.Option.REPLICATION_LISTEN_PORT))); return sc; } @@ -861,7 +870,7 @@ public class ReplicationManager implements IReplicationManager { Set<String> replicasIds = new HashSet<>(); for (Replica replica : replicas.values()) { if (replica.getState() == ReplicaState.DEAD) { - replicasIds.add(replica.getNode().getId()); + replicasIds.add(replica.getId()); } } return replicasIds; @@ -872,7 +881,7 @@ public class ReplicationManager implements IReplicationManager { Set<String> replicasIds = new HashSet<>(); for (Replica replica : replicas.values()) { if (replica.getState() == ReplicaState.ACTIVE) { - replicasIds.add(replica.getNode().getId()); + replicasIds.add(replica.getId()); } } return replicasIds; @@ -970,9 +979,8 @@ public class ReplicationManager implements IReplicationManager { private String getReplicaIdBySocket(SocketChannel socketChannel) { InetSocketAddress socketAddress = NetworkingUtil.getSocketAddress(socketChannel); for (Replica replica : replicas.values()) { - InetSocketAddress replicaAddress = replica.getAddress(replicationProperties); - if (replicaAddress.getHostName().equals(socketAddress.getHostName()) - && replicaAddress.getPort() == socketAddress.getPort()) { + if (replica.getClusterIp().equals(socketAddress.getHostName()) + && ncConfig.getReplicationPublicPort() == socketAddress.getPort()) { return replica.getId(); } } @@ -1176,21 +1184,21 @@ public class ReplicationManager implements IReplicationManager { buffer.reset(); } } - //move the buffer position to the sent limit + //move the bufeer position to the sent limit buffer.position(buffer.limit()); } @Override public void register(IPartitionReplica replica) { // find the replica node based on ip and replication port - final Optional<Node> replicaNode = ClusterProperties.INSTANCE.getCluster().getNode().stream() + Optional<Replica> replicaNode = replicationStrategy.getRemoteReplicasAndSelf(nodeId).stream() .filter(node -> node.getClusterIp().equals(replica.getIdentifier().getLocation().getHostString()) - && node.getReplicationPort().intValue() == replica.getIdentifier().getLocation().getPort()) + && node.getPort() == replica.getIdentifier().getLocation().getPort()) .findAny(); if (!replicaNode.isPresent()) { throw new IllegalStateException("Couldn't find node for replica: " + replica); } - Replica replicaRef = new Replica(replicaNode.get()); + Replica replicaRef = replicaNode.get(); final String replicaId = replicaRef.getId(); replicas.putIfAbsent(replicaId, replicaRef); replica2PartitionsMap.computeIfAbsent(replicaId, k -> new HashSet<>()); @@ -1353,4 +1361,9 @@ public class ReplicationManager implements IReplicationManager { } } } -} \ No newline at end of file + + @Override + public IReplicationStrategy getReplicationStrategy() { + return replicationStrategy; + } +}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/recovery/RemoteRecoveryManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/recovery/RemoteRecoveryManager.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/recovery/RemoteRecoveryManager.java index f3eea32..54d8562 100644 --- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/recovery/RemoteRecoveryManager.java +++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/recovery/RemoteRecoveryManager.java @@ -32,11 +32,12 @@ import java.util.stream.Collectors; import org.apache.asterix.common.api.IDatasetLifecycleManager; import org.apache.asterix.common.api.INcApplicationContext; import org.apache.asterix.common.cluster.ClusterPartition; -import org.apache.asterix.common.config.ClusterProperties; import org.apache.asterix.common.config.ReplicationProperties; import org.apache.asterix.common.exceptions.ACIDException; import org.apache.asterix.common.replication.IRemoteRecoveryManager; import org.apache.asterix.common.replication.IReplicationManager; +import org.apache.asterix.common.replication.IReplicationStrategy; +import org.apache.asterix.common.replication.Replica; import org.apache.asterix.common.transactions.ILogManager; import org.apache.asterix.common.transactions.IRecoveryManager; import org.apache.asterix.common.utils.StorageConstants; @@ -51,33 +52,35 @@ public class RemoteRecoveryManager implements IRemoteRecoveryManager { private final INcApplicationContext runtimeContext; private final ReplicationProperties replicationProperties; private Map<String, Set<String>> failbackRecoveryReplicas; + private IReplicationStrategy replicationStrategy; public RemoteRecoveryManager(IReplicationManager replicationManager, INcApplicationContext runtimeContext, ReplicationProperties replicationProperties) { this.replicationManager = replicationManager; this.runtimeContext = runtimeContext; this.replicationProperties = replicationProperties; + this.replicationStrategy = replicationManager.getReplicationStrategy(); } private Map<String, Set<String>> constructRemoteRecoveryPlan() { //1. identify which replicas reside in this node String localNodeId = runtimeContext.getTransactionSubsystem().getId(); - Set<String> nodes = replicationProperties.getNodeReplicasIds(localNodeId); + Set<Replica> replicas = replicationStrategy.getRemoteReplicasAndSelf(localNodeId); Map<String, Set<String>> recoveryCandidates = new HashMap<>(); Map<String, Integer> candidatesScore = new HashMap<>(); //2. identify which nodes has backup per lost node data - for (String node : nodes) { - Set<String> locations = replicationProperties.getNodeReplicasIds(node); + for (Replica node : replicas) { + Set<Replica> locations = replicationStrategy.getRemoteReplicasAndSelf(node.getId()); //since the local node just started, remove it from candidates - locations.remove(localNodeId); + locations.remove(new Replica(localNodeId, "", -1)); //remove any dead replicas Set<String> deadReplicas = replicationManager.getDeadReplicasIds(); for (String deadReplica : deadReplicas) { - locations.remove(deadReplica); + locations.remove(new Replica(deadReplica, "", -1)); } //no active replicas to recover from @@ -85,14 +88,15 @@ public class RemoteRecoveryManager implements IRemoteRecoveryManager { throw new IllegalStateException("Could not find any ACTIVE replica to recover " + node + " data."); } - for (String location : locations) { + for (Replica locationRep : locations) { + String location = locationRep.getId(); if (candidatesScore.containsKey(location)) { candidatesScore.put(location, candidatesScore.get(location) + 1); } else { candidatesScore.put(location, 1); } } - recoveryCandidates.put(node, locations); + recoveryCandidates.put(node.getId(), locations.stream().map(Replica::getId).collect(Collectors.toSet())); } Map<String, Set<String>> recoveryList = new HashMap<>(); @@ -156,8 +160,8 @@ public class RemoteRecoveryManager implements IRemoteRecoveryManager { replayReplicaPartitionLogs(partitionsToTakeover, false); //mark these partitions as active in this node - PersistentLocalResourceRepository resourceRepository = - (PersistentLocalResourceRepository) runtimeContext.getLocalResourceRepository(); + PersistentLocalResourceRepository resourceRepository = (PersistentLocalResourceRepository) runtimeContext + .getLocalResourceRepository(); for (Integer patitionId : partitions) { resourceRepository.addActivePartition(patitionId); } @@ -166,8 +170,8 @@ public class RemoteRecoveryManager implements IRemoteRecoveryManager { @Override public void startFailbackProcess() { int maxRecoveryAttempts = replicationProperties.getMaxRemoteRecoveryAttempts(); - PersistentLocalResourceRepository resourceRepository = - (PersistentLocalResourceRepository) runtimeContext.getLocalResourceRepository(); + PersistentLocalResourceRepository resourceRepository = (PersistentLocalResourceRepository) runtimeContext + .getLocalResourceRepository(); IDatasetLifecycleManager datasetLifeCycleManager = runtimeContext.getDatasetLifecycleManager(); Map<String, ClusterPartition[]> nodePartitions = runtimeContext.getMetadataProperties().getNodePartitions(); @@ -223,8 +227,8 @@ public class RemoteRecoveryManager implements IRemoteRecoveryManager { @Override public void completeFailbackProcess() throws IOException, InterruptedException { ILogManager logManager = runtimeContext.getTransactionSubsystem().getLogManager(); - ReplicaResourcesManager replicaResourcesManager = - (ReplicaResourcesManager) runtimeContext.getReplicaResourcesManager(); + ReplicaResourcesManager replicaResourcesManager = (ReplicaResourcesManager) runtimeContext + .getReplicaResourcesManager(); Map<String, ClusterPartition[]> nodePartitions = runtimeContext.getMetadataProperties().getNodePartitions(); /* @@ -278,8 +282,8 @@ public class RemoteRecoveryManager implements IRemoteRecoveryManager { @Override public void doRemoteRecoveryPlan(Map<String, Set<Integer>> recoveryPlan) throws HyracksDataException { int maxRecoveryAttempts = replicationProperties.getMaxRemoteRecoveryAttempts(); - PersistentLocalResourceRepository resourceRepository = - (PersistentLocalResourceRepository) runtimeContext.getLocalResourceRepository(); + PersistentLocalResourceRepository resourceRepository = (PersistentLocalResourceRepository) runtimeContext + .getLocalResourceRepository(); IDatasetLifecycleManager datasetLifeCycleManager = runtimeContext.getDatasetLifecycleManager(); ILogManager logManager = runtimeContext.getTransactionSubsystem().getLogManager(); while (true) { @@ -320,4 +324,4 @@ public class RemoteRecoveryManager implements IRemoteRecoveryManager { } } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/recovery/ReplicaSynchronizer.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/recovery/ReplicaSynchronizer.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/recovery/ReplicaSynchronizer.java index 5c88460..1fa3246 100644 --- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/recovery/ReplicaSynchronizer.java +++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/recovery/ReplicaSynchronizer.java @@ -51,8 +51,7 @@ public class ReplicaSynchronizer { final ReplicaFilesSynchronizer fileSync = new ReplicaFilesSynchronizer(appCtx, replica); fileSync.sync(); // flush replicated dataset to generate disk component for any remaining in-memory components - final ReplicationProperties repl = appCtx.getReplicationProperties(); - final IReplicationStrategy replStrategy = repl.getReplicationStrategy(); + final IReplicationStrategy replStrategy = appCtx.getReplicationManager().getReplicationStrategy(); appCtx.getDatasetLifecycleManager().flushDataset(replStrategy); // sync any newly generated files fileSync.sync(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-replication/src/test/resources/scripts/kill_cc_and_nc.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-replication/src/test/resources/scripts/kill_cc_and_nc.sh b/asterixdb/asterix-replication/src/test/resources/scripts/kill_cc_and_nc.sh index 4b876be..9c6b9f5 100755 --- a/asterixdb/asterix-replication/src/test/resources/scripts/kill_cc_and_nc.sh +++ b/asterixdb/asterix-replication/src/test/resources/scripts/kill_cc_and_nc.sh @@ -16,3 +16,4 @@ # specific language governing permissions and limitations # under the License. ps -ef | awk '/java.*org\.apache\.hyracks\.control\.[cn]c\.[CN]CDriver/ {print $2}' | xargs -n 1 kill -9 +ps -ef | awk '/java.*org\.apache\.hyracks\.control\.nc\.NCService/ {print $2}' | xargs -n 1 kill -9 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/message/ReplicaEventMessage.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/message/ReplicaEventMessage.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/message/ReplicaEventMessage.java index fe230b4..8ae1bd7 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/message/ReplicaEventMessage.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/message/ReplicaEventMessage.java @@ -22,7 +22,6 @@ import org.apache.asterix.common.api.INcApplicationContext; import org.apache.asterix.common.messaging.api.INcAddressedMessage; import org.apache.asterix.common.replication.Replica; import org.apache.asterix.common.replication.ReplicaEvent; -import org.apache.asterix.event.schema.cluster.Node; import org.apache.hyracks.api.application.IClusterLifecycleListener.ClusterEventType; import org.apache.hyracks.api.exceptions.HyracksDataException; @@ -32,10 +31,12 @@ public class ReplicaEventMessage implements INcAddressedMessage { private final String nodeId; private final ClusterEventType event; private final String nodeIPAddress; + private final int nodePort; - public ReplicaEventMessage(String nodeId, String nodeIPAddress, ClusterEventType event) { + public ReplicaEventMessage(String nodeId, String nodeIPAddress, int nodePort, ClusterEventType event) { this.nodeId = nodeId; this.nodeIPAddress = nodeIPAddress; + this.nodePort = nodePort; this.event = event; } @@ -53,10 +54,7 @@ public class ReplicaEventMessage implements INcAddressedMessage { @Override public void handle(INcApplicationContext appContext) throws HyracksDataException, InterruptedException { - Node node = new Node(); - node.setId(nodeId); - node.setClusterIp(nodeIPAddress); - Replica replica = new Replica(node); + Replica replica = new Replica(nodeId, nodeIPAddress, nodePort); appContext.getReplicationManager().reportReplicaEvent(new ReplicaEvent(replica, event)); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java index 10926e0..ead5f53 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java @@ -35,14 +35,11 @@ import java.util.logging.Logger; import org.apache.asterix.common.api.IClusterManagementWork.ClusterState; import org.apache.asterix.common.cluster.ClusterPartition; import org.apache.asterix.common.cluster.IClusterStateManager; -import org.apache.asterix.common.config.ClusterProperties; -import org.apache.asterix.common.dataflow.ICcApplicationContext; import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.common.replication.IFaultToleranceStrategy; +import org.apache.asterix.common.dataflow.ICcApplicationContext; import org.apache.asterix.common.transactions.IResourceIdManager; -import org.apache.asterix.event.schema.cluster.Cluster; -import org.apache.asterix.event.schema.cluster.Node; import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.config.IOption; @@ -71,7 +68,6 @@ public class ClusterStateManager implements IClusterStateManager { private static final Logger LOGGER = Logger.getLogger(ClusterStateManager.class.getName()); private final Map<String, Map<IOption, Object>> ncConfigMap = new HashMap<>(); private Set<String> pendingRemoval = new HashSet<>(); - private final Cluster cluster; private ClusterState state = ClusterState.UNUSABLE; private AlgebricksAbsolutePartitionConstraint clusterPartitionConstraint; private Map<String, ClusterPartition[]> node2PartitionsMap; @@ -83,10 +79,6 @@ public class ClusterStateManager implements IClusterStateManager { private IFaultToleranceStrategy ftStrategy; private ICcApplicationContext appCtx; - public ClusterStateManager() { - cluster = ClusterProperties.INSTANCE.getCluster(); - } - @Override public void setCcAppCtx(ICcApplicationContext appCtx) { this.appCtx = appCtx; @@ -179,8 +171,8 @@ public class ClusterStateManager implements IClusterStateManager { } resetClusterPartitionConstraint(); // if the cluster has no registered partitions or all partitions are pending activation -> UNUSABLE - if (clusterPartitions.isEmpty() || clusterPartitions.values().stream() - .allMatch(ClusterPartition::isPendingActivation)) { + if (clusterPartitions.isEmpty() + || clusterPartitions.values().stream().allMatch(ClusterPartition::isPendingActivation)) { LOGGER.info("Cluster does not have any registered partitions"); setState(ClusterState.UNUSABLE); return; @@ -262,12 +254,6 @@ public class ClusterStateManager implements IClusterStateManager { } @Override - public synchronized Node getAvailableSubstitutionNode() { - List<Node> subNodes = cluster.getSubstituteNodes() == null ? null : cluster.getSubstituteNodes().getNode(); - return subNodes == null || subNodes.isEmpty() ? null : subNodes.get(0); - } - - @Override public synchronized Set<String> getParticipantNodes() { return new HashSet<>(participantNodes); } @@ -302,10 +288,6 @@ public class ClusterStateManager implements IClusterStateManager { @Override public synchronized boolean isClusterActive() { - if (cluster == null) { - // this is a virtual cluster - return true; - } return state == ClusterState.ACTIVE; } @@ -456,6 +438,11 @@ public class ClusterStateManager implements IClusterStateManager { } } + @Override + public Map<String, Map<IOption, Object>> getActiveNcConfiguration() { + return ncConfigMap; + } + public synchronized Set<String> getNodesPendingRemoval() { return new HashSet<>(pendingRemoval); } @@ -470,4 +457,8 @@ public class ClusterStateManager implements IClusterStateManager { }); } + public String getStoragePathPrefix() { + return appCtx.getNodeProperties().getStorageSubdir(); + } + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-server/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml index cdd680b..7780f8b 100644 --- a/asterixdb/asterix-server/pom.xml +++ b/asterixdb/asterix-server/pom.xml @@ -20,6 +20,16 @@ <modelVersion>4.0.0</modelVersion> <artifactId>asterix-server</artifactId> <name>asterix-server</name> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <appendedResourcesDirectory>${basedir}/src/main/appended-resources</appendedResourcesDirectory> + <failsafe.test.excludes>**/DmlRecoveryIT.java</failsafe.test.excludes> + <cluster.test.excludes>**/AsterixClusterLifeCycleIT.java</cluster.test.excludes> + <cluster.extest.excludes>**/ClusterExecutionIT.java</cluster.extest.excludes> + <appendedResourcesDirectory>${basedir}/../src/main/appended-resources</appendedResourcesDirectory> + </properties> + <parent> <groupId>org.apache.asterix</groupId> <artifactId>apache-asterixdb</artifactId> @@ -35,10 +45,6 @@ </license> </licenses> - <properties> - <appendedResourcesDirectory>${basedir}/../src/main/appended-resources</appendedResourcesDirectory> - </properties> - <build> <plugins> <plugin> @@ -334,6 +340,27 @@ </executions> </plugin> <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + <configuration> + <excludes combine.children="append"> + <exclude>${failsafe.test.excludes}</exclude> + <exclude>${cluster.test.excludes}</exclude> + <exclude>${cluster.extest.excludes}</exclude> + </excludes> + <executions> + <execution> + <id>run-tests</id> + <phase>integration-test</phase> + <goals> + <goal>integration-test</goal> + <goal>verify</goal> + </goals> + </execution> + </executions> + </configuration> + </plugin> + <plugin> <artifactId>maven-assembly-plugin</artifactId> <executions> <execution> @@ -552,6 +579,16 @@ <classifier>assembly</classifier> </dependency> <dependency> + <groupId>org.apache.asterix</groupId> + <artifactId>asterix-external-data</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.codehaus.plexus</groupId> + <artifactId>plexus-utils</artifactId> + <version>3.0.24</version> + </dependency> + <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf b/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf index bec2122..cc2d9bd 100644 --- a/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf +++ b/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf @@ -15,13 +15,13 @@ ; specific language governing permissions and limitations ; under the License. -[nc/red] +[nc/asterix_nc1] txn.log.dir=data/red/txnlog core.dump.dir=data/red/coredump iodevices=data/red nc.api.port=19004 -[nc/blue] +[nc/asterix_nc2] ncservice.port=9091 txn.log.dir=data/blue/txnlog core.dump.dir=data/blue/coredump http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/AbstractExecutionIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/AbstractExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/AbstractExecutionIT.java new file mode 100644 index 0000000..6c14aa0 --- /dev/null +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/AbstractExecutionIT.java @@ -0,0 +1,164 @@ +/* + * Copyright 2009-2013 by The Regents of the University of California + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * you may obtain a copy of the License from + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.asterix.server.test; + +import org.apache.asterix.external.util.ExternalDataConstants; +import org.apache.asterix.external.util.IdentitiyResolverFactory; +import org.apache.asterix.test.base.RetainLogsRule; +import org.apache.asterix.test.common.TestExecutor; +import org.apache.asterix.testframework.context.TestCaseContext; +import org.apache.asterix.testframework.context.TestFileContext; +import org.apache.asterix.testframework.xml.TestCase.CompilationUnit; +import org.apache.asterix.testframework.xml.TestGroup; +import org.apache.commons.lang3.StringUtils; +import org.apache.hyracks.util.file.FileUtil; +import org.codehaus.plexus.util.FileUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static org.apache.hyracks.util.file.FileUtil.joinPath; + +/** + * Runs the runtime test cases under 'asterix-app/src/test/resources/runtimets'. + */ +@RunWith(Parameterized.class) +public abstract class AbstractExecutionIT { + + protected static final Logger LOGGER = Logger.getLogger(AbstractExecutionIT.class.getName()); + + protected static final String PATH_ACTUAL = joinPath("target", "ittest"); + protected static final String PATH_BASE = joinPath("..", "asterix-app", "src", "test", "resources", "runtimets"); + + protected static final String HDFS_BASE = "../asterix-app/"; + + protected static final TestExecutor testExecutor = new TestExecutor(); + + private static final String EXTERNAL_LIBRARY_TEST_GROUP = "lib"; + + private static final List<String> badTestCases = new ArrayList<>(); + + private static String reportPath = new File(joinPath("target", "failsafe-reports")).getAbsolutePath(); + + @Rule + public TestRule retainLogs = new RetainLogsRule(NCServiceExecutionIT.LOG_DIR, reportPath, this); + + @BeforeClass + public static void setUp() throws Exception { + System.out.println("Starting setup"); + if (LOGGER.isLoggable(Level.INFO)) { + LOGGER.info("Starting setup"); + } + File outdir = new File(PATH_ACTUAL); + outdir.mkdirs(); + + File externalTestsJar = + new File(StringUtils.join(new String[] { "..", "asterix-external-data", "target" }, File.separator)) + .listFiles((dir, name) -> name.matches("asterix-external-data-.*-tests.jar"))[0]; + + FileUtils.copyFile(externalTestsJar, + new File(NCServiceExecutionIT.APP_HOME + "/repo", externalTestsJar.getName())); + + NCServiceExecutionIT.setUp(); + + FileUtils.copyDirectoryStructure(new File(joinPath("..", "asterix-app", "data")), + new File(NCServiceExecutionIT.ASTERIX_APP_DIR + "/clusters/local/working_dir/data")); + + FileUtils.copyDirectoryStructure(new File(joinPath("..", "asterix-app", "target", "data")), + new File(NCServiceExecutionIT.ASTERIX_APP_DIR + "/clusters/local/working_dir/target/data")); + + // Set the node resolver to be the identity resolver that expects node names + // to be node controller ids; a valid assumption in test environment. + System.setProperty(ExternalDataConstants.NODE_RESOLVER_FACTORY_PROPERTY, + IdentitiyResolverFactory.class.getName()); + + reportPath = new File(joinPath("target", "failsafe-reports")).getAbsolutePath(); + } + + @AfterClass + public static void tearDown() throws Exception { + File outdir = new File(PATH_ACTUAL); + File[] files = outdir.listFiles(); + if ((files == null) || (files.length == 0)) { + outdir.delete(); + } + //AsterixLifecycleIT.tearDown(); + NCServiceExecutionIT.tearDown(); + if (!badTestCases.isEmpty()) { + System.out.println("The following test cases left some data"); + for (String testCase : badTestCases) { + System.out.println(testCase); + } + } + } + + @Parameters + public static Collection<Object[]> tests() throws Exception { + Collection<Object[]> testArgs = new ArrayList<Object[]>(); + TestCaseContext.Builder b = new TestCaseContext.Builder(); + for (TestCaseContext ctx : b.build(new File(PATH_BASE))) { + testArgs.add(new Object[] { ctx }); + } + return testArgs; + } + + private TestCaseContext tcCtx; + + public AbstractExecutionIT(TestCaseContext tcCtx) { + this.tcCtx = tcCtx; + } + + @Test + public void test() throws Exception { + if (skip()) { + return; + } + testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false); + testExecutor.cleanup(tcCtx.toString(), badTestCases); + } + + protected boolean skip() { + // If the test case contains library commands, we skip them + List<CompilationUnit> cUnits = tcCtx.getTestCase().getCompilationUnit(); + for (CompilationUnit cUnit : cUnits) { + List<TestFileContext> testFileCtxs = tcCtx.getTestFiles(cUnit); + for (TestFileContext ctx : testFileCtxs) { + if (ctx.getType().equals(EXTERNAL_LIBRARY_TEST_GROUP)) { + return true; + } + } + } + // For now we skip api tests. + for (TestGroup group : tcCtx.getTestGroups()) { + if (group != null && "api".equals(group.getName())) { + LOGGER.info("Skipping test: " + tcCtx.toString()); + return true; + } + } + return false; + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/DmlRecoveryIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/DmlRecoveryIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/DmlRecoveryIT.java new file mode 100644 index 0000000..bc209c3 --- /dev/null +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/DmlRecoveryIT.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.server.test; + +import java.io.File; +import java.io.FilenameFilter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; +import java.util.logging.Logger; + +import org.apache.asterix.test.base.RetainLogsRule; +import org.apache.asterix.test.common.TestExecutor; +import org.apache.asterix.testframework.context.TestCaseContext; +import org.apache.commons.io.FileUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class DmlRecoveryIT { + + // variable to indicate whether this test will be executed + + private static final Logger LOGGER = Logger.getLogger(RecoveryIT.class.getName()); + private static final String PATH_ACTUAL = "target" + File.separator + "rttest" + File.separator; + + private static final String TESTSUITE_PATH_BASE = "../asterix-app/src/test/resources/runtimets/"; + + private TestCaseContext tcCtx; + private static File asterixInstallerPath; + private static File installerTargetPath; + private static String managixHomeDirName; + private static String managixHomePath; + private static String scriptHomePath; + private static String reportPath; + private static ProcessBuilder pb; + private static Map<String, String> env; + private final TestExecutor testExecutor = new TestExecutor(); + + @Rule + public TestRule retainLogs = new RetainLogsRule(managixHomePath, reportPath, this); + + @BeforeClass + public static void setUp() throws Exception { + File outdir = new File(PATH_ACTUAL); + outdir.mkdirs(); + + asterixInstallerPath = new File(System.getProperty("user.dir")); + installerTargetPath = new File(asterixInstallerPath, "target"); + reportPath = new File(installerTargetPath, "failsafe-reports").getAbsolutePath(); + managixHomeDirName = installerTargetPath.list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return new File(dir, name).isDirectory() && name.startsWith("asterix-installer") + && name.endsWith("binary-assembly"); + } + })[0]; + managixHomePath = new File(installerTargetPath, managixHomeDirName).getAbsolutePath(); + LOGGER.info("MANAGIX_HOME=" + managixHomePath); + + pb = new ProcessBuilder(); + env = pb.environment(); + env.put("MANAGIX_HOME", managixHomePath); + scriptHomePath = asterixInstallerPath + File.separator + "src" + File.separator + "test" + File.separator + + "resources" + File.separator + "transactionts" + File.separator + "scripts"; + env.put("SCRIPT_HOME", scriptHomePath); + + TestExecutor.executeScript(pb, + scriptHomePath + File.separator + "dml_recovery" + File.separator + "configure_and_validate.sh"); + TestExecutor.executeScript(pb, + scriptHomePath + File.separator + "dml_recovery" + File.separator + "stop_and_delete.sh"); + + TestExecutor.executeScript(pb, + scriptHomePath + File.separator + "dml_recovery" + File.separator + "create_and_start.sh"); + + } + + @AfterClass + public static void tearDown() throws Exception { + File outdir = new File(PATH_ACTUAL); + FileUtils.deleteDirectory(outdir); + TestExecutor.executeScript(pb, + scriptHomePath + File.separator + "dml_recovery" + File.separator + "stop_and_delete.sh"); + TestExecutor.executeScript(pb, + scriptHomePath + File.separator + "dml_recovery" + File.separator + "shutdown.sh"); + + } + + @Parameters + public static Collection<Object[]> tests() throws Exception { + + Collection<Object[]> testArgs = new ArrayList<Object[]>(); + TestCaseContext.Builder b = new TestCaseContext.Builder(); + for (TestCaseContext ctx : b.build(new File(TESTSUITE_PATH_BASE))) { + if (ctx.getTestCase().getFilePath().equals("dml")) { + testArgs.add(new Object[] { ctx }); + } + } + return testArgs; + } + + public DmlRecoveryIT(TestCaseContext tcCtx) { + this.tcCtx = tcCtx; + } + + @Test + public void test() throws Exception { + testExecutor.executeTest(PATH_ACTUAL, tcCtx, pb, true); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/MetadataReplicationIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/MetadataReplicationIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/MetadataReplicationIT.java new file mode 100644 index 0000000..445f53f --- /dev/null +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/MetadataReplicationIT.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.server.test; + +import java.io.File; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +import org.apache.asterix.test.base.RetainLogsRule; +import org.apache.asterix.test.common.TestExecutor; +import org.apache.asterix.testframework.context.TestCaseContext; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hyracks.server.process.HyracksCCProcess; +import org.apache.hyracks.server.process.HyracksNCServiceProcess; +import org.apache.hyracks.server.process.HyracksVirtualCluster; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class MetadataReplicationIT { + + // Important paths and files for this test. + + // The "target" subdirectory of asterix-server. All outputs go here. + public static final String TARGET_DIR = StringUtils.join(new String[] { "../asterix-server/target" }, + File.separator); + + // Directory where the NCs create and store all data, as configured by + // src/test/resources/NCServiceExecutionIT/cc.conf. + public static final String INSTANCE_DIR = StringUtils.join(new String[] { TARGET_DIR, "tmp" }, File.separator); + + // The log directory, where all CC, NCService, and NC logs are written. CC and + // NCService logs are configured on the HyracksVirtualCluster below. NC logs + // are configured in src/test/resources/NCServiceExecutionIT/ncservice*.conf. + public static final String LOG_DIR = StringUtils.join(new String[] { TARGET_DIR, "failsafe-reports" }, + File.separator); + + // Directory where *.conf files are located. + public static final String CONF_DIR = StringUtils + .join(new String[] { TARGET_DIR, "test-classes", "MetadataReplicationIT" }, File.separator); + + // The app.home specified for HyracksVirtualCluster. The NCService expects + // to find the NC startup script in ${app.home}/bin. + public static final String APP_HOME = StringUtils.join(new String[] { TARGET_DIR, "appassembler" }, File.separator); + + // Path to the asterix-app directory. This is used as the current working + // directory for the CC and NCService processes, which allows relative file + // paths in "load" statements in test queries to find the right data. It is + // also used for HDFSCluster. + public static final String ASTERIX_APP_DIR = StringUtils.join(new String[] { "..", "asterix-app" }, File.separator); + + // Path to the actual AQL test files, which we borrow from asterix-app. This is + // passed to TestExecutor. + protected static final String TESTS_DIR = StringUtils + .join(new String[] { ASTERIX_APP_DIR, "src", "test", "resources", "runtimets" }, File.separator); + + // Path that actual results are written to. We create and clean this directory + // here, and also pass it to TestExecutor which writes the test output there. + public static final String ACTUAL_RESULTS_DIR = StringUtils.join(new String[] { TARGET_DIR, "ittest" }, + File.separator); + private static final String PATH_BASE = Paths + .get("src", "test", "resources", "integrationts", "metadata_only_replication").toString() + File.separator; + private static final String PATH_ACTUAL = "target" + File.separator + "ittest" + File.separator; + + private static final Logger LOGGER = Logger.getLogger(MetadataReplicationIT.class.getName()); + private static String reportPath = new File( + StringUtils.join(new String[] { "target", "failsafe-reports" }, File.separator)).getAbsolutePath(); + + private final TestExecutor testExecutor = new TestExecutor(); + private TestCaseContext tcCtx; + private static String scriptHomePath; + private static File asterixInstallerPath; + private static ProcessBuilder pb; + private static Map<String, String> env; + + private static HyracksCCProcess cc; + private static HyracksNCServiceProcess nc1; + private static HyracksNCServiceProcess nc2; + private static HyracksVirtualCluster cluster; + + public MetadataReplicationIT(TestCaseContext tcCtx) { + this.tcCtx = tcCtx; + } + + @Rule + public TestRule retainLogs = new RetainLogsRule(NCServiceExecutionIT.ASTERIX_APP_DIR, reportPath, this); + + @Before + public void before() throws Exception { + LOGGER.info("Creating new instance..."); + File instanceDir = new File(INSTANCE_DIR); + if (instanceDir.isDirectory()) { + FileUtils.deleteDirectory(instanceDir); + } + + // HDFSCluster requires the input directory to end with a file separator. + + cluster = new HyracksVirtualCluster(new File(APP_HOME), new File(ASTERIX_APP_DIR)); + nc1 = cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), new File(LOG_DIR, "ncservice1.log")); + + nc2 = cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), new File(LOG_DIR, "ncservice2.log")); + + // Start CC + cc = cluster.start(new File(CONF_DIR, "cc.conf"), new File(LOG_DIR, "cc.log")); + + LOGGER.info("Instance created."); + testExecutor.waitForClusterActive(30, TimeUnit.SECONDS); + LOGGER.info("Instance is in ACTIVE state."); + + } + + @After + public void after() throws Exception { + LOGGER.info("Destroying instance..."); + cluster.stop(); + LOGGER.info("Instance destroyed."); + } + + @Test + public void test() throws Exception { + testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false); + } + + @Parameterized.Parameters(name = "MetadataReplicationIT {index}: {0}") + public static Collection<Object[]> tests() throws Exception { + Collection<Object[]> testArgs = buildTestsInXml(TestCaseContext.DEFAULT_TESTSUITE_XML_NAME); + if (testArgs.size() == 0) { + testArgs = buildTestsInXml(TestCaseContext.DEFAULT_TESTSUITE_XML_NAME); + } + return testArgs; + } + + protected static Collection<Object[]> buildTestsInXml(String xmlfile) throws Exception { + Collection<Object[]> testArgs = new ArrayList<>(); + TestCaseContext.Builder b = new TestCaseContext.Builder(); + for (TestCaseContext ctx : b.build(new File(PATH_BASE), xmlfile)) { + testArgs.add(new Object[] { ctx }); + } + return testArgs; + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NCServiceExecutionIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NCServiceExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NCServiceExecutionIT.java index 7b926a6..93b26dc 100644 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NCServiceExecutionIT.java +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NCServiceExecutionIT.java @@ -54,48 +54,42 @@ public class NCServiceExecutionIT { // Important paths and files for this test. // The "target" subdirectory of asterix-server. All outputs go here. - private static final String TARGET_DIR = StringUtils - .join(new String[] { "target" }, File.separator); + public static final String TARGET_DIR = StringUtils.join(new String[] { "../asterix-server/target" }, + File.separator); // Directory where the NCs create and store all data, as configured by // src/test/resources/NCServiceExecutionIT/cc.conf. - private static final String INSTANCE_DIR = StringUtils - .join(new String[] { TARGET_DIR, "tmp" }, File.separator); + public static final String INSTANCE_DIR = StringUtils.join(new String[] { TARGET_DIR, "tmp" }, File.separator); // The log directory, where all CC, NCService, and NC logs are written. CC and // NCService logs are configured on the HyracksVirtualCluster below. NC logs // are configured in src/test/resources/NCServiceExecutionIT/ncservice*.conf. - private static final String LOG_DIR = StringUtils - .join(new String[] { TARGET_DIR, "failsafe-reports" }, File.separator); + public static final String LOG_DIR = StringUtils.join(new String[] { TARGET_DIR, "failsafe-reports" }, + File.separator); // Directory where *.conf files are located. - private static final String CONF_DIR = StringUtils - .join(new String[] { TARGET_DIR, "test-classes", "NCServiceExecutionIT" }, - File.separator); + public static final String CONF_DIR = StringUtils + .join(new String[] { TARGET_DIR, "test-classes", "NCServiceExecutionIT" }, File.separator); // The app.home specified for HyracksVirtualCluster. The NCService expects // to find the NC startup script in ${app.home}/bin. - private static final String APP_HOME = StringUtils - .join(new String[] { TARGET_DIR, "appassembler" }, File.separator); + public static final String APP_HOME = StringUtils.join(new String[] { TARGET_DIR, "appassembler" }, File.separator); // Path to the asterix-app directory. This is used as the current working // directory for the CC and NCService processes, which allows relative file // paths in "load" statements in test queries to find the right data. It is // also used for HDFSCluster. - private static final String ASTERIX_APP_DIR = StringUtils - .join(new String[] { "..", "asterix-app" }, - File.separator); + public static final String ASTERIX_APP_DIR = StringUtils.join(new String[] { "..", "asterix-app" }, File.separator); // Path to the actual AQL test files, which we borrow from asterix-app. This is // passed to TestExecutor. protected static final String TESTS_DIR = StringUtils - .join(new String[] { ASTERIX_APP_DIR, "src", "test", "resources", "runtimets" }, - File.separator); + .join(new String[] { ASTERIX_APP_DIR, "src", "test", "resources", "runtimets" }, File.separator); // Path that actual results are written to. We create and clean this directory // here, and also pass it to TestExecutor which writes the test output there. - private static final String ACTUAL_RESULTS_DIR = StringUtils - .join(new String[] { TARGET_DIR, "ittest" }, File.separator); + public static final String ACTUAL_RESULTS_DIR = StringUtils.join(new String[] { TARGET_DIR, "ittest" }, + File.separator); private static final Logger LOGGER = Logger.getLogger(NCServiceExecutionIT.class.getName()); @@ -138,18 +132,12 @@ public class NCServiceExecutionIT { HDFSCluster.getInstance().setup(ASTERIX_APP_DIR + File.separator); cluster = new HyracksVirtualCluster(new File(APP_HOME), new File(ASTERIX_APP_DIR)); - nc1 = cluster.addNCService( - new File(CONF_DIR, "ncservice1.conf"), - new File(LOG_DIR, "ncservice1.log")); + nc1 = cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), new File(LOG_DIR, "ncservice1.log")); - nc2 = cluster.addNCService( - new File(CONF_DIR, "ncservice2.conf"), - new File(LOG_DIR, "ncservice2.log")); + nc2 = cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), new File(LOG_DIR, "ncservice2.log")); // Start CC - cc = cluster.start( - new File(CONF_DIR, "cc.conf"), - new File(LOG_DIR, "cc.log")); + cc = cluster.start(new File(CONF_DIR, "cc.conf"), new File(LOG_DIR, "cc.log")); testExecutor.waitForClusterActive(30, TimeUnit.SECONDS); clusterActive = true; @@ -184,7 +172,7 @@ public class NCServiceExecutionIT { // let's kill something every 50 tests if (testArgs.size() % 50 == 0) { final KillCommand killCommand = KillCommand.values()[random.nextInt(KillCommand.values().length)]; - testArgs.add(new Object[] { killCommand, null, killCommand}); + testArgs.add(new Object[] { killCommand, null, killCommand }); } } return testArgs; @@ -211,8 +199,7 @@ public class NCServiceExecutionIT { private static boolean skip(TestCaseContext tcCtx) { // For now we skip feeds tests, external-library, and api tests. for (TestGroup group : tcCtx.getTestGroups()) { - if (group.getName().startsWith("external-") - || group.getName().equals("feeds") + if (group.getName().startsWith("external-") || group.getName().equals("feeds") || group.getName().equals("api")) { LOGGER.info("Skipping test: " + tcCtx.toString()); return true; @@ -251,6 +238,7 @@ public class NCServiceExecutionIT { nc1.stop(); // we can't kill due to ASTERIXDB-1941 testExecutor.waitForClusterState("UNUSABLE", 60, TimeUnit.SECONDS); // wait for missed heartbeats... nc1.start(); // this restarts the NC service + testExecutor.startNC("asterix_nc1"); break; case NC2: @@ -258,6 +246,7 @@ public class NCServiceExecutionIT { nc2.stop(); // we can't kill due to ASTERIXDB-1941 testExecutor.waitForClusterState("UNUSABLE", 60, TimeUnit.SECONDS); // wait for missed heartbeats... nc2.start(); // this restarts the NC service + testExecutor.startNC("asterix_nc2"); break; default: http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/RecoveryIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/RecoveryIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/RecoveryIT.java new file mode 100644 index 0000000..21e3068 --- /dev/null +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/RecoveryIT.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.server.test; + +import java.io.File; +import java.io.FilenameFilter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; +import java.util.logging.Logger; + +import org.apache.asterix.test.base.RetainLogsRule; +import org.apache.asterix.test.common.TestExecutor; +import org.apache.asterix.test.runtime.HDFSCluster; +import org.apache.asterix.testframework.context.TestCaseContext; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class RecoveryIT { + + private static final Logger LOGGER = Logger.getLogger(RecoveryIT.class.getName()); + private static final String PATH_ACTUAL = "target" + File.separator + "rttest" + File.separator; + private static final String PATH_BASE = "src/test/resources/transactionts/"; + private static final String HDFS_BASE = "../asterix-app/"; + private TestCaseContext tcCtx; + private static File asterixInstallerPath; + private static File installerTargetPath; + private static String ncServiceHomeDirName; + private static String ncServiceHomePath; + private static String scriptHomePath; + private static String reportPath; + private static ProcessBuilder pb; + private static Map<String, String> env; + private final TestExecutor testExecutor = new TestExecutor(); + + @Rule + public TestRule retainLogs = new RetainLogsRule(ncServiceHomePath, reportPath, this); + + @BeforeClass + public static void setUp() throws Exception { + File outdir = new File(PATH_ACTUAL); + outdir.mkdirs(); + + File externalTestsJar = new File( + StringUtils.join(new String[] { "..", "asterix-external-data", "target" }, File.separator)) + .listFiles((dir, name) -> name.matches("asterix-external-data-.*-tests.jar"))[0]; + + asterixInstallerPath = new File(System.getProperty("user.dir")); + installerTargetPath = new File(new File(asterixInstallerPath.getParentFile(), "asterix-server"), "target"); + reportPath = new File(installerTargetPath, "failsafe-reports").getAbsolutePath(); + ncServiceHomeDirName = installerTargetPath.list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return new File(dir, name).isDirectory() && name.startsWith("asterix-server") + && name.endsWith("binary-assembly"); + } + })[0]; + ncServiceHomePath = new File(installerTargetPath, ncServiceHomeDirName).getAbsolutePath(); + + LOGGER.info("NCSERVICE_HOME=" + ncServiceHomePath); + + FileUtils.copyFile(externalTestsJar, new File(ncServiceHomePath + "/repo", externalTestsJar.getName())); + + pb = new ProcessBuilder(); + env = pb.environment(); + env.put("NCSERVICE_HOME", ncServiceHomePath); + env.put("JAVA_HOME", System.getProperty("java.home")); + scriptHomePath = asterixInstallerPath + File.separator + "src" + File.separator + "test" + File.separator + + "resources" + File.separator + "transactionts" + File.separator + "scripts"; + env.put("SCRIPT_HOME", scriptHomePath); + + TestExecutor.executeScript(pb, + scriptHomePath + File.separator + "setup_teardown" + File.separator + "configure_and_validate.sh"); + TestExecutor.executeScript(pb, + scriptHomePath + File.separator + "setup_teardown" + File.separator + "stop_and_delete.sh"); + HDFSCluster.getInstance().setup(HDFS_BASE); + } + + @AfterClass + public static void tearDown() throws Exception { + File outdir = new File(PATH_ACTUAL); + FileUtils.deleteDirectory(outdir); + File dataCopyDir = new File( + ncServiceHomePath + File.separator + ".." + File.separator + ".." + File.separator + "data"); + FileUtils.deleteDirectory(dataCopyDir); + TestExecutor.executeScript(pb, + scriptHomePath + File.separator + "setup_teardown" + File.separator + "stop_and_delete.sh"); + HDFSCluster.getInstance().cleanup(); + } + + @Parameters(name = "RecoveryIT {index}: {0}") + public static Collection<Object[]> tests() throws Exception { + Collection<Object[]> testArgs = new ArrayList<Object[]>(); + TestCaseContext.Builder b = new TestCaseContext.Builder(); + for (TestCaseContext ctx : b.build(new File(PATH_BASE))) { + testArgs.add(new Object[] { ctx }); + } + return testArgs; + } + + public RecoveryIT(TestCaseContext tcCtx) { + this.tcCtx = tcCtx; + } + + @Test + public void test() throws Exception { + testExecutor.executeTest(PATH_ACTUAL, tcCtx, pb, false); + } + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/ReplicationIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/ReplicationIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/ReplicationIT.java new file mode 100644 index 0000000..6ebb632 --- /dev/null +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/ReplicationIT.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.server.test; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +import org.apache.asterix.test.base.RetainLogsRule; +import org.apache.asterix.test.common.TestExecutor; +import org.apache.asterix.testframework.context.TestCaseContext; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hyracks.server.process.HyracksCCProcess; +import org.apache.hyracks.server.process.HyracksNCServiceProcess; +import org.apache.hyracks.server.process.HyracksVirtualCluster; +import org.apache.hyracks.util.file.FileUtil; +import org.junit.*; +import org.junit.rules.TestRule; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import static org.apache.asterix.server.test.NCServiceExecutionIT.APP_HOME; +import static org.apache.asterix.server.test.NCServiceExecutionIT.INSTANCE_DIR; +import static org.apache.asterix.server.test.NCServiceExecutionIT.TARGET_DIR; +import static org.apache.asterix.server.test.NCServiceExecutionIT.ASTERIX_APP_DIR; +import static org.apache.asterix.server.test.NCServiceExecutionIT.LOG_DIR; + +@RunWith(Parameterized.class) +public class ReplicationIT { + + private static final String PATH_BASE = FileUtil.joinPath("src", "test", "resources", "integrationts", + "replication"); + public static final String CONF_DIR = StringUtils.join(new String[] { TARGET_DIR, "test-classes", "ReplicationIT" }, + File.separator); + private static final String PATH_ACTUAL = FileUtil.joinPath("target", "ittest"); + private static final Logger LOGGER = Logger.getLogger(ReplicationIT.class.getName()); + private static String reportPath = new File(FileUtil.joinPath("target", "failsafe-reports")).getAbsolutePath(); + + private final TestExecutor testExecutor = new TestExecutor(); + private TestCaseContext tcCtx; + private static ProcessBuilder pb; + + private static HyracksCCProcess cc; + private static HyracksNCServiceProcess nc1; + private static HyracksNCServiceProcess nc2; + private static HyracksVirtualCluster cluster; + + public ReplicationIT(TestCaseContext tcCtx) { + this.tcCtx = tcCtx; + } + + @Rule + public TestRule retainLogs = new RetainLogsRule(NCServiceExecutionIT.ASTERIX_APP_DIR, reportPath, this); + + @Before + public void before() throws Exception { + LOGGER.info("Creating new instance..."); + File instanceDir = new File(INSTANCE_DIR); + if (instanceDir.isDirectory()) { + FileUtils.deleteDirectory(instanceDir); + } + + // HDFSCluster requires the input directory to end with a file separator. + + cluster = new HyracksVirtualCluster(new File(APP_HOME), new File(ASTERIX_APP_DIR)); + nc1 = cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), new File(LOG_DIR, "ncservice1.log")); + + nc2 = cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), new File(LOG_DIR, "ncservice2.log")); + + // Start CC + cc = cluster.start(new File(CONF_DIR, "cc.conf"), new File(LOG_DIR, "cc.log")); + + LOGGER.info("Instance created."); + testExecutor.waitForClusterActive(30, TimeUnit.SECONDS); + LOGGER.info("Instance is in ACTIVE state."); + + } + + @After + public void after() throws Exception { + LOGGER.info("Destroying instance..."); + cluster.stop(); + LOGGER.info("Instance destroyed."); + } + + @Test + public void test() throws Exception { + testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false); + } + + @Parameterized.Parameters(name = "ReplicationIT {index}: {0}") + public static Collection<Object[]> tests() throws Exception { + Collection<Object[]> testArgs = buildTestsInXml(TestCaseContext.DEFAULT_TESTSUITE_XML_NAME); + if (testArgs.size() == 0) { + testArgs = buildTestsInXml(TestCaseContext.DEFAULT_TESTSUITE_XML_NAME); + } + return testArgs; + } + + protected static Collection<Object[]> buildTestsInXml(String xmlfile) throws Exception { + Collection<Object[]> testArgs = new ArrayList<>(); + TestCaseContext.Builder b = new TestCaseContext.Builder(); + for (TestCaseContext ctx : b.build(new File(PATH_BASE), xmlfile)) { + testArgs.add(new Object[] { ctx }); + } + return testArgs; + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java index 0beb38d..b6cb030 100644 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java @@ -79,10 +79,11 @@ public class SampleLocalClusterIT { String installerZip = joinPath(pathElements); TestHelper.unzip(installerZip, OUTPUT_DIR); + } private static List<File> findLogFiles(File directory, List<File> fileList) { - File [] match = directory.listFiles(pathname -> pathname.isDirectory() || pathname.toString().endsWith(".log")); + File[] match = directory.listFiles(pathname -> pathname.isDirectory() || pathname.toString().endsWith(".log")); if (match != null) { for (File file : match) { if (file.isDirectory()) { @@ -118,8 +119,7 @@ public class SampleLocalClusterIT { public void test1_sanityQuery() throws Exception { TestExecutor testExecutor = new TestExecutor(); InputStream resultStream = testExecutor.executeQuery("1+1", OutputFormat.ADM, - new URI("http", null, "127.0.0.1", 19002, Servlets.AQL_QUERY, null, null), - Collections.emptyList()); + new URI("http", null, "127.0.0.1", 19002, Servlets.AQL_QUERY, null, null), Collections.emptyList()); StringWriter sw = new StringWriter(); IOUtils.copy(resultStream, sw); Assert.assertEquals("2", sw.toString().trim()); @@ -127,8 +127,8 @@ public class SampleLocalClusterIT { @Test public void test2_stopCluster() throws Exception { - Process process = - new ProcessBuilder(joinPath(LOCAL_SAMPLES_DIR, "bin/stop-sample-cluster.sh")).inheritIO().start(); + Process process = new ProcessBuilder(joinPath(LOCAL_SAMPLES_DIR, "bin/stop-sample-cluster.sh")).inheritIO() + .start(); Assert.assertEquals(0, process.waitFor()); try { new URL("http://127.0.0.1:19002";).openConnection().connect(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SqlppExecutionIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SqlppExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SqlppExecutionIT.java new file mode 100644 index 0000000..066ddd9 --- /dev/null +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SqlppExecutionIT.java @@ -0,0 +1,54 @@ +/* + * Copyright 2009-2013 by The Regents of the University of California + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * you may obtain a copy of the License from + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.asterix.server.test; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; + +import org.apache.asterix.testframework.context.TestCaseContext; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Runs the runtime test cases under 'asterix-app/src/test/resources/runtimets'. + */ +@RunWith(Parameterized.class) +public class SqlppExecutionIT extends AbstractExecutionIT { + + @Parameters(name = "SqlppExecutionIT {index}: {0}") + public static Collection<Object[]> tests() throws Exception { + Collection<Object[]> testArgs = buildTestsInXml("only_sqlpp.xml"); + if (testArgs.size() == 0) { + testArgs = buildTestsInXml("testsuite_sqlpp.xml"); + } + return testArgs; + } + + protected static Collection<Object[]> buildTestsInXml(String xmlfile) throws Exception { + Collection<Object[]> testArgs = new ArrayList<Object[]>(); + TestCaseContext.Builder b = new TestCaseContext.Builder(); + for (TestCaseContext ctx : b.build(new File(PATH_BASE), xmlfile)) { + testArgs.add(new Object[] { ctx }); + } + return testArgs; + + } + + public SqlppExecutionIT(TestCaseContext tcCtx) { + super(tcCtx); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-server/src/test/resources/MetadataReplicationIT/cc.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/resources/MetadataReplicationIT/cc.conf b/asterixdb/asterix-server/src/test/resources/MetadataReplicationIT/cc.conf new file mode 100644 index 0000000..32e38ae --- /dev/null +++ b/asterixdb/asterix-server/src/test/resources/MetadataReplicationIT/cc.conf @@ -0,0 +1,52 @@ +; Licensed to the Apache Software Foundation (ASF) under one +; or more contributor license agreements. See the NOTICE file +; distributed with this work for additional information +; regarding copyright ownership. The ASF licenses this file +; to you under the Apache License, Version 2.0 (the +; "License"); you may not use this file except in compliance +; with the License. You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an +; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +; KIND, either express or implied. See the License for the +; specific language governing permissions and limitations +; under the License. + +[nc/asterix_nc1] +txn.log.dir=../asterix-server/target/tmp/asterix_nc1/txnlog +core.dump.dir=../asterix-server/target/tmp/asterix_nc1/coredump +iodevices=../asterix-server/target/tmp/asterix_nc1/iodevice1,../asterix-server/target/tmp/asterix_nc1/iodevice2 +jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006 +replication.listen.port=2001 +nc.api.port=19004 + +[nc/asterix_nc2] +ncservice.port=9091 +txn.log.dir=../asterix-server/target/tmp/asterix_nc2/txnlog +core.dump.dir=../asterix-server/target/tmp/asterix_nc2/coredump +iodevices=../asterix-server/target/tmp/asterix_nc2/iodevice1,../asterix-server/target/tmp/asterix_nc2/iodevice2 +jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5007 +replication.listen.port=2002 +nc.api.port=19005 + +[nc] +address=127.0.0.1 +command=asterixnc +app.class=org.apache.asterix.hyracks.bootstrap.NCApplication +jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory" +storage.subdir=test_storage +storage.memorycomponent.globalbudget = 1073741824 + +[cc] +address = 127.0.0.1 +app.class=org.apache.asterix.hyracks.bootstrap.CCApplication +heartbeat.period=2000 + +[common] +log.level = INFO +replication.enabled=true +replication.factor=2 +replication.strategy=metadata_only http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-server/src/test/resources/MetadataReplicationIT/ncservice1.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/resources/MetadataReplicationIT/ncservice1.conf b/asterixdb/asterix-server/src/test/resources/MetadataReplicationIT/ncservice1.conf new file mode 100644 index 0000000..ba10142 --- /dev/null +++ b/asterixdb/asterix-server/src/test/resources/MetadataReplicationIT/ncservice1.conf @@ -0,0 +1,20 @@ +; Licensed to the Apache Software Foundation (ASF) under one +; or more contributor license agreements. See the NOTICE file +; distributed with this work for additional information +; regarding copyright ownership. The ASF licenses this file +; to you under the Apache License, Version 2.0 (the +; "License"); you may not use this file except in compliance +; with the License. You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an +; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +; KIND, either express or implied. See the License for the +; specific language governing permissions and limitations +; under the License. + +[ncservice] +logdir=../asterix-server/target/failsafe-reports + http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-server/src/test/resources/MetadataReplicationIT/ncservice2.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/resources/MetadataReplicationIT/ncservice2.conf b/asterixdb/asterix-server/src/test/resources/MetadataReplicationIT/ncservice2.conf new file mode 100644 index 0000000..2036584 --- /dev/null +++ b/asterixdb/asterix-server/src/test/resources/MetadataReplicationIT/ncservice2.conf @@ -0,0 +1,21 @@ +; Licensed to the Apache Software Foundation (ASF) under one +; or more contributor license agreements. See the NOTICE file +; distributed with this work for additional information +; regarding copyright ownership. The ASF licenses this file +; to you under the Apache License, Version 2.0 (the +; "License"); you may not use this file except in compliance +; with the License. You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an +; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +; KIND, either express or implied. See the License for the +; specific language governing permissions and limitations +; under the License. + +[ncservice] +logdir=../asterix-server/target/failsafe-reports +port=9091 +
