http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/AsterixEventService.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/AsterixEventService.java b/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/AsterixEventService.java deleted file mode 100644 index f95b934..0000000 --- a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/AsterixEventService.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.event.service; - -import java.io.File; -import java.io.FileFilter; - -import org.apache.asterix.event.error.OutputHandler; -import org.apache.asterix.event.management.AsterixEventServiceClient; -import org.apache.asterix.event.schema.cluster.Cluster; -import org.apache.asterix.installer.schema.conf.Configuration; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - -public class AsterixEventService { - - private static final Logger LOGGER = Logger.getLogger(AsterixEventService.class.getName()); - private static Configuration configuration; - private static String asterixDir; - private static String asterixZip; - private static String eventHome; - - public static void initialize(Configuration configuration, String asterixDir, String eventHome) throws Exception { - AsterixEventService.configuration = configuration; - AsterixEventService.asterixDir = asterixDir; - AsterixEventService.asterixZip = initBinary("asterix-server"); - AsterixEventService.eventHome = eventHome; - } - - public static AsterixEventServiceClient getAsterixEventServiceClient(Cluster cluster, boolean transferArtifacts, - boolean dryRun) throws Exception { - return new AsterixEventServiceClient(configuration, eventHome, cluster, transferArtifacts, dryRun, - OutputHandler.INSTANCE); - } - - public static AsterixEventServiceClient getAsterixEventServiceClient(Cluster cluster) throws Exception { - return new AsterixEventServiceClient(configuration, eventHome, cluster, false, false, OutputHandler.INSTANCE); - } - - private static String initBinary(final String fileNamePattern) { - File file = new File(asterixDir); - File[] zipFiles = file.listFiles(new FileFilter() { - public boolean accept(File arg0) { - return arg0.getAbsolutePath().contains(fileNamePattern) && arg0.isFile(); - } - }); - if (zipFiles.length == 0) { - String msg = " Binary not found at " + asterixDir; - LOGGER.log(Level.FATAL, msg); - throw new IllegalStateException(msg); - } - if (zipFiles.length > 1) { - String msg = " Multiple binaries found at " + asterixDir; - LOGGER.log(Level.FATAL, msg); - throw new IllegalStateException(msg); - } - - return zipFiles[0].getAbsolutePath(); - } - - public static Configuration getConfiguration() { - return configuration; - } - - public static String getAsterixZip() { - return asterixZip; - } - - public static String getAsterixDir() { - return asterixDir; - } - - public static String getEventHome() { - return eventHome; - } -}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/AsterixEventServiceUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/AsterixEventServiceUtil.java b/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/AsterixEventServiceUtil.java deleted file mode 100644 index b48e311..0000000 --- a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/AsterixEventServiceUtil.java +++ /dev/null @@ -1,514 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.event.service; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileFilter; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.StringWriter; -import java.net.InetAddress; -import java.util.ArrayList; -import java.util.Enumeration; -import java.util.HashSet; -import java.util.List; -import java.util.Random; -import java.util.Set; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; -import java.util.jar.JarOutputStream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; - -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Marshaller; - -import org.apache.asterix.common.configuration.AsterixConfiguration; -import org.apache.asterix.common.configuration.Coredump; -import org.apache.asterix.common.configuration.Store; -import org.apache.asterix.common.configuration.TransactionLogDir; -import org.apache.asterix.event.driver.EventDriver; -import org.apache.asterix.event.error.EventException; -import org.apache.asterix.event.management.EventUtil; -import org.apache.asterix.event.model.AsterixInstance; -import org.apache.asterix.event.model.AsterixInstance.State; -import org.apache.asterix.event.schema.cluster.Cluster; -import org.apache.asterix.event.schema.cluster.Env; -import org.apache.asterix.event.schema.cluster.Node; -import org.apache.asterix.event.schema.cluster.Property; -import org.apache.commons.io.IOUtils; - -public class AsterixEventServiceUtil { - - public static final String TXN_LOG_DIR = "txnLogs"; - public static final String ASTERIX_CONFIGURATION_FILE = "asterix-configuration.xml"; - public static final String CLUSTER_CONFIGURATION_FILE = "cluster.xml"; - public static final String EVENT_DIR = "events"; - public static final int CLUSTER_NET_PORT_DEFAULT = 1098; - public static final int CLIENT_NET_PORT_DEFAULT = 1099; - public static final int HTTP_PORT_DEFAULT = 8888; - private static final String EMPTY_STRING = ""; - - public static final String MANAGIX_CONF_XML = "conf" + File.separator + "managix-conf.xml"; - - private static final int BUFFER_SIZE = 4096; - - public static AsterixInstance createAsterixInstance(String asterixInstanceName, Cluster cluster, - AsterixConfiguration asterixConfiguration) throws IOException { - Node metadataNode = getMetadataNode(asterixInstanceName, cluster); - String asterixZipName = asterixZipName(); - String asterixVersion = asterixZipName.substring("asterix-server-".length(), - asterixZipName.indexOf("-binary-assembly")); - return new AsterixInstance(asterixInstanceName, cluster, asterixConfiguration, - metadataNode.getId(), asterixVersion); - } - - public static void createAsterixZip(AsterixInstance asterixInstance) - throws IOException, InterruptedException, JAXBException, EventException { - String asterixInstanceDir = asterixInstanceDir(asterixInstance); - unzip(AsterixEventService.getAsterixZip(), asterixInstanceDir); - - injectAsterixPropertyFile(asterixInstanceDir, asterixInstance); - injectAsterixClusterConfigurationFile(asterixInstanceDir, asterixInstance); - - final String asterixZipPath = asterixInstanceDir + File.separator + asterixZipName(); - zipDir(new File(asterixInstanceDir), new File(asterixZipPath)); - } - - public static void createClusterProperties(Cluster cluster, AsterixConfiguration asterixConfiguration) { - - String ccJavaOpts = null; - String ncJavaOpts = null; - for (org.apache.asterix.common.configuration.Property property : asterixConfiguration.getProperty()) { - if (property.getName().equalsIgnoreCase(EventUtil.CC_JAVA_OPTS)) { - ccJavaOpts = property.getValue(); - } else if (property.getName().equalsIgnoreCase(EventUtil.NC_JAVA_OPTS)) { - ncJavaOpts = property.getValue(); - } - } - - poulateClusterEnvironmentProperties(cluster, ccJavaOpts, ncJavaOpts); - } - - public static void poulateClusterEnvironmentProperties(Cluster cluster, String ccJavaOpts, String ncJavaOpts) { - List<Property> clusterProperties = null; - if (cluster.getEnv() != null && cluster.getEnv().getProperty() != null) { - clusterProperties = cluster.getEnv().getProperty(); - clusterProperties.clear(); - } else { - clusterProperties = new ArrayList<Property>(); - } - - clusterProperties.add(new Property(EventUtil.CC_JAVA_OPTS, ccJavaOpts)); - clusterProperties.add(new Property(EventUtil.NC_JAVA_OPTS, ncJavaOpts)); - clusterProperties - .add(new Property("ASTERIX_HOME", cluster.getWorkingDir().getDir() + File.separator + "asterix")); - clusterProperties.add(new Property("LOG_DIR", cluster.getLogDir())); - clusterProperties.add(new Property("JAVA_HOME", cluster.getJavaHome())); - clusterProperties.add(new Property("WORKING_DIR", cluster.getWorkingDir().getDir())); - clusterProperties.add(new Property("CLIENT_NET_IP", cluster.getMasterNode().getClientIp())); - clusterProperties.add(new Property("CLUSTER_NET_IP", cluster.getMasterNode().getClusterIp())); - - int clusterNetPort = cluster.getMasterNode().getClusterPort() != null - ? cluster.getMasterNode().getClusterPort().intValue() : CLUSTER_NET_PORT_DEFAULT; - int clientNetPort = cluster.getMasterNode().getClientPort() != null - ? cluster.getMasterNode().getClientPort().intValue() : CLIENT_NET_PORT_DEFAULT; - int httpPort = cluster.getMasterNode().getHttpPort() != null ? cluster.getMasterNode().getHttpPort().intValue() - : HTTP_PORT_DEFAULT; - - clusterProperties.add(new Property("CLIENT_NET_PORT", "" + clientNetPort)); - clusterProperties.add(new Property("CLUSTER_NET_PORT", "" + clusterNetPort)); - clusterProperties.add(new Property("HTTP_PORT", "" + httpPort)); - - //pass Cluster optional parameters - if (cluster.getHeartbeatPeriod() != null) { - clusterProperties - .add(new Property("HEARTBEAT_PERIOD", String.valueOf(cluster.getHeartbeatPeriod().intValue()))); - } - if (cluster.getMaxHeartbeatLapsePeriods() != null) { - clusterProperties.add(new Property("MAX_HEARTBEAT_LAPSE_PERIODS", - String.valueOf(cluster.getMaxHeartbeatLapsePeriods().intValue()))); - } - if (cluster.getProfileDumpPeriod() != null) { - clusterProperties.add( - new Property("PROFILE_DUMP_PERIOD", String.valueOf(cluster.getProfileDumpPeriod().intValue()))); - } - if (cluster.getJobHistorySize() != null) { - clusterProperties - .add(new Property("JOB_HISTORY_SIZE", String.valueOf(cluster.getJobHistorySize().intValue()))); - } - if (cluster.getResultTimeToLive() != null) { - clusterProperties.add( - new Property("RESULT_TIME_TO_LIVE", String.valueOf(cluster.getResultTimeToLive().longValue()))); - } - if (cluster.getResultSweepThreshold() != null) { - clusterProperties.add(new Property("RESULT_SWEEP_THRESHOLD", - String.valueOf(cluster.getResultSweepThreshold().longValue()))); - } - if (cluster.getCcRoot() != null) { - clusterProperties.add(new Property("CC_ROOT", cluster.getCcRoot())); - } - cluster.setEnv(new Env(clusterProperties)); - } - - private static String asterixZipName() { - return AsterixEventService.getAsterixZip() - .substring(AsterixEventService.getAsterixZip().lastIndexOf(File.separator) + 1); - } - - private static String asterixJarPath(AsterixInstance asterixInstance, String asterixInstanceDir) { - return asterixInstanceDir + File.separator + "repo" + File.separator + "asterix-app-" - + asterixInstance.getAsterixVersion() + ".jar"; - } - - private static String asterixInstanceDir(AsterixInstance asterixInstance) { - return AsterixEventService.getAsterixDir() + File.separator + asterixInstance.getName(); - } - - private static void injectAsterixPropertyFile(String asterixInstanceDir, AsterixInstance asterixInstance) - throws IOException, JAXBException { - writeAsterixConfigurationFile(asterixInstance); - - File sourceJar = new File(asterixJarPath(asterixInstance, asterixInstanceDir)); - File replacementFile = new File(asterixInstanceDir + File.separator + ASTERIX_CONFIGURATION_FILE); - replaceInJar(sourceJar, ASTERIX_CONFIGURATION_FILE, replacementFile); - new File(asterixInstanceDir + File.separator + ASTERIX_CONFIGURATION_FILE).delete(); - } - - private static void injectAsterixClusterConfigurationFile(String asterixInstanceDir, - AsterixInstance asterixInstance) throws IOException, EventException, JAXBException { - File sourceJar = new File(asterixJarPath(asterixInstance, asterixInstanceDir)); - writeAsterixClusterConfigurationFile(asterixInstance); - - File replacementFile = new File(asterixInstanceDir + File.separator + "cluster.xml"); - replaceInJar(sourceJar, CLUSTER_CONFIGURATION_FILE, replacementFile); - - new File(asterixInstanceDir + File.separator + CLUSTER_CONFIGURATION_FILE).delete(); - } - - private static void writeAsterixClusterConfigurationFile(AsterixInstance asterixInstance) - throws IOException, EventException, JAXBException { - String asterixInstanceName = asterixInstance.getName(); - Cluster cluster = asterixInstance.getCluster(); - - JAXBContext ctx = JAXBContext.newInstance(Cluster.class); - Marshaller marshaller = ctx.createMarshaller(); - marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); - marshaller.marshal(cluster, new FileOutputStream(AsterixEventService.getAsterixDir() + File.separator - + asterixInstanceName + File.separator + "cluster.xml")); - } - - public static void addLibraryToAsterixZip(AsterixInstance asterixInstance, String dataverseName, String libraryName, - String libraryPath) throws IOException { - File instanceDir = new File(asterixInstanceDir(asterixInstance)); - if (!instanceDir.exists()) { - instanceDir.mkdirs(); - } - String asterixZipName = asterixZipName(); - - String sourceZip = instanceDir.getAbsolutePath() + File.separator + asterixZipName; - unzip(sourceZip, instanceDir.getAbsolutePath()); - File libraryPathInZip = new File(instanceDir.getAbsolutePath() + File.separator + "external" + File.separator - + "library" + dataverseName + File.separator + "to-add" + File.separator + libraryName); - libraryPathInZip.mkdirs(); - Runtime.getRuntime().exec("cp" + " " + libraryPath + " " + libraryPathInZip.getAbsolutePath()); - Runtime.getRuntime().exec("rm " + sourceZip); - String destZip = AsterixEventService.getAsterixDir() + File.separator + asterixInstance.getName() - + File.separator + asterixZipName; - zipDir(instanceDir, new File(destZip)); - Runtime.getRuntime().exec("mv" + " " + destZip + " " + sourceZip); - } - - private static Node getMetadataNode(String asterixInstanceName, Cluster cluster) { - Node metadataNode = null; - if (cluster.getMetadataNode() != null) { - for (Node node : cluster.getNode()) { - if (node.getId().equals(cluster.getMetadataNode())) { - metadataNode = node; - break; - } - } - } else { - Random random = new Random(); - int nNodes = cluster.getNode().size(); - metadataNode = cluster.getNode().get(random.nextInt(nNodes)); - } - return metadataNode; - } - - private static void writeAsterixConfigurationFile(AsterixInstance asterixInstance) - throws IOException, JAXBException { - String asterixInstanceName = asterixInstance.getName(); - Cluster cluster = asterixInstance.getCluster(); - String metadataNodeId = asterixInstance.getMetadataNodeId(); - - AsterixConfiguration configuration = asterixInstance.getAsterixConfiguration(); - configuration.setInstanceName(asterixInstanceName); - configuration.setMetadataNode(asterixInstanceName + "_" + metadataNodeId); - List<Store> stores = new ArrayList<Store>(); - String storeDir = cluster.getStore().trim(); - for (Node node : cluster.getNode()) { - String iodevices = node.getIodevices() == null ? cluster.getIodevices() : node.getIodevices(); - String[] nodeIdDevice = iodevices.split(","); - StringBuilder nodeStores = new StringBuilder(); - for (int i = 0; i < nodeIdDevice.length; i++) { - nodeStores.append(nodeIdDevice[i] + File.separator + storeDir + ","); - } - //remove last comma - nodeStores.deleteCharAt(nodeStores.length() - 1); - stores.add(new Store(asterixInstanceName + "_" + node.getId(), nodeStores.toString())); - } - configuration.setStore(stores); - List<Coredump> coredump = new ArrayList<Coredump>(); - List<TransactionLogDir> txnLogDirs = new ArrayList<TransactionLogDir>(); - for (Node node : cluster.getNode()) { - String coredumpdir = node.getLogDir() == null ? cluster.getLogDir() : node.getLogDir(); - coredump.add(new Coredump(asterixInstanceName + "_" + node.getId(), - coredumpdir + File.separator + asterixInstanceName + "_" + node.getId())); - - String txnLogDir = node.getTxnLogDir() == null ? cluster.getTxnLogDir() : node.getTxnLogDir(); - txnLogDirs.add(new TransactionLogDir(asterixInstanceName + "_" + node.getId(), txnLogDir)); - } - configuration.setCoredump(coredump); - configuration.setTransactionLogDir(txnLogDirs); - File asterixConfDir = new File(AsterixEventService.getAsterixDir() + File.separator + asterixInstanceName); - asterixConfDir.mkdirs(); - - JAXBContext ctx = JAXBContext.newInstance(AsterixConfiguration.class); - Marshaller marshaller = ctx.createMarshaller(); - marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); - FileOutputStream os = new FileOutputStream(asterixConfDir + File.separator + ASTERIX_CONFIGURATION_FILE); - marshaller.marshal(configuration, os); - os.close(); - } - - public static void unzip(String sourceFile, String destDir) throws IOException { - final FileInputStream fis = new FileInputStream(sourceFile); - final ZipInputStream zis = new ZipInputStream(new BufferedInputStream(fis)); - final File destDirFile = new File(destDir); - final byte [] data = new byte[BUFFER_SIZE]; - - ZipEntry entry; - Set<String> visitedDirs = new HashSet<>(); - createDir(destDir); - while ((entry = zis.getNextEntry()) != null) { - createDir(destDirFile, entry, visitedDirs); - if (entry.isDirectory()) { - continue; - } - int count; - - // write the file to the disk - File dst = new File(destDir, entry.getName()); - FileOutputStream fos = new FileOutputStream(dst); - BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER_SIZE); - while ((count = zis.read(data, 0, BUFFER_SIZE)) != -1) { - dest.write(data, 0, count); - } - // close the output streams - dest.flush(); - dest.close(); - } - - zis.close(); - } - - public static void zipDir(File sourceDir, File destFile) throws IOException { - FileOutputStream fos = new FileOutputStream(destFile); - ZipOutputStream zos = new ZipOutputStream(fos); - zipDir(EMPTY_STRING, sourceDir, destFile, zos); - zos.close(); - } - - private static void zipDir(String prefix, File sourceDir, final File destFile, ZipOutputStream zos) - throws IOException { - File[] dirList = sourceDir.listFiles(f->!f.getName().endsWith(destFile.getName())); - String newPrefix = prefix.equals(EMPTY_STRING) ? prefix : prefix + File.separator; - for (int i = 0; i < dirList.length; i++) { - File f = dirList[i]; - if (f.isDirectory()) { - zipDir(newPrefix + f.getName(), f, destFile, zos); - } else { - int bytesIn = 0; - byte[] readBuffer = new byte[2156]; - FileInputStream fis = new FileInputStream(f); - ZipEntry entry = new ZipEntry(newPrefix + f.getName()); - zos.putNextEntry(entry); - while ((bytesIn = fis.read(readBuffer)) != -1) { - zos.write(readBuffer, 0, bytesIn); - } - fis.close(); - } - } - } - - private static void replaceInJar(File sourceJar, String origFile, File replacementFile) throws IOException { - String srcJarAbsPath = sourceJar.getAbsolutePath(); - String srcJarSuffix = srcJarAbsPath.substring(srcJarAbsPath.lastIndexOf(File.separator) + 1); - String srcJarName = srcJarSuffix.split(".jar")[0]; - - String destJarName = srcJarName + "-managix"; - String destJarSuffix = destJarName + ".jar"; - File destJar = new File(sourceJar.getParentFile().getAbsolutePath() + File.separator + destJarSuffix); - // File destJar = new File(sourceJar.getAbsolutePath() + ".modified"); - JarFile sourceJarFile = new JarFile(sourceJar); - Enumeration<JarEntry> entries = sourceJarFile.entries(); - JarOutputStream jos = new JarOutputStream(new FileOutputStream(destJar)); - byte[] buffer = new byte[2048]; - int read; - while (entries.hasMoreElements()) { - JarEntry entry = (JarEntry) entries.nextElement(); - String name = entry.getName(); - if (name.equals(origFile)) { - continue; - } - InputStream jarIs = sourceJarFile.getInputStream(entry); - jos.putNextEntry(entry); - while ((read = jarIs.read(buffer)) != -1) { - jos.write(buffer, 0, read); - } - jarIs.close(); - } - sourceJarFile.close(); - JarEntry entry = new JarEntry(origFile); - jos.putNextEntry(entry); - FileInputStream fis = new FileInputStream(replacementFile); - while ((read = fis.read(buffer)) != -1) { - jos.write(buffer, 0, read); - } - fis.close(); - jos.close(); - sourceJar.delete(); - destJar.renameTo(sourceJar); - destJar.setExecutable(true); - } - - public static void dumpToFile(String dest, String content) throws IOException { - FileWriter writer = new FileWriter(dest); - writer.write(content); - writer.close(); - } - - private static void createDir(File destDirectory, ZipEntry entry, Set<String> visitedDirs) { - String name = entry.getName(); - int index = name.lastIndexOf(File.separator); - if (index != -1) { - String dirSequence = name.substring(0, index); - if (visitedDirs.add(dirSequence)) { - File newDirs = new File(destDirectory, dirSequence); - newDirs.mkdirs(); - } - } - } - - private static void createDir(String destDirectory) { - File newDirs = new File(destDirectory + File.separator); - newDirs.mkdirs(); - } - - public static AsterixInstance validateAsterixInstanceExists(String name, State... permissibleStates) - throws Exception { - AsterixInstance instance = ServiceProvider.INSTANCE.getLookupService().getAsterixInstance(name); - if (instance == null) { - throw new EventException("Asterix instance by name " + name + " does not exist."); - } - boolean valid = false; - for (State state : permissibleStates) { - if (state.equals(instance.getState())) { - valid = true; - break; - } - } - if (!valid) { - throw new EventException( - "Asterix instance by the name " + name + " is in " + instance.getState() + " state "); - } - return instance; - } - - public static void validateAsterixInstanceNotExists(String name) throws Exception { - AsterixInstance instance = ServiceProvider.INSTANCE.getLookupService().getAsterixInstance(name); - if (instance != null) { - throw new EventException("Asterix instance by name " + name + " already exists."); - } - } - - public static void evaluateConflictWithOtherInstances(AsterixInstance instance) throws Exception { - List<AsterixInstance> existingInstances = ServiceProvider.INSTANCE.getLookupService().getAsterixInstances(); - List<String> usedIps = new ArrayList<String>(); - String masterIp = instance.getCluster().getMasterNode().getClusterIp(); - for (Node node : instance.getCluster().getNode()) { - usedIps.add(node.getClusterIp()); - } - usedIps.add(instance.getCluster().getMasterNode().getClusterIp()); - boolean conflictFound = false; - AsterixInstance conflictingInstance = null; - for (AsterixInstance existing : existingInstances) { - if (existing.getState().equals(State.INACTIVE)) { - continue; - } - InetAddress extantAddress = InetAddress.getByName(existing.getCluster().getMasterNode().getClusterIp()); - InetAddress masterAddress = InetAddress.getByName(masterIp); - if (extantAddress.equals(masterAddress)) { - conflictingInstance = existing; - break; - } - for (Node n : existing.getCluster().getNode()) { - if (usedIps.contains(n.getClusterIp())) { - conflictFound = true; - conflictingInstance = existing; - break; - } - } - } - if (conflictFound) { - throw new Exception("Cluster definition conflicts with an existing instance of Asterix: " - + conflictingInstance.getName()); - } - } - - public static void deleteDirectory(String path) throws IOException { - Runtime.getRuntime().exec("rm -rf " + path); - } - - public static String executeLocalScript(String path, List<String> args) throws Exception { - List<String> pargs = new ArrayList<String>(); - pargs.add("/bin/bash"); - pargs.add(path); - if (args != null) { - pargs.addAll(args); - } - ProcessBuilder pb = new ProcessBuilder(pargs); - pb.environment().putAll(EventDriver.getEnvironment()); - pb.environment().put("IP_LOCATION", EventDriver.CLIENT_NODE.getClusterIp()); - Process p = pb.start(); - BufferedInputStream bis = new BufferedInputStream(p.getInputStream()); - StringWriter writer = new StringWriter(); - IOUtils.copy(bis, writer, "UTF-8"); - return writer.toString(); - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/ClusterStateWatcher.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/ClusterStateWatcher.java b/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/ClusterStateWatcher.java deleted file mode 100644 index c51e2cf..0000000 --- a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/ClusterStateWatcher.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright 2009-2013 by The Regents of the University of California - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * you may obtain a copy of the License from - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.asterix.event.service; - -import java.io.File; -import java.util.List; - -import org.apache.asterix.common.api.IClusterManagementWork.ClusterState; -import org.apache.log4j.Logger; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.ZooKeeper; - -//A zookeeper watcher that watches the change in the state of the cluster -public class ClusterStateWatcher implements Watcher { - private final Object mutex = new Object(); - private final ZooKeeper zk; - private String clusterStatePath; - private boolean done = false; - private ClusterState clusterState = ClusterState.UNUSABLE; - private boolean failed = false; - private Exception failureCause = null; - private static Logger LOGGER = Logger.getLogger(ClusterStateWatcher.class.getName()); - - public ClusterStateWatcher(ZooKeeper zk, String clusterName) { - this.clusterStatePath = ZooKeeperService.ASTERIX_INSTANCE_BASE_PATH + File.separator + clusterName - + ZooKeeperService.ASTERIX_INSTANCE_STATE_PATH; - this.zk = zk; - } - - public ClusterState waitForClusterStart() throws Exception { - while (true) { - synchronized (mutex) { - if (done) { - if (failed) { - LOGGER.error("An error took place in the startup sequence. Check the CC logs."); - throw failureCause; - } else { - return clusterState; - } - } else { - mutex.wait(); - } - } - } - } - - private void monitorStateChange() { - try { - while (true) { - synchronized (mutex) { - // Get the cluster state - List<String> list = zk.getChildren(clusterStatePath, this); - if (list.size() == 0) { - // Cluster state not found, wait to be awaken by Zookeeper - mutex.wait(); - } else { - // Cluster state found - byte[] b = zk.getData(clusterStatePath + ZooKeeperService.ASTERIX_INSTANCE_STATE_REPORT, false, - null); - zk.delete(clusterStatePath + ZooKeeperService.ASTERIX_INSTANCE_STATE_REPORT, 0); - clusterState = ClusterState.values()[(int) b[0]]; - done = true; - mutex.notifyAll(); - return; - } - } - } - } catch (Exception e) { - // Exception was thrown, let Managix know that a failure took place - failed = true; - done = true; - failureCause = e; - } - } - - public void startMonitoringThread() { - Runnable monitoringThread = new Runnable() { - @Override - public void run() { - monitorStateChange(); - } - }; - // Start the monitoring thread - (new Thread(monitoringThread)).start(); - } - - @Override - synchronized public void process(WatchedEvent event) { - synchronized (mutex) { - mutex.notifyAll(); - } - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/ILookupService.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/ILookupService.java b/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/ILookupService.java deleted file mode 100644 index 34252e3..0000000 --- a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/ILookupService.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.event.service; - -import java.util.List; - -import org.apache.asterix.common.api.IClusterManagementWork.ClusterState; -import org.apache.asterix.event.model.AsterixInstance; -import org.apache.asterix.installer.schema.conf.Configuration; - -public interface ILookupService { - - public void writeAsterixInstance(AsterixInstance asterixInstance) throws Exception; - - public AsterixInstance getAsterixInstance(String name) throws Exception; - - public boolean isRunning(Configuration conf) throws Exception; - - public void startService(Configuration conf) throws Exception; - - public void stopService(Configuration conf) throws Exception; - - public boolean exists(String name) throws Exception; - - public void removeAsterixInstance(String name) throws Exception; - - public List<AsterixInstance> getAsterixInstances() throws Exception; - - public void updateAsterixInstance(AsterixInstance updatedInstance) throws Exception; - - public void reportClusterState(String instanceName, ClusterState active) throws Exception; - - public ClusterStateWatcher startWatchingClusterState(String asterixInstanceName); -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/ServiceProvider.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/ServiceProvider.java b/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/ServiceProvider.java deleted file mode 100644 index ab421fd..0000000 --- a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/ServiceProvider.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.event.service; - -public class ServiceProvider { - - public static ServiceProvider INSTANCE = new ServiceProvider(); - private static ILookupService lookupService = new ZooKeeperService(); - - private ServiceProvider() { - } - - public ILookupService getLookupService() { - return lookupService; - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/ZooKeeperService.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/ZooKeeperService.java b/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/ZooKeeperService.java deleted file mode 100644 index 5e102e7..0000000 --- a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/ZooKeeperService.java +++ /dev/null @@ -1,329 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.event.service; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.io.StringWriter; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -import org.apache.asterix.common.api.IClusterManagementWork.ClusterState; -import org.apache.asterix.event.error.EventException; -import org.apache.asterix.event.model.AsterixInstance; -import org.apache.asterix.installer.schema.conf.Configuration; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.log4j.Logger; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.Watcher.Event.KeeperState; -import org.apache.zookeeper.ZooDefs.Ids; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.data.Stat; - -public class ZooKeeperService implements ILookupService { - - private static final Logger LOGGER = Logger.getLogger(ZooKeeperService.class.getName()); - - private static final int ZOOKEEPER_LEADER_CONN_PORT = 2222; - private static final int ZOOKEEPER_LEADER_ELEC_PORT = 2223; - private static final int ZOOKEEPER_SESSION_TIME_OUT = 40 * 1000; //milliseconds - private static final String ZOOKEEPER_HOME = AsterixEventService.getEventHome() + File.separator + "zookeeper"; - private static final String ZOO_KEEPER_CONFIG = ZOOKEEPER_HOME + File.separator + "zk.cfg"; - - private boolean isRunning = false; - private ZooKeeper zk; - private String zkConnectionString; - public static final String ASTERIX_INSTANCE_BASE_PATH = File.separator + "Asterix"; - public static final String ASTERIX_INSTANCE_STATE_PATH = File.separator + "state"; - public static final String ASTERIX_INSTANCE_STATE_REPORT = File.separator + "clusterState"; - public static final int DEFAULT_NODE_VERSION = -1; - private LinkedBlockingQueue<String> msgQ = new LinkedBlockingQueue<String>(); - private ZooKeeperWatcher watcher = new ZooKeeperWatcher(msgQ); - - @Override - public boolean isRunning(Configuration conf) throws Exception { - List<String> servers = conf.getZookeeper().getServers().getServer(); - int clientPort = conf.getZookeeper().getClientPort().intValue(); - StringBuilder connectionString = new StringBuilder(); - for (String serverAddress : servers) { - connectionString.append(serverAddress); - connectionString.append(":"); - connectionString.append(clientPort); - connectionString.append(","); - } - if (connectionString.length() > 0) { - connectionString.deleteCharAt(connectionString.length() - 1); - } - zkConnectionString = connectionString.toString(); - - zk = new ZooKeeper(zkConnectionString, ZOOKEEPER_SESSION_TIME_OUT, watcher); - try { - zk.exists("/dummy", watcher); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("ZooKeeper running at " + connectionString); - } - createRootIfNotExist(); - isRunning = true; - } catch (KeeperException ke) { - isRunning = false; - } - return isRunning; - } - - @Override - public void startService(Configuration conf) throws Exception { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Starting ZooKeeper at " + zkConnectionString); - } - ZookeeperUtil.writeConfiguration(ZOO_KEEPER_CONFIG, conf, ZOOKEEPER_LEADER_CONN_PORT, - ZOOKEEPER_LEADER_ELEC_PORT); - String initScript = ZOOKEEPER_HOME + File.separator + "bin" + File.separator + "zk.init"; - StringBuilder cmdBuffer = new StringBuilder(); - cmdBuffer.append(initScript + " "); - cmdBuffer.append(conf.getZookeeper().getHomeDir() + " "); - cmdBuffer.append(conf.getZookeeper().getServers().getJavaHome() + " "); - List<String> zkServers = conf.getZookeeper().getServers().getServer(); - for (String zkServer : zkServers) { - cmdBuffer.append(zkServer + " "); - } - //TODO: Create a better way to interact with zookeeper - Process zkProcess = Runtime.getRuntime().exec(cmdBuffer.toString()); - int exitCode = zkProcess.waitFor(); - Pair<CharSequence, CharSequence> outputs = getProcessStreams(zkProcess); - if (exitCode != 0) { - StringBuilder msg = new StringBuilder("Error starting zookeeper server; output code = "); - msg.append(exitCode); - appendNonEmptyStreams(outputs, msg); - throw new Exception(msg.toString()); - } - zk = new ZooKeeper(zkConnectionString, ZOOKEEPER_SESSION_TIME_OUT, watcher); - String head = msgQ.poll(60, TimeUnit.SECONDS); - if (head == null) { - StringBuilder msg = new StringBuilder("Unable to start Zookeeper Service. This could be because of the" - + " following reasons.\n1) Managix is incorrectly configured. Please run " - + "managix validate to run a validation test and correct the errors reported.\n" - + "2) If validation in (1) is successful, ensure that java_home parameter is set correctly" - + " in Managix configuration (" + AsterixEventServiceUtil.MANAGIX_CONF_XML + ")"); - appendNonEmptyStreams(outputs, msg); - throw new Exception(msg.toString()); - } - msgQ.take(); - createRootIfNotExist(); - } - - private void appendNonEmptyStreams(Pair<CharSequence, CharSequence> outputs, StringBuilder msg) { - appendIfNotEmpty(msg, outputs.getLeft(), "stdout"); - appendIfNotEmpty(msg, outputs.getRight(), "stderr"); - } - - private Pair<CharSequence, CharSequence> getProcessStreams(Process process) throws IOException { - StringWriter stdout = new StringWriter(); - StringWriter stderr = new StringWriter(); - IOUtils.copy(process.getInputStream(), stdout, Charset.defaultCharset()); - IOUtils.copy(process.getErrorStream(), stderr, Charset.defaultCharset()); - return new ImmutablePair<>(stdout.getBuffer(), stderr.getBuffer()); - } - - private void appendIfNotEmpty(StringBuilder msg, CharSequence output, String outputName) { - if (output.length() > 0) { - msg.append(", ").append(outputName).append(" = ").append(output); - } - } - - @Override - public void stopService(Configuration conf) throws Exception { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Stopping ZooKeeper running at " + zkConnectionString); - } - String stopScript = ZOOKEEPER_HOME + File.separator + "bin" + File.separator + "stop_zk"; - StringBuilder cmdBuffer = new StringBuilder(); - cmdBuffer.append(stopScript + " "); - cmdBuffer.append(conf.getZookeeper().getHomeDir() + " "); - List<String> zkServers = conf.getZookeeper().getServers().getServer(); - for (String zkServer : zkServers) { - cmdBuffer.append(zkServer + " "); - } - Runtime.getRuntime().exec(cmdBuffer.toString()); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Stopped ZooKeeper service at " + zkConnectionString); - } - } - - @Override - public void writeAsterixInstance(AsterixInstance asterixInstance) throws Exception { - String instanceBasePath = ASTERIX_INSTANCE_BASE_PATH + File.separator + asterixInstance.getName(); - ByteArrayOutputStream b = new ByteArrayOutputStream(); - ObjectOutputStream o = new ObjectOutputStream(b); - o.writeObject(asterixInstance); - zk.create(instanceBasePath, b.toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - // Create a place to put the state of the cluster in - String instanceStatePath = instanceBasePath + ASTERIX_INSTANCE_STATE_PATH; - zk.create(instanceStatePath, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } - - private void createRootIfNotExist() throws Exception { - try { - Stat stat = zk.exists(ASTERIX_INSTANCE_BASE_PATH, false); - if (stat == null) { - zk.create(ASTERIX_INSTANCE_BASE_PATH, "root".getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } - } catch (Exception e) { - // Is this the right way to handle the exception (try again? forever?) - LOGGER.error("An error took place when creating the root in Zookeeper"); - e.printStackTrace(); - createRootIfNotExist(); - } - } - - @Override - public AsterixInstance getAsterixInstance(String name) throws Exception { - String path = ASTERIX_INSTANCE_BASE_PATH + File.separator + name; - Stat stat = zk.exists(ASTERIX_INSTANCE_BASE_PATH + File.separator + name, false); - if (stat == null) { - return null; - } - byte[] asterixInstanceBytes = zk.getData(path, false, new Stat()); - return readAsterixInstanceObject(asterixInstanceBytes); - } - - @Override - public boolean exists(String path) throws Exception { - return zk.exists(ASTERIX_INSTANCE_BASE_PATH + File.separator + path, false) != null; - } - - @Override - public void removeAsterixInstance(String name) throws Exception { - if (!exists(name)) { - throw new EventException("Asterix instance by name " + name + " does not exists."); - } - if (exists(name + ASTERIX_INSTANCE_STATE_PATH)) { - if (exists(name + ASTERIX_INSTANCE_STATE_PATH + File.separator + "clusterState")) { - zk.delete(ASTERIX_INSTANCE_BASE_PATH + File.separator + name + ASTERIX_INSTANCE_STATE_PATH - + ASTERIX_INSTANCE_STATE_REPORT, DEFAULT_NODE_VERSION); - } - zk.delete(ASTERIX_INSTANCE_BASE_PATH + File.separator + name + ASTERIX_INSTANCE_STATE_PATH, - DEFAULT_NODE_VERSION); - } - zk.delete(ASTERIX_INSTANCE_BASE_PATH + File.separator + name, DEFAULT_NODE_VERSION); - } - - @Override - public List<AsterixInstance> getAsterixInstances() throws Exception { - List<String> instanceNames = zk.getChildren(ASTERIX_INSTANCE_BASE_PATH, false); - List<AsterixInstance> asterixInstances = new ArrayList<AsterixInstance>(); - String path; - for (String instanceName : instanceNames) { - path = ASTERIX_INSTANCE_BASE_PATH + File.separator + instanceName; - byte[] asterixInstanceBytes = zk.getData(path, false, new Stat()); - asterixInstances.add(readAsterixInstanceObject(asterixInstanceBytes)); - } - return asterixInstances; - } - - private AsterixInstance readAsterixInstanceObject(byte[] asterixInstanceBytes) - throws IOException, ClassNotFoundException { - ByteArrayInputStream b = new ByteArrayInputStream(asterixInstanceBytes); - ObjectInputStream ois = new ObjectInputStream(b); - return (AsterixInstance) ois.readObject(); - } - - @Override - public void updateAsterixInstance(AsterixInstance updatedInstance) throws Exception { - removeAsterixInstance(updatedInstance.getName()); - writeAsterixInstance(updatedInstance); - } - - @Override - public ClusterStateWatcher startWatchingClusterState(String instanceName) { - ClusterStateWatcher watcher = new ClusterStateWatcher(zk, instanceName); - watcher.startMonitoringThread(); - return watcher; - } - - @Override - public void reportClusterState(String instanceName, ClusterState state) throws Exception { - String clusterStatePath = ZooKeeperService.ASTERIX_INSTANCE_BASE_PATH + File.separator + instanceName - + ASTERIX_INSTANCE_STATE_PATH; - Integer value = state.ordinal(); - byte[] stateValue = new byte[] { value.byteValue() }; - // Create a place to put the state of the cluster in - zk.create(clusterStatePath + ASTERIX_INSTANCE_STATE_REPORT, stateValue, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - return; - } - -} - -class ZooKeeperWatcher implements Watcher { - - private boolean isRunning = true; - private LinkedBlockingQueue<String> msgQ; - - public ZooKeeperWatcher(LinkedBlockingQueue<String> msgQ) { - this.msgQ = msgQ; - } - - @Override - public void process(WatchedEvent wEvent) { - if (wEvent.getState() == KeeperState.SyncConnected) { - msgQ.add("connected"); - } - } - - public boolean isRunning() { - return isRunning; - } - -} - -class ZookeeperUtil { - - public static void writeConfiguration(String zooKeeperConfigPath, Configuration conf, int leaderConnPort, - int leaderElecPort) throws IOException { - - StringBuilder buffer = new StringBuilder(); - buffer.append("tickTime=1000" + "\n"); - buffer.append("dataDir=" + conf.getZookeeper().getHomeDir() + File.separator + "data" + "\n"); - buffer.append("clientPort=" + conf.getZookeeper().getClientPort().intValue() + "\n"); - buffer.append("initLimit=" + 2 + "\n"); - buffer.append("syncLimit=" + 2 + "\n"); - - List<String> servers = conf.getZookeeper().getServers().getServer(); - int serverId = 1; - for (String server : servers) { - buffer.append( - "server" + "." + serverId + "=" + server + ":" + leaderConnPort + ":" + leaderElecPort + "\n"); - serverId++; - } - AsterixEventServiceUtil.dumpToFile(zooKeeperConfigPath, buffer.toString()); - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/util/PatternCreator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/util/PatternCreator.java b/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/util/PatternCreator.java deleted file mode 100644 index c0f8f5f..0000000 --- a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/util/PatternCreator.java +++ /dev/null @@ -1,585 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.event.util; - -import java.io.File; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.asterix.event.driver.EventDriver; -import org.apache.asterix.event.error.VerificationUtil; -import org.apache.asterix.event.model.AsterixInstance; -import org.apache.asterix.event.model.BackupInfo; -import org.apache.asterix.event.model.BackupInfo.BackupType; -import org.apache.asterix.event.schema.cluster.Cluster; -import org.apache.asterix.event.schema.cluster.Node; -import org.apache.asterix.event.schema.pattern.Delay; -import org.apache.asterix.event.schema.pattern.Event; -import org.apache.asterix.event.schema.pattern.Nodeid; -import org.apache.asterix.event.schema.pattern.Pattern; -import org.apache.asterix.event.schema.pattern.Patterns; -import org.apache.asterix.event.schema.pattern.Value; -import org.apache.asterix.event.service.AsterixEventService; -import org.apache.asterix.event.service.AsterixEventServiceUtil; -import org.apache.asterix.installer.schema.conf.Backup; - -public class PatternCreator { - - public static final PatternCreator INSTANCE = new PatternCreator(); - - private PatternCreator() { - } - - private void addInitialDelay(Pattern p, int delay, String unit) { - Delay d = new Delay(new Value(null, Integer.toString(delay)), unit); - p.setDelay(d); - } - - public Patterns getAsterixBinaryTransferPattern(String asterixInstanceName, Cluster cluster) throws Exception { - String ccLocationIp = cluster.getMasterNode().getClusterIp(); - String destDir = cluster.getWorkingDir().getDir() + File.separator + "asterix"; - List<Pattern> ps = new ArrayList<>(); - - Pattern copyHyracks = createCopyHyracksPattern(asterixInstanceName, cluster, ccLocationIp, destDir); - ps.add(copyHyracks); - - boolean copyHyracksToNC = !cluster.getWorkingDir().isNFS(); - - for (Node node : cluster.getNode()) { - if (copyHyracksToNC) { - Pattern copyHyracksForNC = - createCopyHyracksPattern(asterixInstanceName, cluster, node.getClusterIp(), destDir); - ps.add(copyHyracksForNC); - } - } - ps.addAll(createHadoopLibraryTransferPattern(cluster).getPattern()); - return new Patterns(ps); - } - - public Patterns getStartAsterixPattern(String asterixInstanceName, Cluster cluster, boolean createCommand) - throws Exception { - String ccLocationId = cluster.getMasterNode().getId(); - List<Pattern> ps = new ArrayList<>(); - - Pattern createCC = createCCStartPattern(ccLocationId); - addInitialDelay(createCC, 3, "sec"); - ps.add(createCC); - - for (Node node : cluster.getNode()) { - String iodevices = node.getIodevices() == null ? cluster.getIodevices() : node.getIodevices(); - Pattern createNC = createNCStartPattern(cluster.getMasterNode().getClusterIp(), node.getId(), - asterixInstanceName + "_" + node.getId(), iodevices, createCommand); - addInitialDelay(createNC, 5, "sec"); - ps.add(createNC); - } - - return new Patterns(ps); - } - - public Patterns getBackUpAsterixPattern(AsterixInstance instance, Backup backupConf) throws Exception { - BackupType backupType = BackupInfo.getBackupType(backupConf); - Patterns patterns = null; - switch (backupType) { - case HDFS: - patterns = getHDFSBackUpAsterixPattern(instance, backupConf); - break; - case LOCAL: - patterns = getLocalBackUpAsterixPattern(instance, backupConf); - break; - } - return patterns; - } - - public Patterns getRestoreAsterixPattern(AsterixInstance instance, BackupInfo backupInfo) throws Exception { - BackupType backupType = backupInfo.getBackupType(); - Patterns patterns = null; - switch (backupType) { - case HDFS: - patterns = getHDFSRestoreAsterixPattern(instance, backupInfo); - break; - case LOCAL: - patterns = getLocalRestoreAsterixPattern(instance, backupInfo); - break; - } - return patterns; - } - - private Patterns getHDFSBackUpAsterixPattern(AsterixInstance instance, Backup backupConf) throws Exception { - Cluster cluster = instance.getCluster(); - String hdfsUrl = backupConf.getHdfs().getUrl(); - String hadoopVersion = backupConf.getHdfs().getVersion(); - String hdfsBackupDir = backupConf.getBackupDir(); - VerificationUtil.verifyBackupRestoreConfiguration(hdfsUrl, hadoopVersion, hdfsBackupDir); - String workingDir = cluster.getWorkingDir().getDir(); - String backupId = Integer.toString(instance.getBackupInfo().size()); - String store; - String pargs; - String iodevices; - store = cluster.getStore(); - List<Pattern> patternList = new ArrayList<>(); - for (Node node : cluster.getNode()) { - Nodeid nodeid = new Nodeid(new Value(null, node.getId())); - iodevices = node.getIodevices() == null ? instance.getCluster().getIodevices() : node.getIodevices(); - pargs = workingDir + " " + instance.getName() + " " + iodevices + " " + store + " " + " " - + AsterixEventServiceUtil.TXN_LOG_DIR + " " + backupId + " " + hdfsBackupDir + " " + "hdfs" + " " - + node.getId() + " " + hdfsUrl + " " + hadoopVersion; - Event event = new Event("backup", nodeid, pargs); - patternList.add(new Pattern(null, 1, null, event)); - } - return new Patterns(patternList); - } - - private Patterns getLocalBackUpAsterixPattern(AsterixInstance instance, Backup backupConf) throws Exception { - Cluster cluster = instance.getCluster(); - String backupDir = backupConf.getBackupDir(); - String workingDir = cluster.getWorkingDir().getDir(); - String backupId = Integer.toString(instance.getBackupInfo().size()); - String iodevices; - String txnLogDir; - String store; - String pargs; - store = cluster.getStore(); - List<Pattern> patternList = new ArrayList<>(); - for (Node node : cluster.getNode()) { - Nodeid nodeid = new Nodeid(new Value(null, node.getId())); - iodevices = node.getIodevices() == null ? instance.getCluster().getIodevices() : node.getIodevices(); - txnLogDir = node.getTxnLogDir() == null ? instance.getCluster().getTxnLogDir() : node.getTxnLogDir(); - pargs = workingDir + " " + instance.getName() + " " + iodevices + " " + store + " " + " " + txnLogDir + " " - + backupId + " " + backupDir + " " + "local" + " " + node.getId(); - Event event = new Event("backup", nodeid, pargs); - patternList.add(new Pattern(null, 1, null, event)); - } - return new Patterns(patternList); - } - - public Patterns getHDFSRestoreAsterixPattern(AsterixInstance instance, BackupInfo backupInfo) throws Exception { - Cluster cluster = instance.getCluster(); - String clusterStore = instance.getCluster().getStore(); - String hdfsUrl = backupInfo.getBackupConf().getHdfs().getUrl(); - String hadoopVersion = backupInfo.getBackupConf().getHdfs().getVersion(); - String hdfsBackupDir = backupInfo.getBackupConf().getBackupDir(); - VerificationUtil.verifyBackupRestoreConfiguration(hdfsUrl, hadoopVersion, hdfsBackupDir); - String workingDir = cluster.getWorkingDir().getDir(); - int backupId = backupInfo.getId(); - String pargs; - List<Pattern> patternList = new ArrayList<>(); - for (Node node : cluster.getNode()) { - Nodeid nodeid = new Nodeid(new Value(null, node.getId())); - String iodevices = node.getIodevices() == null ? cluster.getIodevices() : node.getIodevices(); - pargs = workingDir + " " + instance.getName() + " " + iodevices + " " + clusterStore + " " + " " - + AsterixEventServiceUtil.TXN_LOG_DIR + " " + backupId + " " + " " + hdfsBackupDir + " " + "hdfs" - + " " + node.getId() + " " + hdfsUrl + " " + hadoopVersion; - Event event = new Event("restore", nodeid, pargs); - patternList.add(new Pattern(null, 1, null, event)); - } - return new Patterns(patternList); - } - - public Patterns getLocalRestoreAsterixPattern(AsterixInstance instance, BackupInfo backupInfo) throws Exception { - Cluster cluster = instance.getCluster(); - String clusterStore = instance.getCluster().getStore(); - String backupDir = backupInfo.getBackupConf().getBackupDir(); - String workingDir = cluster.getWorkingDir().getDir(); - int backupId = backupInfo.getId(); - String pargs; - List<Pattern> patternList = new ArrayList<>(); - for (Node node : cluster.getNode()) { - Nodeid nodeid = new Nodeid(new Value(null, node.getId())); - String iodevices = node.getIodevices() == null ? cluster.getIodevices() : node.getIodevices(); - pargs = workingDir + " " + instance.getName() + " " + iodevices + " " + clusterStore + " " + " " - + AsterixEventServiceUtil.TXN_LOG_DIR + " " + backupId + " " + backupDir + " " + "local" + " " - + node.getId(); - Event event = new Event("restore", nodeid, pargs); - patternList.add(new Pattern(null, 1, null, event)); - } - return new Patterns(patternList); - } - - public Patterns createHadoopLibraryTransferPattern(Cluster cluster) throws Exception { - List<Pattern> patternList = new ArrayList<>(); - String workingDir = cluster.getWorkingDir().getDir(); - String hadoopVersion = AsterixEventService.getConfiguration().getBackup().getHdfs().getVersion(); - File hadoopDir = new File(AsterixEventService.getEventHome() + File.separator + "hadoop-" + hadoopVersion); - if (!hadoopDir.exists()) { - throw new IllegalStateException("Hadoop version :" + hadoopVersion + " not supported"); - } - - Nodeid nodeid = new Nodeid(new Value(null, EventDriver.CLIENT_NODE.getId())); - String username = cluster.getUsername() != null ? cluster.getUsername() : System.getProperty("user.name"); - String pargs = username + " " + hadoopDir.getAbsolutePath() + " " + cluster.getMasterNode().getClusterIp() + " " - + workingDir; - Event event = new Event("directory_transfer", nodeid, pargs); - Pattern p = new Pattern(null, 1, null, event); - addInitialDelay(p, 2, "sec"); - patternList.add(p); - - boolean copyToNC = !cluster.getWorkingDir().isNFS(); - if (copyToNC) { - for (Node node : cluster.getNode()) { - nodeid = new Nodeid(new Value(null, node.getId())); - pargs = cluster.getUsername() + " " + hadoopDir.getAbsolutePath() + " " + node.getClusterIp() + " " - + workingDir; - event = new Event("directory_transfer", nodeid, pargs); - p = new Pattern(null, 1, null, event); - addInitialDelay(p, 2, "sec"); - patternList.add(p); - } - } - return new Patterns(patternList); - } - - public Patterns createDeleteInstancePattern(AsterixInstance instance) throws Exception { - List<Pattern> patternList = new ArrayList<>(); - patternList.addAll(createRemoveAsterixStoragePattern(instance).getPattern()); - if (instance.getBackupInfo() != null && !instance.getBackupInfo().isEmpty()) { - List<BackupInfo> backups = instance.getBackupInfo(); - Set<String> removedBackupDirsHDFS = new HashSet<>(); - Set<String> removedBackupDirsLocal = new HashSet<>(); - - String backupDir; - for (BackupInfo binfo : backups) { - backupDir = binfo.getBackupConf().getBackupDir(); - switch (binfo.getBackupType()) { - case HDFS: - if (removedBackupDirsHDFS.contains(backupDir)) { - continue; - } - patternList.addAll(createRemoveHDFSBackupPattern(instance, backupDir).getPattern()); - removedBackupDirsHDFS.add(backupDir); - break; - - case LOCAL: - if (removedBackupDirsLocal.contains(backupDir)) { - continue; - } - patternList.addAll(createRemoveLocalBackupPattern(instance, backupDir).getPattern()); - removedBackupDirsLocal.add(backupDir); - break; - } - - } - } - patternList.addAll(createRemoveAsterixLogDirPattern(instance).getPattern()); - patternList.addAll(createRemoveAsterixTxnLogs(instance).getPattern()); - return new Patterns(patternList); - } - - private Patterns createRemoveAsterixTxnLogs(AsterixInstance instance) throws Exception { - List<Pattern> patternList = new ArrayList<>(); - Cluster cluster = instance.getCluster(); - Nodeid nodeid; - Event event; - for (Node node : cluster.getNode()) { - String txnLogDir = node.getTxnLogDir() == null ? cluster.getTxnLogDir() : node.getTxnLogDir(); - nodeid = new Nodeid(new Value(null, node.getId())); - event = new Event("file_delete", nodeid, txnLogDir); - patternList.add(new Pattern(null, 1, null, event)); - } - - return new Patterns(patternList); - } - - private Patterns createRemoveHDFSBackupPattern(AsterixInstance instance, String hdfsBackupDir) throws Exception { - List<Pattern> patternList = new ArrayList<>(); - Cluster cluster = instance.getCluster(); - String hdfsUrl = AsterixEventService.getConfiguration().getBackup().getHdfs().getUrl(); - String hadoopVersion = AsterixEventService.getConfiguration().getBackup().getHdfs().getVersion(); - String workingDir = cluster.getWorkingDir().getDir(); - Node launchingNode = cluster.getNode().get(0); - Nodeid nodeid = new Nodeid(new Value(null, launchingNode.getId())); - String pathToDelete = hdfsBackupDir + File.separator + instance.getName(); - String pargs = workingDir + " " + hadoopVersion + " " + hdfsUrl + " " + pathToDelete; - Event event = new Event("hdfs_delete", nodeid, pargs); - patternList.add(new Pattern(null, 1, null, event)); - return new Patterns(patternList); - } - - private Patterns createRemoveLocalBackupPattern(AsterixInstance instance, String localBackupDir) throws Exception { - List<Pattern> patternList = new ArrayList<>(); - Cluster cluster = instance.getCluster(); - - String pathToDelete = localBackupDir + File.separator + instance.getName(); - String pargs = pathToDelete; - List<String> removedBackupDirs = new ArrayList<>(); - for (Node node : cluster.getNode()) { - if (removedBackupDirs.contains(node.getClusterIp())) { - continue; - } - Nodeid nodeid = new Nodeid(new Value(null, node.getId())); - Event event = new Event("file_delete", nodeid, pargs); - patternList.add(new Pattern(null, 1, null, event)); - removedBackupDirs.add(node.getClusterIp()); - } - - return new Patterns(patternList); - } - - public Patterns createRemoveAsterixWorkingDirPattern(AsterixInstance instance) throws Exception { - List<Pattern> patternList = new ArrayList<>(); - Cluster cluster = instance.getCluster(); - String workingDir = cluster.getWorkingDir().getDir(); - String pargs = workingDir; - Nodeid nodeid = new Nodeid(new Value(null, cluster.getMasterNode().getId())); - Event event = new Event("file_delete", nodeid, pargs); - patternList.add(new Pattern(null, 1, null, event)); - - if (!cluster.getWorkingDir().isNFS()) { - for (Node node : cluster.getNode()) { - nodeid = new Nodeid(new Value(null, node.getId())); - event = new Event("file_delete", nodeid, pargs); - patternList.add(new Pattern(null, 1, null, event)); - } - } - return new Patterns(patternList); - } - - public Patterns getLibraryInstallPattern(AsterixInstance instance, String dataverse, String libraryName, - String libraryPath) throws Exception { - List<Pattern> patternList = new ArrayList<>(); - Cluster cluster = instance.getCluster(); - Nodeid nodeid = new Nodeid(new Value(null, EventDriver.CLIENT_NODE.getId())); - String username = cluster.getUsername() != null ? cluster.getUsername() : System.getProperty("user.name"); - String workingDir = cluster.getWorkingDir().getDir(); - String destDir = - workingDir + File.separator + "library" + File.separator + dataverse + File.separator + libraryName; - String fileToTransfer = new File(libraryPath).getAbsolutePath(); - - Iterator<Node> installTargets = cluster.getNode().iterator(); - Node installNode = installTargets.next(); - String destinationIp = installNode.getClusterIp(); - String pargs = username + " " + fileToTransfer + " " + destinationIp + " " + destDir + " " + "unpack"; - Event event = new Event("file_transfer", nodeid, pargs); - Pattern p = new Pattern(null, 1, null, event); - patternList.add(p); - - if (!cluster.getWorkingDir().isNFS()) { - while (installTargets.hasNext()) { - Node node = installTargets.next(); - pargs = username + " " + fileToTransfer + " " + node.getClusterIp() + " " + destDir + " " + "unpack"; - event = new Event("file_transfer", nodeid, pargs); - p = new Pattern(null, 1, null, event); - patternList.add(p); - } - - pargs = username + " " + fileToTransfer + " " + cluster.getMasterNode().getClusterIp() + " " + destDir + " " - + "unpack"; - event = new Event("file_transfer", nodeid, pargs); - p = new Pattern(null, 1, null, event); - patternList.add(p); - } - return new Patterns(patternList); - } - - public Patterns getLibraryUninstallPattern(AsterixInstance instance, String dataverse, String libraryName) - throws Exception { - List<Pattern> patternList = new ArrayList<>(); - Cluster cluster = instance.getCluster(); - String workingDir = cluster.getWorkingDir().getDir(); - String destFile = dataverse + "." + libraryName; - String pargs = workingDir + File.separator + "uninstall" + " " + destFile; - - String metadataNodeId = instance.getMetadataNodeId(); - Nodeid nodeid = new Nodeid(new Value(null, metadataNodeId)); - Event event = new Event("file_create", nodeid, pargs); - Pattern p = new Pattern(null, 1, null, event); - patternList.add(p); - - Iterator<Node> uninstallTargets = cluster.getNode().iterator(); - String libDir = - workingDir + File.separator + "library" + File.separator + dataverse + File.separator + libraryName; - Node uninstallNode = uninstallTargets.next(); - nodeid = new Nodeid(new Value(null, uninstallNode.getId())); - event = new Event("file_delete", nodeid, libDir); - p = new Pattern(null, 1, null, event); - patternList.add(p); - pargs = libDir; - - if (!cluster.getWorkingDir().isNFS()) { - while (uninstallTargets.hasNext()) { - uninstallNode = uninstallTargets.next(); - nodeid = new Nodeid(new Value(null, uninstallNode.getId())); - event = new Event("file_delete", nodeid, pargs); - p = new Pattern(null, 1, null, event); - patternList.add(p); - } - - nodeid = new Nodeid(new Value(null, cluster.getMasterNode().getId())); - event = new Event("file_delete", nodeid, pargs); - p = new Pattern(null, 1, null, event); - patternList.add(p); - - } - return new Patterns(patternList); - } - - private Patterns createRemoveAsterixLogDirPattern(AsterixInstance instance) throws Exception { - List<Pattern> patternList = new ArrayList<>(); - Cluster cluster = instance.getCluster(); - String pargs = instance.getCluster().getLogDir(); - Nodeid nodeid = new Nodeid(new Value(null, cluster.getMasterNode().getId())); - Event event = new Event("file_delete", nodeid, pargs); - patternList.add(new Pattern(null, 1, null, event)); - - for (Node node : cluster.getNode()) { - nodeid = new Nodeid(new Value(null, node.getId())); - if (node.getLogDir() != null) { - pargs = node.getLogDir(); - } - event = new Event("file_delete", nodeid, pargs); - patternList.add(new Pattern(null, 1, null, event)); - } - - return new Patterns(patternList); - } - - private Patterns createRemoveAsterixStoragePattern(AsterixInstance instance) throws Exception { - List<Pattern> patternList = new ArrayList<>(); - Cluster cluster = instance.getCluster(); - String pargs; - - for (Node node : cluster.getNode()) { - Nodeid nodeid = new Nodeid(new Value(null, node.getId())); - String[] nodeIODevices; - String iodevices = node.getIodevices() == null ? cluster.getIodevices() : node.getIodevices(); - nodeIODevices = iodevices.trim().split(","); - String nodeStore = cluster.getStore().trim(); - for (String nodeIODevice : nodeIODevices) { - pargs = nodeIODevice.trim() + File.separator + nodeStore; - Event event = new Event("file_delete", nodeid, pargs); - patternList.add(new Pattern(null, 1, null, event)); - } - } - return new Patterns(patternList); - } - - private Pattern createCopyHyracksPattern(String instanceName, Cluster cluster, String destinationIp, - String destDir) { - Nodeid nodeid = new Nodeid(new Value(null, EventDriver.CLIENT_NODE.getId())); - String username = cluster.getUsername() != null ? cluster.getUsername() : System.getProperty("user.name"); - String asterixZipName = AsterixEventService.getAsterixZip() - .substring(AsterixEventService.getAsterixZip().lastIndexOf(File.separator) + 1); - String fileToTransfer = new File( - AsterixEventService.getAsterixDir() + File.separator + instanceName + File.separator + asterixZipName) - .getAbsolutePath(); - String pargs = username + " " + fileToTransfer + " " + destinationIp + " " + destDir + " " + "unpack"; - Event event = new Event("file_transfer", nodeid, pargs); - return new Pattern(null, 1, null, event); - } - - private Pattern createCCStartPattern(String hostId) { - Nodeid nodeid = new Nodeid(new Value(null, hostId)); - Event event = new Event("cc_start", nodeid, ""); - return new Pattern(null, 1, null, event); - } - - public Pattern createCCStopPattern(String hostId) { - Nodeid nodeid = new Nodeid(new Value(null, hostId)); - Event event = new Event("cc_failure", nodeid, null); - return new Pattern(null, 1, null, event); - } - - public Pattern createNCStartPattern(String ccHost, String hostId, String nodeControllerId, String iodevices, - boolean isInitialRun) { - Nodeid nodeid = new Nodeid(new Value(null, hostId)); - String pargs = ccHost + " " + nodeControllerId + " " + iodevices; - if (isInitialRun) { - pargs += " " + "-initial-run"; - } - Event event = new Event("node_join", nodeid, pargs); - return new Pattern(null, 1, null, event); - } - - public Pattern createNCStopPattern(String hostId, String nodeControllerId) { - Nodeid nodeid = new Nodeid(new Value(null, hostId)); - Event event = new Event("node_failure", nodeid, nodeControllerId); - return new Pattern(null, 1, null, event); - } - - public Patterns createPrepareNodePattern(String instanceName, Cluster cluster, Node nodeToBeAdded) { - List<Pattern> ps = new ArrayList<>(); - boolean workingDirOnNFS = cluster.getWorkingDir().isNFS(); - if (!workingDirOnNFS) { - String ccLocationIp = cluster.getMasterNode().getClusterIp(); - String destDir = cluster.getWorkingDir().getDir() + File.separator + "asterix"; - Pattern copyHyracks = createCopyHyracksPattern(instanceName, cluster, ccLocationIp, destDir); - ps.add(copyHyracks); - - String workingDir = cluster.getWorkingDir().getDir(); - String hadoopVersion = AsterixEventService.getConfiguration().getBackup().getHdfs().getVersion(); - File hadoopDir = new File(AsterixEventService.getEventHome() + File.separator + "hadoop-" + hadoopVersion); - if (!hadoopDir.exists()) { - throw new IllegalStateException("Hadoop version :" + hadoopVersion + " not supported"); - } - - Nodeid nodeid = new Nodeid(new Value(null, EventDriver.CLIENT_NODE.getId())); - String username = cluster.getUsername() != null ? cluster.getUsername() : System.getProperty("user.name"); - String pargs = username + " " + hadoopDir.getAbsolutePath() + " " + cluster.getMasterNode().getClusterIp() - + " " + workingDir; - Event event = new Event("directory_transfer", nodeid, pargs); - Pattern p = new Pattern(null, 1, null, event); - addInitialDelay(p, 2, "sec"); - ps.add(p); - - nodeid = new Nodeid(new Value(null, nodeToBeAdded.getId())); - pargs = cluster.getUsername() + " " + hadoopDir.getAbsolutePath() + " " + nodeToBeAdded.getClusterIp() + " " - + workingDir; - event = new Event("directory_transfer", nodeid, pargs); - p = new Pattern(null, 1, null, event); - addInitialDelay(p, 2, "sec"); - ps.add(p); - } - - return new Patterns(ps); - } - - public Patterns getGenerateLogPattern(Cluster cluster, String outputDir) { - List<Pattern> patternList = new ArrayList<>(); - Map<String, String> nodeLogs = new HashMap<>(); - - String username = cluster.getUsername() == null ? System.getProperty("user.name") : cluster.getUsername(); - String srcHost = cluster.getMasterNode().getClientIp(); - Nodeid nodeid = new Nodeid(new Value(null, EventDriver.CLIENT_NODE.getId())); - String srcDir = - cluster.getMasterNode().getLogDir() == null ? cluster.getLogDir() : cluster.getMasterNode().getLogDir(); - String destDir = outputDir + File.separator + "cc"; - String pargs = username + " " + srcHost + " " + srcDir + " " + destDir; - Event event = new Event("directory_copy", nodeid, pargs); - Pattern p = new Pattern(null, 1, null, event); - patternList.add(p); - nodeLogs.put(cluster.getMasterNode().getClusterIp(), srcDir); - for (Node node : cluster.getNode()) { - srcHost = node.getClusterIp(); - srcDir = node.getLogDir() == null ? cluster.getLogDir() : node.getLogDir(); - if (nodeLogs.get(node.getClusterIp()) != null && nodeLogs.get(node.getClusterIp()).equals(srcDir)) { - continue; - } - destDir = outputDir + File.separator + node.getId(); - pargs = username + " " + srcHost + " " + srcDir + " " + destDir; - event = new Event("directory_copy", nodeid, pargs); - p = new Pattern(null, 1, null, event); - patternList.add(p); - } - return new Patterns(patternList); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-events/src/main/resources/events/backup/backup.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-events/src/main/resources/events/backup/backup.sh b/asterixdb/asterix-events/src/main/resources/events/backup/backup.sh deleted file mode 100755 index 155cb86..0000000 --- a/asterixdb/asterix-events/src/main/resources/events/backup/backup.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -WORKING_DIR=$1 -ASTERIX_INSTANCE_NAME=$2 -ASTERIX_IODEVICES=$3 -NODE_STORE=$4 -ASTERIX_ROOT_METADATA_DIR=$5 -TXN_LOG_DIR=$6 -BACKUP_ID=$7 -BACKUP_DIR=$8 -BACKUP_TYPE=$9 -NODE_ID=${10} - -nodeIODevices=$(echo $ASTERIX_IODEVICES | tr "," "\n") - -if [ $BACKUP_TYPE == "hdfs" ]; -then - HDFS_URL=${11} - HADOOP_VERSION=${12} - export HADOOP_HOME=$WORKING_DIR/hadoop-$HADOOP_VERSION - index=1 - for nodeIODevice in $nodeIODevices - do - STORE_DIR=$nodeIODevice/$NODE_STORE - NODE_BACKUP_DIR=$BACKUP_DIR/$ASTERIX_INSTANCE_NAME/$BACKUP_ID/$NODE_ID/ - - # make the destination directory - $HADOOP_HOME/bin/hadoop fs -mkdir $STORE_DIR $HDFS_URL/$NODE_BACKUP_DIR - - # copy store directory - $HADOOP_HOME/bin/hadoop fs -copyFromLocal $STORE_DIR $HDFS_URL/$NODE_BACKUP_DIR/ - - # copy asterix root metadata directory and log directory from the primary(first) iodevice - if [ $index -eq 1 ]; - then - # copy asterix root metadata directory - $HADOOP_HOME/bin/hadoop fs -copyFromLocal $nodeIODevice/$ASTERIX_ROOT_METADATA_DIR $HDFS_URL/$NODE_BACKUP_DIR/ - - # copy log directory - $HADOOP_HOME/bin/hadoop fs -copyFromLocal $TXN_LOG_DIR $HDFS_URL/$NODE_BACKUP_DIR/ - fi - - index=`expr $index + 1` - done -else - index=1 - for nodeIODevice in $nodeIODevices - do - STORE_DIR=$nodeIODevice/$NODE_STORE - NODE_BACKUP_DIR=$BACKUP_DIR/$ASTERIX_INSTANCE_NAME/$BACKUP_ID/$NODE_ID - - # create the backup directory, if it does not exists - if [ ! -d $NODE_BACKUP_DIR ]; - then - mkdir -p $NODE_BACKUP_DIR - fi - - # copy store directory - cp -r $STORE_DIR $NODE_BACKUP_DIR/ - - # copy asterix root metadata directory and log directory from the primary(first) iodevice - if [ $index -eq 1 ]; - then - cp -r $nodeIODevice/$ASTERIX_ROOT_METADATA_DIR $NODE_BACKUP_DIR/ - - # copy log directory - cp -r $TXN_LOG_DIR $NODE_BACKUP_DIR/ - fi - - index=`expr $index + 1` - done -fi http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5dcf139e/asterixdb/asterix-events/src/main/resources/events/cc_failure/cc_failure.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-events/src/main/resources/events/cc_failure/cc_failure.sh b/asterixdb/asterix-events/src/main/resources/events/cc_failure/cc_failure.sh deleted file mode 100755 index e8bc326..0000000 --- a/asterixdb/asterix-events/src/main/resources/events/cc_failure/cc_failure.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -#/* -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -CC_PARENT_ID=`ps -ef | grep asterix | grep cc_start | grep -v ssh | awk '{ print $2 }'` -CC_ID=`ps -ef | grep asterix | grep $CC_PARENT_ID | grep -v bash | awk "\\\$3 == $CC_PARENT_ID { print \\\$2 }"` -kill -9 $CC_ID
