http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java ---------------------------------------------------------------------- diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java new file mode 100644 index 0000000..71ef0ea --- /dev/null +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -0,0 +1,778 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo; + +import com.google.common.base.Charsets; +import com.google.common.io.Closeables; +import com.google.common.io.Files; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.*; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.util.ShutdownHookManager; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.tajo.catalog.*; +import org.apache.tajo.client.TajoClient; +import org.apache.tajo.client.TajoClientImpl; +import org.apache.tajo.client.TajoClientUtil; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.engine.planner.global.rewriter.GlobalPlanTestRuleProvider; +import org.apache.tajo.master.TajoMaster; +import org.apache.tajo.plan.rewrite.LogicalPlanTestRuleProvider; +import org.apache.tajo.querymaster.Query; +import org.apache.tajo.querymaster.QueryMasterTask; +import org.apache.tajo.querymaster.Stage; +import org.apache.tajo.querymaster.StageState; +import org.apache.tajo.service.ServiceTrackerFactory; +import org.apache.tajo.storage.FileTablespace; +import org.apache.tajo.storage.TablespaceManager; +import org.apache.tajo.util.CommonTestingUtil; +import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.NetUtils; +import org.apache.tajo.util.Pair; +import org.apache.tajo.worker.TajoWorker; + +import java.io.File; +import java.io.IOException; +import java.io.Writer; +import java.net.InetSocketAddress; +import java.net.URI; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.TimeZone; +import java.util.UUID; + +public class TajoTestingCluster { + private static Log LOG = LogFactory.getLog(TajoTestingCluster.class); + private TajoConf conf; + private FileSystem defaultFS; + private MiniDFSCluster dfsCluster; + private MiniCatalogServer catalogServer; + private HBaseTestClusterUtil hbaseUtil; + + private TajoMaster tajoMaster; + private List<TajoWorker> tajoWorkers = new ArrayList<TajoWorker>(); + private boolean isDFSRunning = false; + private boolean isTajoClusterRunning = false; + private boolean isCatalogServerRunning = false; + + private File clusterTestBuildDir = null; + + /** + * Default parent directory for test output. + */ + public static final String DEFAULT_TEST_DIRECTORY = "target/" + + System.getProperty("tajo.test.data.dir", "test-data"); + + /** + * True If HiveCatalogStore is used. Otherwise, it is FALSE. + */ + public Boolean isHiveCatalogStoreUse = false; + + private static final String LOG_LEVEL; + + static { + LOG_LEVEL = System.getProperty("LOG_LEVEL"); + } + + public TajoTestingCluster() { + this(false); + } + + public TajoTestingCluster(boolean masterHaEMode) { + this.conf = new TajoConf(); + this.conf.setBoolVar(ConfVars.TAJO_MASTER_HA_ENABLE, masterHaEMode); + + initTestDir(); + setTestingFlagProperties(); + initPropertiesAndConfigs(); + } + + void setTestingFlagProperties() { + System.setProperty(CommonTestingUtil.TAJO_TEST_KEY, CommonTestingUtil.TAJO_TEST_TRUE); + conf.set(CommonTestingUtil.TAJO_TEST_KEY, CommonTestingUtil.TAJO_TEST_TRUE); + } + + void initPropertiesAndConfigs() { + + // Set time zone + TimeZone testDefaultTZ = TimeZone.getTimeZone(TajoConstants.DEFAULT_SYSTEM_TIMEZONE); + conf.setSystemTimezone(testDefaultTZ); + TimeZone.setDefault(testDefaultTZ); + + // Injection of equality testing code of logical plan (de)serialization + conf.setClassVar(ConfVars.LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, LogicalPlanTestRuleProvider.class); + conf.setClassVar(ConfVars.GLOBAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, GlobalPlanTestRuleProvider.class); + + conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES.varname, 4); + conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_MEMORY_MB.varname, 2000); + conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_DISK_PARALLEL_NUM.varname, 3); + conf.setInt(ConfVars.SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM.varname, 2); + + // Client API RPC + conf.setIntVar(ConfVars.RPC_CLIENT_WORKER_THREAD_NUM, 2); + + //Client API service RPC Server + conf.setIntVar(ConfVars.MASTER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM, 2); + conf.setIntVar(ConfVars.WORKER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM, 2); + conf.setIntVar(ConfVars.REST_SERVICE_RPC_SERVER_WORKER_THREAD_NUM, 2); + + // Internal RPC Client + conf.setIntVar(ConfVars.INTERNAL_RPC_CLIENT_WORKER_THREAD_NUM, 2); + conf.setIntVar(ConfVars.SHUFFLE_RPC_CLIENT_WORKER_THREAD_NUM, 2); + + // Internal RPC Server + conf.setIntVar(ConfVars.MASTER_RPC_SERVER_WORKER_THREAD_NUM, 2); + conf.setIntVar(ConfVars.QUERY_MASTER_RPC_SERVER_WORKER_THREAD_NUM, 2); + conf.setIntVar(ConfVars.WORKER_RPC_SERVER_WORKER_THREAD_NUM, 2); + conf.setIntVar(ConfVars.CATALOG_RPC_SERVER_WORKER_THREAD_NUM, 2); + conf.setIntVar(ConfVars.SHUFFLE_RPC_SERVER_WORKER_THREAD_NUM, 2); + + // Memory cache termination + conf.setIntVar(ConfVars.WORKER_HISTORY_EXPIRE_PERIOD, 1); + + // Python function path + conf.setStrings(ConfVars.PYTHON_CODE_DIR.varname, getClass().getResource("/python").toString()); + + /* Since Travis CI limits the size of standard output log up to 4MB */ + if (!StringUtils.isEmpty(LOG_LEVEL)) { + Level defaultLevel = Logger.getRootLogger().getLevel(); + Logger.getLogger("org.apache.tajo").setLevel(Level.toLevel(LOG_LEVEL.toUpperCase(), defaultLevel)); + Logger.getLogger("org.apache.hadoop").setLevel(Level.toLevel(LOG_LEVEL.toUpperCase(), defaultLevel)); + Logger.getLogger("org.apache.zookeeper").setLevel(Level.toLevel(LOG_LEVEL.toUpperCase(), defaultLevel)); + Logger.getLogger("BlockStateChange").setLevel(Level.toLevel(LOG_LEVEL.toUpperCase(), defaultLevel)); + Logger.getLogger("org.mortbay.log").setLevel(Level.toLevel(LOG_LEVEL.toUpperCase(), defaultLevel)); + } + } + + public TajoConf getConfiguration() { + return this.conf; + } + + public void initTestDir() { + if (clusterTestBuildDir == null) { + clusterTestBuildDir = setupClusterTestBuildDir(); + } + } + + /** + * @return Where to write test data on local filesystem; usually + * {@link #DEFAULT_TEST_DIRECTORY} + * @see #setupClusterTestBuildDir() + */ + public File getTestDir() { + return clusterTestBuildDir; + } + + /** + * @param subdirName + * @return Path to a subdirectory named <code>subdirName</code> under + * {@link #getTestDir()}. + * @see #setupClusterTestBuildDir() + */ + public static File getTestDir(final String subdirName) { + return new File(new File(DEFAULT_TEST_DIRECTORY), subdirName); + } + + public static File setupClusterTestBuildDir() { + String randomStr = UUID.randomUUID().toString(); + String dirStr = getTestDir(randomStr).toString(); + File dir = new File(dirStr).getAbsoluteFile(); + // Have it cleaned up on exit + dir.deleteOnExit(); + return dir; + } + + //////////////////////////////////////////////////////// + // HDFS Section + //////////////////////////////////////////////////////// + /** + * Start a minidfscluster. + * @param servers How many DNs to start. + * @throws Exception + * @see {@link #shutdownMiniDFSCluster()} + * @return The mini dfs cluster created. + */ + public MiniDFSCluster startMiniDFSCluster(int servers) throws Exception { + return startMiniDFSCluster(servers, null, null); + } + + /** + * Start a minidfscluster. + * Can only create one. + * @param servers How many DNs to start. + * @param dir Where to home your dfs cluster. + * @param hosts hostnames DNs to run on. + * @throws Exception + * @see {@link #shutdownMiniDFSCluster()} + * @return The mini dfs cluster created. + * @throws java.io.IOException + */ + public MiniDFSCluster startMiniDFSCluster(int servers, + File dir, + final String hosts[]) + throws IOException { + + conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dir.toString()); + conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); + conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, false); + MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(new HdfsConfiguration(conf)); + builder.hosts(hosts); + builder.numDataNodes(servers); + builder.format(true); + builder.manageNameDfsDirs(true); + builder.manageDataDfsDirs(true); + builder.waitSafeMode(true); + this.dfsCluster = builder.build(); + + // Set this just-started cluster as our filesystem. + this.defaultFS = this.dfsCluster.getFileSystem(); + this.conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultFS.getUri().toString()); + this.conf.setVar(TajoConf.ConfVars.ROOT_DIR, defaultFS.getUri() + "/tajo"); + isDFSRunning = true; + return this.dfsCluster; + } + + public void shutdownMiniDFSCluster() throws Exception { + if (this.dfsCluster != null) { + try { + FileSystem fs = this.dfsCluster.getFileSystem(); + if (fs != null) fs.close(); + } catch (IOException e) { + System.err.println("error closing file system: " + e); + } + // The below throws an exception per dn, AsynchronousCloseException. + this.dfsCluster.shutdown(); + } + } + + public boolean isRunningDFSCluster() { + return this.defaultFS != null; + } + + public MiniDFSCluster getMiniDFSCluster() { + return this.dfsCluster; + } + + public FileSystem getDefaultFileSystem() { + return this.defaultFS; + } + + public HBaseTestClusterUtil getHBaseUtil() { + return hbaseUtil; + } + + //////////////////////////////////////////////////////// + // Catalog Section + //////////////////////////////////////////////////////// + public MiniCatalogServer startCatalogCluster() throws Exception { + if(isCatalogServerRunning) throw new IOException("Catalog Cluster already running"); + + TajoConf c = getConfiguration(); + + conf.set(CatalogConstants.STORE_CLASS, "org.apache.tajo.catalog.store.MemStore"); + conf.set(CatalogConstants.CATALOG_URI, "jdbc:derby:" + clusterTestBuildDir.getAbsolutePath() + "/db"); + LOG.info("Apache Derby repository is set to " + conf.get(CatalogConstants.CATALOG_URI)); + conf.setVar(ConfVars.CATALOG_ADDRESS, "localhost:0"); + + catalogServer = new MiniCatalogServer(conf); + CatalogServer catServer = catalogServer.getCatalogServer(); + InetSocketAddress sockAddr = catServer.getBindAddress(); + c.setVar(ConfVars.CATALOG_ADDRESS, NetUtils.normalizeInetSocketAddress(sockAddr)); + isCatalogServerRunning = true; + return this.catalogServer; + } + + public void shutdownCatalogCluster() { + if (catalogServer != null) { + this.catalogServer.shutdown(); + } + isCatalogServerRunning = false; + } + + public MiniCatalogServer getMiniCatalogCluster() { + return this.catalogServer; + } + + public boolean isHiveCatalogStoreRunning() { + return isHiveCatalogStoreUse; + } + + //////////////////////////////////////////////////////// + // Tajo Cluster Section + //////////////////////////////////////////////////////// + private void startMiniTajoCluster(File testBuildDir, + final int numSlaves, + boolean local) throws Exception { + TajoConf c = getConfiguration(); + c.setVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, "localhost:0"); + c.setVar(ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, "localhost:0"); + c.setVar(ConfVars.RESOURCE_TRACKER_RPC_ADDRESS, "localhost:0"); + c.setVar(ConfVars.WORKER_PEER_RPC_ADDRESS, "localhost:0"); + c.setVar(ConfVars.WORKER_TEMPORAL_DIR, "file://" + testBuildDir.getAbsolutePath() + "/tajo-localdir"); + c.setIntVar(ConfVars.REST_SERVICE_PORT, 0); + + LOG.info("derby repository is set to "+conf.get(CatalogConstants.CATALOG_URI)); + + if (!local) { + String tajoRootDir = getMiniDFSCluster().getFileSystem().getUri().toString() + "/tajo"; + c.setVar(ConfVars.ROOT_DIR, tajoRootDir); + + URI defaultTsUri = TajoConf.getWarehouseDir(c).toUri(); + FileTablespace defaultTableSpace = + new FileTablespace(TablespaceManager.DEFAULT_TABLESPACE_NAME, defaultTsUri); + defaultTableSpace.init(conf); + TablespaceManager.addTableSpaceForTest(defaultTableSpace); + + } else { + c.setVar(ConfVars.ROOT_DIR, "file://" + testBuildDir.getAbsolutePath() + "/tajo"); + } + + setupCatalogForTesting(c, testBuildDir); + + tajoMaster = new TajoMaster(); + tajoMaster.init(c); + tajoMaster.start(); + + this.conf.setVar(ConfVars.WORKER_PEER_RPC_ADDRESS, c.getVar(ConfVars.WORKER_PEER_RPC_ADDRESS)); + this.conf.setVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, c.getVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS)); + + InetSocketAddress tajoMasterAddress = tajoMaster.getContext().getTajoMasterService().getBindAddress(); + + this.conf.setVar(ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, + tajoMasterAddress.getHostName() + ":" + tajoMasterAddress.getPort()); + this.conf.setVar(ConfVars.RESOURCE_TRACKER_RPC_ADDRESS, c.getVar(ConfVars.RESOURCE_TRACKER_RPC_ADDRESS)); + this.conf.setVar(ConfVars.CATALOG_ADDRESS, c.getVar(ConfVars.CATALOG_ADDRESS)); + + InetSocketAddress tajoRestAddress = tajoMaster.getContext().getRestServer().getBindAddress(); + + this.conf.setIntVar(ConfVars.REST_SERVICE_PORT, tajoRestAddress.getPort()); + + startTajoWorkers(numSlaves); + + isTajoClusterRunning = true; + LOG.info("Mini Tajo cluster is up"); + LOG.info("===================================================================================="); + LOG.info("= MiniTajoCluster starts up ="); + LOG.info("===================================================================================="); + LOG.info("= * Master Address: " + tajoMaster.getMasterName()); + LOG.info("= * CatalogStore: " + tajoMaster.getCatalogServer().getStoreClassName()); + LOG.info("------------------------------------------------------------------------------------"); + LOG.info("= * Warehouse Dir: " + TajoConf.getWarehouseDir(c)); + LOG.info("= * Worker Tmp Dir: " + c.getVar(ConfVars.WORKER_TEMPORAL_DIR)); + LOG.info("===================================================================================="); + } + + private void setupCatalogForTesting(TajoConf c, File testBuildDir) throws IOException { + final String HIVE_CATALOG_CLASS_NAME = "org.apache.tajo.catalog.store.HiveCatalogStore"; + boolean hiveCatalogClassExists = false; + try { + getClass().getClassLoader().loadClass(HIVE_CATALOG_CLASS_NAME); + hiveCatalogClassExists = true; + } catch (ClassNotFoundException e) { + LOG.info("HiveCatalogStore is not available."); + } + String driverClass = System.getProperty(CatalogConstants.STORE_CLASS); + + if (hiveCatalogClassExists && + driverClass != null && driverClass.equals(HIVE_CATALOG_CLASS_NAME)) { + try { + getClass().getClassLoader().loadClass(HIVE_CATALOG_CLASS_NAME); + String jdbcUri = "jdbc:derby:;databaseName="+ testBuildDir.toURI().getPath() + "/metastore_db;create=true"; + c.set("hive.metastore.warehouse.dir", TajoConf.getWarehouseDir(c).toString() + "/default"); + c.set("javax.jdo.option.ConnectionURL", jdbcUri); + c.set(TajoConf.ConfVars.WAREHOUSE_DIR.varname, conf.getVar(ConfVars.WAREHOUSE_DIR)); + c.set(CatalogConstants.STORE_CLASS, HIVE_CATALOG_CLASS_NAME); + Path defaultDatabasePath = new Path(TajoConf.getWarehouseDir(c).toString() + "/default"); + FileSystem fs = defaultDatabasePath.getFileSystem(c); + if (!fs.exists(defaultDatabasePath)) { + fs.mkdirs(defaultDatabasePath); + } + isHiveCatalogStoreUse = true; + } catch (ClassNotFoundException cnfe) { + throw new IOException(cnfe); + } + } else { // for derby + c.set(CatalogConstants.STORE_CLASS, "org.apache.tajo.catalog.store.MemStore"); + c.set(CatalogConstants.CATALOG_URI, "jdbc:derby:" + testBuildDir.getAbsolutePath() + "/db"); + } + c.setVar(ConfVars.CATALOG_ADDRESS, "localhost:0"); + } + + private void startTajoWorkers(int numSlaves) throws Exception { + for(int i = 0; i < 1; i++) { + TajoWorker tajoWorker = new TajoWorker(); + + TajoConf workerConf = new TajoConf(this.conf); + + workerConf.setVar(ConfVars.WORKER_INFO_ADDRESS, "localhost:0"); + workerConf.setVar(ConfVars.WORKER_CLIENT_RPC_ADDRESS, "localhost:0"); + workerConf.setVar(ConfVars.WORKER_PEER_RPC_ADDRESS, "localhost:0"); + + workerConf.setVar(ConfVars.WORKER_QM_RPC_ADDRESS, "localhost:0"); + + tajoWorker.startWorker(workerConf, new String[0]); + + LOG.info("MiniTajoCluster Worker #" + (i + 1) + " started."); + tajoWorkers.add(tajoWorker); + } + } + + public TajoMaster getMaster() { + return this.tajoMaster; + } + + public List<TajoWorker> getTajoWorkers() { + return this.tajoWorkers; + } + + public void shutdownMiniTajoCluster() { + if(this.tajoMaster != null) { + this.tajoMaster.stop(); + } + for(TajoWorker eachWorker: tajoWorkers) { + eachWorker.stopWorkerForce(); + } + tajoWorkers.clear(); + this.tajoMaster= null; + } + + //////////////////////////////////////////////////////// + // Meta Cluster Section + //////////////////////////////////////////////////////// + /** + * @throws java.io.IOException If a cluster -- dfs or engine -- already running. + */ + void isRunningCluster() throws IOException { + if (!isTajoClusterRunning && !isCatalogServerRunning && !isDFSRunning) return; + throw new IOException("Cluster already running at " + + this.clusterTestBuildDir); + } + + /** + * This method starts up a tajo cluster with a given number of clusters in + * distributed mode. + * + * @param numSlaves the number of tajo cluster to start up + * @throws Exception + */ + public void startMiniCluster(final int numSlaves) + throws Exception { + startMiniCluster(numSlaves, null); + } + + public void startMiniCluster(final int numSlaves, final String [] dataNodeHosts) throws Exception { + + int numDataNodes = numSlaves; + if(dataNodeHosts != null && dataNodeHosts.length != 0) { + numDataNodes = dataNodeHosts.length; + } + + LOG.info("Starting up minicluster with 1 master(s) and " + + numSlaves + " worker(s) and " + numDataNodes + " datanode(s)"); + + // If we already bring up the cluster, fail. + isRunningCluster(); + if (clusterTestBuildDir != null) { + LOG.info("Using passed path: " + clusterTestBuildDir); + } + + startMiniDFSCluster(numDataNodes, clusterTestBuildDir, dataNodeHosts); + this.dfsCluster.waitClusterUp(); + + conf.setInt("hbase.hconnection.threads.core", 5); + conf.setInt("hbase.hconnection.threads.max", 50); + hbaseUtil = new HBaseTestClusterUtil(conf, clusterTestBuildDir); + + startMiniTajoCluster(this.clusterTestBuildDir, numSlaves, false); + } + + public void startMiniClusterInLocal(final int numSlaves) throws Exception { + isRunningCluster(); + + if (clusterTestBuildDir != null) { + LOG.info("Using passed path: " + clusterTestBuildDir); + } + + startMiniTajoCluster(this.clusterTestBuildDir, numSlaves, true); + } + + public void shutdownMiniCluster() throws IOException { + LOG.info("========================================"); + LOG.info("Minicluster is stopping"); + LOG.info("========================================"); + + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + shutdownMiniTajoCluster(); + + if(this.catalogServer != null) { + shutdownCatalogCluster(); + isCatalogServerRunning = false; + } + + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + if(this.dfsCluster != null) { + try { + FileSystem fs = this.dfsCluster.getFileSystem(); + if (fs != null) fs.close(); + this.dfsCluster.shutdown(); + } catch (IOException e) { + System.err.println("error closing file system: " + e); + } + isDFSRunning = false; + } + + if(this.clusterTestBuildDir != null && this.clusterTestBuildDir.exists()) { + if(!ShutdownHookManager.get().isShutdownInProgress()) { + //TODO clean test dir when ShutdownInProgress + LocalFileSystem localFS = LocalFileSystem.getLocal(conf); + localFS.delete(new Path(clusterTestBuildDir.toString()), true); + localFS.close(); + } + this.clusterTestBuildDir = null; + } + + if(hbaseUtil != null) { + hbaseUtil.stopZooKeeperCluster(); + hbaseUtil.stopHBaseCluster(); + } + + LOG.info("Minicluster is down"); + isTajoClusterRunning = false; + } + + public TajoClient newTajoClient() throws Exception { + return new TajoClientImpl(ServiceTrackerFactory.get(getConfiguration())); + } + + public static ResultSet run(String[] names, + Schema[] schemas, + KeyValueSet tableOption, + String[][] tables, + String query, + TajoClient client) throws Exception { + TajoTestingCluster util = TpchTestBase.getInstance().getTestingCluster(); + + FileSystem fs = util.getDefaultFileSystem(); + Path rootDir = TajoConf.getWarehouseDir(util.getConfiguration()); + fs.mkdirs(rootDir); + for (int i = 0; i < names.length; i++) { + createTable(names[i], schemas[i], tableOption, tables[i]); + } + Thread.sleep(1000); + ResultSet res = client.executeQueryAndGetResult(query); + return res; + } + + public static ResultSet run(String[] names, + Schema[] schemas, + KeyValueSet tableOption, + String[][] tables, + String query) throws Exception { + TpchTestBase instance = TpchTestBase.getInstance(); + TajoTestingCluster util = instance.getTestingCluster(); + while(true) { + if(util.getMaster().isMasterRunning()) { + break; + } + Thread.sleep(1000); + } + TajoConf conf = util.getConfiguration(); + TajoClient client = new TajoClientImpl(ServiceTrackerFactory.get(conf)); + + try { + return run(names, schemas, tableOption, tables, query, client); + } finally { + client.close(); + } + } + + public static TajoClient newTajoClient(TajoTestingCluster util) throws SQLException, InterruptedException { + while(true) { + if(util.getMaster().isMasterRunning()) { + break; + } + Thread.sleep(1000); + } + TajoConf conf = util.getConfiguration(); + return new TajoClientImpl(ServiceTrackerFactory.get(conf)); + } + + public static void createTable(String tableName, Schema schema, + KeyValueSet tableOption, String[] tableDatas) throws Exception { + createTable(tableName, schema, tableOption, tableDatas, 1); + } + + public static void createTable(String tableName, Schema schema, + KeyValueSet tableOption, String[] tableDatas, int numDataFiles) throws Exception { + TpchTestBase instance = TpchTestBase.getInstance(); + TajoTestingCluster util = instance.getTestingCluster(); + TajoClient client = newTajoClient(util); + try { + FileSystem fs = util.getDefaultFileSystem(); + Path rootDir = TajoConf.getWarehouseDir(util.getConfiguration()); + if (!fs.exists(rootDir)) { + fs.mkdirs(rootDir); + } + Path tablePath; + if (CatalogUtil.isFQTableName(tableName)) { + Pair<String, String> name = CatalogUtil.separateQualifierAndName(tableName); + tablePath = new Path(rootDir, new Path(name.getFirst(), name.getSecond())); + } else { + tablePath = new Path(rootDir, tableName); + } + + fs.mkdirs(tablePath); + if (tableDatas.length > 0) { + int recordPerFile = tableDatas.length / numDataFiles; + if (recordPerFile == 0) { + recordPerFile = 1; + } + FSDataOutputStream out = null; + for (int j = 0; j < tableDatas.length; j++) { + if (out == null || j % recordPerFile == 0) { + if (out != null) { + out.close(); + } + Path dfsPath = new Path(tablePath, tableName + j + ".tbl"); + out = fs.create(dfsPath); + } + out.write((tableDatas[j] + "\n").getBytes()); + } + if (out != null) { + out.close(); + } + } + TableMeta meta = CatalogUtil.newTableMeta("TEXT", tableOption); + client.createExternalTable(tableName, schema, tablePath.toUri(), meta); + } finally { + client.close(); + } + } + + /** + * Write lines to a file. + * + * @param file File to write lines to + * @param lines Strings written to the file + * @throws java.io.IOException + */ + private static void writeLines(File file, String... lines) + throws IOException { + Writer writer = Files.newWriter(file, Charsets.UTF_8); + try { + for (String line : lines) { + writer.write(line); + writer.write('\n'); + } + } finally { + Closeables.closeQuietly(writer); + } + } + + public void setAllTajoDaemonConfValue(String key, String value) { + tajoMaster.getContext().getConf().set(key, value); + setAllWorkersConfValue(key, value); + } + + public void setAllWorkersConfValue(String key, String value) { + for (TajoWorker eachWorker: tajoWorkers) { + eachWorker.getConfig().set(key, value); + } + } + + public void waitForQuerySubmitted(QueryId queryId) throws Exception { + waitForQuerySubmitted(queryId, 50); + } + + public void waitForQuerySubmitted(QueryId queryId, int delay) throws Exception { + QueryMasterTask qmt = null; + + int i = 0; + while (qmt == null || TajoClientUtil.isQueryWaitingForSchedule(qmt.getState())) { + try { + Thread.sleep(delay); + + if (qmt == null) { + qmt = getQueryMasterTask(queryId); + } + } catch (InterruptedException e) { + } + if (++i > 200) { + throw new IOException("Timed out waiting for query to start"); + } + } + } + + public void waitForQueryState(Query query, TajoProtos.QueryState expected, int delay) throws Exception { + int i = 0; + while (query == null || query.getSynchronizedState() != expected) { + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + } + if (++i > 200) { + throw new IOException("Timed out waiting. expected: " + expected + + ", actual: " + query != null ? String.valueOf(query.getSynchronizedState()) : String.valueOf(query)); + } + } + } + + public void waitForStageState(Stage stage, StageState expected, int delay) throws Exception { + + int i = 0; + while (stage == null || stage.getSynchronizedState() != expected) { + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + } + if (++i > 200) { + throw new IOException("Timed out waiting"); + } + } + } + + public QueryMasterTask getQueryMasterTask(QueryId queryId) { + QueryMasterTask qmt = null; + for (TajoWorker worker : getTajoWorkers()) { + qmt = worker.getWorkerContext().getQueryMaster().getQueryMasterTask(queryId, true); + if (qmt != null && queryId.equals(qmt.getQueryId())) { + break; + } + } + return qmt; + } +}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java ---------------------------------------------------------------------- diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java new file mode 100644 index 0000000..055dd02 --- /dev/null +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo; + +import com.google.common.collect.Maps; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.benchmark.TPCH; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.util.FileUtil; +import org.apache.tajo.util.KeyValueSet; + +import java.io.File; +import java.io.IOException; +import java.sql.ResultSet; +import java.util.Map; + +public class TpchTestBase { + private static final Log LOG = LogFactory.getLog(TpchTestBase.class); + + String [] names; + String [] paths; + String [][] tables; + Schema[] schemas; + Map<String, Integer> nameMap = Maps.newHashMap(); + protected TPCH tpch; + protected LocalTajoTestingUtility util; + + private static TpchTestBase testBase; + + static { + try { + testBase = new TpchTestBase(); + testBase.setUp(); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + } + + private TpchTestBase() throws IOException { + names = new String[] {"customer", "lineitem", "nation", "orders", "part", "partsupp", "region", "supplier", "empty_orders"}; + paths = new String[names.length]; + for (int i = 0; i < names.length; i++) { + nameMap.put(names[i], i); + } + + tpch = new TPCH(); + tpch.loadSchemas(); + tpch.loadQueries(); + + schemas = new Schema[names.length]; + for (int i = 0; i < names.length; i++) { + schemas[i] = tpch.getSchema(names[i]); + } + + tables = new String[names.length][]; + File file; + for (int i = 0; i < names.length; i++) { + file = TPCH.getDataFile(names[i]); + tables[i] = FileUtil.readTextFile(file).split("\n"); + paths[i] = file.getAbsolutePath(); + } + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + private void setUp() throws Exception { + util = new LocalTajoTestingUtility(); + KeyValueSet opt = new KeyValueSet(); + opt.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + util.setup(names, paths, schemas, opt); + } + + public static TpchTestBase getInstance() { + return testBase; + } + + public ResultSet execute(String query) throws Exception { + return util.execute(query); + } + + public TajoTestingCluster getTestingCluster() { + return util.getTestingCluster(); + } + + public void tearDown() throws IOException { + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + } + util.shutdown(); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-core-tests/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-core-tests/pom.xml b/tajo-core-tests/pom.xml new file mode 100644 index 0000000..8f2051d --- /dev/null +++ b/tajo-core-tests/pom.xml @@ -0,0 +1,356 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>tajo-project</artifactId> + <groupId>org.apache.tajo</groupId> + <version>0.11.0-SNAPSHOT</version> + <relativePath>../tajo-project</relativePath> + </parent> + <artifactId>tajo-core-tests</artifactId> + <packaging>jar</packaging> + <name>Tajo Core Tests</name> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> + </properties> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <executions> + <execution> + <phase>verify</phase> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + <configuration> + <excludes> + <exclude>derby.log</exclude> + <exclude>benchmark/**</exclude> + <exclude>src/test/tpch/**</exclude> + <exclude>src/test/resources/dataset/**</exclude> + <exclude>src/test/resources/queries/**</exclude> + <exclude>src/test/resources/results/**</exclude> + <exclude>src/main/resources/META-INF/services/*</exclude> + <exclude>src/main/resources/webapps/static/js/*</exclude> + </excludes> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <systemProperties> + <tajo.test>TRUE</tajo.test> + </systemProperties> + <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8</argLine> + </configuration> + </plugin> + <plugin> + <artifactId>maven-deploy-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>2.3.1</version> + <configuration> + <excludes> + <exclude>LICENSE</exclude> + </excludes> + <archive> + <addMavenDescriptor>false</addMavenDescriptor> + </archive> + </configuration> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy-dependencies</id> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <includeScope>runtime</includeScope> + <outputDirectory>${project.build.directory}/lib</outputDirectory> + <overWriteReleases>false</overWriteReleases> + <overWriteSnapshots>false</overWriteSnapshots> + <overWriteIfNewer>true</overWriteIfNewer> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + + <dependencies> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-core</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-algebra</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-catalog-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-plan</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-client</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-catalog-client</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-catalog-server</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-hdfs</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-hbase</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-pullserver</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-rpc-protobuf</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-thirdparty-asm</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-ws-rs</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-metrics</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-cluster-tests</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>commons-el</groupId> + <artifactId>commons-el</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-runtime</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-compiler</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp-2.1-jetty</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey.jersey-test-framework</groupId> + <artifactId>jersey-test-framework-grizzly2</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>commons-el</groupId> + <artifactId>commons-el</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-runtime</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-compiler</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp-2.1-jetty</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey.jersey-test-framework</groupId> + <artifactId>jersey-test-framework-grizzly2</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-common</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-common</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-server</artifactId> + <version>${hbase.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-server</artifactId> + <version>${hbase.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-hadoop-compat</artifactId> + <version>${hbase.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-hadoop2-compat</artifactId> + <version>${hbase.version}</version> + <type>test-jar</type> + <scope>test</scope> + <exclusions> + <exclusion> + <artifactId>jdk.tools</artifactId> + <groupId>jdk.tools</groupId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>com.github.stephenc.jcip</groupId> + <artifactId>jcip-annotations</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <profiles> + <profile> + <id>parallel-test</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <executions> + <execution> + <phase>test</phase> + <goals> + <goal>test</goal> + </goals> + </execution> + </executions> + <configuration combine.self="override"> + <forkCount>${maven.fork.count}</forkCount> + <reuseForks>true</reuseForks> + <trimStackTrace>false</trimStackTrace> + <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8</argLine> + <useSystemClassLoader>true</useSystemClassLoader> + <useManifestOnlyJar>true</useManifestOnlyJar> + <systemProperties> + <tajo.test>TRUE</tajo.test> + <tajo.test.data.dir>test-data${surefire.forkNumber}</tajo.test.data.dir> + </systemProperties> + </configuration> + </plugin> + </plugins> + </build> + </profile> + </profiles> +</project> http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-core-tests/src/test/java/log4j.properties ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/log4j.properties b/tajo-core-tests/src/test/java/log4j.properties new file mode 100644 index 0000000..48f9d8e --- /dev/null +++ b/tajo-core-tests/src/test/java/log4j.properties @@ -0,0 +1,28 @@ +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# log4j configuration used during build and unit tests + +log4j.rootLogger=info,stdout +log4j.threshhold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p: %c (%M(%L)) - %m%n + +log4j.logger.org.apache.hadoop=WARN +log4j.logger.org.apache.hadoop.conf=ERROR http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-core-tests/src/test/java/org/apache/tajo/TestQueryIdFactory.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/TestQueryIdFactory.java b/tajo-core-tests/src/test/java/org/apache/tajo/TestQueryIdFactory.java new file mode 100644 index 0000000..8dc95de --- /dev/null +++ b/tajo-core-tests/src/test/java/org/apache/tajo/TestQueryIdFactory.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo; + +import org.apache.tajo.engine.planner.global.MasterPlan; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +public class TestQueryIdFactory { + + @Before + public void setup() { + } + + @Test + public void testNewQueryId() { + QueryId qid1 = LocalTajoTestingUtility.newQueryId(); + QueryId qid2 = LocalTajoTestingUtility.newQueryId(); + assertTrue(qid1.compareTo(qid2) < 0); + } + + @Test + public void testNewStageId() { + QueryId qid = LocalTajoTestingUtility.newQueryId(); + MasterPlan plan = new MasterPlan(qid, null, null); + ExecutionBlockId stageId1 = plan.newExecutionBlockId(); + ExecutionBlockId stageId2 = plan.newExecutionBlockId(); + assertTrue(stageId1.compareTo(stageId2) < 0); + } + + @Test + public void testNewTaskId() { + QueryId qid = LocalTajoTestingUtility.newQueryId(); + MasterPlan plan = new MasterPlan(qid, null, null); + ExecutionBlockId subid = plan.newExecutionBlockId(); + TaskId quid1 = QueryIdFactory.newTaskId(subid); + TaskId quid2 = QueryIdFactory.newTaskId(subid); + assertTrue(quid1.compareTo(quid2) < 0); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-core-tests/src/test/java/org/apache/tajo/TestTajoIds.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/TestTajoIds.java b/tajo-core-tests/src/test/java/org/apache/tajo/TestTajoIds.java new file mode 100644 index 0000000..d15e282 --- /dev/null +++ b/tajo-core-tests/src/test/java/org/apache/tajo/TestTajoIds.java @@ -0,0 +1,168 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.tajo.engine.planner.global.MasterPlan; +import org.apache.tajo.util.TajoIdUtils; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class TestTajoIds { + @Test + public void testQueryId() { + long ts1 = 1315890136000l; + long ts2 = 1315890136001l; + + QueryId j1 = createQueryId(ts1, 2); + QueryId j2 = createQueryId(ts1, 1); + QueryId j3 = createQueryId(ts2, 1); + QueryId j4 = createQueryId(ts1, 2); + + assertTrue(j1.equals(j4)); + assertFalse(j1.equals(j2)); + assertFalse(j1.equals(j3)); + + assertTrue(j1.compareTo(j4) == 0); + assertTrue(j1.compareTo(j2) > 0); + assertTrue(j1.compareTo(j3) < 0); + + assertTrue(j1.hashCode() == j4.hashCode()); + assertFalse(j1.hashCode() == j2.hashCode()); + assertFalse(j1.hashCode() == j3.hashCode()); + + QueryId j5 = createQueryId(ts1, 231415); + assertEquals("q_" + ts1 + "_0002", j1.toString()); + assertEquals("q_" + ts1 + "_231415", j5.toString()); + } + + @Test + public void testQueryIds() { + long timeId = 1315890136000l; + + QueryId queryId = createQueryId(timeId, 1); + assertEquals("q_" + timeId + "_0001", queryId.toString()); + + ExecutionBlockId subId = QueryIdFactory.newExecutionBlockId(queryId, 2); + assertEquals("eb_" + timeId +"_0001_000002", subId.toString()); + + TaskId qId = new TaskId(subId, 5); + assertEquals("t_" + timeId + "_0001_000002_000005", qId.toString()); + + TaskAttemptId attemptId = new TaskAttemptId(qId, 4); + assertEquals("ta_" + timeId + "_0001_000002_000005_04", attemptId.toString()); + } + + @Test + public void testEqualsObject() { + long timeId = System.currentTimeMillis(); + + QueryId queryId1 = createQueryId(timeId, 1); + QueryId queryId2 = createQueryId(timeId, 2); + assertNotSame(queryId1, queryId2); + QueryId queryId3 = createQueryId(timeId, 1); + assertEquals(queryId1, queryId3); + + ExecutionBlockId sid1 = QueryIdFactory.newExecutionBlockId(queryId1, 1); + ExecutionBlockId sid2 = QueryIdFactory.newExecutionBlockId(queryId1, 2); + assertNotSame(sid1, sid2); + ExecutionBlockId sid3 = QueryIdFactory.newExecutionBlockId(queryId1, 1); + assertEquals(sid1, sid3); + + TaskId qid1 = new TaskId(sid1, 9); + TaskId qid2 = new TaskId(sid1, 10); + assertNotSame(qid1, qid2); + TaskId qid3 = new TaskId(sid1, 9); + assertEquals(qid1, qid3); + } + + @Test + public void testCompareTo() { + long time = System.currentTimeMillis(); + + QueryId queryId1 = createQueryId(time, 1); + QueryId queryId2 = createQueryId(time, 2); + QueryId queryId3 = createQueryId(time, 1); + assertEquals(-1, queryId1.compareTo(queryId2)); + assertEquals(1, queryId2.compareTo(queryId1)); + assertEquals(0, queryId3.compareTo(queryId1)); + + ExecutionBlockId sid1 = QueryIdFactory.newExecutionBlockId(queryId1, 1); + ExecutionBlockId sid2 = QueryIdFactory.newExecutionBlockId(queryId1, 2); + ExecutionBlockId sid3 = QueryIdFactory.newExecutionBlockId(queryId1, 1); + assertEquals(-1, sid1.compareTo(sid2)); + assertEquals(1, sid2.compareTo(sid1)); + assertEquals(0, sid3.compareTo(sid1)); + + TaskId qid1 = new TaskId(sid1, 9); + TaskId qid2 = new TaskId(sid1, 10); + TaskId qid3 = new TaskId(sid1, 9); + assertEquals(-1, qid1.compareTo(qid2)); + assertEquals(1, qid2.compareTo(qid1)); + assertEquals(0, qid3.compareTo(qid1)); + } + + @Test + public void testConstructFromString() { + QueryId qid1 = LocalTajoTestingUtility.newQueryId(); + QueryId qid2 = TajoIdUtils.parseQueryId(qid1.toString()); + assertEquals(qid1, qid2); + + MasterPlan plan1 = new MasterPlan(qid1, null, null); + ExecutionBlockId sub1 = plan1.newExecutionBlockId(); + ExecutionBlockId sub2 = TajoIdUtils.createExecutionBlockId(sub1.toString()); + assertEquals(sub1, sub2); + + TaskId u1 = QueryIdFactory.newTaskId(sub1); + TaskId u2 = new TaskId(u1.getProto()); + assertEquals(u1, u2); + + TaskAttemptId attempt1 = new TaskAttemptId(u1, 1); + TaskAttemptId attempt2 = new TaskAttemptId(attempt1.getProto()); + assertEquals(attempt1, attempt2); + } + + @Test + public void testConstructFromPB() { + QueryId qid1 = LocalTajoTestingUtility.newQueryId(); + QueryId qid2 = new QueryId(qid1.getProto()); + assertEquals(qid1, qid2); + + MasterPlan plan = new MasterPlan(qid1, null, null); + ExecutionBlockId sub1 = plan.newExecutionBlockId(); + ExecutionBlockId sub2 = TajoIdUtils.createExecutionBlockId(sub1.toString()); + assertEquals(sub1, sub2); + + TaskId u1 = QueryIdFactory.newTaskId(sub1); + TaskId u2 = new TaskId(u1.getProto()); + assertEquals(u1, u2); + + TaskAttemptId attempt1 = new TaskAttemptId(u1, 1); + TaskAttemptId attempt2 = new TaskAttemptId(attempt1.getProto()); + assertEquals(attempt1, attempt2); + } + + public static QueryId createQueryId(long timestamp, int id) { + ApplicationId appId = BuilderUtils.newApplicationId(timestamp, id); + + return QueryIdFactory.newQueryId(appId.toString()); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-core-tests/src/test/java/org/apache/tajo/benchmark/TestTPCH.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/benchmark/TestTPCH.java b/tajo-core-tests/src/test/java/org/apache/tajo/benchmark/TestTPCH.java new file mode 100644 index 0000000..53d4350 --- /dev/null +++ b/tajo-core-tests/src/test/java/org/apache/tajo/benchmark/TestTPCH.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.benchmark; + +import org.apache.tajo.IntegrationTest; +import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.TajoConstants; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(IntegrationTest.class) +public class TestTPCH extends QueryTestCaseBase { + + public TestTPCH() { + super(TajoConstants.DEFAULT_DATABASE_NAME); + } + + @Test + @Option(withExplain = true, withExplainGlobal = true) + @SimpleTest + public void testQ1OrderBy() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = true, withExplainGlobal = true) + @SimpleTest + public void testQ2FourJoins() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = true, withExplainGlobal = true) + @SimpleTest + public void testTPCH14Expr() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = true, withExplainGlobal = true) + @SimpleTest + public void testTPCHQ5() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = true, withExplainGlobal = true) + @SimpleTest + public void testFirstJoinInQ7() throws Exception { + runSimpleTests(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java new file mode 100644 index 0000000..06a54c4 --- /dev/null +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java @@ -0,0 +1,133 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.cli.tools; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.compress.GzipCodec; +import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.partition.PartitionMethodDesc; +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.util.FileUtil; +import org.junit.Test; + +import static org.junit.Assert.*; + + +public class TestDDLBuilder { + private static final Schema schema1; + private static final TableMeta meta1; + private static final PartitionMethodDesc partitionMethod1; + + static { + schema1 = new Schema(); + schema1.addColumn("name", TajoDataTypes.Type.BLOB); + schema1.addColumn("addr", TajoDataTypes.Type.TEXT); + + meta1 = CatalogUtil.newTableMeta("TEXT"); + meta1.putOption(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + meta1.putOption(StorageConstants.COMPRESSION_CODEC, GzipCodec.class.getName()); + + Schema expressionSchema = new Schema(); + expressionSchema.addColumn("key", TajoDataTypes.Type.INT4); + expressionSchema.addColumn("key2", TajoDataTypes.Type.TEXT); + partitionMethod1 = new PartitionMethodDesc( + "db1", + "table1", + CatalogProtos.PartitionType.COLUMN, + "key,key2", + expressionSchema); + } + + @Test + public void testBuildDDLForExternalTable() throws Exception { + TableDesc desc = new TableDesc("db1.table1", schema1, meta1, new Path("/table1").toUri()); + desc.setPartitionMethod(partitionMethod1); + desc.setExternal(true); + assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLForExternalTable.result"), + DDLBuilder.buildDDLForExternalTable(desc)); + } + + @Test + public void testBuildDDLQuotedTableName() throws Exception { + Schema schema2 = new Schema(); + schema2.addColumn("name", TajoDataTypes.Type.BLOB); + schema2.addColumn("addr", TajoDataTypes.Type.TEXT); + schema2.addColumn("FirstName", TajoDataTypes.Type.TEXT); + schema2.addColumn("LastName", TajoDataTypes.Type.TEXT); + schema2.addColumn("with", TajoDataTypes.Type.TEXT); + + Schema expressionSchema2 = new Schema(); + expressionSchema2.addColumn("BirthYear", TajoDataTypes.Type.INT4); + + PartitionMethodDesc partitionMethod2 = new PartitionMethodDesc( + "db1", + "table1", + CatalogProtos.PartitionType.COLUMN, + "key,key2", + expressionSchema2); + + TableDesc desc = new TableDesc("db1.TABLE2", schema2, meta1, new Path("/table1").toUri()); + desc.setPartitionMethod(partitionMethod2); + desc.setExternal(true); + assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLQuotedTableName1.result"), + DDLBuilder.buildDDLForExternalTable(desc)); + + desc = new TableDesc("db1.TABLE1", schema2, meta1, new Path("/table1").toUri()); + desc.setPartitionMethod(partitionMethod2); + desc.setExternal(false); + assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLQuotedTableName2.result"), + DDLBuilder.buildDDLForBaseTable(desc)); + } + + @Test + public void testBuildDDLForBaseTable() throws Exception { + TableDesc desc = new TableDesc("db1.table2", schema1, meta1, new Path("/table1").toUri()); + assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLForBaseTable.result"), + DDLBuilder.buildDDLForBaseTable(desc)); + } + + @Test + public void testBuildColumn() throws Exception { + String [] tobeUnquoted = { + "column_name", + "columnname", + "column_1", + }; + + for (String columnName : tobeUnquoted) { + assertFalse(CatalogUtil.isShouldBeQuoted(columnName)); + } + + String [] quoted = { + "Column_Name", + "COLUMN_NAME", + "컬ë¼", + "$column_name", + "Column_Name1", + "with", + "when" + }; + + for (String columnName : quoted) { + assertTrue(CatalogUtil.isShouldBeQuoted(columnName)); + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java new file mode 100644 index 0000000..aa8070e --- /dev/null +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.cli.tools; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.auth.UserRoleInfo; +import org.apache.tajo.storage.StorageUtil; +import org.apache.tajo.storage.TablespaceManager; +import org.apache.tajo.util.FileUtil; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.PrintWriter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestTajoDump extends QueryTestCaseBase { + + @Test + public void testDump1() throws Exception { + if (!testingCluster.isHiveCatalogStoreRunning()) { + executeString("CREATE TABLE \"" + getCurrentDatabase() + + "\".\"TableName1\" (\"Age\" int, \"FirstName\" TEXT, lastname TEXT)"); + + try { + UserRoleInfo userInfo = UserRoleInfo.getCurrentUser(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintWriter printWriter = new PrintWriter(bos); + TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter); + printWriter.flush(); + printWriter.close(); + assertStrings(new String(bos.toByteArray())); + bos.close(); + } finally { + executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName1\""); + } + } + } + + @Test + public void testDump2() throws Exception { + if (!testingCluster.isHiveCatalogStoreRunning()) { + executeString("CREATE TABLE \"" + getCurrentDatabase() + + "\".\"TableName2\" (\"Age\" int, \"Name\" Record (\"FirstName\" TEXT, lastname TEXT))"); + + try { + UserRoleInfo userInfo = UserRoleInfo.getCurrentUser(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintWriter printWriter = new PrintWriter(bos); + TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter); + printWriter.flush(); + printWriter.close(); + assertStrings(new String(bos.toByteArray())); + bos.close(); + } finally { + executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName2\""); + } + } + } + + @Test + public void testDump3() throws Exception { + if (!testingCluster.isHiveCatalogStoreRunning()) { + executeString("CREATE TABLE \"" + getCurrentDatabase() + + "\".\"TableName1\" (\"Age\" int, \"FirstName\" TEXT, lastname TEXT)"); + + executeString("CREATE INDEX test_idx on \"" + getCurrentDatabase() + + "\".\"TableName1\" ( \"Age\" asc null first, \"FirstName\" desc null last )"); + + try { + UserRoleInfo userInfo = UserRoleInfo.getCurrentUser(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintWriter printWriter = new PrintWriter(bos); + TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter); + printWriter.flush(); + printWriter.close(); + assertOutputResult("testDump3.result", new String(bos.toByteArray()), new String[]{"${index.path}"}, + new String[]{TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "test_idx").toString()}); + bos.close(); + } finally { + executeString("DROP INDEX test_idx"); + executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName1\""); + } + } + } + + private void assertOutputResult(String expectedResultFile, String actual, String[] paramKeys, String[] paramValues) + throws Exception { + FileSystem fs = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration()); + Path resultFile = StorageUtil.concatPath(currentResultPath, expectedResultFile); + assertTrue(resultFile.toString() + " existence check", fs.exists(resultFile)); + + String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri())); + + if (paramKeys != null) { + for (int i = 0; i < paramKeys.length; i++) { + if (i < paramValues.length) { + expectedResult = expectedResult.replace(paramKeys[i], paramValues[i]); + } + } + } + assertEquals(expectedResult.trim(), actual.trim()); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java new file mode 100644 index 0000000..3b53c60 --- /dev/null +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java @@ -0,0 +1,179 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.cli.tsql; + +import org.apache.hadoop.fs.Path; +import org.apache.tajo.TajoTestingCluster; +import org.apache.tajo.TpchTestBase; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.datum.Float8Datum; +import org.apache.tajo.datum.Int4Datum; +import org.apache.tajo.datum.TextDatum; +import org.apache.tajo.jdbc.MetaDataTuple; +import org.apache.tajo.jdbc.TajoMetaDataResultSet; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.net.URL; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class TestDefaultCliOutputFormatter { + protected static final TpchTestBase testBase; + protected static final TajoTestingCluster cluster; + + /** the base path of result directories */ + protected static final Path resultBasePath; + static { + testBase = TpchTestBase.getInstance(); + cluster = testBase.getTestingCluster(); + URL resultBaseURL = ClassLoader.getSystemResource("results"); + resultBasePath = new Path(resultBaseURL.toString()); + } + + private TajoConf conf; + private TajoCli tajoCli; + private TajoCli.TajoCliContext cliContext; + + @Before + public void setUp() throws Exception { + conf = cluster.getConfiguration(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + tajoCli = new TajoCli(conf, new String[]{}, System.in, out); + cliContext = tajoCli.getContext(); + } + + @After + public void tearDown() { + if (tajoCli != null) { + tajoCli.close(); + } + } + + + @Test + public void testParseErrorMessage() { + String message = "java.sql.SQLException: ERROR: no such a table: table1"; + assertEquals("ERROR: no such a table: table1", DefaultTajoCliOutputFormatter.parseErrorMessage(message)); + + String multiLineMessage = + "ERROR: java.sql.SQLException: ERROR: no such a table: table1\n" + + "com.google.protobuf.ServiceException: java.sql.SQLException: ERROR: no such a table: table1\n" + + "\tat org.apache.tajo.client.TajoClient.getTableDesc(TajoClient.java:777)\n" + + "\tat org.apache.tajo.cli.tsql.commands.DescTableCommand.invoke(DescTableCommand.java:43)\n" + + "\tat org.apache.tajo.cli.tsql.TajoCli.executeMetaCommand(TajoCli.java:300)\n" + + "\tat org.apache.tajo.cli.tsql.TajoCli.executeParsedResults(TajoCli.java:280)\n" + + "\tat org.apache.tajo.cli.tsql.TajoCli.runShell(TajoCli.java:271)\n" + + "\tat org.apache.tajo.cli.tsql.TajoCli.main(TajoCli.java:420)\n" + + "Caused by: java.sql.SQLException: ERROR: no such a table: table1\n" + + "\t... 6 more"; + + assertEquals(multiLineMessage, DefaultTajoCliOutputFormatter.parseErrorMessage(multiLineMessage)); + + String noPrefixMessage = "RTFM please"; + assertEquals("ERROR: "+noPrefixMessage, DefaultTajoCliOutputFormatter.parseErrorMessage(noPrefixMessage)); + + String errorMessageWithLine = "ERROR: syntax error at or near '('\n" + + "LINE 1:7 select (*) from tc\n" + + " ^"; + assertEquals(errorMessageWithLine, DefaultTajoCliOutputFormatter.parseErrorMessage(errorMessageWithLine)); + } + + @Test + public void testPrintResultInsertStatement() throws Exception { + DefaultTajoCliOutputFormatter outputFormatter = new DefaultTajoCliOutputFormatter(); + outputFormatter.init(cliContext); + + float responseTime = 10.1f; + long numBytes = 102; + long numRows = 30; + + TableDesc tableDesc = new TableDesc(); + TableStats stats = new TableStats(); + stats.setNumBytes(102); + stats.setNumRows(numRows); + tableDesc.setStats(stats); + + StringWriter stringWriter = new StringWriter(); + PrintWriter writer = new PrintWriter(stringWriter); + outputFormatter.printResult(writer, null, tableDesc, responseTime, null); + + String expectedOutput = "(" + numRows + " rows, " + responseTime + " sec, " + numBytes + " B inserted)\n"; + assertEquals(expectedOutput, stringWriter.toString()); + } + + @Test + public void testPrintResultSelectStatement() throws Exception { + DefaultTajoCliOutputFormatter outputFormatter = new DefaultTajoCliOutputFormatter(); + outputFormatter.init(cliContext); + + float responseTime = 10.1f; + long numBytes = 102; + long numRows = 30; + + TableDesc tableDesc = new TableDesc(); + TableStats stats = new TableStats(); + stats.setNumBytes(102); + stats.setNumRows(numRows); + tableDesc.setStats(stats); + + final List<MetaDataTuple> resultTables = new ArrayList<MetaDataTuple>(); + + String expectedOutput = "col1, col2, col3\n"; + expectedOutput += "-------------------------------\n"; + + String prefix = ""; + for (int i = 0; i < numRows; i++) { + MetaDataTuple tuple = new MetaDataTuple(3); + + int index = 0; + + tuple.put(index++, new TextDatum("row_" + i)); + tuple.put(index++, new Int4Datum(i)); + tuple.put(index++, new Float8Datum(i)); + + expectedOutput += prefix + "row_" + i + ", " + (new Int4Datum(i)) + ", " + (new Float8Datum(i)); + prefix = "\n"; + resultTables.add(tuple); + } + expectedOutput += "\n(" + numRows + " rows, " + responseTime + " sec, " + numBytes + " B selected)\n"; + + ResultSet resultSet = new TajoMetaDataResultSet( + Arrays.asList("col1", "col2", "col3"), + Arrays.asList(TajoDataTypes.Type.TEXT, TajoDataTypes.Type.INT4, TajoDataTypes.Type.FLOAT8), + resultTables); + + StringWriter stringWriter = new StringWriter(); + PrintWriter writer = new PrintWriter(stringWriter); + outputFormatter.printResult(writer, null, tableDesc, responseTime, resultSet); + + assertEquals(expectedOutput, stringWriter.toString()); + } +}
