Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/55e46301 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/55e46301 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/55e46301 Branch: refs/heads/index_support Commit: 55e46301ebe92d2bf65aaab26688f957f1cf03c7 Parents: 1c1dfa0 161ee9e Author: Jihoon Son <[email protected]> Authored: Fri Feb 6 17:35:22 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Fri Feb 6 17:35:22 2015 +0900 ---------------------------------------------------------------------- CHANGES | 16 + pom.xml | 1 + .../org/apache/tajo/algebra/AlterTable.java | 34 +- .../apache/tajo/algebra/AlterTableOpType.java | 2 +- tajo-cli/pom.xml | 337 +++++++++ .../org/apache/tajo/cli/tools/TajoAdmin.java | 459 ++++++++++++ .../org/apache/tajo/cli/tools/TajoDump.java | 202 ++++++ .../org/apache/tajo/cli/tools/TajoGetConf.java | 161 +++++ .../org/apache/tajo/cli/tools/TajoHAAdmin.java | 210 ++++++ .../cli/tsql/DefaultTajoCliOutputFormatter.java | 211 ++++++ .../cli/tsql/InvalidStatementException.java | 25 + .../org/apache/tajo/cli/tsql/ParsedResult.java | 53 ++ .../org/apache/tajo/cli/tsql/SimpleParser.java | 387 ++++++++++ .../java/org/apache/tajo/cli/tsql/TajoCli.java | 686 ++++++++++++++++++ .../tajo/cli/tsql/TajoCliOutputFormatter.java | 97 +++ .../apache/tajo/cli/tsql/TajoFileHistory.java | 41 ++ .../tsql/commands/ConnectDatabaseCommand.java | 72 ++ .../cli/tsql/commands/CopyrightCommand.java | 65 ++ .../cli/tsql/commands/DescFunctionCommand.java | 136 ++++ .../cli/tsql/commands/DescTableCommand.java | 155 ++++ .../tsql/commands/ExecExternalShellCommand.java | 124 ++++ .../tajo/cli/tsql/commands/ExitCommand.java | 52 ++ .../tajo/cli/tsql/commands/HdfsCommand.java | 58 ++ .../tajo/cli/tsql/commands/HelpCommand.java | 133 ++++ .../cli/tsql/commands/ListDatabaseCommand.java | 50 ++ .../tajo/cli/tsql/commands/SetCommand.java | 127 ++++ .../cli/tsql/commands/TajoAdminCommand.java | 58 ++ .../cli/tsql/commands/TajoGetConfCommand.java | 58 ++ .../cli/tsql/commands/TajoHAAdminCommand.java | 58 ++ .../cli/tsql/commands/TajoShellCommand.java | 129 ++++ .../tajo/cli/tsql/commands/UnsetCommand.java | 53 ++ .../tajo/cli/tsql/commands/VersionCommand.java | 49 ++ .../org/apache/tajo/cli/tools/TajoAdmin.java | 457 ------------ .../org/apache/tajo/cli/tools/TajoDump.java | 202 ------ .../org/apache/tajo/cli/tools/TajoGetConf.java | 161 ----- .../org/apache/tajo/cli/tools/TajoHAAdmin.java | 211 ------ .../cli/tsql/DefaultTajoCliOutputFormatter.java | 211 ------ .../cli/tsql/InvalidStatementException.java | 25 - .../org/apache/tajo/cli/tsql/ParsedResult.java | 53 -- .../org/apache/tajo/cli/tsql/SimpleParser.java | 388 ---------- .../java/org/apache/tajo/cli/tsql/TajoCli.java | 701 ------------------- .../tajo/cli/tsql/TajoCliOutputFormatter.java | 98 --- .../apache/tajo/cli/tsql/TajoFileHistory.java | 41 -- .../tsql/commands/ConnectDatabaseCommand.java | 72 -- .../cli/tsql/commands/CopyrightCommand.java | 65 -- .../cli/tsql/commands/DescFunctionCommand.java | 136 ---- .../cli/tsql/commands/DescTableCommand.java | 155 ---- .../tsql/commands/ExecExternalShellCommand.java | 124 ---- .../tajo/cli/tsql/commands/ExitCommand.java | 52 -- .../tajo/cli/tsql/commands/HdfsCommand.java | 58 -- .../tajo/cli/tsql/commands/HelpCommand.java | 133 ---- .../cli/tsql/commands/ListDatabaseCommand.java | 50 -- .../tajo/cli/tsql/commands/SetCommand.java | 127 ---- .../cli/tsql/commands/TajoAdminCommand.java | 58 -- .../cli/tsql/commands/TajoGetConfCommand.java | 58 -- .../cli/tsql/commands/TajoHAAdminCommand.java | 58 -- .../cli/tsql/commands/TajoShellCommand.java | 129 ---- .../tajo/cli/tsql/commands/UnsetCommand.java | 53 -- .../tajo/cli/tsql/commands/VersionCommand.java | 49 -- .../apache/tajo/client/TajoHAClientUtil.java | 87 --- .../java/org/apache/tajo/conf/TajoConf.java | 5 +- tajo-core/pom.xml | 4 + .../org/apache/tajo/engine/parser/SQLLexer.g4 | 2 + .../org/apache/tajo/engine/parser/SQLParser.g4 | 12 + .../tajo/engine/function/builtin/Lead.java | 92 +++ .../tajo/engine/function/builtin/LeadDate.java | 49 ++ .../engine/function/builtin/LeadDouble.java | 49 ++ .../tajo/engine/function/builtin/LeadFloat.java | 49 ++ .../tajo/engine/function/builtin/LeadInt.java | 49 ++ .../tajo/engine/function/builtin/LeadLong.java | 49 ++ .../engine/function/builtin/LeadString.java | 49 ++ .../tajo/engine/function/builtin/LeadTime.java | 49 ++ .../engine/function/builtin/LeadTimestamp.java | 49 ++ .../apache/tajo/engine/function/window/Lag.java | 82 +++ .../tajo/engine/function/window/LagDate.java | 42 ++ .../tajo/engine/function/window/LagDouble.java | 42 ++ .../tajo/engine/function/window/LagFloat.java | 42 ++ .../tajo/engine/function/window/LagInt.java | 42 ++ .../tajo/engine/function/window/LagLong.java | 42 ++ .../tajo/engine/function/window/LagString.java | 42 ++ .../tajo/engine/function/window/LagTime.java | 42 ++ .../engine/function/window/LagTimestamp.java | 42 ++ .../apache/tajo/engine/parser/SQLAnalyzer.java | 82 ++- .../engine/planner/physical/WindowAggExec.java | 2 +- .../apache/tajo/querymaster/QueryMaster.java | 15 +- .../tajo/webapp/QueryExecutorServlet.java | 2 - .../tajo/worker/ExecutionBlockContext.java | 28 +- .../main/java/org/apache/tajo/worker/Task.java | 47 +- .../apache/tajo/worker/TaskAttemptContext.java | 2 +- .../apache/tajo/worker/TaskRunnerManager.java | 10 +- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 23 + .../engine/function/TestBuiltinFunctions.java | 38 + .../tajo/engine/parser/TestSQLAnalyzer.java | 236 +++++-- .../tajo/engine/query/TestWindowQuery.java | 98 +++ .../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 42 ++ .../apache/tajo/querymaster/TestKillQuery.java | 34 + .../queries/TestWindowQuery/testLag1.sql | 18 + .../TestWindowQuery/testLagWithDefault.sql | 18 + .../TestWindowQuery/testLagWithNoArgs.sql | 18 + .../queries/TestWindowQuery/testLead1.sql | 18 + .../TestWindowQuery/testLeadWithDefault.sql | 18 + .../TestWindowQuery/testLeadWithNoArgs.sql | 18 + .../default/alter_table_add_partition_1.sql | 1 + .../default/alter_table_add_partition_2.sql | 1 + .../default/alter_table_add_partition_3.sql | 2 + .../default/alter_table_add_partition_4.sql | 1 + .../default/alter_table_drop_partition_1.sql | 1 + .../default/alter_table_drop_partition_2.sql | 1 + .../default/alter_table_drop_partition_3.sql | 1 + .../testAlterTableAddPartition.result | 2 + .../testAlterTableDropPartition.result | 2 + .../results/TestWindowQuery/testLag1.result | 7 + .../TestWindowQuery/testLagWithDefault.result | 7 + .../TestWindowQuery/testLagWithNoArgs.result | 7 + .../results/TestWindowQuery/testLead1.result | 7 + .../TestWindowQuery/testLeadWithDefault.result | 7 + .../TestWindowQuery/testLeadWithNoArgs.result | 7 + tajo-dist/pom.xml | 1 + .../configuration/catalog_configuration.rst | 12 +- .../org/apache/tajo/plan/ExprAnnotator.java | 6 +- .../org/apache/tajo/plan/LogicalPlanner.java | 4 +- .../org/apache/tajo/plan/TypeDeterminant.java | 4 + .../org/apache/tajo/plan/expr/EvalTreeUtil.java | 22 + .../plan/verifier/PreLogicalPlanVerifier.java | 12 + tajo-project/pom.xml | 5 + 125 files changed, 5985 insertions(+), 4138 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java ---------------------------------------------------------------------- diff --cc tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java index 0000000,f48a5b4..2395043 mode 000000,100644..100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java @@@ -1,0 -1,686 +1,686 @@@ + /** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package org.apache.tajo.cli.tsql; + + import com.google.common.annotations.VisibleForTesting; + import com.google.common.base.Preconditions; + import com.google.protobuf.ServiceException; + import jline.UnsupportedTerminal; + import jline.console.ConsoleReader; + import org.apache.commons.cli.*; + import org.apache.tajo.*; + import org.apache.tajo.TajoProtos.QueryState; + import org.apache.tajo.catalog.TableDesc; + import org.apache.tajo.cli.tsql.ParsedResult.StatementType; + import org.apache.tajo.cli.tsql.SimpleParser.ParsingState; + import org.apache.tajo.cli.tsql.commands.*; + import org.apache.tajo.client.*; + import org.apache.tajo.conf.TajoConf; + import org.apache.tajo.conf.TajoConf.ConfVars; + import org.apache.tajo.ipc.ClientProtos; + import org.apache.tajo.util.FileUtil; + + import java.io.*; + import java.lang.reflect.Constructor; + import java.sql.ResultSet; + import java.sql.SQLException; + import java.util.Collection; + import java.util.List; + import java.util.Map; + import java.util.TreeMap; + + public class TajoCli { + public static final String ERROR_PREFIX = "ERROR: "; + public static final String KILL_PREFIX = "KILL: "; + + private final TajoConf conf; + private TajoClient client; + private final TajoCliContext context; + + // Jline and Console related things + private final ConsoleReader reader; + private final InputStream sin; + private final PrintWriter sout; + private TajoFileHistory history; + + // Current States + private String currentDatabase; + + private TajoCliOutputFormatter displayFormatter; + + private boolean wasError = false; + + private static final Class [] registeredCommands = { + DescTableCommand.class, + DescFunctionCommand.class, + HelpCommand.class, + ExitCommand.class, + CopyrightCommand.class, + VersionCommand.class, + ConnectDatabaseCommand.class, + ListDatabaseCommand.class, + SetCommand.class, + UnsetCommand.class, + ExecExternalShellCommand.class, + HdfsCommand.class, + TajoAdminCommand.class, + TajoGetConfCommand.class, + TajoHAAdminCommand.class + }; + private final Map<String, TajoShellCommand> commands = new TreeMap<String, TajoShellCommand>(); + + protected static final Options options; + private static final String HOME_DIR = System.getProperty("user.home"); + private static final String HISTORY_FILE = ".tajo_history"; + + static { + options = new Options(); + options.addOption("c", "command", true, "execute only single command, then exit"); + options.addOption("f", "file", true, "execute commands from file, then exit"); + options.addOption("h", "host", true, "Tajo server host"); + options.addOption("p", "port", true, "Tajo server port"); + options.addOption("B", "background", false, "execute as background process"); + options.addOption("conf", "conf", true, "configuration value"); + options.addOption("param", "param", true, "parameter value in SQL file"); + options.addOption("help", "help", false, "help"); + } + + public class TajoCliContext extends OverridableConf { + public TajoCliContext(TajoConf conf) { + super(conf, ConfigKey.ConfigType.SESSION); + } + + public TajoClient getTajoClient() { + return client; + } + + public void setCurrentDatabase(String databasae) { + currentDatabase = databasae; + } + + public String getCurrentDatabase() { + return currentDatabase; + } + + public PrintWriter getOutput() { + return sout; + } + + public TajoConf getConf() { + return conf; + } + + @VisibleForTesting + public String getCliSideVar(String key) { + if (SessionVars.exists(key)) { + ConfigKey configKey = SessionVars.get(key); + return get(configKey); + } else { + return get(key); + } + } + + public void setCliSideVar(String key, String value) { + Preconditions.checkNotNull(key); + Preconditions.checkNotNull(value); + + boolean shouldReloadFormatter = false; + + if (SessionVars.exists(key)) { + SessionVars configKey = SessionVars.get(key); + put(configKey, value); + shouldReloadFormatter = configKey.getMode() == SessionVars.VariableMode.CLI_SIDE_VAR; + } else { + set(key, value); + + // It is hard to recognize it is a client side variable. So, we always reload formatter. + shouldReloadFormatter = true; + } + + if (shouldReloadFormatter) { + try { + initFormatter(); + } catch (Exception e) { + System.err.println(ERROR_PREFIX + e.getMessage()); + } + } + } + + public Map<String, TajoShellCommand> getCommands() { + return commands; + } + } + + public TajoCli(TajoConf c, String [] args, InputStream in, OutputStream out) throws Exception { + CommandLineParser parser = new PosixParser(); + CommandLine cmd = parser.parse(options, args); + + this.conf = new TajoConf(c); + context = new TajoCliContext(conf); + this.sin = in; + if (cmd.hasOption("B")) { + this.reader = new ConsoleReader(sin, out, new UnsupportedTerminal()); + } else { + this.reader = new ConsoleReader(sin, out); + } + + this.reader.setExpandEvents(false); + this.sout = new PrintWriter(reader.getOutput()); + initFormatter(); + + if (cmd.hasOption("help")) { + printUsage(); + System.exit(0); + } + + String hostName = null; + Integer port = null; + if (cmd.hasOption("h")) { + hostName = cmd.getOptionValue("h"); + } + if (cmd.hasOption("p")) { + port = Integer.parseInt(cmd.getOptionValue("p")); + } + + String baseDatabase = null; + if (cmd.getArgList().size() > 0) { + baseDatabase = (String) cmd.getArgList().get(0); + } + + if (cmd.getOptionValues("conf") != null) { + processConfVarCommand(cmd.getOptionValues("conf")); + } + + // if there is no "-h" option, + if(hostName == null) { + if (conf.getVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) { + // it checks if the client service address is given in configuration and distributed mode. + // if so, it sets entryAddr. + hostName = conf.getVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[0]; + } + } + if (port == null) { + if (conf.getVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) { + // it checks if the client service address is given in configuration and distributed mode. + // if so, it sets entryAddr. + port = Integer.parseInt(conf.getVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[1]); + } + } + + if ((hostName == null) ^ (port == null)) { + System.err.println(ERROR_PREFIX + "cannot find valid Tajo server address"); + throw new RuntimeException("cannot find valid Tajo server address"); + } else if (hostName != null && port != null) { + conf.setVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, hostName+":"+port); + client = new TajoClientImpl(conf, baseDatabase); + } else if (hostName == null && port == null) { + client = new TajoClientImpl(conf, baseDatabase); + } + + try { + context.setCurrentDatabase(client.getCurrentDatabase()); + initHistory(); + initCommands(); + + if (cmd.getOptionValues("conf") != null) { + processSessionVarCommand(cmd.getOptionValues("conf")); + } + + if (cmd.hasOption("c")) { + displayFormatter.setScriptMode(); + int exitCode = executeScript(cmd.getOptionValue("c")); + sout.flush(); + System.exit(exitCode); + } + if (cmd.hasOption("f")) { + displayFormatter.setScriptMode(); + cmd.getOptionValues(""); + File sqlFile = new File(cmd.getOptionValue("f")); + if (sqlFile.exists()) { + String script = FileUtil.readTextFile(new File(cmd.getOptionValue("f"))); + script = replaceParam(script, cmd.getOptionValues("param")); + int exitCode = executeScript(script); + sout.flush(); + System.exit(exitCode); + } else { + System.err.println(ERROR_PREFIX + "No such a file \"" + cmd.getOptionValue("f") + "\""); + System.exit(-1); + } + } + } catch (Exception e) { + System.err.println(ERROR_PREFIX + "Exception was thrown. Caused by " + e.getMessage()); + + if (client != null) { + client.close(); + } + + throw e; + } + + addShutdownHook(); + } + + private void processConfVarCommand(String[] confCommands) throws ServiceException { + for (String eachParam: confCommands) { + String[] tokens = eachParam.split("="); + if (tokens.length != 2) { + continue; + } + + if (!SessionVars.exists(tokens[0])) { + conf.set(tokens[0], tokens[1]); + } + } + } + + private void processSessionVarCommand(String[] confCommands) throws ServiceException { + for (String eachParam: confCommands) { + String[] tokens = eachParam.split("="); + if (tokens.length != 2) { + continue; + } + + if (SessionVars.exists(tokens[0])) { + ((SetCommand)commands.get("\\set")).set(tokens[0], tokens[1]); + } + } + } + + private void initFormatter() throws Exception { + Class formatterClass = context.getClass(SessionVars.CLI_FORMATTER_CLASS); + if (displayFormatter == null || !displayFormatter.getClass().equals(formatterClass)) { + displayFormatter = (TajoCliOutputFormatter)formatterClass.newInstance(); + } + displayFormatter.init(context); + } + + public TajoCliContext getContext() { + return context; + } + + protected static String replaceParam(String script, String[] params) { + if (params == null || params.length == 0) { + return script; + } + + for (String eachParam: params) { + String[] tokens = eachParam.split("="); + if (tokens.length != 2) { + continue; + } + script = script.replace("${" + tokens[0] + "}", tokens[1]); + } + + return script; + } + + private void initHistory() { + try { + String historyPath = HOME_DIR + File.separator + HISTORY_FILE; + if ((new File(HOME_DIR)).exists()) { + history = new TajoFileHistory(new File(historyPath)); + history.setAutoTrim(false); + history.setIgnoreDuplicates(false); + reader.setHistory(history); + } else { + System.err.println(ERROR_PREFIX + "home directory : '" + HOME_DIR +"' does not exist."); + } + } catch (Exception e) { + System.err.println(ERROR_PREFIX + e.getMessage()); + } + } + + private void initCommands() { + for (Class clazz : registeredCommands) { + TajoShellCommand cmd = null; + try { + Constructor cons = clazz.getConstructor(new Class[] {TajoCliContext.class}); + cmd = (TajoShellCommand) cons.newInstance(context); + } catch (Exception e) { + System.err.println(e.getMessage()); + throw new RuntimeException(e.getMessage()); + } + commands.put(cmd.getCommand(), cmd); + for (String alias : cmd.getAliases()) { + commands.put(alias, cmd); + } + } + } + + private void addShutdownHook() { + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + @Override + public void run() { + try { + history.flush(); + } catch (IOException e) { + } + client.close(); + } + })); + } + + private String updatePrompt(ParsingState state) throws ServiceException { + if (state == ParsingState.WITHIN_QUOTE) { + return "'"; + } else if (state == ParsingState.TOK_START) { + return context.getCurrentDatabase(); + } else { + return ""; + } + } + + public int runShell() throws Exception { + String line; + String currentPrompt = context.getCurrentDatabase(); + int exitCode; + ParsingState latestState = SimpleParser.START_STATE; + + sout.write("Try \\? for help.\n"); + + SimpleParser parser = new SimpleParser(); + + try { + while((line = reader.readLine(currentPrompt + "> ")) != null) { + if (line.equals("")) { + continue; + } + wasError = false; + if (line.startsWith("{")) { + executeJsonQuery(line); + } else { + List<ParsedResult> parsedResults = parser.parseLines(line); + + if (latestState != ParsingState.TOK_START && parsedResults.size() > 0) { + // Add multi-line statements to history in addition to individual lines. + ParsedResult parsed = parsedResults.get(0); + history.add(parsed.getHistoryStatement() + (parsed.getType() == StatementType.STATEMENT ? ";" : "")); + } + + exitCode = executeParsedResults(parsedResults); + latestState = parser.getState(); + currentPrompt = updatePrompt(latestState); + + // if at least one failed + if (exitCode != 0) { + return exitCode; + } + } + } + } catch (Exception e) { + System.err.println(ERROR_PREFIX + "Exception was thrown. Casued by " + e.getMessage()); + + if (client != null) { + client.close(); + } + + throw e; + } + return 0; + } + + private int executeParsedResults(Collection<ParsedResult> parsedResults) throws Exception { + int exitCode; + for (ParsedResult parsedResult : parsedResults) { + if (parsedResult.getType() == StatementType.META) { + exitCode = executeMetaCommand(parsedResult.getStatement()); + } else { + exitCode = executeQuery(parsedResult.getStatement()); + } + + if (exitCode != 0 && context.getBool(SessionVars.ON_ERROR_STOP)) { + return exitCode; + } + } + + return 0; + } + + public int executeMetaCommand(String line) throws Exception { + String [] metaCommands = line.split(";"); + for (String metaCommand : metaCommands) { + String arguments [] = metaCommand.split(" "); + + TajoShellCommand invoked = commands.get(arguments[0]); + if (invoked == null) { + printInvalidCommand(arguments[0]); + wasError = true; + return -1; + } + + try { + invoked.invoke(arguments); + } catch (IllegalArgumentException ige) { + displayFormatter.printErrorMessage(sout, ige); + wasError = true; + return -1; + } catch (Exception e) { + displayFormatter.printErrorMessage(sout, e); + wasError = true; + return -1; + } finally { + context.getOutput().flush(); + } + + if (wasError && context.getBool(SessionVars.ON_ERROR_STOP)) { + break; + } + } + + return 0; + } + + private void executeJsonQuery(String json) throws ServiceException, IOException { + + long startTime = System.currentTimeMillis(); + ClientProtos.SubmitQueryResponse response = client.executeQueryWithJson(json); + if (response == null) { + displayFormatter.printErrorMessage(sout, "response is null"); + wasError = true; - } else if (response.getResultCode() == ClientProtos.ResultCode.OK) { ++ } else if (response.getResult().getResultCode() == ClientProtos.ResultCode.OK) { + if (response.getIsForwarded()) { + QueryId queryId = new QueryId(response.getQueryId()); + waitForQueryCompleted(queryId); + } else { + if (!response.hasTableDesc() && !response.hasResultSet()) { + displayFormatter.printMessage(sout, "OK"); + wasError = true; + } else { + localQueryCompleted(response, startTime); + } + } + } else { - if (response.hasErrorMessage()) { - displayFormatter.printErrorMessage(sout, response.getErrorMessage()); ++ if (response.getResult().hasErrorMessage()) { ++ displayFormatter.printErrorMessage(sout, response.getResult().getErrorMessage()); + wasError = true; + } + } + } + + private int executeQuery(String statement) throws ServiceException, IOException { + + long startTime = System.currentTimeMillis(); + ClientProtos.SubmitQueryResponse response = null; + try{ + response = client.executeQuery(statement); + } catch (ServiceException e){ + displayFormatter.printErrorMessage(sout, e.getMessage()); + wasError = true; + } catch(Throwable te){ + displayFormatter.printErrorMessage(sout, te); + wasError = true; + } + + if (response == null) { + displayFormatter.printErrorMessage(sout, "response is null"); + wasError = true; - } else if (response.getResultCode() == ClientProtos.ResultCode.OK) { ++ } else if (response.getResult().getResultCode() == ClientProtos.ResultCode.OK) { + if (response.getIsForwarded()) { + QueryId queryId = new QueryId(response.getQueryId()); + waitForQueryCompleted(queryId); + } else { + if (!response.hasTableDesc() && !response.hasResultSet()) { + displayFormatter.printMessage(sout, "OK"); + } else { + localQueryCompleted(response, startTime); + } + } + } else { - if (response.hasErrorMessage()) { - displayFormatter.printErrorMessage(sout, response.getErrorMessage()); ++ if (response.getResult().hasErrorMessage()) { ++ displayFormatter.printErrorMessage(sout, response.getResult().getErrorMessage()); + wasError = true; + } + } + + return wasError ? -1 : 0; + } + + private void localQueryCompleted(ClientProtos.SubmitQueryResponse response, long startTime) { + ResultSet res = null; + try { + QueryId queryId = new QueryId(response.getQueryId()); + float responseTime = ((float)(System.currentTimeMillis() - startTime) / 1000.0f); + TableDesc desc = new TableDesc(response.getTableDesc()); + + // non-forwarded INSERT INTO query does not have any query id. + // In this case, it just returns succeeded query information without printing the query results. + if (response.getMaxRowNum() < 0 && queryId.equals(QueryIdFactory.NULL_QUERY_ID)) { + displayFormatter.printResult(sout, sin, desc, responseTime, res); + } else { + res = TajoClientUtil.createResultSet(conf, client, response); + displayFormatter.printResult(sout, sin, desc, responseTime, res); + } + } catch (Throwable t) { + displayFormatter.printErrorMessage(sout, t); + wasError = true; + } finally { + if (res != null) { + try { + res.close(); + } catch (SQLException e) { + } + } + } + } + + private void waitForQueryCompleted(QueryId queryId) { + // if query is empty string + if (queryId.equals(QueryIdFactory.NULL_QUERY_ID)) { + return; + } + + // query execute + ResultSet res = null; + QueryStatus status = null; + try { + + int initRetries = 0; + int progressRetries = 0; + while (true) { + // TODO - configurable + status = client.getQueryStatus(queryId); + if(TajoClientUtil.isQueryWaitingForSchedule(status.getState())) { + Thread.sleep(Math.min(20 * initRetries, 1000)); + initRetries++; + continue; + } + + if (TajoClientUtil.isQueryRunning(status.getState()) || status.getState() == QueryState.QUERY_SUCCEEDED) { + displayFormatter.printProgress(sout, status); + } + + if (TajoClientUtil.isQueryComplete(status.getState()) && status.getState() != QueryState.QUERY_KILL_WAIT) { + break; + } else { + Thread.sleep(Math.min(200 * progressRetries, 1000)); + progressRetries += 2; + } + } + + if (status.getState() == QueryState.QUERY_ERROR || status.getState() == QueryState.QUERY_FAILED) { + displayFormatter.printErrorMessage(sout, status); + wasError = true; + } else if (status.getState() == QueryState.QUERY_KILLED) { + displayFormatter.printKilledMessage(sout, queryId); + wasError = true; + } else { + if (status.getState() == QueryState.QUERY_SUCCEEDED) { + float responseTime = ((float)(status.getFinishTime() - status.getSubmitTime()) / 1000.0f); + ClientProtos.GetQueryResultResponse response = client.getResultResponse(queryId); + if (status.hasResult()) { + res = TajoClientUtil.createResultSet(conf, client, queryId, response); + TableDesc desc = new TableDesc(response.getTableDesc()); + displayFormatter.printResult(sout, sin, desc, responseTime, res); + } else { + TableDesc desc = new TableDesc(response.getTableDesc()); + displayFormatter.printResult(sout, sin, desc, responseTime, res); + } + } + } + } catch (Throwable t) { + displayFormatter.printErrorMessage(sout, t); + wasError = true; + } finally { + if (res != null) { + try { + res.close(); + } catch (SQLException e) { + } + } else { + if (status != null && status.getQueryId() != null) { + client.closeQuery(status.getQueryId()); + } + } + } + } + + public int executeScript(String script) throws Exception { + wasError = false; + List<ParsedResult> results = SimpleParser.parseScript(script); + return executeParsedResults(results); + } + + private void printUsage() { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("tsql [options] [database]", options); + } + + private void printInvalidCommand(String command) { + sout.println("Invalid command " + command + ". Try \\? for help."); + } + + @VisibleForTesting + public void close() { + //for testcase + if (client != null) { + client.close(); + } + + if (reader != null) { + reader.shutdown(); + } + } + + public static void main(String [] args) throws Exception { + TajoConf conf = new TajoConf(); + TajoCli shell = new TajoCli(conf, args, System.in, System.out); + System.out.println(); + System.exit(shell.runShell()); + } + } http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java ---------------------------------------------------------------------- diff --cc tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java index 0000000,b7d9334..cae7018 mode 000000,100644..100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java @@@ -1,0 -1,137 +1,155 @@@ + /** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package org.apache.tajo.cli.tsql.commands; + + import org.apache.commons.lang.CharUtils; + import org.apache.commons.lang.StringEscapeUtils; + import org.apache.tajo.TajoConstants; + import org.apache.tajo.catalog.CatalogUtil; + import org.apache.tajo.catalog.Column; + import org.apache.tajo.catalog.TableDesc; + import org.apache.tajo.catalog.partition.PartitionMethodDesc; ++import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; ++import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto; + import org.apache.tajo.cli.tsql.TajoCli; + import org.apache.tajo.util.FileUtil; + import org.apache.tajo.util.TUtil; + + import java.util.List; + import java.util.Map; + + public class DescTableCommand extends TajoShellCommand { + public DescTableCommand(TajoCli.TajoCliContext context) { + super(context); + } + + @Override + public String getCommand() { + return "\\d"; + } + + @Override + public void invoke(String[] cmd) throws Exception { + if (cmd.length == 2) { + String tableName = cmd[1]; + tableName = tableName.replace("\"", ""); + TableDesc desc = client.getTableDesc(tableName); + if (desc == null) { + context.getOutput().println("Did not find any relation named \"" + tableName + "\""); + } else { + context.getOutput().println(toFormattedString(desc)); ++ // If there exists any indexes for the table, print index information ++ if (client.hasIndexes(tableName)) { ++ StringBuilder sb = new StringBuilder(); ++ sb.append("Indexes:\n"); ++ for (IndexDescProto index : client.getIndexes(tableName)) { ++ sb.append("\"").append(index.getIndexName()).append("\" "); ++ sb.append(index.getIndexMethod()).append(" ("); ++ for (SortSpecProto key : index.getKeySortSpecsList()) { ++ sb.append(CatalogUtil.extractSimpleName(key.getColumn().getName())); ++ sb.append(key.getAscending() ? " ASC" : " DESC"); ++ sb.append(key.getNullFirst() ? " NULLS FIRST, " : " NULLS LAST, "); ++ } ++ sb.delete(sb.length()-2, sb.length()-1).append(")\n"); ++ } ++ context.getOutput().println(sb.toString()); ++ } + } + } else if (cmd.length == 1) { + List<String> tableList = client.getTableList(null); + if (tableList.size() == 0) { + context.getOutput().println("No Relation Found"); + } + for (String table : tableList) { + context.getOutput().println(table); + } + } else { + throw new IllegalArgumentException(); + } + } + + @Override + public String getUsage() { + return "[table_name]"; + } + + @Override + public String getDescription() { + return "show table description"; + } + + protected String toFormattedString(TableDesc desc) { + StringBuilder sb = new StringBuilder(); + sb.append("\ntable name: ").append(desc.getName()).append("\n"); + sb.append("table path: ").append(desc.getPath()).append("\n"); + sb.append("store type: ").append(CatalogUtil.getStoreTypeString(desc.getMeta().getStoreType())).append("\n"); + if (desc.getStats() != null) { + + long row = desc.getStats().getNumRows(); + String rowText = row == TajoConstants.UNKNOWN_ROW_NUMBER ? "unknown" : row + ""; + sb.append("number of rows: ").append(rowText).append("\n"); + sb.append("volume: ").append( + FileUtil.humanReadableByteCount(desc.getStats().getNumBytes(), + true)).append("\n"); + } + sb.append("Options: \n"); + for(Map.Entry<String, String> entry : desc.getMeta().toMap().entrySet()){ + + /* + * Checks whether the character is ASCII 7 bit printable. + * For example, a printable unicode '\u007c' become the character â|â. + * + * Control-chars : ctrl-a(\u0001), tab(\u0009) .. + * Printable-chars : '|'(\u007c), ','(\u002c) .. + * */ + + String value = entry.getValue(); + String unescaped = StringEscapeUtils.unescapeJava(value); + if (unescaped.length() == 1 && CharUtils.isAsciiPrintable(unescaped.charAt(0))) { + value = unescaped; + } + sb.append("\t").append("'").append(entry.getKey()).append("'").append("=") + .append("'").append(value).append("'").append("\n"); + } + sb.append("\n"); + sb.append("schema: \n"); + + for(int i = 0; i < desc.getSchema().size(); i++) { + Column col = desc.getSchema().getColumn(i); + sb.append(col.getSimpleName()).append("\t").append(col.getDataType().getType()); + if (col.getDataType().hasLength()) { + sb.append("(").append(col.getDataType().getLength()).append(")"); + } + sb.append("\n"); + } + + sb.append("\n"); + if (desc.getPartitionMethod() != null) { + PartitionMethodDesc partition = desc.getPartitionMethod(); + sb.append("Partitions: \n"); + + sb.append("type:").append(partition.getPartitionType().name()).append("\n"); + + sb.append("columns:").append(":"); + sb.append(TUtil.arrayToString(partition.getExpressionSchema().toArray())); + } + + return sb.toString(); + } + } http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-core/src/main/java/org/apache/tajo/worker/Task.java ---------------------------------------------------------------------- diff --cc tajo-core/src/main/java/org/apache/tajo/worker/Task.java index 14b0d1a,8f84a9d..859b3ae --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@@ -120,12 -129,27 +129,15 @@@ public class Task request.getFragments().toArray(new FragmentProto[request.getFragments().size()]), taskDir); this.context.setDataChannel(request.getDataChannel()); this.context.setEnforcer(request.getEnforcer()); + this.context.setState(TaskAttemptState.TA_PENDING); this.inputStats = new TableStats(); + } + public void initPlan() throws IOException { plan = LogicalNodeDeserializer.deserialize(queryContext, request.getPlan()); - LogicalNode [] scanNode = PlannerUtil.findAllNodes(plan, NodeType.SCAN); - if (scanNode != null) { - for (LogicalNode node : scanNode) { - ScanNode scan = (ScanNode) node; - descs.put(scan.getCanonicalName(), scan.getTableDesc()); - } - } - - LogicalNode [] partitionScanNode = PlannerUtil.findAllNodes(plan, NodeType.PARTITIONS_SCAN); - if (partitionScanNode != null) { - for (LogicalNode node : partitionScanNode) { - PartitionedTableScanNode scan = (PartitionedTableScanNode) node; - descs.put(scan.getCanonicalName(), scan.getTableDesc()); - } - } + updateDescsForScanNodes(NodeType.SCAN); + updateDescsForScanNodes(NodeType.PARTITIONS_SCAN); + updateDescsForScanNodes(NodeType.INDEX_SCAN); interQuery = request.getProto().getInterQuery(); if (interQuery) { @@@ -167,18 -189,9 +177,20 @@@ LOG.info("=================================="); } + private void updateDescsForScanNodes(NodeType nodeType) { + assert nodeType == NodeType.SCAN || nodeType == NodeType.PARTITIONS_SCAN || nodeType == NodeType.INDEX_SCAN; + LogicalNode[] scanNodes = PlannerUtil.findAllNodes(plan, nodeType); + if (scanNodes != null) { + for (LogicalNode node : scanNodes) { + ScanNode scanNode = (ScanNode) node; + descs.put(scanNode.getCanonicalName(), scanNode.getTableDesc()); + } + } + } + public void init() throws IOException { + initPlan(); + if (context.getState() == TaskAttemptState.TA_PENDING) { // initialize a task temporal dir FileSystem localFS = executionBlockContext.getLocalFS(); http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java ----------------------------------------------------------------------
