TAJO-1269: Separate cli from tajo-client. Closes #374
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/ad4fda7c Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/ad4fda7c Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/ad4fda7c Branch: refs/heads/branch-0.10.0 Commit: ad4fda7cefe4f86c78296a71d74d9f6b8f2e2b56 Parents: a6eb3d6 Author: Hyunsik Choi <[email protected]> Authored: Thu Feb 5 00:58:43 2015 -0800 Committer: Hyunsik Choi <[email protected]> Committed: Thu Feb 5 01:04:19 2015 -0800 ---------------------------------------------------------------------- CHANGES | 2 + pom.xml | 1 + tajo-cli/pom.xml | 337 +++++++++ .../org/apache/tajo/cli/tools/TajoAdmin.java | 459 ++++++++++++ .../org/apache/tajo/cli/tools/TajoDump.java | 202 ++++++ .../org/apache/tajo/cli/tools/TajoGetConf.java | 161 +++++ .../org/apache/tajo/cli/tools/TajoHAAdmin.java | 210 ++++++ .../cli/tsql/DefaultTajoCliOutputFormatter.java | 211 ++++++ .../cli/tsql/InvalidStatementException.java | 25 + .../org/apache/tajo/cli/tsql/ParsedResult.java | 53 ++ .../org/apache/tajo/cli/tsql/SimpleParser.java | 387 ++++++++++ .../java/org/apache/tajo/cli/tsql/TajoCli.java | 686 ++++++++++++++++++ .../tajo/cli/tsql/TajoCliOutputFormatter.java | 97 +++ .../apache/tajo/cli/tsql/TajoFileHistory.java | 41 ++ .../tsql/commands/ConnectDatabaseCommand.java | 72 ++ .../cli/tsql/commands/CopyrightCommand.java | 65 ++ .../cli/tsql/commands/DescFunctionCommand.java | 136 ++++ .../cli/tsql/commands/DescTableCommand.java | 137 ++++ .../tsql/commands/ExecExternalShellCommand.java | 124 ++++ .../tajo/cli/tsql/commands/ExitCommand.java | 52 ++ .../tajo/cli/tsql/commands/HdfsCommand.java | 58 ++ .../tajo/cli/tsql/commands/HelpCommand.java | 133 ++++ .../cli/tsql/commands/ListDatabaseCommand.java | 50 ++ .../tajo/cli/tsql/commands/SetCommand.java | 127 ++++ .../cli/tsql/commands/TajoAdminCommand.java | 58 ++ .../cli/tsql/commands/TajoGetConfCommand.java | 58 ++ .../cli/tsql/commands/TajoHAAdminCommand.java | 58 ++ .../cli/tsql/commands/TajoShellCommand.java | 129 ++++ .../tajo/cli/tsql/commands/UnsetCommand.java | 53 ++ .../tajo/cli/tsql/commands/VersionCommand.java | 49 ++ .../org/apache/tajo/cli/tools/TajoAdmin.java | 457 ------------ .../org/apache/tajo/cli/tools/TajoDump.java | 202 ------ .../org/apache/tajo/cli/tools/TajoGetConf.java | 161 ----- .../org/apache/tajo/cli/tools/TajoHAAdmin.java | 211 ------ .../cli/tsql/DefaultTajoCliOutputFormatter.java | 211 ------ .../cli/tsql/InvalidStatementException.java | 25 - .../org/apache/tajo/cli/tsql/ParsedResult.java | 53 -- .../org/apache/tajo/cli/tsql/SimpleParser.java | 388 ---------- .../java/org/apache/tajo/cli/tsql/TajoCli.java | 701 ------------------- .../tajo/cli/tsql/TajoCliOutputFormatter.java | 98 --- .../apache/tajo/cli/tsql/TajoFileHistory.java | 41 -- .../tsql/commands/ConnectDatabaseCommand.java | 72 -- .../cli/tsql/commands/CopyrightCommand.java | 65 -- .../cli/tsql/commands/DescFunctionCommand.java | 136 ---- .../cli/tsql/commands/DescTableCommand.java | 137 ---- .../tsql/commands/ExecExternalShellCommand.java | 124 ---- .../tajo/cli/tsql/commands/ExitCommand.java | 52 -- .../tajo/cli/tsql/commands/HdfsCommand.java | 58 -- .../tajo/cli/tsql/commands/HelpCommand.java | 133 ---- .../cli/tsql/commands/ListDatabaseCommand.java | 50 -- .../tajo/cli/tsql/commands/SetCommand.java | 127 ---- .../cli/tsql/commands/TajoAdminCommand.java | 58 -- .../cli/tsql/commands/TajoGetConfCommand.java | 58 -- .../cli/tsql/commands/TajoHAAdminCommand.java | 58 -- .../cli/tsql/commands/TajoShellCommand.java | 129 ---- .../tajo/cli/tsql/commands/UnsetCommand.java | 53 -- .../tajo/cli/tsql/commands/VersionCommand.java | 49 -- .../apache/tajo/client/TajoHAClientUtil.java | 87 --- tajo-core/pom.xml | 4 + .../tajo/webapp/QueryExecutorServlet.java | 2 - tajo-dist/pom.xml | 1 + tajo-project/pom.xml | 5 + 62 files changed, 4241 insertions(+), 3996 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/ad4fda7c/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 2e4849f..5a551b6 100644 --- a/CHANGES +++ b/CHANGES @@ -29,6 +29,8 @@ Release 0.10.0 - unreleased IMPROVEMENT + TAJO-1269: Separate cli from tajo-client. (hyunsik) + TAJO-1328: Fix deprecated property names in the catalog configuration document. (jihun) http://git-wip-us.apache.org/repos/asf/tajo/blob/ad4fda7c/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 8e5dd5e..7793550 100644 --- a/pom.xml +++ b/pom.xml @@ -92,6 +92,7 @@ <module>tajo-pullserver</module> <module>tajo-dist</module> <module>tajo-thirdparty/asm</module> + <module>tajo-cli</module> </modules> <build> http://git-wip-us.apache.org/repos/asf/tajo/blob/ad4fda7c/tajo-cli/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-cli/pom.xml b/tajo-cli/pom.xml new file mode 100644 index 0000000..36e6118 --- /dev/null +++ b/tajo-cli/pom.xml @@ -0,0 +1,337 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>tajo-project</artifactId> + <groupId>org.apache.tajo</groupId> + <version>0.10.0-SNAPSHOT</version> + <relativePath>../tajo-project</relativePath> + </parent> + <artifactId>tajo-cli</artifactId> + <packaging>jar</packaging> + <name>Tajo CLI tools</name> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> + <metrics.version>3.0.1</metrics.version> + </properties> + + <repositories> + <repository> + <id>repository.jboss.org</id> + <url>https://repository.jboss.org/nexus/content/repositories/releases/ + </url> + <snapshots> + <enabled>false</enabled> + </snapshots> + </repository> + </repositories> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <source>1.6</source> + <target>1.6</target> + <encoding>${project.build.sourceEncoding}</encoding> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <executions> + <execution> + <phase>verify</phase> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <systemProperties> + <tajo.test>TRUE</tajo.test> + </systemProperties> + <argLine>-Xms512m -Xmx1024m -Dfile.encoding=UTF-8</argLine> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>2.2</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy-dependencies</id> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <includeScope>runtime</includeScope> + <outputDirectory>${project.build.directory}/lib</outputDirectory> + <overWriteReleases>false</overWriteReleases> + <overWriteSnapshots>false</overWriteSnapshots> + <overWriteIfNewer>true</overWriteIfNewer> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-pmd-plugin</artifactId> + <version>2.7.1</version> + </plugin> + </plugins> + </build> + + + <dependencies> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-client</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-catalog-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-hdfs</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-rpc</artifactId> + </dependency> + <dependency> + <groupId>commons-cli</groupId> + <artifactId>commons-cli</artifactId> + <version>1.2</version> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <scope>provided</scope> + <exclusions> + <exclusion> + <artifactId>protobuf-java</artifactId> + <groupId>com.google.protobuf</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-mapreduce-client-app</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-yarn-api</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-mapreduce-client-jobclient</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <version>2.4</version> + </dependency> + <dependency> + <groupId>jline</groupId> + <artifactId>jline</artifactId> + </dependency> + </dependencies> + + + <profiles> + <profile> + <id>all-dependencies</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <exclusions> + <exclusion> + <artifactId>protobuf-java</artifactId> + <groupId>com.google.protobuf</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-mapreduce-client-app</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-yarn-api</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-mapreduce-client-jobclient</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + <exclusion> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + </exclusions> + </dependency> + </dependencies> + </profile> + + + <profile> + <id>docs</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <executions> + <execution> + <!-- build javadoc jars per jar for publishing to maven --> + <id>module-javadocs</id> + <phase>package</phase> + <goals> + <goal>jar</goal> + </goals> + <configuration> + <destDir>${project.build.directory}</destDir> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> + <id>dist</id> + <activation> + <activeByDefault>false</activeByDefault> + <property> + <name>tar|rpm|deb</name> + </property> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>dist</id> + <phase>package</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <echo file="${project.build.directory}/dist-layout-stitching.sh"> + run() { + echo "\$ ${@}" + "${@}" + res=$? + if [ $res != 0 ]; then + echo + echo "Failed!" + echo + exit $res + fi + } + + ROOT=`cd ${basedir}/..;pwd` + echo + echo "Current directory `pwd`" + echo + run rm -rf ${project.artifactId}-${project.version} + run mkdir ${project.artifactId}-${project.version} + run cd ${project.artifactId}-${project.version} + run cp -r ${basedir}/target/${project.artifactId}-${project.version}*.jar . + echo + echo "Tajo Client dist layout available at: ${project.build.directory}/${project.artifactId}-${project.version}" + echo + </echo> + <exec executable="sh" dir="${project.build.directory}" failonerror="true"> + <arg line="./dist-layout-stitching.sh" /> + </exec> + </target> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> + + <reporting> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-project-info-reports-plugin</artifactId> + <version>2.4</version> + <configuration> + <dependencyLocationsEnabled>false</dependencyLocationsEnabled> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + </plugin> + </plugins> + </reporting> +</project> + http://git-wip-us.apache.org/repos/asf/tajo/blob/ad4fda7c/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java ---------------------------------------------------------------------- diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java new file mode 100644 index 0000000..18b2d99 --- /dev/null +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java @@ -0,0 +1,459 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.cli.tools; + +import com.google.protobuf.ServiceException; +import org.apache.commons.cli.*; +import org.apache.commons.lang.StringUtils; +import org.apache.tajo.QueryId; +import org.apache.tajo.TajoProtos; +import org.apache.tajo.client.*; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.ipc.ClientProtos.BriefQueryInfo; +import org.apache.tajo.ipc.ClientProtos.WorkerResourceInfo; +import org.apache.tajo.service.ServiceTracker; +import org.apache.tajo.util.NetUtils; +import org.apache.tajo.ha.HAServiceUtil; +import org.apache.tajo.util.TajoIdUtils; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Writer; +import java.net.InetSocketAddress; +import java.sql.SQLException; +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; + +public class TajoAdmin { + private static final org.apache.commons.cli.Options options; + private static DecimalFormat decimalF = new DecimalFormat("###.0"); + private enum WorkerStatus { + RUNNING, + LOST, + DECOMMISSIONED + } + + final static String DASHLINE_LEN5 = "-----"; + final static String DASHLINE_LEN10 = "----------"; + final static String DASHLINE_LEN12 = "------------"; + final static String DASHLINE_LEN25 = "-------------------------"; + final static String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; + + static { + options = new Options(); + options.addOption("h", "host", true, "Tajo server host"); + options.addOption("p", "port", true, "Tajo server port"); + options.addOption("list", null, false, "Show Tajo query list"); + options.addOption("cluster", null, false, "Show Cluster Info"); + options.addOption("showmasters", null, false, "gets list of tajomasters in the cluster"); + options.addOption("desc", null, false, "Show Query Description"); + options.addOption("kill", null, true, "Kill a running query"); + } + + private TajoConf tajoConf; + private TajoClient tajoClient; + private ServiceTracker serviceTracker; + private Writer writer; + + public TajoAdmin(TajoConf tajoConf, Writer writer) { + this(tajoConf, writer, null); + } + + public TajoAdmin(TajoConf tajoConf, Writer writer, TajoClient tajoClient) { + this.tajoConf = tajoConf; + this.writer = writer; + this.tajoClient = tajoClient; + } + + private void printUsage() { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp( "admin [options]", options ); + } + + public void runCommand(String[] args) throws Exception { + CommandLineParser parser = new PosixParser(); + CommandLine cmd = parser.parse(options, args); + + String param = ""; + int cmdType = 0; + + String hostName = null; + Integer port = null; + if (cmd.hasOption("h")) { + hostName = cmd.getOptionValue("h"); + } + if (cmd.hasOption("p")) { + port = Integer.parseInt(cmd.getOptionValue("p")); + } + + String queryId = null; + + if (cmd.hasOption("list")) { + cmdType = 1; + } else if (cmd.hasOption("desc")) { + cmdType = 2; + } else if (cmd.hasOption("cluster")) { + cmdType = 3; + } else if (cmd.hasOption("kill")) { + cmdType = 4; + queryId = cmd.getOptionValue("kill"); + } else if (cmd.hasOption("showmasters")) { + cmdType = 5; + } + + // if there is no "-h" option, + if(hostName == null) { + if (tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) { + // it checks if the client service address is given in configuration and distributed mode. + // if so, it sets entryAddr. + hostName = tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[0]; + } + } + if (port == null) { + if (tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) { + // it checks if the client service address is given in configuration and distributed mode. + // if so, it sets entryAddr. + port = Integer.parseInt(tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[1]); + } + } + + if (cmdType == 0) { + printUsage(); + return; + } + + + if ((hostName == null) ^ (port == null)) { + System.err.println("ERROR: cannot find valid Tajo server address"); + return; + } else if (hostName != null && port != null) { + tajoConf.setVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, hostName + ":" + port); + tajoClient = new TajoClientImpl(tajoConf); + } else if (hostName == null && port == null) { + tajoClient = new TajoClientImpl(tajoConf); + } + + switch (cmdType) { + case 1: + processList(writer); + break; + case 2: + processDesc(writer); + break; + case 3: + processCluster(writer); + break; + case 4: + processKill(writer, queryId); + break; + case 5: + processMasters(writer); + break; + default: + printUsage(); + break; + } + + writer.flush(); + } + + private void processDesc(Writer writer) throws ParseException, IOException, + ServiceException, SQLException { + + List<BriefQueryInfo> queryList = tajoClient.getRunningQueryList(); + SimpleDateFormat df = new SimpleDateFormat(DATE_FORMAT); + int id = 1; + for (BriefQueryInfo queryInfo : queryList) { + String queryId = String.format("q_%s_%04d", + queryInfo.getQueryId().getId(), + queryInfo.getQueryId().getSeq()); + + writer.write("Id: " + id); + writer.write("\n"); + id++; + writer.write("Query Id: " + queryId); + writer.write("\n"); + writer.write("Started Time: " + df.format(queryInfo.getStartTime())); + writer.write("\n"); + + writer.write("Query State: " + queryInfo.getState().name()); + writer.write("\n"); + long end = queryInfo.getFinishTime(); + long start = queryInfo.getStartTime(); + String executionTime = decimalF.format((end-start) / 1000) + " sec"; + if (TajoClientUtil.isQueryComplete(queryInfo.getState())) { + writer.write("Finished Time: " + df.format(queryInfo.getFinishTime())); + writer.write("\n"); + } + writer.write("Execution Time: " + executionTime); + writer.write("\n"); + writer.write("Query Progress: " + queryInfo.getProgress()); + writer.write("\n"); + writer.write("Query Statement:"); + writer.write("\n"); + writer.write(queryInfo.getQuery()); + writer.write("\n"); + writer.write("\n"); + } + } + + private void processCluster(Writer writer) throws ParseException, IOException, + ServiceException, SQLException { + + List<WorkerResourceInfo> workerList = tajoClient.getClusterInfo(); + + int runningQueryMasterTasks = 0; + + List<WorkerResourceInfo> liveWorkers = new ArrayList<WorkerResourceInfo>(); + List<WorkerResourceInfo> deadWorkers = new ArrayList<WorkerResourceInfo>(); + List<WorkerResourceInfo> decommissionWorkers = new ArrayList<WorkerResourceInfo>(); + + List<WorkerResourceInfo> liveQueryMasters = new ArrayList<WorkerResourceInfo>(); + List<WorkerResourceInfo> deadQueryMasters = new ArrayList<WorkerResourceInfo>(); + + for (WorkerResourceInfo eachWorker : workerList) { + if(eachWorker.getQueryMasterMode() == true) { + if(eachWorker.getWorkerStatus().equals(WorkerStatus.RUNNING.toString())) { + liveQueryMasters.add(eachWorker); + runningQueryMasterTasks += eachWorker.getNumQueryMasterTasks(); + } + if(eachWorker.getWorkerStatus().equals(WorkerStatus.LOST.toString())) { + deadQueryMasters.add(eachWorker); + } + } + + if(eachWorker.getTaskRunnerMode() == true) { + if(eachWorker.getWorkerStatus().equals(WorkerStatus.RUNNING.toString())) { + liveWorkers.add(eachWorker); + } else if(eachWorker.getWorkerStatus().equals(WorkerStatus.LOST.toString())) { + deadWorkers.add(eachWorker); + } else if(eachWorker.getWorkerStatus().equals(WorkerStatus.DECOMMISSIONED.toString())) { + decommissionWorkers.add(eachWorker); + } + } + } + + String fmtInfo = "%1$-5s %2$-5s %3$-5s%n"; + String infoLine = String.format(fmtInfo, "Live", "Dead", "Tasks"); + + writer.write("Query Master\n"); + writer.write("============\n\n"); + writer.write(infoLine); + String line = String.format(fmtInfo, DASHLINE_LEN5, DASHLINE_LEN5, DASHLINE_LEN5); + writer.write(line); + + line = String.format(fmtInfo, liveQueryMasters.size(), + deadQueryMasters.size(), runningQueryMasterTasks); + writer.write(line); + writer.write("\n"); + + writer.write("Live QueryMasters\n"); + writer.write("=================\n\n"); + + if (liveQueryMasters.isEmpty()) { + writer.write("No Live QueryMasters\n"); + } else { + String fmtQueryMasterLine = "%1$-25s %2$-5s %3$-5s %4$-10s %5$-10s%n"; + line = String.format(fmtQueryMasterLine, "QueryMaster", "Port", "Query", + "Heap", "Status"); + writer.write(line); + line = String.format(fmtQueryMasterLine, DASHLINE_LEN25, DASHLINE_LEN5, + DASHLINE_LEN5, DASHLINE_LEN10, DASHLINE_LEN10); + writer.write(line); + for (WorkerResourceInfo queryMaster : liveQueryMasters) { + TajoProtos.WorkerConnectionInfoProto connInfo = queryMaster.getConnectionInfo(); + String queryMasterHost = String.format("%s:%d", connInfo.getHost(), connInfo.getQueryMasterPort()); + String heap = String.format("%d MB", queryMaster.getMaxHeap() / 1024 / 1024); + line = String.format(fmtQueryMasterLine, + queryMasterHost, + connInfo.getClientPort(), + queryMaster.getNumQueryMasterTasks(), + heap, + queryMaster.getWorkerStatus()); + writer.write(line); + } + + writer.write("\n\n"); + } + + if (!deadQueryMasters.isEmpty()) { + writer.write("Dead QueryMasters\n"); + writer.write("=================\n\n"); + + String fmtQueryMasterLine = "%1$-25s %2$-5s %3$-10s%n"; + line = String.format(fmtQueryMasterLine, "QueryMaster", "Port", "Status"); + writer.write(line); + line = String.format(fmtQueryMasterLine, DASHLINE_LEN25, DASHLINE_LEN5, DASHLINE_LEN10); + writer.write(line); + + for (WorkerResourceInfo queryMaster : deadQueryMasters) { + TajoProtos.WorkerConnectionInfoProto connInfo = queryMaster.getConnectionInfo(); + String queryMasterHost = String.format("%s:%d", connInfo.getHost(), connInfo.getQueryMasterPort()); + line = String.format(fmtQueryMasterLine, + queryMasterHost, + connInfo.getClientPort(), + queryMaster.getWorkerStatus()); + writer.write(line); + } + + writer.write("\n\n"); + } + + writer.write("Worker\n"); + writer.write("======\n\n"); + + String fmtWorkerInfo = "%1$-5s %2$-5s%n"; + String workerInfoLine = String.format(fmtWorkerInfo, "Live", "Dead"); + writer.write(workerInfoLine); + line = String.format(fmtWorkerInfo, DASHLINE_LEN5, DASHLINE_LEN5); + writer.write(line); + + line = String.format(fmtWorkerInfo, liveWorkers.size(), deadWorkers.size()); + writer.write(line); + writer.write("\n"); + + writer.write("Live Workers\n"); + writer.write("============\n\n"); + if(liveWorkers.isEmpty()) { + writer.write("No Live Workers\n\n"); + } else { + writeWorkerInfo(writer, liveWorkers); + } + + writer.write("Dead Workers\n"); + writer.write("============\n\n"); + if(deadWorkers.isEmpty()) { + writer.write("No Dead Workers\n\n"); + } else { + writeWorkerInfo(writer, deadWorkers); + } + } + + private void writeWorkerInfo(Writer writer, List<WorkerResourceInfo> workers) throws ParseException, + IOException, ServiceException, SQLException { + String fmtWorkerLine = "%1$-25s %2$-5s %3$-5s %4$-10s %5$-10s %6$-12s %7$-10s%n"; + String line = String.format(fmtWorkerLine, + "Worker", "Port", "Tasks", + "Mem", "Disk", + "Heap", "Status"); + writer.write(line); + line = String.format(fmtWorkerLine, + DASHLINE_LEN25, DASHLINE_LEN5, DASHLINE_LEN5, + DASHLINE_LEN10, DASHLINE_LEN10, + DASHLINE_LEN12, DASHLINE_LEN10); + writer.write(line); + + for (WorkerResourceInfo worker : workers) { + TajoProtos.WorkerConnectionInfoProto connInfo = worker.getConnectionInfo(); + String workerHost = String.format("%s:%d", connInfo.getHost(), connInfo.getPeerRpcPort()); + String mem = String.format("%d/%d", worker.getUsedMemoryMB(), + worker.getMemoryMB()); + String disk = String.format("%.2f/%.2f", worker.getUsedDiskSlots(), + worker.getDiskSlots()); + String heap = String.format("%d/%d MB", worker.getFreeHeap()/1024/1024, + worker.getMaxHeap()/1024/1024); + + line = String.format(fmtWorkerLine, workerHost, + connInfo.getPullServerPort(), + worker.getNumRunningTasks(), + mem, disk, heap, worker.getWorkerStatus()); + writer.write(line); + } + writer.write("\n\n"); + } + + private void processList(Writer writer) throws ParseException, IOException, + ServiceException, SQLException { + + List<BriefQueryInfo> queryList = tajoClient.getRunningQueryList(); + SimpleDateFormat df = new SimpleDateFormat(DATE_FORMAT); + StringBuilder builder = new StringBuilder(); + + /* print title */ + builder.append(StringUtils.rightPad("QueryId", 21)); + builder.append(StringUtils.rightPad("State", 20)); + builder.append(StringUtils.rightPad("StartTime", 20)); + builder.append(StringUtils.rightPad("Query", 30)).append("\n"); + + builder.append(StringUtils.rightPad(StringUtils.repeat("-", 20), 21)); + builder.append(StringUtils.rightPad(StringUtils.repeat("-", 19), 20)); + builder.append(StringUtils.rightPad(StringUtils.repeat("-", 19), 20)); + builder.append(StringUtils.rightPad(StringUtils.repeat("-", 29), 30)).append("\n"); + writer.write(builder.toString()); + + builder = new StringBuilder(); + for (BriefQueryInfo queryInfo : queryList) { + builder.append(StringUtils.rightPad(new QueryId(queryInfo.getQueryId()).toString(), 21)); + builder.append(StringUtils.rightPad(queryInfo.getState().name(), 20)); + builder.append(StringUtils.rightPad(df.format(queryInfo.getStartTime()), 20)); + builder.append(StringUtils.abbreviate(queryInfo.getQuery(), 30)).append("\n"); + } + writer.write(builder.toString()); + } + + public void processKill(Writer writer, String queryIdStr) + throws IOException, ServiceException { + QueryStatus status = tajoClient.killQuery(TajoIdUtils.parseQueryId(queryIdStr)); + if (status.getState() == TajoProtos.QueryState.QUERY_KILLED) { + writer.write(queryIdStr + " is killed successfully.\n"); + } else if (status.getState() == TajoProtos.QueryState.QUERY_KILL_WAIT) { + writer.write(queryIdStr + " will be finished after a while.\n"); + } else { + writer.write("ERROR:" + status.getErrorMessage()); + } + } + + private void processMasters(Writer writer) throws ParseException, IOException, + ServiceException, SQLException { + + if (tajoConf.getBoolVar(TajoConf.ConfVars.TAJO_MASTER_HA_ENABLE)) { + + List<String> list = HAServiceUtil.getMasters(tajoConf); + int i = 0; + for (String master : list) { + if (i > 0) { + writer.write(" "); + } + writer.write(master); + i++; + } + writer.write("\n"); + } else { + String confMasterServiceAddr = tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS); + InetSocketAddress masterAddress = NetUtils.createSocketAddr(confMasterServiceAddr); + writer.write(masterAddress.getHostName()); + writer.write("\n"); + } + } + + public static void main(String [] args) throws Exception { + TajoConf conf = new TajoConf(); + + Writer writer = new PrintWriter(System.out); + try { + TajoAdmin admin = new TajoAdmin(conf, writer); + admin.runCommand(args); + } finally { + writer.close(); + System.exit(0); + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/ad4fda7c/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java ---------------------------------------------------------------------- diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java new file mode 100644 index 0000000..7f38a5d --- /dev/null +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java @@ -0,0 +1,202 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.cli.tools; + +import com.google.protobuf.ServiceException; + +import org.apache.commons.cli.*; +import org.apache.tajo.auth.UserRoleInfo; +import org.apache.tajo.catalog.CatalogConstants; +import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.DDLBuilder; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; +import org.apache.tajo.client.TajoClient; +import org.apache.tajo.client.TajoClientImpl; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.util.Pair; +import org.apache.tajo.util.TUtil; + +import java.io.IOException; +import java.io.PrintWriter; +import java.sql.SQLException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collections; +import java.util.List; + +public class TajoDump { + private static final org.apache.commons.cli.Options options; + + static { + options = new Options(); + options.addOption("h", "host", true, "Tajo server host"); + options.addOption("p", "port", true, "Tajo server port"); + options.addOption("a", "all", false, "dump all table DDLs"); + } + + private static void printUsage() { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp( "tajo-dump [options] [database name]", options); + } + + private static Pair<String, Integer> getConnectionAddr(TajoConf conf, CommandLine cmd) { + String hostName = null; + Integer port = null; + if (cmd.hasOption("h")) { + hostName = cmd.getOptionValue("h"); + } + if (cmd.hasOption("p")) { + port = Integer.parseInt(cmd.getOptionValue("p")); + } + + if(hostName == null) { + if (conf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) { + hostName = conf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[0]; + } + } + if (port == null) { + if (conf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) { + port = Integer.parseInt(conf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[1]); + } + } + return new Pair<String, Integer>(hostName, port); + } + + public static void main(String [] args) throws ParseException, IOException, ServiceException, SQLException { + final TajoConf conf = new TajoConf(); + final CommandLineParser parser = new PosixParser(); + final CommandLine cmd = parser.parse(options, args); + final Pair<String, Integer> hostAndPort = getConnectionAddr(conf, cmd); + final String hostName = hostAndPort.getFirst(); + final Integer port = hostAndPort.getSecond(); + final UserRoleInfo userInfo = UserRoleInfo.getCurrentUser(); + + String baseDatabaseName = null; + if (cmd.getArgList().size() > 0) { + baseDatabaseName = (String) cmd.getArgList().get(0); + } + + boolean isDumpingAllDatabases = cmd.hasOption('a'); + + // Neither two choices + if (!isDumpingAllDatabases && baseDatabaseName == null) { + printUsage(); + System.exit(-1); + } + + TajoClient client = null; + if ((hostName == null) ^ (port == null)) { + System.err.println("ERROR: cannot find any TajoMaster rpc address in arguments and tajo-site.xml."); + System.exit(-1); + } else if (hostName != null && port != null) { + conf.setVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, hostName+":"+port); + client = new TajoClientImpl(conf); + } else { + client = new TajoClientImpl(conf); + } + + PrintWriter writer = new PrintWriter(System.out); + dump(client, userInfo, baseDatabaseName, isDumpingAllDatabases, true, true, writer); + + System.exit(0); + } + + private static boolean isAcceptableDumpingDatabase(String databaseName) { + return (databaseName == null || !databaseName.equalsIgnoreCase(CatalogConstants.INFORMATION_SCHEMA_DB_NAME)); + } + + public static void dump(TajoClient client, UserRoleInfo userInfo, String baseDatabaseName, + boolean isDumpingAllDatabases, boolean includeUserName, boolean includeDate, PrintWriter out) + throws SQLException, ServiceException { + printHeader(out, userInfo, includeUserName, includeDate); + + if (isDumpingAllDatabases) { + // sort database names in an ascending lexicographic order of the names. + List<String> sorted = new ArrayList<String>(client.getAllDatabaseNames()); + Collections.sort(sorted); + + for (String databaseName : sorted) { + if (isAcceptableDumpingDatabase(databaseName)) { + dumpDatabase(client, databaseName, out); + } + } + } else { + dumpDatabase(client, baseDatabaseName, out); + } + out.flush(); + } + + private static void printHeader(PrintWriter writer, UserRoleInfo userInfo, boolean includeUSerName, + boolean includeDate) { + writer.write("--\n"); + writer.write("-- Tajo database dump\n"); + if (includeUSerName) { + writer.write("--\n-- Dump user: " + userInfo.getUserName() + "\n"); + } + if (includeDate) { + writer.write("--\n-- Dump date: " + toDateString() + "\n"); + } + writer.write("--\n"); + writer.write("\n"); + } + + private static void dumpDatabase(TajoClient client, String databaseName, PrintWriter writer) + throws SQLException, ServiceException { + writer.write("\n"); + writer.write("--\n"); + writer.write(String.format("-- Database name: %s%n", CatalogUtil.denormalizeIdentifier(databaseName))); + writer.write("--\n"); + writer.write("\n"); + writer.write(String.format("CREATE DATABASE IF NOT EXISTS %s;", CatalogUtil.denormalizeIdentifier(databaseName))); + writer.write("\n\n"); + + // returned list is immutable. + List<String> tableNames = TUtil.newList(client.getTableList(databaseName)); + Collections.sort(tableNames); + for (String tableName : tableNames) { + try { + TableDesc table = client.getTableDesc(CatalogUtil.buildFQName(databaseName, tableName)); + + if (table.getMeta().getStoreType() == StoreType.SYSTEM) { + continue; + } + + if (table.isExternal()) { + writer.write(DDLBuilder.buildDDLForExternalTable(table)); + } else { + writer.write(DDLBuilder.buildDDLForBaseTable(table)); + } + writer.write("\n\n"); + } catch (Exception e) { + // dump for each table can throw any exception. We need to skip the exception case. + // here, the error message prints out via stderr. + System.err.println("ERROR:" + tableName + "," + e.getMessage()); + } + } + } + + private static String toDateString() { + DateFormat df = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss"); + java.util.Date today = Calendar.getInstance().getTime(); + return df.format(today); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/ad4fda7c/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoGetConf.java ---------------------------------------------------------------------- diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoGetConf.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoGetConf.java new file mode 100644 index 0000000..aa7620b --- /dev/null +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoGetConf.java @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.cli.tools; + +import com.google.protobuf.ServiceException; +import org.apache.commons.cli.*; +import org.apache.tajo.client.TajoClient; +import org.apache.tajo.client.TajoClientImpl; +import org.apache.tajo.conf.TajoConf; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Writer; +import java.sql.SQLException; + +public class TajoGetConf { + private static final org.apache.commons.cli.Options options; + + static { + options = new Options(); + options.addOption("h", "host", true, "Tajo server host"); + options.addOption("p", "port", true, "Tajo server port"); + } + + private TajoConf tajoConf; + private TajoClient tajoClient; + private Writer writer; + + public final static String defaultLeftPad = " "; + public final static String defaultDescPad = " "; + + public TajoGetConf(TajoConf tajoConf, Writer writer) { + this(tajoConf, writer, null); + } + + public TajoGetConf(TajoConf tajoConf, Writer writer, TajoClient tajoClient) { + this.tajoConf = tajoConf; + this.writer = writer; + this.tajoClient = tajoClient; + } + + private void printUsage(boolean tsqlMode) { + if (!tsqlMode) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp( "getconf <key> [options]", options ); + } + System.out.println(defaultLeftPad + "key" + defaultDescPad + "gets a specific key from the configuration"); + } + + public void runCommand(String[] args) throws Exception { + runCommand(args, true); + } + + public void runCommand(String[] args, boolean tsqlMode) throws Exception { + CommandLineParser parser = new PosixParser(); + + if (args.length == 0) { + printUsage(tsqlMode); + return; + } + + CommandLine cmd = parser.parse(options, args); + + String hostName = null; + Integer port = null; + if (cmd.hasOption("h")) { + hostName = cmd.getOptionValue("h"); + } + if (cmd.hasOption("p")) { + port = Integer.parseInt(cmd.getOptionValue("p")); + } + + String param; + if (cmd.getArgs().length > 1) { + printUsage(tsqlMode); + return; + } else { + param = cmd.getArgs()[0]; + } + + // if there is no "-h" option, + if(hostName == null) { + if (tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) { + // it checks if the client service address is given in configuration and distributed mode. + // if so, it sets entryAddr. + hostName = tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[0]; + } + } + if (port == null) { + if (tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) { + // it checks if the client service address is given in configuration and distributed mode. + // if so, it sets entryAddr. + port = Integer.parseInt(tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[1]); + } + } + + if ((hostName == null) ^ (port == null)) { + return; + } else if (hostName != null && port != null) { + tajoConf.setVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, hostName + ":" + port); + tajoClient = new TajoClientImpl(tajoConf); + } else if (hostName == null && port == null) { + tajoClient = new TajoClientImpl(tajoConf); + } + + processConfKey(writer, param); + writer.flush(); + } + + private void processConfKey(Writer writer, String param) throws ParseException, IOException, + ServiceException, SQLException { + String value = tajoConf.getTrimmed(param); + + // If there is no value in the configuration file, we need to find all ConfVars. + if (value == null) { + for(TajoConf.ConfVars vars : TajoConf.ConfVars.values()) { + if (vars.varname.equalsIgnoreCase(param)) { + value = tajoConf.getVar(vars); + break; + } + } + } + + if (value != null) { + writer.write(value); + } else { + writer.write("Configuration " + param + " is missing."); + } + + writer.write("\n"); + } + + public static void main(String [] args) throws Exception { + TajoConf conf = new TajoConf(); + + Writer writer = new PrintWriter(System.out); + try { + TajoGetConf admin = new TajoGetConf(conf, writer); + admin.runCommand(args, false); + } finally { + writer.close(); + System.exit(0); + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/ad4fda7c/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoHAAdmin.java ---------------------------------------------------------------------- diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoHAAdmin.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoHAAdmin.java new file mode 100644 index 0000000..4f2d024 --- /dev/null +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoHAAdmin.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.cli.tools; + +import com.google.protobuf.ServiceException; +import org.apache.commons.cli.*; +import org.apache.tajo.client.TajoClient; +import org.apache.tajo.client.TajoClientImpl; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.ha.HAServiceUtil; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Writer; + +public class TajoHAAdmin { + private static final Options options; + + static { + options = new Options(); + options.addOption("h", "host", true, "Tajo server host"); + options.addOption("p", "port", true, "Tajo server port"); + options.addOption("transitionToActive", null, true, "Transitions the master into Active state"); + options.addOption("transitionToBackup", null, true, "Transitions the master into Backup state"); + options.addOption("getState", null, true, "Returns the state of the master"); + options.addOption("formatHA", null, false, "Format HA status on share storage"); + } + + private TajoConf tajoConf; + private TajoClient tajoClient; + private Writer writer; + + public TajoHAAdmin(TajoConf tajoConf, Writer writer) { + this(tajoConf, writer, null); + } + + public TajoHAAdmin(TajoConf tajoConf, Writer writer, TajoClient tajoClient) { + this.tajoConf = tajoConf; + this.writer = writer; + this.tajoClient = tajoClient; + } + + private void printUsage() { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp( "haadmin [options]", options ); + } + + public void runCommand(String[] args) throws Exception { + if(args.length == 1 && + (args[0].equalsIgnoreCase("-transitionToActive") + || args[0].equalsIgnoreCase("-transitionToBackup") + || args[0].equalsIgnoreCase("-getState"))) { + writer.write("Not enough arguments: expected 1 but got 0\n"); + writer.flush(); + return; + } + + CommandLineParser parser = new PosixParser(); + CommandLine cmd = parser.parse(options, args); + + String param = ""; + int cmdType = 0; + + String hostName = null; + Integer port = null; + if (cmd.hasOption("h")) { + hostName = cmd.getOptionValue("h"); + } + if (cmd.hasOption("p")) { + port = Integer.parseInt(cmd.getOptionValue("p")); + } + + if (cmd.hasOption("transitionToActive")) { + cmdType = 1; + param = cmd.getOptionValue("transitionToActive"); + } else if (cmd.hasOption("transitionToBackup")) { + cmdType = 2; + param = cmd.getOptionValue("transitionToBackup"); + } else if (cmd.hasOption("getState")) { + cmdType = 3; + param = cmd.getOptionValue("getState"); + } else if (cmd.hasOption("formatHA")) { + cmdType = 4; + } + + // if there is no "-h" option, + if(hostName == null) { + if (tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) { + // it checks if the client service address is given in configuration and distributed mode. + // if so, it sets entryAddr. + hostName = tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[0]; + } + } + if (port == null) { + if (tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) { + // it checks if the client service address is given in configuration and distributed mode. + // if so, it sets entryAddr. + port = Integer.parseInt(tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[1]); + } + } + + if (cmdType == 0) { + printUsage(); + return; + } + + + if ((hostName == null) ^ (port == null)) { + System.err.println("ERROR: cannot find valid Tajo server address"); + return; + } else if (hostName != null && port != null) { + tajoConf.setVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, hostName + ":" + port); + tajoClient = new TajoClientImpl(tajoConf); + } else if (hostName == null && port == null) { + tajoClient = new TajoClientImpl(tajoConf); + } + + if (!tajoConf.getBoolVar(TajoConf.ConfVars.TAJO_MASTER_HA_ENABLE)) { + writer.write("HA is not enabled for this tajo cluster."); + } else { + switch (cmdType) { + case 1: + writer.write("Not Yet Implemented\n"); + break; + case 2: + writer.write("Not Yet Implemented\n"); + break; + case 3: + getState(writer, param); + break; + case 4: + formatHA(writer); + break; + default: + printUsage(); + break; + } + } + + writer.flush(); + } + + private void getState(Writer writer, String param) throws ParseException, IOException, + ServiceException { + + int retValue = HAServiceUtil.getState(param, tajoConf); + + switch (retValue) { + case 1: + writer.write("The master is active.\n"); + break; + case 0: + writer.write("The master is backup.\n"); + break; + case -1: + writer.write("Finding failed. - master:" + param + "\n"); + break; + default: + writer.write("Cannot find the master. - master:" + param + "\n"); + break; + } + } + + private void formatHA(Writer writer) throws ParseException, IOException, + ServiceException { + int retValue = HAServiceUtil.formatHA(tajoConf); + + switch (retValue) { + case 1: + writer.write("Formatting finished successfully.\n"); + break; + case 0: + writer.write("If you want to format the ha information, you must shutdown tajo masters " + + " before formatting.\n"); + break; + default: + writer.write("Cannot format ha information.\n"); + break; + } + } + + public static void main(String [] args) throws Exception { + TajoConf conf = new TajoConf(); + + Writer writer = new PrintWriter(System.out); + try { + TajoHAAdmin admin = new TajoHAAdmin(conf, writer); + admin.runCommand(args); + } finally { + writer.close(); + System.exit(0); + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/ad4fda7c/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java ---------------------------------------------------------------------- diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java new file mode 100644 index 0000000..5cbe77b --- /dev/null +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java @@ -0,0 +1,211 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.cli.tsql; + +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.tajo.QueryId; +import org.apache.tajo.SessionVars; +import org.apache.tajo.TajoConstants; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.client.QueryStatus; +import org.apache.tajo.util.FileUtil; + +import java.io.InputStream; +import java.io.PrintWriter; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; + +public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter { + private int printPauseRecords; + private boolean printPause; + private boolean printErrorTrace; + private String nullChar; + public static char QUIT_COMMAND = 'q'; + + @Override + public void init(TajoCli.TajoCliContext context) { + this.printPause = context.getBool(SessionVars.CLI_PAGING_ENABLED); + this.printPauseRecords = context.getInt(SessionVars.CLI_PAGE_ROWS); + this.printErrorTrace = context.getBool(SessionVars.CLI_DISPLAY_ERROR_TRACE); + this.nullChar = context.get(SessionVars.CLI_NULL_CHAR); + } + + @Override + public void setScriptMode() { + this.printPause = false; + } + + private String getQuerySuccessMessage(TableDesc tableDesc, float responseTime, int totalPrintedRows, String postfix, + boolean endOfTuple) { + TableStats stat = tableDesc.getStats(); + String volume = stat == null ? (endOfTuple ? "0 B" : "unknown bytes") : + FileUtil.humanReadableByteCount(stat.getNumBytes(), false); + long resultRows = stat == null ? TajoConstants.UNKNOWN_ROW_NUMBER : stat.getNumRows(); + + String displayRowNum; + if (resultRows == TajoConstants.UNKNOWN_ROW_NUMBER) { + + if (endOfTuple) { + displayRowNum = totalPrintedRows + " rows"; + } else { + displayRowNum = "unknown row number"; + } + + } else { + displayRowNum = resultRows + " rows"; + } + return "(" + displayRowNum + ", " + getResponseTimeReadable(responseTime) + ", " + volume + " " + postfix + ")"; + } + + protected String getResponseTimeReadable(float responseTime) { + return responseTime + " sec"; + } + + @Override + public void printResult(PrintWriter sout, InputStream sin, TableDesc tableDesc, + float responseTime, ResultSet res) throws Exception { + long resultRows = tableDesc.getStats() == null ? -1 : tableDesc.getStats().getNumRows(); + if (resultRows == -1) { + resultRows = Integer.MAX_VALUE; + } + + if (res == null) { + sout.println(getQuerySuccessMessage(tableDesc, responseTime, 0, "inserted", true)); + return; + } + ResultSetMetaData rsmd = res.getMetaData(); + int numOfColumns = rsmd.getColumnCount(); + for (int i = 1; i <= numOfColumns; i++) { + if (i > 1) sout.print(", "); + String columnName = rsmd.getColumnName(i); + sout.print(columnName); + } + sout.println("\n-------------------------------"); + + int numOfPrintedRows = 0; + int totalPrintedRows = 0; + boolean endOfTuple = true; + while (res.next()) { + for (int i = 1; i <= numOfColumns; i++) { + if (i > 1) sout.print(", "); + String columnValue = res.getString(i); + if(res.wasNull()){ + sout.print(nullChar); + } else { + sout.print(columnValue); + } + } + sout.println(); + sout.flush(); + numOfPrintedRows++; + totalPrintedRows++; + if (printPause && printPauseRecords > 0 && totalPrintedRows < resultRows && numOfPrintedRows >= printPauseRecords) { + if (resultRows < Integer.MAX_VALUE) { + sout.print("(" + totalPrintedRows + "/" + resultRows + " rows, continue... 'q' is quit)"); + } else { + sout.print("(" + totalPrintedRows + " rows, continue... 'q' is quit)"); + } + sout.flush(); + if (sin != null) { + if (sin.read() == QUIT_COMMAND) { + endOfTuple = false; + sout.println(); + break; + } + } + numOfPrintedRows = 0; + sout.println(); + } + } + sout.println(getQuerySuccessMessage(tableDesc, responseTime, totalPrintedRows, "selected", endOfTuple)); + sout.flush(); + } + + @Override + public void printNoResult(PrintWriter sout) { + sout.println("(0 rows)"); + sout.flush(); + } + + @Override + public void printProgress(PrintWriter sout, QueryStatus status) { + sout.println("Progress: " + (int)(status.getProgress() * 100.0f) + + "%, response time: " + + getResponseTimeReadable((float)((status.getFinishTime() - status.getSubmitTime()) / 1000.0))); + sout.flush(); + } + + @Override + public void printMessage(PrintWriter sout, String message) { + sout.println(message); + sout.flush(); + } + + @Override + public void printErrorMessage(PrintWriter sout, Throwable t) { + sout.println(parseErrorMessage(t.getMessage())); + if (printErrorTrace) { + sout.println(ExceptionUtils.getStackTrace(t)); + } + sout.flush(); + } + + @Override + public void printErrorMessage(PrintWriter sout, String message) { + sout.println(parseErrorMessage(message)); + sout.flush(); + } + + @Override + public void printKilledMessage(PrintWriter sout, QueryId queryId) { + sout.println(TajoCli.KILL_PREFIX + queryId); + sout.flush(); + } + + @Override + public void printErrorMessage(PrintWriter sout, QueryStatus status) { + if (status.getErrorMessage() != null && !status.getErrorMessage().isEmpty()) { + printErrorMessage(sout, parseErrorMessage(status.getErrorMessage())); + } else { + printErrorMessage(sout, "No error message"); + } + if (printErrorTrace && status.getErrorTrace() != null && !status.getErrorTrace().isEmpty()) { + sout.println(status.getErrorTrace()); + } + sout.flush(); + } + + public static String parseErrorMessage(String message) { + if (message == null) { + return TajoCli.ERROR_PREFIX + "No error message"; + } + String[] lines = message.split("\n"); + message = lines[0]; + + int index = message.lastIndexOf(TajoCli.ERROR_PREFIX); + if (index < 0) { + message = TajoCli.ERROR_PREFIX + message; + } else { + message = message.substring(index); + } + + return message; + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/ad4fda7c/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/InvalidStatementException.java ---------------------------------------------------------------------- diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/InvalidStatementException.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/InvalidStatementException.java new file mode 100644 index 0000000..2fed9fe --- /dev/null +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/InvalidStatementException.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.cli.tsql; + +public class InvalidStatementException extends Exception { + public InvalidStatementException(String message) { + super(message); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/ad4fda7c/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/ParsedResult.java ---------------------------------------------------------------------- diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/ParsedResult.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/ParsedResult.java new file mode 100644 index 0000000..7894731 --- /dev/null +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/ParsedResult.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.cli.tsql; + + +public class ParsedResult { + public static enum StatementType { + META, + STATEMENT + } + + private final StatementType type; + private final String historyStatement; + private final String statement; + + public ParsedResult(StatementType type, String statement, String historyStatement) { + this.type = type; + this.statement = statement; + this.historyStatement = historyStatement; + } + + public StatementType getType() { + return type; + } + + public String getHistoryStatement() { + return historyStatement.trim(); + } + + public String getStatement() { + return statement.trim(); + } + + public String toString() { + return "(" + type.name() + ") " + historyStatement; + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/ad4fda7c/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/SimpleParser.java ---------------------------------------------------------------------- diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/SimpleParser.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/SimpleParser.java new file mode 100644 index 0000000..cc772a3 --- /dev/null +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/SimpleParser.java @@ -0,0 +1,387 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.cli.tsql; + +import org.apache.tajo.cli.tsql.ParsedResult.StatementType; + +import java.util.ArrayList; +import java.util.List; + +/** + * This is a parser used in tsql to parse multiple SQL lines into SQL statements. + * It helps tsql recognizes the termination of each SQL statement and quotation mark (') while + * parses multiple separate lines. + */ +public class SimpleParser { + + public static enum ParsingState { + TOK_START, // Start State + META, // Meta Command + STATEMENT, // Statement + WITHIN_QUOTE, // Within Quote + INVALID, // Invalid Statement + STATEMENT_EOS, // End State (End of Statement) + META_EOS // End State (End of Statement) + } + + ParsingState state = START_STATE; + int lineNum; + + /** + * It will be used to store a query statement into Jline history. + * the query statement for history does not include unnecessary white spaces and new line. + */ + private StringBuilder historyAppender = new StringBuilder(); + /** + * It will be used to submit a query statement to the TajoMaster. It just contains a raw query statement string. + */ + private StringBuilder rawAppender = new StringBuilder(); + + public static final ParsingState START_STATE = ParsingState.TOK_START; + + /** + * <h2>State Machine</h2> + * All whitespace are ignored in all cases except for + * + * <pre> + * (start) TOK_START --> META ---------------------> META_EOS + * | + * | + * | + * |-----------> STATEMENT ----------> STMT_EOS + * \ ^ + * \ / + * \-> WITHIN_QUOTE + * \ ^ + * \---/ + * </pre> + */ + + public static List<ParsedResult> parseScript(String str) throws InvalidStatementException { + SimpleParser parser = new SimpleParser(); + List<ParsedResult> parsedResults = new ArrayList<ParsedResult>(); + parsedResults.addAll(parser.parseLines(str)); + parsedResults.addAll(parser.EOF()); + return parsedResults; + } + + public List<ParsedResult> parseLines(String str) throws InvalidStatementException { + List<ParsedResult> statements = new ArrayList<ParsedResult>(); + int lineStartIdx; + int idx = 0; + char [] chars = str.toCharArray(); + + // if parsing continues, it means that the previous line is broken by '\n'. + // So, we should add new line to rawAppender. + if (isStatementContinue()) { + rawAppender.append("\n"); + } + + while(idx < str.length()) { + + // initialization for new statement + if (state == ParsingState.TOK_START) { + lineNum = 0; + + // ignore all whitespace before start + if (Character.isWhitespace(chars[idx])) { + idx++; + continue; + } + } + + //////////////////////////// + // TOK_START --> META + //////////////////////////// + + lineStartIdx = idx; + + if (state == ParsingState.TOK_START && chars[idx] == '\\') { + state = ParsingState.META; + + //////////////////////////// + // META --> TOK_EOS + //////////////////////////// + while (state != ParsingState.META_EOS && idx < chars.length) { + char character = chars[idx++]; + + if (isEndOfMeta(character)) { + state = ParsingState.META_EOS; + } else if (Character.isWhitespace(character)) { + // skip + } + } + + if (state == ParsingState.META_EOS) { + historyAppender.append(str.subSequence(lineStartIdx, idx - 1).toString()); + appendToRawStatement(str.subSequence(lineStartIdx, idx - 1).toString(), true); + } else { + historyAppender.append(str.subSequence(lineStartIdx, idx).toString()); + appendToRawStatement(str.subSequence(lineStartIdx, idx).toString(), true); + } + + } else if (isInlineCommentStart(chars, idx)) { + idx = consumeInlineComment(chars, idx); + appendToRawStatement(str.subSequence(lineStartIdx, idx).toString(), true); + + ///////////////////////////////// + // TOK_START -> STATEMENT + // or TOK_STATEMENT -> STATEMENT + //////////////////////////////// + } else if (isStatementContinue() || isStatementStart(chars[idx])) { + if (!isStatementContinue()) { // TOK_START -> STATEMENT + state = ParsingState.STATEMENT; + rawAppender.append("\n"); + } + + while (!isTerminateState(state) && idx < chars.length) { + char character = chars[idx++]; + + /////////////////////////////////////////////////////// + // in-statement loop BEGIN + /////////////////////////////////////////////////////// + if (isEndOfStatement(character)) { + state = ParsingState.STATEMENT_EOS; + + } else if (state == ParsingState.STATEMENT && character == '\n') { + appendToBothStatements(chars, lineStartIdx, idx, 1); // omit new line chacter '\n' from history statement + lineStartIdx = idx; + + } else if (state == ParsingState.STATEMENT && character == '\'') { // TOK_STATEMENT -> WITHIN_QUOTE + state = ParsingState.WITHIN_QUOTE; + + if (idx < chars.length) { + character = chars[idx++]; + } else { + continue; + } + + + // idx points the characters followed by the current character. So, we should use 'idx - 1' + // in order to point the current character. + } else if (state == ParsingState.STATEMENT && idx < chars.length && isInlineCommentStart(chars, idx - 1)) { + idx++; + appendToBothStatements(chars, lineStartIdx, idx, 2); // omit two dash characters '--' from history statement + int commentStartIdx = idx; + idx = consumeInlineComment(chars, idx); + appendToRawStatement(str.subSequence(commentStartIdx, idx).toString(), true); + lineStartIdx = idx; + } + /////////////////////////////////////////////////////// + // in-statement loop END + /////////////////////////////////////////////////////// + + if (state == ParsingState.WITHIN_QUOTE) { + while(idx < chars.length) { + /////////////////////////////// + // WITHIN_QUOTE --> STATEMENT + /////////////////////////////// + if (character == '\'') { + state = ParsingState.STATEMENT; + break; + } + character = chars[idx++]; + } + if (state == ParsingState.WITHIN_QUOTE && character == '\'') { + state = ParsingState.STATEMENT; + } + } + } + + // After all characters are consumed + + if (state == ParsingState.STATEMENT_EOS) { // If one query statement is terminated + appendToBothStatements(chars, lineStartIdx, idx - 1); // skip semicolon (;) + } else { + appendToBothStatements(chars, lineStartIdx, idx); + + // if it is not within quote and there is no space between lines, adds a space. + if (state == ParsingState.STATEMENT && (historyAppender.charAt(historyAppender.length() - 1) != ' ')) { + historyAppender.append(" "); + rawAppender.append("\n"); + } + } + } else { // skip unknown character + idx++; + } + + lineNum++; + statements.addAll(doProcessEndOfStatement(state == ParsingState.META)); + } + + return statements; + } + + /** + * Append the range of characters into a given StringBuilder instance. + * + * @param chars Characters + * @param fromIdx start character index + * @param toIdx end character index + */ + private void appendToStatement(StringBuilder builder, char[] chars, int fromIdx, int toIdx) { + builder.append(chars, fromIdx, toIdx - fromIdx); + } + + /** + * Append the range of characters into both history and raw appenders. It omits the number of characters specified by + * <code>omitCharNums</code>. + * + * + * @param chars Characters + * @param fromIdx start character index + * @param toIdx end character index + * @param omitCharNums how many characters will be omitted from history statement + */ + private void appendToBothStatements(char[] chars, int fromIdx, int toIdx, int omitCharNums) { + appendToStatement(historyAppender, chars, fromIdx, toIdx - omitCharNums); + if (historyAppender.charAt(historyAppender.length() - 1) != ' ') { + historyAppender.append(" "); + } + appendToStatement(rawAppender, chars, fromIdx, toIdx); + } + + /** + * Append the range of characters into both history and raw appenders. + * + * + * @param chars Characters + * @param fromIdx start character index + * @param toIdx end character index + */ + private void appendToBothStatements(char[] chars, int fromIdx, int toIdx) { + historyAppender.append(chars, fromIdx, toIdx - fromIdx); + rawAppender.append(chars, fromIdx, toIdx - fromIdx); + } + + private int consumeInlineComment(char [] chars, int currentIdx) { + currentIdx++; + while (currentIdx < chars.length && !isNewLine(chars[currentIdx])) { + currentIdx++; + } + return currentIdx; + } + + private void appendToRawStatement(String str, boolean addLF) { + if (!str.isEmpty() && !"\n".equals(str) && + rawAppender.length() > 0 && addLF && rawAppender.charAt(rawAppender.length() - 1) != '\n') { + rawAppender.append(str); + } else { + rawAppender.append(str); + } + } + + private static boolean isEndOfMeta(char character) { + return character == ';' || character == '\n'; + } + + private static boolean isEndOfStatement(char character) { + return character == ';'; + } + + /** + * It checks if inline comment '--' begins. + * @param chars + * @param idx + * @return + */ + private boolean isInlineCommentStart(char[] chars, int idx) { + if (idx >= chars.length - 1) { + return false; + } + return (state == ParsingState.STATEMENT || state == ParsingState.TOK_START) && + (chars[idx] == '-' && chars[idx + 1] == '-'); + } + + private boolean isNewLine(char character) { + return character == '\n'; + } + + private boolean isStatementStart(char character) { + return state == ParsingState.TOK_START && (Character.isLetterOrDigit(character)); + } + + private boolean isStatementContinue() { + return state == ParsingState.WITHIN_QUOTE || state == ParsingState.STATEMENT; + } + + /** + * process all parsed statements so far and return a list of parsed results. + * + * @param endOfFile TRUE if the end of file. + * @return the list of parsed results, each of result contains one query statement or meta command. + * @throws InvalidStatementException + */ + private List<ParsedResult> doProcessEndOfStatement(boolean endOfFile) throws InvalidStatementException { + List<ParsedResult> parsedResults = new ArrayList<ParsedResult>(); + String errorMessage = ""; + if (endOfFile) { + if (state == ParsingState.META) { + state = ParsingState.META_EOS; + } else if (state == ParsingState.STATEMENT) { + state = ParsingState.STATEMENT_EOS; + } else if (state == ParsingState.WITHIN_QUOTE) { + state = ParsingState.INVALID; + errorMessage = "unterminated quoted string at LINE " + lineNum; + } + } + + if (isTerminateState(state)) { + String historyStatement = historyAppender.toString(); + String rawStatement = rawAppender.toString(); + if (state == ParsingState.META_EOS) { + parsedResults.add(new ParsedResult(StatementType.META, rawStatement, historyStatement)); + state = ParsingState.TOK_START; + } else if (state == ParsingState.STATEMENT_EOS) { + parsedResults.add(new ParsedResult(StatementType.STATEMENT, rawStatement, historyStatement)); + } else { + throw new InvalidStatementException("ERROR: " + errorMessage); + } + + // reset all states + historyAppender.delete(0, historyAppender.length()); + rawAppender.delete(0, rawAppender.length()); + state = START_STATE; + } + + return parsedResults; + } + + /** + * It manually triggers the end of file. + * + * @return the list of parsed results, each of result contains one query statement or meta command. + * @throws InvalidStatementException + */ + public List<ParsedResult> EOF() throws InvalidStatementException { + return doProcessEndOfStatement(true); + } + + private static boolean isTerminateState(ParsingState state) { + return (state == ParsingState.META_EOS || state == ParsingState.STATEMENT_EOS || state == ParsingState.INVALID); + } + + public ParsingState getState() { + return state; + } + + public String toString() { + return "[" + state.name() + "]: " + historyAppender.toString(); + } +}
