Repository: tajo Updated Branches: refs/heads/master 242d6ad65 -> 912df6f08
TAJO-990: Implement a tool to find tajo configurations. (jaehwa) Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/912df6f0 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/912df6f0 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/912df6f0 Branch: refs/heads/master Commit: 912df6f081b91785a56cb4b312e94b71228ecfd2 Parents: 242d6ad Author: Jaehwa Jung <[email protected]> Authored: Sun Aug 10 15:03:07 2014 +0900 Committer: Jaehwa Jung <[email protected]> Committed: Sun Aug 10 15:03:07 2014 +0900 ---------------------------------------------------------------------- .../main/java/org/apache/tajo/cli/TajoCli.java | 3 +- .../org/apache/tajo/cli/TajoGetConfCommand.java | 57 +++++++ .../java/org/apache/tajo/client/TajoAdmin.java | 16 ++ .../org/apache/tajo/client/TajoGetConf.java | 171 +++++++++++++++++++ .../java/org/apache/tajo/cli/TestTajoCli.java | 31 ++++ tajo-dist/src/main/bin/tajo | 5 + 6 files changed, 282 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/912df6f0/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java index 98a5bf4..c20e44b 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java @@ -79,7 +79,8 @@ public class TajoCli { UnsetCommand.class, ExecExternalShellCommand.class, HdfsCommand.class, - TajoAdminCommand.class + TajoAdminCommand.class, + TajoGetConfCommand.class }; private final Map<String, TajoShellCommand> commands = new TreeMap<String, TajoShellCommand>(); http://git-wip-us.apache.org/repos/asf/tajo/blob/912df6f0/tajo-client/src/main/java/org/apache/tajo/cli/TajoGetConfCommand.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoGetConfCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoGetConfCommand.java new file mode 100644 index 0000000..83ba4dd --- /dev/null +++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoGetConfCommand.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.cli; + +import org.apache.tajo.client.TajoGetConf; + +public class TajoGetConfCommand extends TajoShellCommand { + private TajoGetConf getconf; + + public TajoGetConfCommand(TajoCli.TajoCliContext context) { + super(context); + getconf = new TajoGetConf(context.getConf(), context.getOutput(), context.getTajoClient()); + } + + @Override + public String getCommand() { + return "\\getconf"; + } + + @Override + public void invoke(String[] command) throws Exception { + try { + String[] getConfCommands = new String[command.length - 1]; + System.arraycopy(command, 1, getConfCommands, 0, getConfCommands.length); + + getconf.runCommand(getConfCommands); + } catch (Exception e) { + context.getOutput().println("ERROR: " + e.getMessage()); + } + } + + @Override + public String getUsage() { + return "<command> [options]"; + } + + @Override + public String getDescription() { + return "execute a tajo getconf command."; + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/912df6f0/tajo-client/src/main/java/org/apache/tajo/client/TajoAdmin.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoAdmin.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoAdmin.java index ad42675..4f38858 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/TajoAdmin.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoAdmin.java @@ -26,11 +26,13 @@ import org.apache.tajo.TajoProtos; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.ipc.ClientProtos.BriefQueryInfo; import org.apache.tajo.ipc.ClientProtos.WorkerResourceInfo; +import org.apache.tajo.util.NetUtils; import org.apache.tajo.util.TajoIdUtils; import java.io.IOException; import java.io.PrintWriter; import java.io.Writer; +import java.net.InetSocketAddress; import java.sql.SQLException; import java.text.DecimalFormat; import java.text.SimpleDateFormat; @@ -58,6 +60,7 @@ public class TajoAdmin { options.addOption("p", "port", true, "Tajo server port"); options.addOption("list", null, false, "Show Tajo query list"); options.addOption("cluster", null, false, "Show Cluster Info"); + options.addOption("showmasters", null, false, "gets list of tajomasters in the cluster"); options.addOption("desc", null, false, "Show Query Description"); options.addOption("kill", null, true, "Kill a running query"); } @@ -108,6 +111,8 @@ public class TajoAdmin { } else if (cmd.hasOption("kill")) { cmdType = 4; queryId = cmd.getOptionValue("kill"); + } else if (cmd.hasOption("showmasters")) { + cmdType = 5; } // if there is no "-h" option, @@ -155,6 +160,9 @@ public class TajoAdmin { case 4: processKill(writer, queryId); break; + case 5: + processMasters(writer); + break; default: printUsage(); break; @@ -406,6 +414,14 @@ public class TajoAdmin { } } + private void processMasters(Writer writer) throws ParseException, IOException, + ServiceException, SQLException { + String confMasterServiceAddr = tajoClient.getConf().getVar(TajoConf.ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS); + InetSocketAddress masterAddress = NetUtils.createSocketAddr(confMasterServiceAddr); + writer.write(masterAddress.getHostName()); + writer.write("\n"); + } + public static void main(String [] args) throws Exception { TajoConf conf = new TajoConf(); http://git-wip-us.apache.org/repos/asf/tajo/blob/912df6f0/tajo-client/src/main/java/org/apache/tajo/client/TajoGetConf.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoGetConf.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoGetConf.java new file mode 100644 index 0000000..32e6382 --- /dev/null +++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoGetConf.java @@ -0,0 +1,171 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.client; + +import com.google.protobuf.ServiceException; +import org.apache.commons.cli.*; +import org.apache.commons.lang.StringUtils; +import org.apache.tajo.QueryId; +import org.apache.tajo.TajoProtos; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.ipc.ClientProtos.BriefQueryInfo; +import org.apache.tajo.ipc.ClientProtos.WorkerResourceInfo; +import org.apache.tajo.util.NetUtils; +import org.apache.tajo.util.TajoIdUtils; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Writer; +import java.net.InetSocketAddress; +import java.sql.SQLException; +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; + +public class TajoGetConf { + private static final org.apache.commons.cli.Options options; + + static { + options = new Options(); + options.addOption("h", "host", true, "Tajo server host"); + options.addOption("p", "port", true, "Tajo server port"); + } + + private TajoConf tajoConf; + private TajoClient tajoClient; + private Writer writer; + + public final static String defaultLeftPad = " "; + public final static String defaultDescPad = " "; + + public TajoGetConf(TajoConf tajoConf, Writer writer) { + this(tajoConf, writer, null); + } + + public TajoGetConf(TajoConf tajoConf, Writer writer, TajoClient tajoClient) { + this.tajoConf = tajoConf; + this.writer = writer; + this.tajoClient = tajoClient; + } + + private void printUsage(boolean tsqlMode) { + if (!tsqlMode) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp( "getconf <key> [options]", options ); + } + System.out.println(defaultLeftPad + "key" + defaultDescPad + "gets a specific key from the configuration"); + } + + public void runCommand(String[] args) throws Exception { + runCommand(args, true); + } + + public void runCommand(String[] args, boolean tsqlMode) throws Exception { + CommandLineParser parser = new PosixParser(); + + if (args.length == 0) { + printUsage(tsqlMode); + return; + } + + CommandLine cmd = parser.parse(options, args); + + String hostName = null; + Integer port = null; + if (cmd.hasOption("h")) { + hostName = cmd.getOptionValue("h"); + } + if (cmd.hasOption("p")) { + port = Integer.parseInt(cmd.getOptionValue("p")); + } + + String param; + if (cmd.getArgs().length > 1) { + printUsage(tsqlMode); + return; + } else { + param = cmd.getArgs()[0]; + } + + // if there is no "-h" option, + if(hostName == null) { + if (tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) { + // it checks if the client service address is given in configuration and distributed mode. + // if so, it sets entryAddr. + hostName = tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[0]; + } + } + if (port == null) { + if (tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) { + // it checks if the client service address is given in configuration and distributed mode. + // if so, it sets entryAddr. + port = Integer.parseInt(tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[1]); + } + } + + if ((hostName == null) ^ (port == null)) { + return; + } else if (hostName != null && port != null) { + tajoConf.setVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, hostName + ":" + port); + tajoClient = new TajoClient(tajoConf); + } else if (hostName == null && port == null) { + tajoClient = new TajoClient(tajoConf); + } + + processConfKey(writer, param); + writer.flush(); + } + + private void processConfKey(Writer writer, String param) throws ParseException, IOException, + ServiceException, SQLException { + String value = tajoClient.getConf().getTrimmed(param); + + // If there is no value in the configuration file, we need to find all ConfVars. + if (value == null) { + for(TajoConf.ConfVars vars : TajoConf.ConfVars.values()) { + if (vars.varname.equalsIgnoreCase(param)) { + value = tajoClient.getConf().getVar(vars); + break; + } + } + } + + if (value != null) { + writer.write(value); + } else { + writer.write("Configuration " + param + " is missing."); + } + + writer.write("\n"); + } + + public static void main(String [] args) throws Exception { + TajoConf conf = new TajoConf(); + + Writer writer = new PrintWriter(System.out); try { + System.out.println("### 1000 ###"); + TajoGetConf admin = new TajoGetConf(conf, writer); + admin.runCommand(args, false); + } finally { + writer.close(); + System.exit(0); + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/912df6f0/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java index f862cb1..0631b6e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java @@ -30,6 +30,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.util.FileUtil; +import org.apache.tajo.util.NetUtils; import org.junit.After; import org.junit.Rule; import org.junit.Test; @@ -292,6 +293,36 @@ public class TestTajoCli { assertOutputResult(consoleResult); } + @Test + public void testGetConf() throws Exception { + TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); + tajoConf.setVar(ConfVars.CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out); + tajoCli.executeMetaCommand("\\getconf tajo.rootdir"); + + String consoleResult = new String(out.toByteArray()); + assertEquals(consoleResult, tajoCli.getContext().getConf().getVar(ConfVars.ROOT_DIR) + "\n"); + } + + @Test + public void testShowMasters() throws Exception { + TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); + tajoConf.setVar(ConfVars.CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out); + tajoCli.executeMetaCommand("\\admin -showmasters"); + + String consoleResult = new String(out.toByteArray()); + + String masterAddress = tajoCli.getContext().getConf().getVar(ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS); + String host = masterAddress.split(":")[0]; + + assertEquals(consoleResult, host + "\n"); + } + public static class TajoCliOutputTestFormatter extends DefaultTajoCliOutputFormatter { @Override protected String getResponseTimeReadable(float responseTime) { http://git-wip-us.apache.org/repos/asf/tajo/blob/912df6f0/tajo-dist/src/main/bin/tajo ---------------------------------------------------------------------- diff --git a/tajo-dist/src/main/bin/tajo b/tajo-dist/src/main/bin/tajo index a05c65c..b6acdd8 100755 --- a/tajo-dist/src/main/bin/tajo +++ b/tajo-dist/src/main/bin/tajo @@ -68,6 +68,7 @@ if [ $# = 0 ]; then echo " catutil catalog utility" echo " cli run the tajo cli" echo " admin run the tajo admin util" + echo " getconf print tajo configuration" echo " jar <jar> run a jar file" echo " benchmark run the benchmark driver" echo " or" @@ -348,6 +349,10 @@ elif [ "$COMMAND" = "admin" ] ; then CLASS='org.apache.tajo.client.TajoAdmin' TAJO_ROOT_LOGGER_APPENDER="${TAJO_ROOT_LOGGER_APPENDER:-NullAppender}" TAJO_OPTS="$TAJO_OPTS $TAJO_CLI_OPTS" +elif [ "$COMMAND" = "getconf" ] ; then + CLASS='org.apache.tajo.client.TajoGetConf' + TAJO_ROOT_LOGGER_APPENDER="${TAJO_ROOT_LOGGER_APPENDER:-NullAppender}" + TAJO_OPTS="$TAJO_OPTS $TAJO_CLI_OPTS" elif [ "$COMMAND" = "dump" ] ; then CLASS='org.apache.tajo.client.TajoDump' TAJO_ROOT_LOGGER_APPENDER="${TAJO_ROOT_LOGGER_APPENDER:-NullAppender}"
