This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch branch-1.7
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/branch-1.7 by this push:
new 36ddb0db1 [KYUUBI #4965] [BEELINE] Support `--python-mode` option and
remove comments for non-python mode
36ddb0db1 is described below
commit 36ddb0db15c17275295ed58ee61a409f720205cc
Author: fwang12 <[email protected]>
AuthorDate: Fri Jun 16 17:34:24 2023 +0800
[KYUUBI #4965] [BEELINE] Support `--python-mode` option and remove comments
for non-python mode
Close #4803
The beeline has regression because of
https://github.com/apache/kyuubi/commit/70590f71ef32f5d3903acd75e72b78e5420b384c#diff-993fdbefe9fe3d1c91fcedba99362a8c8d9b94793ec16cbfbc989e750367ea89
- [x] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run
test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #4965 from turboFei/revert_beeline_python_change.
Closes #4965
856d92391 [fwang12] trim for non python mdoe
f8464606b [fwang12] Revert "[KYUUBI #4619] Fix beeline with -e When there
are other SQL statements before the source statement, the source statement
cannot be executed normally"
e5e3c31b3 [fwang12] revert trim
bec09c254 [fwang12] migration guide
585da6fc1 [fwang12] fix'
f3fcfe97e [fwang12] save
8cb8cb9d0 [fwang12] save
e1539775a [fwang12] comments
814c970a2 [fwang12] save
b1baa773b [fwang12] save
3337ca8fa [fwang12] options
Authored-by: fwang12 <[email protected]>
Signed-off-by: fwang12 <[email protected]>
---
docs/deployment/migration-guide.md | 4 ++
.../org/apache/hive/beeline/KyuubiBeeLine.java | 36 +++++++++--
.../org/apache/hive/beeline/KyuubiCommands.java | 24 +++++--
.../org/apache/hive/beeline/KyuubiBeeLineTest.java | 73 ++++++++++++++++++++++
.../apache/hive/beeline/KyuubiCommandsTest.java | 18 ++++++
5 files changed, 144 insertions(+), 11 deletions(-)
diff --git a/docs/deployment/migration-guide.md
b/docs/deployment/migration-guide.md
index fc916048c..3240fdce9 100644
--- a/docs/deployment/migration-guide.md
+++ b/docs/deployment/migration-guide.md
@@ -17,6 +17,10 @@
# Kyuubi Migration Guide
+## Upgrading from Kyuubi 1.7.1 to 1.7.2
+
+* Since Kyuubi 1.7.2, for Kyuubi BeeLine, please use `--python-mode` option to
run python code or script.
+
## Upgrading from Kyuubi 1.7.0 to 1.7.1
* Since Kyuubi 1.7.1, `protocolVersion` is removed from the request parameters
of the REST API `Open(create) a session`. All removed or unknown parameters
will be silently ignored and affects nothing.
diff --git
a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiBeeLine.java
b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiBeeLine.java
index 3908a44f1..f5336816c 100644
---
a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiBeeLine.java
+++
b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiBeeLine.java
@@ -22,10 +22,7 @@ import java.io.InputStream;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.sql.Driver;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
+import java.util.*;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
@@ -41,6 +38,9 @@ public class KyuubiBeeLine extends BeeLine {
private static final int ERRNO_ARGS = 1;
private static final int ERRNO_OTHER = 2;
+ private static final String PYTHON_MODE_PREFIX = "--python-mode";
+ private boolean pythonMode = false;
+
public KyuubiBeeLine() {
this(true);
}
@@ -68,6 +68,22 @@ public class KyuubiBeeLine extends BeeLine {
}
}
+ @Override
+ void usage() {
+ super.usage();
+ output("Usage: java \" + KyuubiBeeLine.class.getCanonicalName()");
+ output(" --python-mode Execute python code/script.");
+ }
+
+ public boolean isPythonMode() {
+ return pythonMode;
+ }
+
+ // Visible for testing
+ public void setPythonMode(boolean pythonMode) {
+ this.pythonMode = pythonMode;
+ }
+
/** Starts the program. */
public static void main(String[] args) throws IOException {
mainWithInputRedirection(args, null);
@@ -127,7 +143,17 @@ public class KyuubiBeeLine extends BeeLine {
optionsField.setAccessible(true);
Options options = (Options) optionsField.get(this);
- beelineParser = new BeelineParser();
+ beelineParser =
+ new BeelineParser() {
+ @Override
+ protected void processOption(String arg, ListIterator iter) throws
ParseException {
+ if (PYTHON_MODE_PREFIX.equals(arg)) {
+ pythonMode = true;
+ } else {
+ super.processOption(arg, iter);
+ }
+ }
+ };
cl = beelineParser.parse(options, args);
Method connectUsingArgsMethod =
diff --git
a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiCommands.java
b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiCommands.java
index 6580d2490..56ec1de6d 100644
---
a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiCommands.java
+++
b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiCommands.java
@@ -24,6 +24,7 @@ import java.io.*;
import java.sql.*;
import java.util.*;
import org.apache.hive.beeline.logs.KyuubiBeelineInPlaceUpdateStream;
+import org.apache.hive.common.util.HiveStringUtils;
import org.apache.kyuubi.jdbc.hive.KyuubiStatement;
import org.apache.kyuubi.jdbc.hive.Utils;
import org.apache.kyuubi.jdbc.hive.logs.InPlaceUpdateStream;
@@ -44,9 +45,14 @@ public class KyuubiCommands extends Commands {
return execute(line, false, false);
}
+ /** For python mode, keep it as it is. */
+ private String trimForNonPythonMode(String line) {
+ return beeLine.isPythonMode() ? line : line.trim();
+ }
+
/** Extract and clean up the first command in the input. */
private String getFirstCmd(String cmd, int length) {
- return cmd.substring(length).trim();
+ return trimForNonPythonMode(cmd.substring(length));
}
private String[] tokenizeCmd(String cmd) {
@@ -54,14 +60,12 @@ public class KyuubiCommands extends Commands {
}
private boolean isSourceCMD(String cmd) {
- cmd = cmd.trim();
if (cmd == null || cmd.isEmpty()) return false;
String[] tokens = tokenizeCmd(cmd);
return tokens[0].equalsIgnoreCase("source");
}
private boolean sourceFile(String cmd) {
- cmd = cmd.trim();
String[] tokens = tokenizeCmd(cmd);
String cmd_1 = getFirstCmd(cmd, tokens[0].length());
@@ -98,7 +102,7 @@ public class KyuubiCommands extends Commands {
}
String[] cmds = lines.split(beeLine.getOpts().getDelimiter());
for (String c : cmds) {
- c = c.trim();
+ c = trimForNonPythonMode(c);
if (!executeInternal(c, false)) {
return false;
}
@@ -262,9 +266,10 @@ public class KyuubiCommands extends Commands {
beeLine.handleException(e);
}
+ line = trimForNonPythonMode(line);
List<String> cmdList = getCmdList(line, entireLineAsCommand);
for (int i = 0; i < cmdList.size(); i++) {
- String sql = cmdList.get(i);
+ String sql = trimForNonPythonMode(cmdList.get(i));
if (sql.length() != 0) {
if (!executeInternal(sql, call)) {
return false;
@@ -522,6 +527,9 @@ public class KyuubiCommands extends Commands {
? null
: jline.console.ConsoleReader.NULL_MASK;
+ if (!beeLine.isPythonMode()) {
+ line = HiveStringUtils.removeComments(line, startQuote);
+ }
while (isMultiLine(line) && beeLine.getOpts().isAllowMultiLineCommand()) {
StringBuilder prompt = new StringBuilder(beeLine.getPrompt());
if (!beeLine.getOpts().isSilent()) {
@@ -547,6 +555,9 @@ public class KyuubiCommands extends Commands {
if (extra == null) { // it happens when using -f and the line of cmds
does not end with ;
break;
}
+ if (!beeLine.isPythonMode()) {
+ extra = HiveStringUtils.removeComments(extra, startQuote);
+ }
if (!extra.isEmpty()) {
line += "\n" + extra;
}
@@ -558,12 +569,13 @@ public class KyuubiCommands extends Commands {
// console. Used in handleMultiLineCmd method assumes line would never be
null when this method is
// called
private boolean isMultiLine(String line) {
+ line = trimForNonPythonMode(line);
if (line.endsWith(beeLine.getOpts().getDelimiter()) ||
beeLine.isComment(line)) {
return false;
}
// handles the case like line = show tables; --test comment
List<String> cmds = getCmdList(line, false);
- return cmds.isEmpty() || !cmds.get(cmds.size() - 1).startsWith("--");
+ return cmds.isEmpty() || !trimForNonPythonMode(cmds.get(cmds.size() -
1)).startsWith("--");
}
static class KyuubiLogRunnable implements Runnable {
diff --git
a/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiBeeLineTest.java
b/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiBeeLineTest.java
index 96b76373d..9119492b8 100644
---
a/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiBeeLineTest.java
+++
b/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiBeeLineTest.java
@@ -19,7 +19,12 @@
package org.apache.hive.beeline;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.lang.reflect.Field;
import org.junit.Test;
public class KyuubiBeeLineTest {
@@ -47,4 +52,72 @@ public class KyuubiBeeLineTest {
int result3 = kyuubiBeeLine.initArgs(args3);
assertEquals(1, result3);
}
+
+ @Test
+ public void testKyuubiBeeLineCmdUsage() throws ReflectiveOperationException {
+ BufferPrintStream printStream = new BufferPrintStream();
+
+ KyuubiBeeLine kyuubiBeeLine = new KyuubiBeeLine();
+ Field outputStreamField = BeeLine.class.getDeclaredField("outputStream");
+ outputStreamField.setAccessible(true);
+ outputStreamField.set(kyuubiBeeLine, printStream);
+ String[] args1 = {"-h"};
+ kyuubiBeeLine.initArgs(args1);
+ String output = printStream.getOutput();
+ assert output.contains("--python-mode Execute python
code/script.");
+ }
+
+ @Test
+ public void testKyuubiBeeLinePythonMode() {
+ KyuubiBeeLine kyuubiBeeLine = new KyuubiBeeLine();
+ String[] args1 = {"-u", "badUrl", "--python-mode"};
+ kyuubiBeeLine.initArgs(args1);
+ assertTrue(kyuubiBeeLine.isPythonMode());
+ kyuubiBeeLine.setPythonMode(false);
+
+ String[] args2 = {"--python-mode", "-f", "test.sql"};
+ kyuubiBeeLine.initArgs(args2);
+ assertTrue(kyuubiBeeLine.isPythonMode());
+ assert kyuubiBeeLine.getOpts().getScriptFile().equals("test.sql");
+ kyuubiBeeLine.setPythonMode(false);
+
+ String[] args3 = {"-u", "badUrl"};
+ kyuubiBeeLine.initArgs(args3);
+ assertTrue(!kyuubiBeeLine.isPythonMode());
+ kyuubiBeeLine.setPythonMode(false);
+ }
+
+ static class BufferPrintStream extends PrintStream {
+ public StringBuilder stringBuilder = new StringBuilder();
+
+ static OutputStream noOpOutputStream =
+ new OutputStream() {
+ @Override
+ public void write(int b) throws IOException {
+ // do nothing
+ }
+ };
+
+ public BufferPrintStream() {
+ super(noOpOutputStream);
+ }
+
+ public BufferPrintStream(OutputStream outputStream) {
+ super(noOpOutputStream);
+ }
+
+ @Override
+ public void println(String x) {
+ stringBuilder.append(x).append("\n");
+ }
+
+ @Override
+ public void print(String x) {
+ stringBuilder.append(x);
+ }
+
+ public String getOutput() {
+ return stringBuilder.toString();
+ }
+ }
}
diff --git
a/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiCommandsTest.java
b/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiCommandsTest.java
index ecb8d65f5..8cde13d45 100644
---
a/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiCommandsTest.java
+++
b/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiCommandsTest.java
@@ -34,6 +34,7 @@ public class KyuubiCommandsTest {
Mockito.when(reader.readLine()).thenReturn(pythonSnippets);
KyuubiBeeLine beeline = new KyuubiBeeLine();
+ beeline.setPythonMode(true);
beeline.setConsoleReader(reader);
KyuubiCommands commands = new KyuubiCommands(beeline);
String line = commands.handleMultiLineCmd(pythonSnippets);
@@ -42,4 +43,21 @@ public class KyuubiCommandsTest {
assertEquals(cmdList.size(), 1);
assertEquals(cmdList.get(0), pythonSnippets);
}
+
+ @Test
+ public void testHandleMultiLineCmd() throws IOException {
+ ConsoleReader reader = Mockito.mock(ConsoleReader.class);
+ String snippets = "select 1;--comments1\nselect 2;--comments2";
+ Mockito.when(reader.readLine()).thenReturn(snippets);
+
+ KyuubiBeeLine beeline = new KyuubiBeeLine();
+ beeline.setConsoleReader(reader);
+ beeline.setPythonMode(false);
+ KyuubiCommands commands = new KyuubiCommands(beeline);
+ String line = commands.handleMultiLineCmd(snippets);
+ List<String> cmdList = commands.getCmdList(line, false);
+ assertEquals(cmdList.size(), 2);
+ assertEquals(cmdList.get(0), "select 1");
+ assertEquals(cmdList.get(1), "\nselect 2");
+ }
}
