Author: thejas
Date: Wed Nov 27 21:28:31 2013
New Revision: 1546199
URL: http://svn.apache.org/r1546199
Log:
HIVE-4485 : beeline prints null as empty strings (Thejas Nair reviewed by
Ashutosh Chauhan)
Added:
hive/trunk/beeline/src/java/org/apache/hive/beeline/BooleanCompletor.java
Modified:
hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.java
hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
hive/trunk/beeline/src/java/org/apache/hive/beeline/BufferedRows.java
hive/trunk/beeline/src/java/org/apache/hive/beeline/Rows.java
hive/trunk/beeline/src/main/resources/BeeLine.properties
hive/trunk/beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java
Modified: hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.java
URL:
http://svn.apache.org/viewvc/hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.java?rev=1546199&r1=1546198&r2=1546199&view=diff
==============================================================================
--- hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.java (original)
+++ hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.java Wed Nov 27
21:28:31 2013
@@ -224,6 +224,8 @@ public class BeeLine {
null),
new ReflectiveCommandHandler(this, new String[] {"call"},
null),
+ new ReflectiveCommandHandler(this, new String[] {"nullemptystring"},
+ new Completor[] {new BooleanCompletor()}),
};
Modified: hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
URL:
http://svn.apache.org/viewvc/hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java?rev=1546199&r1=1546198&r2=1546199&view=diff
==============================================================================
--- hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
(original)
+++ hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java Wed
Nov 27 21:28:31 2013
@@ -52,6 +52,7 @@ class BeeLineOpts implements Completor {
public static final String PROPERTY_PREFIX = "beeline.";
public static final String PROPERTY_NAME_EXIT =
PROPERTY_PREFIX + "system.exit";
+ public static final String DEFAULT_NULL_STRING = "NULL";
private final BeeLine beeLine;
private boolean autosave = false;
@@ -78,6 +79,9 @@ class BeeLineOpts implements Completor {
private boolean trimScripts = true;
private boolean allowMultiLineCommand = true;
+ //This can be set for old behavior of nulls printed as empty strings
+ private boolean nullEmptyString = false;
+
private final File rcFile = new File(saveDir(), "beeline.properties");
private String historyFile = new File(saveDir(),
"history").getAbsolutePath();
@@ -443,5 +447,22 @@ class BeeLineOpts implements Completor {
this.allowMultiLineCommand = allowMultiLineCommand;
}
+ /**
+ * Use getNullString() to get the null string to be used.
+ * @return true if null representation should be an empty string
+ */
+ public boolean getNullEmptyString() {
+ return nullEmptyString;
+ }
+
+ public void setNullEmptyString(boolean nullStringEmpty) {
+ this.nullEmptyString = nullStringEmpty;
+ }
+
+ public String getNullString(){
+ return nullEmptyString ? "" : DEFAULT_NULL_STRING;
+ }
+
+
}
Added: hive/trunk/beeline/src/java/org/apache/hive/beeline/BooleanCompletor.java
URL:
http://svn.apache.org/viewvc/hive/trunk/beeline/src/java/org/apache/hive/beeline/BooleanCompletor.java?rev=1546199&view=auto
==============================================================================
--- hive/trunk/beeline/src/java/org/apache/hive/beeline/BooleanCompletor.java
(added)
+++ hive/trunk/beeline/src/java/org/apache/hive/beeline/BooleanCompletor.java
Wed Nov 27 21:28:31 2013
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hive.beeline;
+
+import java.util.List;
+
+import jline.Completor;
+import jline.SimpleCompletor;
+
+/**
+ * JLine completor boolean value (true/false)
+ */
+class BooleanCompletor extends SimpleCompletor {
+
+ public BooleanCompletor(){
+ super(new String[] {"true", "false"});
+ }
+
+}
\ No newline at end of file
Modified: hive/trunk/beeline/src/java/org/apache/hive/beeline/BufferedRows.java
URL:
http://svn.apache.org/viewvc/hive/trunk/beeline/src/java/org/apache/hive/beeline/BufferedRows.java?rev=1546199&r1=1546198&r2=1546199&view=diff
==============================================================================
--- hive/trunk/beeline/src/java/org/apache/hive/beeline/BufferedRows.java
(original)
+++ hive/trunk/beeline/src/java/org/apache/hive/beeline/BufferedRows.java Wed
Nov 27 21:28:31 2013
@@ -54,6 +54,11 @@ class BufferedRows extends Rows {
}
@Override
+ public String toString(){
+ return list.toString();
+ }
+
+ @Override
void normalizeWidths() {
int[] max = null;
for (Row row : list) {
Modified: hive/trunk/beeline/src/java/org/apache/hive/beeline/Rows.java
URL:
http://svn.apache.org/viewvc/hive/trunk/beeline/src/java/org/apache/hive/beeline/Rows.java?rev=1546199&r1=1546198&r2=1546199&view=diff
==============================================================================
--- hive/trunk/beeline/src/java/org/apache/hive/beeline/Rows.java (original)
+++ hive/trunk/beeline/src/java/org/apache/hive/beeline/Rows.java Wed Nov 27
21:28:31 2013
@@ -27,19 +27,23 @@ import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.text.DecimalFormat;
import java.text.NumberFormat;
+import java.util.Arrays;
import java.util.Iterator;
/**
* Abstract base class representing a set of rows to be displayed.
+ * Holds column values as strings
*/
abstract class Rows implements Iterator {
private final BeeLine beeLine;
final ResultSetMetaData rsMeta;
final Boolean[] primaryKeys;
final NumberFormat numberFormat;
+ private final String nullStr;
Rows(BeeLine beeLine, ResultSet rs) throws SQLException {
this.beeLine = beeLine;
+ nullStr = beeLine.getOpts().getNullString();
rsMeta = rs.getMetaData();
int count = rsMeta.getColumnCount();
primaryKeys = new Boolean[count];
@@ -125,6 +129,10 @@ abstract class Rows implements Iterator
inserted = false;
}
+ @Override
+ public String toString(){
+ return Arrays.asList(values).toString();
+ }
Row(int size, ResultSet rs) throws SQLException {
isMeta = false;
@@ -157,7 +165,8 @@ abstract class Rows implements Iterator
} else {
values[i] = rs.getString(i + 1);
}
- sizes[i] = values[i] == null ? 1 : values[i].length();
+ values[i] = values[i] == null ? nullStr : values[i];
+ sizes[i] = values[i].length();
}
}
}
Modified: hive/trunk/beeline/src/main/resources/BeeLine.properties
URL:
http://svn.apache.org/viewvc/hive/trunk/beeline/src/main/resources/BeeLine.properties?rev=1546199&r1=1546198&r2=1546199&view=diff
==============================================================================
--- hive/trunk/beeline/src/main/resources/BeeLine.properties (original)
+++ hive/trunk/beeline/src/main/resources/BeeLine.properties Wed Nov 27
21:28:31 2013
@@ -67,6 +67,7 @@ help-tables: List all the tables in the
help-columns: List all the columns for the specified table
help-properties: Connect to the database specified in the properties file(s)
help-outputformat: Set the output format for displaying results
(table,vertical,csv,tsv,xmlattrs,xmlelements)
+help-nullemptystring: Set to true to get historic behavior of printing null as
empty string. Default is false.
jline-missing: The JLine jar was not found. Please ensure it is installed.
@@ -163,6 +164,7 @@ cmd-usage: Usage: java org.apache.hive.c
\ --autosave=[true/false] automatically save preferences\n \
\ --outputformat=[table/vertical/csv/tsv] format mode for result display\n \
\ --isolation=LEVEL set the transaction isolation level\n \
+\ --nullemptystring=[true/false] set to true to get historic behavior of
printing null as empty string\n \
\ --help display this message
Modified:
hive/trunk/beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java
URL:
http://svn.apache.org/viewvc/hive/trunk/beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java?rev=1546199&r1=1546198&r2=1546199&view=diff
==============================================================================
---
hive/trunk/beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java
(original)
+++
hive/trunk/beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java
Wed Nov 27 21:28:31 2013
@@ -23,9 +23,14 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.beeline.BeeLine;
import org.apache.hive.service.server.HiveServer2;
@@ -34,6 +39,8 @@ import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import static org.junit.Assert.*;
+
/**
* TestBeeLineWithArgs - executes tests of the command-line arguments to
BeeLine
*
@@ -42,6 +49,9 @@ import org.junit.Test;
public class TestBeeLineWithArgs {
// Default location of HiveServer2
final private static String JDBC_URL = BeeLine.BEELINE_DEFAULT_JDBC_URL +
"localhost:10000";
+ private static final String tableName = "TestBeelineTable1";
+ private static final String tableComment = "Test table comment";
+
private static HiveServer2 hiveServer2;
@@ -67,6 +77,45 @@ public class TestBeeLineWithArgs {
System.err.println("Starting HiveServer2...");
hiveServer2.start();
Thread.sleep(1000);
+ createTable();
+
+ }
+
+ /**
+ * Create table for use by tests
+ * @throws ClassNotFoundException
+ * @throws SQLException
+ */
+ private static void createTable() throws ClassNotFoundException,
SQLException {
+ Class.forName(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER);
+ Connection con = DriverManager.getConnection(JDBC_URL,"", "");
+
+ assertNotNull("Connection is null", con);
+ assertFalse("Connection should not be closed", con.isClosed());
+ Statement stmt = con.createStatement();
+ assertNotNull("Statement is null", stmt);
+
+ stmt.execute("set hive.support.concurrency = false");
+
+ HiveConf conf = new HiveConf();
+ String dataFileDir = conf.get("test.data.files").replace('\\', '/')
+ .replace("c:", "");
+ Path dataFilePath = new Path(dataFileDir, "kv1.txt");
+ // drop table. ignore error.
+ try {
+ stmt.execute("drop table " + tableName);
+ } catch (Exception ex) {
+ fail(ex.toString());
+ }
+
+ // create table
+ stmt.execute("create table " + tableName
+ + " (under_col int comment 'the under column', value string) comment '"
+ + tableComment + "'");
+
+ // load data
+ stmt.execute("load data local inpath '"
+ + dataFilePath.toString() + "' into table " + tableName);
}
/**
@@ -129,39 +178,12 @@ public class TestBeeLineWithArgs {
argList.add("-f");
argList.add(scriptFile.getAbsolutePath());
- if(shouldMatch) {
- try {
- String output = testCommandLineScript(argList);
- long elapsedTime = (System.currentTimeMillis() - startTime)/1000;
- String time = "(" + elapsedTime + "s)";
- if (output.contains(expectedPattern)) {
- System.out.println(">>> PASSED " + testName + " " + time);
- } else {
- System.err.println("Output: " + output);
- System.err.println(">>> FAILED " + testName + " (ERROR) " + time);
- Assert.fail(testName);
- }
- } catch (Throwable e) {
- e.printStackTrace();
- throw e;
- }
- } else {
- try {
- String output = testCommandLineScript(argList);
- long elapsedTime = (System.currentTimeMillis() - startTime)/1000;
- String time = "(" + elapsedTime + "s)";
- if (output.contains(expectedPattern)) {
- System.err.println("Output: " + output);
- System.err.println(">>> FAILED " + testName + " (ERROR) " + time);
- Assert.fail(testName);
- } else {
- System.out.println(">>> PASSED " + testName + " " + time);
- }
- } catch (Throwable e) {
- System.err.println("Exception: " + e.toString());
- e.printStackTrace();
- throw e;
- }
+ String output = testCommandLineScript(argList);
+ boolean matches = output.contains(expectedPattern);
+ if (shouldMatch != matches) {
+ //failed
+ fail(testName + ": Output" + output + " should" + (shouldMatch ? "" : "
not") +
+ " contain " + expectedPattern);
}
scriptFile.delete();
}
@@ -212,6 +234,72 @@ public class TestBeeLineWithArgs {
testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, false, argList);
}
+
+ /**
+ * Select null from table , check how null is printed
+ * Print PASSED or FAILED
+ */
+ @Test
+ public void testNullDefault() throws Throwable {
+ final String TEST_NAME = "testNullDefault";
+ final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
+ "select null from " + tableName + " limit 1 ;\n";
+ final String EXPECTED_PATTERN = "NULL";
+ testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true,
getBaseArgs(JDBC_URL));
+ }
+
+ /**
+ * Select null from table , check if default null is printed differently
+ * Print PASSED or FAILED
+ */
+ @Test
+ public void testNullNonEmpty() throws Throwable {
+ final String TEST_NAME = "testNullNonDefault";
+ final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
+ "!set nullemptystring false\n select null from " + tableName + " limit
1 ;\n";
+ final String EXPECTED_PATTERN = "NULL";
+ testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true,
getBaseArgs(JDBC_URL));
+ }
+
+ /**
+ * Select null from table , check if setting null to empty string works.
+ * Original beeline/sqlline used to print nulls as empty strings
+ * Print PASSED or FAILED
+ */
+ @Test
+ public void testNullEmpty() throws Throwable {
+ final String TEST_NAME = "testNullNonDefault";
+ final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
+ "!set nullemptystring true\n select 'abc',null,'def' from " +
tableName + " limit 1 ;\n";
+ final String EXPECTED_PATTERN = "'abc','','def'";
+
+ List<String> argList = getBaseArgs(JDBC_URL);
+ argList.add("--outputformat=csv");
+
+ testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ }
+
+ /**
+ * Select null from table , check if setting null to empty string works -
Using beeling cmd line
+ * argument.
+ * Original beeline/sqlline used to print nulls as empty strings
+ * Print PASSED or FAILED
+ */
+ @Test
+ public void testNullEmptyCmdArg() throws Throwable {
+ final String TEST_NAME = "testNullNonDefault";
+ final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
+ "select 'abc',null,'def' from " + tableName + " limit 1 ;\n";
+ //final String EXPECTED_PATTERN = "| abc | | def |";
+ final String EXPECTED_PATTERN = "'abc','','def'";
+
+ List<String> argList = getBaseArgs(JDBC_URL);
+ argList.add("--nullemptystring=true");
+ argList.add("--outputformat=csv");
+
+ testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ }
+
/**
* Attempt to execute a missing script file with the -f option to BeeLine
* Print PASSED or FAILED
@@ -239,7 +327,7 @@ public class TestBeeLineWithArgs {
if (output.contains(EXPECTED_PATTERN)) {
System.err.println("Output: " + output);
System.err.println(">>> FAILED " + TEST_NAME + " (ERROR) " + time);
- Assert.fail(TEST_NAME);
+ fail(TEST_NAME);
} else {
System.out.println(">>> PASSED " + TEST_NAME + " " + time);
}
