This is an automated email from the ASF dual-hosted git repository. dmollitor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new e56a775 HIVE-20447: Add JSON Output Format to beeline (Hunter Logan via David Mollitor) e56a775 is described below commit e56a775c38732da5928c0f6555dc167b5a6d1fa7 Author: Hunter Logan <hunterlo...@outlook.com> AuthorDate: Tue Jul 21 09:10:46 2020 -0400 HIVE-20447: Add JSON Output Format to beeline (Hunter Logan via David Mollitor) --- beeline/pom.xml | 4 + .../src/java/org/apache/hive/beeline/BeeLine.java | 2 + .../apache/hive/beeline/JSONFileOutputFormat.java | 62 ++++++++++ .../org/apache/hive/beeline/JSONOutputFormat.java | 126 +++++++++++++++++++++ beeline/src/main/resources/BeeLine.properties | 4 +- .../hive/beeline/TestJSONFileOutputFormat.java | 106 +++++++++++++++++ .../apache/hive/beeline/TestJSONOutputFormat.java | 108 ++++++++++++++++++ 7 files changed, 410 insertions(+), 2 deletions(-) diff --git a/beeline/pom.xml b/beeline/pom.xml index c8be6a6..46bfe98 100644 --- a/beeline/pom.xml +++ b/beeline/pom.xml @@ -70,6 +70,10 @@ <artifactId>commons-io</artifactId> </dependency> <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </dependency> + <dependency> <groupId>jline</groupId> <artifactId>jline</artifactId> </dependency> diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index cb14013..a86fe5c 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -186,6 +186,8 @@ public class BeeLine implements Closeable { "tsv", new DeprecatedSeparatedValuesOutputFormat(this, '\t'), "xmlattr", new XMLAttributeOutputFormat(this), "xmlelements", new XMLElementOutputFormat(this), + "json", new JSONOutputFormat(this), + "jsonfile", new JSONFileOutputFormat(this), }); private List<String> supportedLocalDriver = diff --git a/beeline/src/java/org/apache/hive/beeline/JSONFileOutputFormat.java b/beeline/src/java/org/apache/hive/beeline/JSONFileOutputFormat.java new file mode 100644 index 0000000..b5f2a84 --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/JSONFileOutputFormat.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import com.fasterxml.jackson.core.util.MinimalPrettyPrinter; + +/** + * OutputFormat for hive JSON file format. + * Removes "{ "resultset": [...] }" wrapping and prints one object per line. + * This output format matches the same format as a Hive table created with JSONFILE file format: + * CREATE TABLE ... STORED AS JSONFILE; + * e.g. + * {"name":"Ritchie Tiger","age":40,"is_employed":true,"college":"RIT"} + * {"name":"Bobby Tables","age":8,"is_employed":false,"college":null} + * ... + * + * Note the lack of "," at the end of lines. + * + */ +public class JSONFileOutputFormat extends JSONOutputFormat { + + + JSONFileOutputFormat(BeeLine beeLine) { + super(beeLine); + this.generator.setPrettyPrinter(new MinimalPrettyPrinter("\n")); + } + + @Override + void printHeader(Rows.Row header) {} + + @Override + void printFooter(Rows.Row header) { + ByteArrayOutputStream buf = (ByteArrayOutputStream) generator.getOutputTarget(); + try { + generator.flush(); + String out = buf.toString(StandardCharsets.UTF_8.name()); + beeLine.output(out); + } catch(IOException e) { + beeLine.handleException(e); + } + buf.reset(); + } +} diff --git a/beeline/src/java/org/apache/hive/beeline/JSONOutputFormat.java b/beeline/src/java/org/apache/hive/beeline/JSONOutputFormat.java new file mode 100644 index 0000000..ef0ddd3 --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/JSONOutputFormat.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This source file is based on code taken from SQLLine 1.9 + * See SQLLine notice in LICENSE + */ +package org.apache.hive.beeline; + +import java.sql.SQLException; +import java.sql.Types; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import com.fasterxml.jackson.core.JsonEncoding; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; + +/** + * OutputFormat for standard JSON. + * {"resultset":[{"String":"a","Int":1,"Decimal":3.14,"Bool":true,"Null":null},{"String":"b","Int":2,"Decimal":2.718,"Bool":false,"Null":null}]} + * + */ +public class JSONOutputFormat extends AbstractOutputFormat { + protected final BeeLine beeLine; + protected JsonGenerator generator; + + /** + * @param beeLine + */ + JSONOutputFormat(BeeLine beeLine) { + this.beeLine = beeLine; + try { + this.generator = new JsonFactory().createGenerator(new ByteArrayOutputStream(), JsonEncoding.UTF8); + } catch(IOException e) { + beeLine.handleException(e); + } + } + + @Override + void printHeader(Rows.Row header) { + try { + generator.writeStartObject(); + generator.writeArrayFieldStart("resultset"); + } catch(IOException e) { + beeLine.handleException(e); + } + } + + @Override + void printFooter(Rows.Row header) { + ByteArrayOutputStream buf = (ByteArrayOutputStream) generator.getOutputTarget(); + try { + generator.writeEndArray(); + generator.writeEndObject(); + generator.flush(); + String out = buf.toString(StandardCharsets.UTF_8.name()); + beeLine.output(out); + } catch(IOException e) { + beeLine.handleException(e); + } + buf.reset(); + } + + @Override + void printRow(Rows rows, Rows.Row header, Rows.Row row) { + String[] head = header.values; + String[] vals = row.values; + + try { + generator.writeStartObject(); + for (int i = 0; (i < head.length) && (i < vals.length); i++) { + generator.writeFieldName(head[i]); + switch(rows.rsMeta.getColumnType(i+1)) { + case Types.TINYINT: + case Types.SMALLINT: + case Types.INTEGER: + case Types.BIGINT: + case Types.REAL: + case Types.FLOAT: + case Types.DOUBLE: + case Types.DECIMAL: + case Types.NUMERIC: + case Types.ROWID: + generator.writeNumber(vals[i]); + break; + case Types.BINARY: + case Types.BLOB: + case Types.VARBINARY: + case Types.LONGVARBINARY: + generator.writeString(vals[i]); + break; + case Types.NULL: + generator.writeNull(); + break; + case Types.BOOLEAN: + generator.writeBoolean(Boolean.parseBoolean(vals[i])); + break; + default: + generator.writeString(vals[i]); + } + } + generator.writeEndObject(); + } catch (IOException e) { + beeLine.handleException(e); + } catch (SQLException e) { + beeLine.handleSQLException(e); + } + } +} diff --git a/beeline/src/main/resources/BeeLine.properties b/beeline/src/main/resources/BeeLine.properties index 1a70e85..f58cf10 100644 --- a/beeline/src/main/resources/BeeLine.properties +++ b/beeline/src/main/resources/BeeLine.properties @@ -68,7 +68,7 @@ help-procedures: List all the procedures help-tables: List all the tables in the database help-columns: List all the columns for the specified table help-properties: Connect to the database specified in the properties file(s) -help-outputformat: Set the output format for displaying results (table,vertical,csv2,dsv,tsv2,xmlattrs,xmlelements, and deprecated formats(csv, tsv)) +help-outputformat: Set the output format for displaying results (table,vertical,csv2,dsv,tsv2,xmlattrs,xmlelements,json,jsonfile and deprecated formats(csv, tsv)) help-delimiterForDSV: Set the delimiter for dsv output format help-nullemptystring: Set to true to get historic behavior of printing null as empty string. Default is false. help-addlocaldriverjar: Add driver jar file in the beeline client side. @@ -194,7 +194,7 @@ cmd-usage: Usage: java org.apache.hive.cli.beeline.BeeLine \n \ \ --silent=[true/false] be more silent\n \ \ --report=[true/false] show number of rows and execution time after query execution\n \ \ --autosave=[true/false] automatically save preferences\n \ -\ --outputformat=[table/vertical/csv2/tsv2/dsv/csv/tsv] format mode for result display\n \ +\ --outputformat=[table/vertical/csv2/tsv2/dsv/csv/tsv/json/jsonfile] format mode for result display\n \ \ Note that csv, and tsv are deprecated - use csv2, tsv2 instead\n \ \ --incremental=[true/false] Defaults to false. When set to false, the entire result set\n \ \ is fetched and buffered before being displayed, yielding optimal\n \ diff --git a/beeline/src/test/org/apache/hive/beeline/TestJSONFileOutputFormat.java b/beeline/src/test/org/apache/hive/beeline/TestJSONFileOutputFormat.java new file mode 100644 index 0000000..7cb85ab --- /dev/null +++ b/beeline/src/test/org/apache/hive/beeline/TestJSONFileOutputFormat.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline; + +import static org.mockito.ArgumentMatchers.anyInt; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; + +import org.junit.Test; +import org.junit.Before; + +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class TestJSONFileOutputFormat { + private final String[][] mockRowData = { + {"aaa","1","3.14","true","","SGVsbG8sIFdvcmxkIQ"}, + {"bbb","2","2.718","false","Null","RWFzdGVyCgllZ2cu"} + }; + + public ResultSet mockResultSet; + public TestBufferedRows.MockRow mockRow; + + @Test + public final void testPrint() throws SQLException { + BeeLine mockBeeline = spy(BeeLine.class); + ArgumentCaptor<String> captureOutput = ArgumentCaptor.forClass(String.class); + Mockito.doNothing().when(mockBeeline).output(captureOutput.capture()); + BufferedRows bfRows = new BufferedRows(mockBeeline, mockResultSet); + JSONFileOutputFormat instance = new JSONFileOutputFormat(mockBeeline); + instance.print(bfRows); + String expResult = "{\"String\":\"aaa\",\"Int\":1,\"Decimal\":3.14,\"Bool\":true,\"Null\":null,\"Binary\":\"SGVsbG8sIFdvcmxkIQ\"}\n{\"String\":\"bbb\",\"Int\":2,\"Decimal\":2.718,\"Bool\":false,\"Null\":null,\"Binary\":\"RWFzdGVyCgllZ2cu\"}"; + assertEquals(expResult, captureOutput.getValue()); + } + + @Before + public void setupMockData() throws SQLException { + mockResultSet = mock(ResultSet.class); + ResultSetMetaData mockResultSetMetaData = mock(ResultSetMetaData.class); + when(mockResultSetMetaData.getColumnCount()).thenReturn(6); + when(mockResultSetMetaData.getColumnLabel(1)).thenReturn("String"); + when(mockResultSetMetaData.getColumnLabel(2)).thenReturn("Int"); + when(mockResultSetMetaData.getColumnLabel(3)).thenReturn("Decimal"); + when(mockResultSetMetaData.getColumnLabel(4)).thenReturn("Bool"); + when(mockResultSetMetaData.getColumnLabel(5)).thenReturn("Null"); + when(mockResultSetMetaData.getColumnLabel(6)).thenReturn("Binary"); + + when(mockResultSetMetaData.getColumnType(1)).thenReturn(Types.VARCHAR); + when(mockResultSetMetaData.getColumnType(2)).thenReturn(Types.INTEGER); + when(mockResultSetMetaData.getColumnType(3)).thenReturn(Types.DECIMAL); + when(mockResultSetMetaData.getColumnType(4)).thenReturn(Types.BOOLEAN); + when(mockResultSetMetaData.getColumnType(5)).thenReturn(Types.NULL); + when(mockResultSetMetaData.getColumnType(6)).thenReturn(Types.BINARY); + + when(mockResultSet.getMetaData()).thenReturn(mockResultSetMetaData); + + mockRow = new TestBufferedRows.MockRow(); + when(mockResultSet.next()).thenAnswer(new Answer<Boolean>() { + private int mockRowDataIndex = 0; + + @Override + public Boolean answer(final InvocationOnMock invocation) { + if (mockRowDataIndex < mockRowData.length) { + mockRow.setCurrentRowData(mockRowData[mockRowDataIndex]); + mockRowDataIndex++; + return true; + } else { + return false; + } + } + }); + + when(mockResultSet.getObject(anyInt())).thenAnswer(new Answer<String>() { + @Override + public String answer(final InvocationOnMock invocation) { + Object[] args = invocation.getArguments(); + int index = ((Integer) args[0]); + return mockRow.getColumn(index); + } + }); + } +} diff --git a/beeline/src/test/org/apache/hive/beeline/TestJSONOutputFormat.java b/beeline/src/test/org/apache/hive/beeline/TestJSONOutputFormat.java new file mode 100644 index 0000000..ac216d7 --- /dev/null +++ b/beeline/src/test/org/apache/hive/beeline/TestJSONOutputFormat.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline; + +import static org.mockito.ArgumentMatchers.anyInt; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; + +import org.junit.Test; +import org.junit.Before; + +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class TestJSONOutputFormat { + private final String[][] mockRowData = { + {"aaa","1","3.14","true","","SGVsbG8sIFdvcmxkIQ"}, + {"bbb","2","2.718","false","Null","RWFzdGVyCgllZ2cu"} + }; + + public ResultSet mockResultSet; + public TestBufferedRows.MockRow mockRow; + + @Test + public final void testPrint() throws SQLException { + BeeLine mockBeeline = spy(BeeLine.class); + ArgumentCaptor<String> captureOutput = ArgumentCaptor.forClass(String.class); + Mockito.doNothing().when(mockBeeline).output(captureOutput.capture()); + //when(mockBeeline.getOpts().getNumberFormat()).thenReturn("default"); + //when(mockBeeline.getOpts().getConvertBinaryArrayToString()).thenReturn(true); + BufferedRows bfRows = new BufferedRows(mockBeeline, mockResultSet); + JSONOutputFormat instance = new JSONOutputFormat(mockBeeline); + instance.print(bfRows); + String expResult = "{\"resultset\":[{\"String\":\"aaa\",\"Int\":1,\"Decimal\":3.14,\"Bool\":true,\"Null\":null,\"Binary\":\"SGVsbG8sIFdvcmxkIQ\"},{\"String\":\"bbb\",\"Int\":2,\"Decimal\":2.718,\"Bool\":false,\"Null\":null,\"Binary\":\"RWFzdGVyCgllZ2cu\"}]}"; + assertEquals(expResult, captureOutput.getValue()); + } + + @Before + public void setupMockData() throws SQLException { + mockResultSet = mock(ResultSet.class); + ResultSetMetaData mockResultSetMetaData = mock(ResultSetMetaData.class); + when(mockResultSetMetaData.getColumnCount()).thenReturn(6); + when(mockResultSetMetaData.getColumnLabel(1)).thenReturn("String"); + when(mockResultSetMetaData.getColumnLabel(2)).thenReturn("Int"); + when(mockResultSetMetaData.getColumnLabel(3)).thenReturn("Decimal"); + when(mockResultSetMetaData.getColumnLabel(4)).thenReturn("Bool"); + when(mockResultSetMetaData.getColumnLabel(5)).thenReturn("Null"); + when(mockResultSetMetaData.getColumnLabel(6)).thenReturn("Binary"); + + when(mockResultSetMetaData.getColumnType(1)).thenReturn(Types.VARCHAR); + when(mockResultSetMetaData.getColumnType(2)).thenReturn(Types.INTEGER); + when(mockResultSetMetaData.getColumnType(3)).thenReturn(Types.DECIMAL); + when(mockResultSetMetaData.getColumnType(4)).thenReturn(Types.BOOLEAN); + when(mockResultSetMetaData.getColumnType(5)).thenReturn(Types.NULL); + when(mockResultSetMetaData.getColumnType(6)).thenReturn(Types.BINARY); + + when(mockResultSet.getMetaData()).thenReturn(mockResultSetMetaData); + + mockRow = new TestBufferedRows.MockRow(); + when(mockResultSet.next()).thenAnswer(new Answer<Boolean>() { + private int mockRowDataIndex = 0; + + @Override + public Boolean answer(final InvocationOnMock invocation) { + if (mockRowDataIndex < mockRowData.length) { + mockRow.setCurrentRowData(mockRowData[mockRowDataIndex]); + mockRowDataIndex++; + return true; + } else { + return false; + } + } + }); + + when(mockResultSet.getObject(anyInt())).thenAnswer(new Answer<String>() { + @Override + public String answer(final InvocationOnMock invocation) { + Object[] args = invocation.getArguments(); + int index = ((Integer) args[0]); + return mockRow.getColumn(index); + } + }); + } +}