PHOENIX-2021 - Implement ARRAY_CAT built in function (Dumindu Buddhika)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/7385899d Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/7385899d Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/7385899d Branch: refs/heads/master Commit: 7385899d966e38cfc798fd509445db24653ad7de Parents: 7175dcb Author: ramkrishna <[email protected]> Authored: Sun Jun 21 22:05:13 2015 +0530 Committer: ramkrishna <[email protected]> Committed: Sun Jun 21 22:14:16 2015 +0530 ---------------------------------------------------------------------- .../phoenix/end2end/ArrayAppendFunctionIT.java | 17 - .../phoenix/end2end/ArrayConcatFunctionIT.java | 578 ++++++++++++++++++ .../phoenix/expression/ExpressionType.java | 4 +- .../function/ArrayAppendFunction.java | 53 +- .../function/ArrayConcatFunction.java | 83 +++ .../function/ArrayModifierFunction.java | 155 ++++- .../function/ArrayPrependFunction.java | 54 +- .../phoenix/schema/types/PArrayDataType.java | 163 +++++- .../expression/ArrayConcatFunctionTest.java | 584 +++++++++++++++++++ 9 files changed, 1543 insertions(+), 148 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/7385899d/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayAppendFunctionIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayAppendFunctionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayAppendFunctionIT.java index 1957b3a..cf45724 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayAppendFunctionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayAppendFunctionIT.java @@ -497,23 +497,6 @@ public class ArrayAppendFunctionIT extends BaseHBaseManagedTimeIT { } @Test - public void testArrayAppendFunctionIntegerWithNull() throws Exception { - Connection conn = DriverManager.getConnection(getUrl()); - initTables(conn); - - ResultSet rs; - rs = conn.createStatement().executeQuery("SELECT ARRAY_APPEND(NULL,NULL) FROM regions WHERE region_name = 'SF Bay Area'"); - assertTrue(rs.next()); - - Integer[] integers = new Integer[]{2345, 46345, 23234, 456}; - - Array array = conn.createArrayOf("INTEGER", integers); - - assertEquals(null, rs.getArray(1)); - assertFalse(rs.next()); - } - - @Test public void testArrayAppendFunctionVarcharWithNull() throws Exception { Connection conn = DriverManager.getConnection(getUrl()); initTables(conn); http://git-wip-us.apache.org/repos/asf/phoenix/blob/7385899d/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayConcatFunctionIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayConcatFunctionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayConcatFunctionIT.java new file mode 100644 index 0000000..247bfb7 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayConcatFunctionIT.java @@ -0,0 +1,578 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import org.apache.phoenix.schema.TypeMismatchException; +import org.junit.Test; + +import java.sql.*; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class ArrayConcatFunctionIT extends BaseHBaseManagedTimeIT { + + private void initTables(Connection conn) throws Exception { + String ddl = "CREATE TABLE regions (region_name VARCHAR PRIMARY KEY,varchars VARCHAR[],integers INTEGER[],doubles DOUBLE[],bigints BIGINT[],chars CHAR(15)[],double1 DOUBLE,char1 CHAR(17),nullcheck INTEGER,chars2 CHAR(15)[])"; + conn.createStatement().execute(ddl); + String dml = "UPSERT INTO regions(region_name,varchars,integers,doubles,bigints,chars,double1,char1,nullcheck,chars2) VALUES('SF Bay Area'," + + "ARRAY['2345','46345','23234']," + + "ARRAY[2345,46345,23234,456]," + + "ARRAY[23.45,46.345,23.234,45.6,5.78]," + + "ARRAY[12,34,56,78,910]," + + "ARRAY['a','bbbb','c','ddd','e']," + + "23.45," + + "'wert'," + + "NULL," + + "ARRAY['a','bbbb','c','ddd','e','foo']" + + ")"; + PreparedStatement stmt = conn.prepareStatement(dml); + stmt.execute(); + conn.commit(); + } + + @Test + public void testArrayConcatFunctionVarchar() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT ARRAY_CAT(varchars,varchars) FROM regions WHERE region_name = 'SF Bay Area'"); + assertTrue(rs.next()); + + String[] strings = new String[]{"2345", "46345", "23234", "2345", "46345", "23234"}; + + Array array = conn.createArrayOf("VARCHAR", strings); + + assertEquals(array, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionInteger() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT ARRAY_CAT(integers,integers) FROM regions WHERE region_name = 'SF Bay Area'"); + assertTrue(rs.next()); + + Integer[] integers = new Integer[]{2345, 46345, 23234, 456, 2345, 46345, 23234, 456}; + + Array array = conn.createArrayOf("INTEGER", integers); + + assertEquals(array, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionDouble() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT ARRAY_CAT(doubles,doubles) FROM regions WHERE region_name = 'SF Bay Area'"); + assertTrue(rs.next()); + + Double[] doubles = new Double[]{23.45, 46.345, 23.234, 45.6, 5.78, 23.45, 46.345, 23.234, 45.6, 5.78}; + + Array array = conn.createArrayOf("DOUBLE", doubles); + + assertEquals(array, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionDouble2() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT ARRAY_CAT(doubles,ARRAY[23]) FROM regions WHERE region_name = 'SF Bay Area'"); + assertTrue(rs.next()); + + Double[] doubles = new Double[]{23.45, 46.345, 23.234, 45.6, 5.78, new Double(23)}; + + Array array = conn.createArrayOf("DOUBLE", doubles); + + assertEquals(array, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionBigint() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT ARRAY_CAT(bigints,bigints) FROM regions WHERE region_name = 'SF Bay Area'"); + assertTrue(rs.next()); + + Long[] longs = new Long[]{12l, 34l, 56l, 78l, 910l, 12l, 34l, 56l, 78l, 910l}; + + Array array = conn.createArrayOf("BIGINT", longs); + + assertEquals(array, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionChar() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT ARRAY_CAT(chars,chars) FROM regions WHERE region_name = 'SF Bay Area'"); + assertTrue(rs.next()); + + String[] strings = new String[]{"a", "bbbb", "c", "ddd", "e", "a", "bbbb", "c", "ddd", "e"}; + + Array array = conn.createArrayOf("CHAR", strings); + + assertEquals(array, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionChar3() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT ARRAY_CAT(chars,chars2) FROM regions WHERE region_name = 'SF Bay Area'"); + assertTrue(rs.next()); + + String[] strings = new String[]{"a", "bbbb", "c", "ddd", "e", "a", "bbbb", "c", "ddd", "e", "foo"}; + + Array array = conn.createArrayOf("CHAR", strings); + + assertEquals(array, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test(expected = TypeMismatchException.class) + public void testArrayConcatFunctionIntToCharArray() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT ARRAY_CAT(varchars,ARRAY[23,45]) FROM regions WHERE region_name = 'SF Bay Area'"); + } + + @Test(expected = TypeMismatchException.class) + public void testArrayConcatFunctionVarcharToIntegerArray() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT ARRAY_CAT(integers,ARRAY['a', 'b']) FROM regions WHERE region_name = 'SF Bay Area'"); + + } + + @Test(expected = SQLException.class) + public void testArrayConcatFunctionChar2() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT ARRAY_CAT(chars,ARRAY['facfacfacfacfacfacfac','facfacfacfacfacfacfac']) FROM regions WHERE region_name = 'SF Bay Area'"); + rs.next(); + rs.getArray(1); + } + + @Test + public void testArrayConcatFunctionIntegerArrayToDoubleArray() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT ARRAY_CAT(doubles,ARRAY[45, 55]) FROM regions WHERE region_name = 'SF Bay Area'"); + assertTrue(rs.next()); + + Double[] doubles = new Double[]{23.45, 46.345, 23.234, 45.6, 5.78, 45.0, 55.0}; + + Array array = conn.createArrayOf("DOUBLE", doubles); + + assertEquals(array, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionWithNestedFunctions1() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT ARRAY_CAT(ARRAY[23,45],ARRAY[integers[1],integers[1]]) FROM regions WHERE region_name = 'SF Bay Area'"); + assertTrue(rs.next()); + + Integer[] integers = new Integer[]{23, 45, 2345, 2345}; + + Array array = conn.createArrayOf("INTEGER", integers); + + assertEquals(array, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionWithNestedFunctions2() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT ARRAY_CAT(integers,ARRAY[ARRAY_ELEM(ARRAY[2,4],1),ARRAY_ELEM(ARRAY[2,4],2)]) FROM regions WHERE region_name = 'SF Bay Area'"); + assertTrue(rs.next()); + + Integer[] integers = new Integer[]{2345, 46345, 23234, 456, 2, 4}; + + Array array = conn.createArrayOf("INTEGER", integers); + + assertEquals(array, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionWithNestedFunctions3() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT ARRAY_CAT(doubles,ARRAY[ARRAY_ELEM(doubles, 1), ARRAY_ELEM(doubles, 1)]) FROM regions WHERE region_name = 'SF Bay Area'"); + assertTrue(rs.next()); + + Double[] doubles = new Double[]{23.45, 46.345, 23.234, 45.6, 5.78, 23.45, 23.45}; + + Array array = conn.createArrayOf("DOUBLE", doubles); + + assertEquals(array, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionWithUpsert1() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + + String ddl = "CREATE TABLE regions (region_name VARCHAR PRIMARY KEY,varchars VARCHAR[])"; + conn.createStatement().execute(ddl); + + String dml = "UPSERT INTO regions(region_name,varchars) VALUES('SF Bay Area',ARRAY_CAT(ARRAY['hello','world'],ARRAY[':-)']))"; + conn.createStatement().execute(dml); + conn.commit(); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT varchars FROM regions WHERE region_name = 'SF Bay Area'"); + assertTrue(rs.next()); + + String[] strings = new String[]{"hello", "world", ":-)"}; + + Array array = conn.createArrayOf("VARCHAR", strings); + + assertEquals(array, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionWithUpsert2() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + + String ddl = "CREATE TABLE regions (region_name VARCHAR PRIMARY KEY,integers INTEGER[])"; + conn.createStatement().execute(ddl); + + String dml = "UPSERT INTO regions(region_name,integers) VALUES('SF Bay Area',ARRAY_CAT(ARRAY[4,5],ARRAY[6, 7]))"; + conn.createStatement().execute(dml); + conn.commit(); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT integers FROM regions WHERE region_name = 'SF Bay Area'"); + assertTrue(rs.next()); + + Integer[] integers = new Integer[]{4, 5, 6, 7}; + + Array array = conn.createArrayOf("INTEGER", integers); + + assertEquals(array, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionWithUpsert3() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + + String ddl = "CREATE TABLE regions (region_name VARCHAR PRIMARY KEY,doubles DOUBLE[])"; + conn.createStatement().execute(ddl); + + String dml = "UPSERT INTO regions(region_name,doubles) VALUES('SF Bay Area',ARRAY_CAT(ARRAY[5.67,7.87],ARRAY[9.0, 8.0]))"; + conn.createStatement().execute(dml); + conn.commit(); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT doubles FROM regions WHERE region_name = 'SF Bay Area'"); + assertTrue(rs.next()); + + Double[] doubles = new Double[]{5.67, 7.87, new Double(9), new Double(8)}; + + Array array = conn.createArrayOf("DOUBLE", doubles); + + assertEquals(array, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionWithUpsertSelect1() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + + String ddl = "CREATE TABLE source (region_name VARCHAR PRIMARY KEY,doubles DOUBLE[])"; + conn.createStatement().execute(ddl); + + ddl = "CREATE TABLE target (region_name VARCHAR PRIMARY KEY,doubles DOUBLE[])"; + conn.createStatement().execute(ddl); + + String dml = "UPSERT INTO source(region_name,doubles) VALUES('SF Bay Area',ARRAY_CAT(ARRAY[5.67,7.87],ARRAY[9.0, 4.0]))"; + conn.createStatement().execute(dml); + + dml = "UPSERT INTO source(region_name,doubles) VALUES('SF Bay Area2',ARRAY_CAT(ARRAY[56.7,7.87],ARRAY[9.2, 3.4]))"; + conn.createStatement().execute(dml); + conn.commit(); + + dml = "UPSERT INTO target(region_name, doubles) SELECT region_name, ARRAY_CAT(doubles,doubles) FROM source"; + conn.createStatement().execute(dml); + conn.commit(); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT doubles FROM target"); + assertTrue(rs.next()); + + Double[] doubles = new Double[]{5.67, 7.87, new Double(9), new Double(4), 5.67, 7.87, new Double(9), new Double(4)}; + Array array = conn.createArrayOf("DOUBLE", doubles); + + assertEquals(array, rs.getArray(1)); + assertTrue(rs.next()); + + doubles = new Double[]{56.7, 7.87, new Double(9.2), new Double(3.4), 56.7, 7.87, new Double(9.2), new Double(3.4)}; + array = conn.createArrayOf("DOUBLE", doubles); + + assertEquals(array, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionWithUpsertSelect2() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + + String ddl = "CREATE TABLE source (region_name VARCHAR PRIMARY KEY,varchars VARCHAR[])"; + conn.createStatement().execute(ddl); + + ddl = "CREATE TABLE target (region_name VARCHAR PRIMARY KEY,varchars VARCHAR[])"; + conn.createStatement().execute(ddl); + + String dml = "UPSERT INTO source(region_name,varchars) VALUES('SF Bay Area',ARRAY_CAT(ARRAY['abcd','b'],ARRAY['c', 'd']))"; + conn.createStatement().execute(dml); + + dml = "UPSERT INTO source(region_name,varchars) VALUES('SF Bay Area2',ARRAY_CAT(ARRAY['d','fgh'],ARRAY['something','something']))"; + conn.createStatement().execute(dml); + conn.commit(); + + dml = "UPSERT INTO target(region_name, varchars) SELECT region_name, ARRAY_CAT(varchars,varchars) FROM source"; + conn.createStatement().execute(dml); + conn.commit(); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT varchars FROM target"); + assertTrue(rs.next()); + + String[] strings = new String[]{"abcd", "b", "c", "d", "abcd", "b", "c", "d"}; + Array array = conn.createArrayOf("VARCHAR", strings); + + assertEquals(array, rs.getArray(1)); + assertTrue(rs.next()); + + strings = new String[]{"d", "fgh", "something", "something", "d", "fgh", "something", "something"}; + array = conn.createArrayOf("VARCHAR", strings); + + assertEquals(array, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionInWhere1() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT region_name FROM regions WHERE ARRAY[2345,46345,23234,456,123]=ARRAY_CAT(integers,ARRAY[123])"); + assertTrue(rs.next()); + + assertEquals("SF Bay Area", rs.getString(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionInWhere2() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT region_name FROM regions WHERE varchars[1]=ANY(ARRAY_CAT(ARRAY['2345','46345','23234'],ARRAY['1234']))"); + assertTrue(rs.next()); + + assertEquals("SF Bay Area", rs.getString(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionInWhere3() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT region_name FROM regions WHERE ARRAY['2345','46345','23234','1234','234']=ARRAY_CAT(ARRAY['2345','46345','23234'],ARRAY['1234', '234'])"); + assertTrue(rs.next()); + + assertEquals("SF Bay Area", rs.getString(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionInWhere4() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT region_name FROM regions WHERE ARRAY[23.45,4634.5,2.3234,123.4,12.0]=ARRAY_CAT(ARRAY[23.45,4634.5,2.3234],ARRAY[123.4,12.0])"); + assertTrue(rs.next()); + + assertEquals("SF Bay Area", rs.getString(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionInWhere5() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT region_name FROM regions WHERE ARRAY['2345','46345','23234','foo','foo']=ARRAY_CAT(varchars,ARRAY['foo','foo'])"); + assertTrue(rs.next()); + + assertEquals("SF Bay Area", rs.getString(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionInWhere6() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT region_name FROM regions WHERE chars2=ARRAY_CAT(chars,ARRAY['foo'])"); + assertTrue(rs.next()); + + assertEquals("SF Bay Area", rs.getString(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionInWhere7() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT region_name FROM regions WHERE ARRAY[2,3,4,5]=ARRAY_CAT(ARRAY[2,3],ARRAY[4,5])"); + assertTrue(rs.next()); + + assertEquals("SF Bay Area", rs.getString(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionWithNulls1() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + PreparedStatement st = conn.prepareStatement("SELECT ARRAY_CAT(?,?) FROM regions WHERE region_name = 'SF Bay Area'"); + Array array1 = conn.createArrayOf("VARCHAR", new Object[]{"a", "b", "c", null}); + st.setArray(1, array1); + Array array2 = conn.createArrayOf("VARCHAR", new Object[]{"a", "b", "c"}); + st.setArray(2, array2); + rs = st.executeQuery(); + assertTrue(rs.next()); + + Array expected = conn.createArrayOf("VARCHAR", new Object[]{"a", "b", "c", null, "a", "b", "c"}); + + assertEquals(expected, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionWithNulls2() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + PreparedStatement st = conn.prepareStatement("SELECT ARRAY_CAT(?,?) FROM regions WHERE region_name = 'SF Bay Area'"); + Array array1 = conn.createArrayOf("VARCHAR", new Object[]{"a", "b", "c"}); + st.setArray(1, array1); + Array array2 = conn.createArrayOf("VARCHAR", new Object[]{null, "a", "b", "c"}); + st.setArray(2, array2); + rs = st.executeQuery(); + assertTrue(rs.next()); + + Array expected = conn.createArrayOf("VARCHAR", new Object[]{"a", "b", "c", null, "a", "b", "c"}); + + assertEquals(expected, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionWithNulls3() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + PreparedStatement st = conn.prepareStatement("SELECT ARRAY_CAT(?,?) FROM regions WHERE region_name = 'SF Bay Area'"); + Array array1 = conn.createArrayOf("VARCHAR", new Object[]{"a", "b", "c", null}); + st.setArray(1, array1); + Array array2 = conn.createArrayOf("VARCHAR", new Object[]{null, "a", "b", "c"}); + st.setArray(2, array2); + rs = st.executeQuery(); + assertTrue(rs.next()); + + Array expected = conn.createArrayOf("VARCHAR", new Object[]{"a", "b", "c", null, null, "a", "b", "c"}); + + assertEquals(expected, rs.getArray(1)); + assertFalse(rs.next()); + } + + @Test + public void testArrayConcatFunctionWithNulls4() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + initTables(conn); + + ResultSet rs; + PreparedStatement st = conn.prepareStatement("SELECT ARRAY_CAT(?,?) FROM regions WHERE region_name = 'SF Bay Area'"); + Array array1 = conn.createArrayOf("VARCHAR", new Object[]{null, "a", null, "b", "c", null, null}); + st.setArray(1, array1); + Array array2 = conn.createArrayOf("VARCHAR", new Object[]{null, null, "a", null, "b", null, "c", null}); + st.setArray(2, array2); + rs = st.executeQuery(); + assertTrue(rs.next()); + + Array expected = conn.createArrayOf("VARCHAR", new Object[]{null, "a", null, "b", "c", null, null, null, null, "a", null, "b", null, "c", null}); + + assertEquals(expected, rs.getArray(1)); + assertFalse(rs.next()); + } +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/7385899d/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java index 4f98cb8..51f4089 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java @@ -23,6 +23,7 @@ import org.apache.phoenix.expression.function.AbsFunction; import org.apache.phoenix.expression.function.ArrayAllComparisonExpression; import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression; import org.apache.phoenix.expression.function.ArrayAppendFunction; +import org.apache.phoenix.expression.function.ArrayConcatFunction; import org.apache.phoenix.expression.function.ArrayElemRefExpression; import org.apache.phoenix.expression.function.ArrayIndexFunction; import org.apache.phoenix.expression.function.ArrayLengthFunction; @@ -245,7 +246,8 @@ public enum ExpressionType { LnFunction(LnFunction.class), LogFunction(LogFunction.class), ExpFunction(ExpFunction.class), - PowerFunction(PowerFunction.class) + PowerFunction(PowerFunction.class), + ArrayConcatFunction(ArrayConcatFunction.class) ; ExpressionType(Class<? extends Expression> clazz) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/7385899d/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayAppendFunction.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayAppendFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayAppendFunction.java index bf6c29f..8c7fa9f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayAppendFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayAppendFunction.java @@ -20,18 +20,13 @@ package org.apache.phoenix.expression.function; import java.util.List; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.phoenix.exception.DataExceedsCapacityException; import org.apache.phoenix.expression.Expression; -import org.apache.phoenix.expression.LiteralExpression; import org.apache.phoenix.parse.FunctionParseNode; -import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.TypeMismatchException; import org.apache.phoenix.schema.types.*; -import org.apache.phoenix.schema.tuple.Tuple; @FunctionParseNode.BuiltInFunction(name = ArrayAppendFunction.NAME, args = { - @FunctionParseNode.Argument(allowedTypes = {PBinaryArray.class, - PVarbinaryArray.class}), + @FunctionParseNode.Argument(allowedTypes = {PBinaryArray.class, PVarbinaryArray.class}), @FunctionParseNode.Argument(allowedTypes = {PVarbinary.class}, defaultValue = "null")}) public class ArrayAppendFunction extends ArrayModifierFunction { @@ -45,54 +40,12 @@ public class ArrayAppendFunction extends ArrayModifierFunction { } @Override - public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { - - if (!getArrayExpr().evaluate(tuple, ptr)) { - return false; - } else if (ptr.getLength() == 0) { - return true; - } - int arrayLength = PArrayDataType.getArrayLength(ptr, getBaseType(), getArrayExpr().getMaxLength()); - - int length = ptr.getLength(); - int offset = ptr.getOffset(); - byte[] arrayBytes = ptr.get(); - - if (!getElementExpr().evaluate(tuple, ptr) || ptr.getLength() == 0) { - ptr.set(arrayBytes, offset, length); - return true; - } - - checkSizeCompatibility(ptr); - coerceBytes(ptr); - return PArrayDataType.appendItemToArray(ptr, length, offset, arrayBytes, getBaseType(), arrayLength, getMaxLength(), getArrayExpr().getSortOrder()); - } - - @Override - public PDataType getDataType() { - return children.get(0).getDataType(); - } - - @Override - public Integer getMaxLength() { - return this.children.get(0).getMaxLength(); - } - - @Override - public SortOrder getSortOrder() { - return getChildren().get(0).getSortOrder(); + protected boolean modifierFunction(ImmutableBytesWritable ptr, int len, int offset, byte[] arrayBytes, PDataType baseDataType, int arrayLength, Integer maxLength, Expression arrayExp) { + return PArrayDataType.appendItemToArray(ptr, len, offset, arrayBytes, baseDataType, arrayLength, getMaxLength(), arrayExp.getSortOrder()); } @Override public String getName() { return NAME; } - - public Expression getArrayExpr() { - return getChildren().get(0); - } - - public Expression getElementExpr() { - return getChildren().get(1); - } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/7385899d/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayConcatFunction.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayConcatFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayConcatFunction.java new file mode 100644 index 0000000..d2b846a --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayConcatFunction.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression.function; + +import java.util.List; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.parse.FunctionParseNode; +import org.apache.phoenix.schema.TypeMismatchException; +import org.apache.phoenix.schema.tuple.Tuple; +import org.apache.phoenix.schema.types.PArrayDataType; +import org.apache.phoenix.schema.types.PBinaryArray; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PVarbinaryArray; + [email protected](name = ArrayConcatFunction.NAME, args = { + @FunctionParseNode.Argument(allowedTypes = {PBinaryArray.class, PVarbinaryArray.class}), + @FunctionParseNode.Argument(allowedTypes = {PBinaryArray.class, PVarbinaryArray.class})}) +public class ArrayConcatFunction extends ArrayModifierFunction { + + public static final String NAME = "ARRAY_CAT"; + + public ArrayConcatFunction() { + } + + public ArrayConcatFunction(List<Expression> children) throws TypeMismatchException { + super(children); + } + + + @Override + public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { + + if (!getLHSExpr().evaluate(tuple, ptr)|| ptr.getLength() == 0){ + return false; + } + + int actualLengthOfArray1 = Math.abs(PArrayDataType.getArrayLength(ptr, getLHSBaseType(), getLHSExpr().getMaxLength())); + int lengthArray1 = ptr.getLength(); + int offsetArray1 = ptr.getOffset(); + byte[] array1Bytes = ptr.get(); + if (!getRHSExpr().evaluate(tuple, ptr)|| ptr.getLength() == 0){ + ptr.set(array1Bytes, offsetArray1, lengthArray1); + return true; + } + + checkSizeCompatibility(ptr, getLHSExpr(), getLHSExpr().getDataType(), getRHSExpr(),getRHSExpr().getDataType()); + + // Coerce array2 to array1 type + coerceBytes(ptr, getLHSExpr(), getLHSExpr().getDataType(), getRHSExpr(),getRHSExpr().getDataType()); + return modifierFunction(ptr, lengthArray1, offsetArray1, array1Bytes, getLHSBaseType(), actualLengthOfArray1, getMaxLength(), getLHSExpr()); + } + + @Override + protected boolean modifierFunction(ImmutableBytesWritable ptr, int len, int offset, + byte[] array1Bytes, PDataType baseDataType, int actualLengthOfArray1, Integer maxLength, + Expression array1Exp) { + int actualLengthOfArray2 = Math.abs(PArrayDataType.getArrayLength(ptr, baseDataType, array1Exp.getMaxLength())); + return PArrayDataType.concatArrays(ptr, len, offset, array1Bytes, baseDataType, actualLengthOfArray1, actualLengthOfArray2); + } + + @Override + public String getName() { + return NAME; + } + +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/7385899d/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayModifierFunction.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayModifierFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayModifierFunction.java index afd10e5..3177c29 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayModifierFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayModifierFunction.java @@ -24,7 +24,9 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.phoenix.exception.DataExceedsCapacityException; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.LiteralExpression; +import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.TypeMismatchException; +import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.*; public abstract class ArrayModifierFunction extends ScalarFunction { @@ -34,42 +36,153 @@ public abstract class ArrayModifierFunction extends ScalarFunction { public ArrayModifierFunction(List<Expression> children) throws TypeMismatchException { super(children); + Expression arrayExpr = null; + PDataType baseDataType = null; + Expression otherExpr = null; + PDataType otherExpressionType = null; + if (getLHSExpr().getDataType().isArrayType()) { + arrayExpr = getLHSExpr(); + baseDataType = getLHSBaseType(); + otherExpr = getRHSExpr(); + otherExpressionType = getRHSBaseType(); + } else { + arrayExpr = getRHSExpr(); + baseDataType = getRHSBaseType(); + otherExpr = getLHSExpr(); + otherExpressionType = getLHSBaseType(); + } + if (getDataType() != null && !(otherExpr instanceof LiteralExpression && otherExpr.isNullable()) && !otherExpressionType.isCoercibleTo(baseDataType)) { + throw TypeMismatchException.newException(baseDataType, otherExpressionType); + } - if (getDataType() != null && !(getElementExpr() instanceof LiteralExpression && getElementExpr().isNullable()) && !getElementDataType().isCoercibleTo(getBaseType())) { - throw TypeMismatchException.newException(getBaseType(), getElementDataType()); + // If the base type of an element is fixed width, make sure the element + // being appended will fit + if (getDataType() != null && otherExpressionType.getByteSize() == null + && otherExpressionType != null && baseDataType.isFixedWidth() + && otherExpressionType.isFixedWidth() && arrayExpr.getMaxLength() != null + && otherExpr.getMaxLength() != null + && otherExpr.getMaxLength() > arrayExpr.getMaxLength()) { + throw new DataExceedsCapacityException("Values are not size compatible"); } + // If the base type has a scale, make sure the element being appended has a + // scale less than or equal to it + if (getDataType() != null && arrayExpr.getScale() != null && otherExpr.getScale() != null + && otherExpr.getScale() > arrayExpr.getScale()) { + throw new DataExceedsCapacityException(baseDataType, arrayExpr.getMaxLength(), + arrayExpr.getScale()); + } + } - // If the base type of an element is fixed width, make sure the element being appended will fit - if (getDataType() != null && getElementExpr().getDataType().getByteSize() == null && getElementDataType() != null && getBaseType().isFixedWidth() && getElementDataType().isFixedWidth() && getArrayExpr().getMaxLength() != null && - getElementExpr().getMaxLength() != null && getElementExpr().getMaxLength() > getArrayExpr().getMaxLength()) { - throw new DataExceedsCapacityException(""); + @Override + public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { + Expression arrayExpr = null; + PDataType baseDataType = null; + Expression otherExpr = null; + PDataType otherExpressionType = null; + if (getLHSExpr().getDataType().isArrayType()) { + arrayExpr = getLHSExpr(); + baseDataType = getLHSBaseType(); + otherExpr = getRHSExpr(); + otherExpressionType = getRHSBaseType(); + } else { + arrayExpr = getRHSExpr(); + baseDataType = getRHSBaseType(); + otherExpr = getLHSExpr(); + otherExpressionType = getLHSBaseType(); } - // If the base type has a scale, make sure the element being appended has a scale less than or equal to it - if (getDataType() != null && getArrayExpr().getScale() != null && getElementExpr().getScale() != null && - getElementExpr().getScale() > getArrayExpr().getScale()) { - throw new DataExceedsCapacityException(getBaseType(), getArrayExpr().getMaxLength(), getArrayExpr().getScale()); + if (!arrayExpr.evaluate(tuple, ptr)) { + return false; + } else if (ptr.getLength() == 0) { + return true; } + int arrayLength = PArrayDataType.getArrayLength(ptr, baseDataType, arrayExpr.getMaxLength()); + + int length = ptr.getLength(); + int offset = ptr.getOffset(); + byte[] arrayBytes = ptr.get(); + + otherExpr.evaluate(tuple, ptr); + + checkSizeCompatibility(ptr, arrayExpr, baseDataType, otherExpr, otherExpressionType); + coerceBytes(ptr, arrayExpr, baseDataType, otherExpr, otherExpressionType); + return modifierFunction(ptr, length, offset, arrayBytes, baseDataType, arrayLength, getMaxLength(), + arrayExpr); } - protected void checkSizeCompatibility(ImmutableBytesWritable ptr) { - if (!getBaseType().isSizeCompatible(ptr, null, getElementDataType(), getElementExpr().getMaxLength(), getElementExpr().getScale(), getArrayExpr().getMaxLength(), getArrayExpr().getScale())) { - throw new DataExceedsCapacityException(""); + // Override this method for various function implementations + protected boolean modifierFunction(ImmutableBytesWritable ptr, int len, int offset, + byte[] arrayBytes, PDataType baseDataType, int arrayLength, Integer maxLength, + Expression arrayExp) { + return false; + } + + protected void checkSizeCompatibility(ImmutableBytesWritable ptr, Expression arrayExpr, + PDataType baseDataType, Expression otherExpr, PDataType otherExpressionType) { + if (!baseDataType.isSizeCompatible(ptr, null, otherExpressionType, + otherExpr.getMaxLength(), otherExpr.getScale(), arrayExpr.getMaxLength(), + arrayExpr.getScale())) { + throw new DataExceedsCapacityException("Values are not size compatible"); } } - protected void coerceBytes(ImmutableBytesWritable ptr) { - getBaseType().coerceBytes(ptr, null, getElementDataType(), getElementExpr().getMaxLength(), getElementExpr().getScale(), getElementExpr().getSortOrder(), getArrayExpr().getMaxLength(), getArrayExpr().getScale(), getArrayExpr().getSortOrder()); + + protected void coerceBytes(ImmutableBytesWritable ptr, Expression arrayExpr, + PDataType baseDataType, Expression otherExpr, PDataType otherExpressionType) { + baseDataType.coerceBytes(ptr, null, otherExpressionType, otherExpr.getMaxLength(), + otherExpr.getScale(), otherExpr.getSortOrder(), arrayExpr.getMaxLength(), + arrayExpr.getScale(), arrayExpr.getSortOrder()); } - public abstract Expression getArrayExpr(); + public Expression getRHSExpr() { + return this.children.get(1); + } - public abstract Expression getElementExpr(); + public Expression getLHSExpr() { + return this.children.get(0); + } - public PDataType getBaseType() { - return PDataType.arrayBaseType(getArrayExpr().getDataType()); + public PDataType getLHSBaseType() { + if (getLHSExpr().getDataType().isArrayType()) { + return PDataType.arrayBaseType(getLHSExpr().getDataType()); + } else { + return getLHSExpr().getDataType(); + } } - public PDataType getElementDataType() { - return getElementExpr().getDataType(); + public PDataType getRHSBaseType() { + if (getRHSExpr().getDataType().isArrayType()) { + return PDataType.arrayBaseType(getRHSExpr().getDataType()); + } else { + return getRHSExpr().getDataType(); + } + } + + @Override + public PDataType getDataType() { + if (getLHSExpr().getDataType().isArrayType()) { + return getLHSExpr().getDataType(); + } else { + return getRHSExpr().getDataType(); + } + } + + + @Override + public Integer getMaxLength() { + if (getLHSExpr().getDataType().isArrayType()) { + return getLHSExpr().getMaxLength(); + } else { + return getRHSExpr().getMaxLength(); + } + } + + @Override + public SortOrder getSortOrder() { + if (getLHSExpr().getDataType().isArrayType()) { + return getLHSExpr().getSortOrder(); + } else { + return getRHSExpr().getSortOrder(); + } } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/7385899d/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayPrependFunction.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayPrependFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayPrependFunction.java index 3cea4df..c2311fb 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayPrependFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayPrependFunction.java @@ -23,16 +23,13 @@ import java.util.List; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.parse.FunctionParseNode; -import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.TypeMismatchException; -import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.*; @FunctionParseNode.BuiltInFunction(name = ArrayPrependFunction.NAME, args = { @FunctionParseNode.Argument(allowedTypes = {PVarbinary.class}), - @FunctionParseNode.Argument(allowedTypes = {PBinaryArray.class, - PVarbinaryArray.class})}) -public class ArrayPrependFunction extends ArrayModifierFunction { + @FunctionParseNode.Argument(allowedTypes = {PBinaryArray.class, PVarbinaryArray.class})}) +public class ArrayPrependFunction extends ArrayModifierFunction { public static final String NAME = "ARRAY_PREPEND"; @@ -44,53 +41,14 @@ public class ArrayPrependFunction extends ArrayModifierFunction { } @Override - public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { - - if (!getArrayExpr().evaluate(tuple, ptr)) { - return false; - } else if (ptr.getLength() == 0) { - return true; - } - int arrayLength = PArrayDataType.getArrayLength(ptr, getBaseType(), getArrayExpr().getMaxLength()); - - int length = ptr.getLength(); - int offset = ptr.getOffset(); - byte[] arrayBytes = ptr.get(); - - getElementExpr().evaluate(tuple, ptr); - - checkSizeCompatibility(ptr); - coerceBytes(ptr); - return PArrayDataType.prependItemToArray(ptr, length, offset, arrayBytes, getBaseType(), arrayLength, getMaxLength(), getArrayExpr().getSortOrder()); - } - - @Override - public PDataType getDataType() { - return children.get(1).getDataType(); - } - - @Override - public Integer getMaxLength() { - return this.children.get(1).getMaxLength(); - } - - @Override - public SortOrder getSortOrder() { - return getChildren().get(1).getSortOrder(); + protected boolean modifierFunction(ImmutableBytesWritable ptr, int len, int offset, + byte[] arrayBytes, PDataType baseDataType, int arrayLength, Integer maxLength, + Expression arrayExp) { + return PArrayDataType.prependItemToArray(ptr, len, offset, arrayBytes, baseDataType, arrayLength, getMaxLength(), arrayExp.getSortOrder()); } @Override public String getName() { return NAME; } - - @Override - public Expression getArrayExpr() { - return getChildren().get(1); - } - - @Override - public Expression getElementExpr() { - return getChildren().get(0); - } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/7385899d/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java index 86f22f7..4e32cc0 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java @@ -21,7 +21,6 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.text.Format; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; @@ -129,6 +128,19 @@ public abstract class PArrayDataType<T> extends PDataType<T> { } return 0; } + + public static int serializeNulls(byte[] bytes, int position, int nulls){ + int nMultiplesOver255 = nulls / 255; + while (nMultiplesOver255-- > 0) { + bytes[position++] = 1; + } + int nRemainingNulls = nulls % 255; + if (nRemainingNulls > 0) { + byte nNullByte = SortOrder.invert((byte)(nRemainingNulls-1)); + bytes[position++] = nNullByte; + } + return position; + } public static void writeEndSeperatorForVarLengthArray(DataOutputStream oStream) throws IOException { oStream.write(QueryConstants.SEPARATOR_BYTE); @@ -246,6 +258,10 @@ public abstract class PArrayDataType<T> extends PDataType<T> { pArr = new PhoenixArray(pArr, desiredMaxLength); } } + //Coerce to new max length when only max lengths differ + if(actualType == desiredType && !pArr.isPrimitiveType() && maxLength != null && maxLength != desiredMaxLength){ + pArr = new PhoenixArray(pArr, desiredMaxLength); + } baseType = desiredBaseType; ptr.set(toBytes(pArr, baseType, expectedModifier)); } else { @@ -460,6 +476,11 @@ public abstract class PArrayDataType<T> extends PDataType<T> { } public static boolean appendItemToArray(ImmutableBytesWritable ptr, int length, int offset, byte[] arrayBytes, PDataType baseType, int arrayLength, Integer maxLength, SortOrder sortOrder) { + if (ptr.getLength() == 0) { + ptr.set(arrayBytes, offset, length); + return true; + } + int elementLength = maxLength == null ? ptr.getLength() : maxLength; //padding @@ -617,16 +638,8 @@ public abstract class PArrayDataType<T> extends PDataType<T> { currentPosition++; newOffsetArrayPosition = offsetArrayPosition + lengthIncrease; - while (nMultiplesOver255-- > 0) { - newArray[currentPosition] = (byte) 1; - currentPosition++; - } - // Write a byte for the remaining null elements - if (nRemainingNulls > 0) { - byte nNullByte = SortOrder.invert((byte) (nRemainingNulls - 1)); - newArray[currentPosition] = nNullByte; // Single byte for repeating nulls - currentPosition++; - } + //serialize nulls at the beginning + currentPosition = serializeNulls(newArray, currentPosition, nulls); } else { if (!useInt) { if (PArrayDataType.useShortForOffsetArray(endElementPosition)) { @@ -702,6 +715,134 @@ public abstract class PArrayDataType<T> extends PDataType<T> { Bytes.putByte(newArray, currentPosition, arrayBytes[offset + length - 1]); } + public static boolean concatArrays(ImmutableBytesWritable ptr, int array1BytesLength, int array1BytesOffset, byte[] array1Bytes, PDataType baseType, int actualLengthOfArray1, int actualLengthOfArray2) { + int array2BytesLength = ptr.getLength(); + int array2BytesOffset = ptr.getOffset(); + byte[] array2Bytes = ptr.get(); + + byte[] newArray; + + if (!baseType.isFixedWidth()) { + int offsetArrayPositionArray1 = Bytes.toInt(array1Bytes, array1BytesOffset + array1BytesLength - Bytes.SIZEOF_INT - Bytes.SIZEOF_INT - Bytes.SIZEOF_BYTE, Bytes.SIZEOF_INT); + int offsetArrayPositionArray2 = Bytes.toInt(array2Bytes, array2BytesOffset + array2BytesLength - Bytes.SIZEOF_INT - Bytes.SIZEOF_INT - Bytes.SIZEOF_BYTE, Bytes.SIZEOF_INT); + int offsetArrayLengthArray1 = array1BytesLength - offsetArrayPositionArray1 - Bytes.SIZEOF_INT - Bytes.SIZEOF_INT - Bytes.SIZEOF_BYTE; + int offsetArrayLengthArray2 = array2BytesLength - offsetArrayPositionArray2 - Bytes.SIZEOF_INT - Bytes.SIZEOF_INT - Bytes.SIZEOF_BYTE; + int newArrayLength = actualLengthOfArray1 + actualLengthOfArray2; + int nullsAtTheEndOfArray1 = 0; + int nullsAtTheBeginningOfArray2 = 0; + //checks whether offset array consists of shorts or integers + boolean useIntArray1 = offsetArrayLengthArray1 / actualLengthOfArray1 == Bytes.SIZEOF_INT; + boolean useIntArray2 = offsetArrayLengthArray2 / actualLengthOfArray2 == Bytes.SIZEOF_INT; + boolean useIntNewArray = false; + //count nulls at the end of array 1 + for (int index = actualLengthOfArray1 - 1; index > -1; index--) { + int offset = getOffset(array1Bytes, index, !useIntArray1, array1BytesOffset + offsetArrayPositionArray1); + if (array1Bytes[array1BytesOffset + offset] == QueryConstants.SEPARATOR_BYTE) { + nullsAtTheEndOfArray1++; + } else { + break; + } + } + //count nulls at the beginning of the array 2 + int array2FirstNonNullElementOffset = 0; + int array2FirstNonNullIndex = 0; + for (int index = 0; index < actualLengthOfArray2; index++) { + int offset = getOffset(array2Bytes, index, !useIntArray2, array2BytesOffset + offsetArrayPositionArray2); + if (array2Bytes[array2BytesOffset + offset] == QueryConstants.SEPARATOR_BYTE) { + nullsAtTheBeginningOfArray2++; + } else { + array2FirstNonNullIndex = index; + array2FirstNonNullElementOffset = offset; + break; + } + } + int nullsInMiddleAfterConcat = nullsAtTheEndOfArray1 + nullsAtTheBeginningOfArray2; + int bytesForNullsBefore = nullsAtTheBeginningOfArray2 / 255 + (nullsAtTheBeginningOfArray2 % 255 == 0 ? 0 : 1); + int bytesForNullsAfter = nullsInMiddleAfterConcat / 255 + (nullsInMiddleAfterConcat % 255 == 0 ? 0 : 1); + //Increase of length required to store nulls + int lengthIncreaseForNulls = bytesForNullsAfter - bytesForNullsBefore; + //Length increase incremented by one when there were no nulls at the beginning of array and when there are + //nulls at the end of array 1 as we need to allocate a byte for separator byte in this case. + lengthIncreaseForNulls += nullsAtTheBeginningOfArray2 == 0 && nullsAtTheEndOfArray1 != 0 ? Bytes.SIZEOF_BYTE : 0; + int newOffsetArrayPosition = offsetArrayPositionArray1 + offsetArrayPositionArray2 + lengthIncreaseForNulls - 2 * Bytes.SIZEOF_BYTE; + int endElementPositionOfArray2 = getOffset(array2Bytes, actualLengthOfArray2 - 1, !useIntArray2, array2BytesOffset + offsetArrayPositionArray2); + int newEndElementPosition = lengthIncreaseForNulls + endElementPositionOfArray2 + offsetArrayPositionArray1 - 2 * Bytes.SIZEOF_BYTE; + //Creates a byre array to store the concatenated array + if (PArrayDataType.useShortForOffsetArray(newEndElementPosition)) { + newArray = new byte[newOffsetArrayPosition + newArrayLength * Bytes.SIZEOF_SHORT + Bytes.SIZEOF_INT + Bytes.SIZEOF_INT + Bytes.SIZEOF_BYTE]; + } else { + useIntNewArray = true; + newArray = new byte[newOffsetArrayPosition + newArrayLength * Bytes.SIZEOF_INT + Bytes.SIZEOF_INT + Bytes.SIZEOF_INT + Bytes.SIZEOF_BYTE]; + } + + int currentPosition = 0; + //Copies all the elements from array 1 to new array + System.arraycopy(array1Bytes, array1BytesOffset, newArray, currentPosition, offsetArrayPositionArray1 - 2 * Bytes.SIZEOF_BYTE); + currentPosition = offsetArrayPositionArray1 - 2 * Bytes.SIZEOF_BYTE; + int array2StartingPosition = currentPosition; + currentPosition += nullsInMiddleAfterConcat != 0 ? 1 : 0; + //Writes nulls in the middle of the array. + currentPosition = serializeNulls(newArray, currentPosition, nullsInMiddleAfterConcat); + //Copies the elements from array 2 beginning from the first non null element. + System.arraycopy(array2Bytes, array2BytesOffset + array2FirstNonNullElementOffset, newArray, currentPosition, offsetArrayPositionArray2 - array2FirstNonNullElementOffset); + currentPosition += offsetArrayPositionArray2 - array2FirstNonNullElementOffset; + + //Writing offset arrays + if (useIntNewArray) { + //offsets for the elements from array 1. Simply copied. + for (int index = 0; index < actualLengthOfArray1; index++) { + int offset = getOffset(array1Bytes, index, !useIntArray1, array1BytesOffset + offsetArrayPositionArray1); + Bytes.putInt(newArray, currentPosition, offset); + currentPosition += Bytes.SIZEOF_INT; + } + //offsets for nulls in the middle + for (int index = 0; index < array2FirstNonNullIndex; index++) { + int offset = getOffset(array2Bytes, index, !useIntArray2, array2BytesOffset + offsetArrayPositionArray2); + Bytes.putInt(newArray, currentPosition, offset + array2StartingPosition); + currentPosition += Bytes.SIZEOF_INT; + } + //offsets for the elements from the first non null element from array 2 + int part2NonNullStartingPosition = array2StartingPosition + bytesForNullsAfter + (bytesForNullsAfter == 0 ? 0 : Bytes.SIZEOF_BYTE); + for (int index = array2FirstNonNullIndex; index < actualLengthOfArray2; index++) { + int offset = getOffset(array2Bytes, index, !useIntArray2, array2BytesOffset + offsetArrayPositionArray2); + Bytes.putInt(newArray, currentPosition, offset - array2FirstNonNullElementOffset + part2NonNullStartingPosition); + currentPosition += Bytes.SIZEOF_INT; + } + } else { + //offsets for the elements from array 1. Simply copied. + for (int index = 0; index < actualLengthOfArray1; index++) { + int offset = getOffset(array1Bytes, index, !useIntArray1, array1BytesOffset + offsetArrayPositionArray1); + Bytes.putShort(newArray, currentPosition, (short) (offset - Short.MAX_VALUE)); + currentPosition += Bytes.SIZEOF_SHORT; + } + //offsets for nulls in the middle + for (int index = 0; index < array2FirstNonNullIndex; index++) { + int offset = getOffset(array2Bytes, index, !useIntArray2, array2BytesOffset + offsetArrayPositionArray2); + Bytes.putShort(newArray, currentPosition, (short) (offset + array2StartingPosition - Short.MAX_VALUE)); + currentPosition += Bytes.SIZEOF_SHORT; + } + //offsets for the elements from the first non null element from array 2 + int part2NonNullStartingPosition = array2StartingPosition + bytesForNullsAfter + (bytesForNullsAfter == 0 ? 0 : Bytes.SIZEOF_BYTE); + for (int index = array2FirstNonNullIndex; index < actualLengthOfArray2; index++) { + int offset = getOffset(array2Bytes, index, !useIntArray2, array2BytesOffset + offsetArrayPositionArray2); + Bytes.putShort(newArray, currentPosition, (short) (offset - array2FirstNonNullElementOffset + part2NonNullStartingPosition - Short.MAX_VALUE)); + currentPosition += Bytes.SIZEOF_SHORT; + } + } + Bytes.putInt(newArray, currentPosition, newOffsetArrayPosition); + currentPosition += Bytes.SIZEOF_INT; + Bytes.putInt(newArray, currentPosition, useIntNewArray ? -newArrayLength : newArrayLength); + currentPosition += Bytes.SIZEOF_INT; + Bytes.putByte(newArray, currentPosition, array1Bytes[array1BytesOffset + array1BytesLength - 1]); + } else { + newArray = new byte[array1BytesLength + array2BytesLength]; + System.arraycopy(array1Bytes, array1BytesOffset, newArray, 0, array1BytesLength); + System.arraycopy(array2Bytes, array2BytesOffset, newArray, array1BytesLength, array2BytesLength); + } + ptr.set(newArray); + return true; + } + public static int serailizeOffsetArrayIntoStream(DataOutputStream oStream, TrustedByteArrayOutputStream byteStream, int noOfElements, int maxOffset, int[] offsetPos) throws IOException { int offsetPosition = (byteStream.size());
