Repository: phoenix Updated Branches: refs/heads/4.5-HBase-0.98 cbf13010d -> 2b63d07fa
PHOENIX-2181: PhoenixHBaseLoader doesn't work with salted tables Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/2b63d07f Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/2b63d07f Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/2b63d07f Branch: refs/heads/4.5-HBase-0.98 Commit: 2b63d07fad2a8b1fc8815422771e1f39bf9856c9 Parents: cbf1301 Author: Ravi Magham <ravi.mag...@bazaarvoice.com> Authored: Fri Aug 14 14:24:14 2015 -0700 Committer: Ravi Magham <ravi.mag...@bazaarvoice.com> Committed: Fri Aug 14 14:24:14 2015 -0700 ---------------------------------------------------------------------- .../org/apache/phoenix/util/PhoenixRuntime.java | 4 +- .../phoenix/pig/PhoenixHBaseLoaderIT.java | 47 ++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/2b63d07f/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java index fe451b5..95f9af0 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java @@ -381,7 +381,9 @@ public class PhoenixRuntime { Set<String> unresolvedColumnNames = new TreeSet<String>(); if (columns == null || columns.isEmpty()) { // use all columns in the table - for(PColumn pColumn : table.getColumns()) { + int offset = (table.getBucketNum() == null ? 0 : 1); + for (int i = offset; i < table.getColumns().size(); i++) { + PColumn pColumn = table.getColumns().get(i); int sqlType = pColumn.getDataType().getSqlType(); columnInfoList.add(new ColumnInfo(pColumn.toString(), sqlType)); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/2b63d07f/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseLoaderIT.java ---------------------------------------------------------------------- diff --git a/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseLoaderIT.java b/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseLoaderIT.java index 5b06eed..4f57f27 100644 --- a/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseLoaderIT.java +++ b/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseLoaderIT.java @@ -584,6 +584,53 @@ public class PhoenixHBaseLoaderIT extends BaseHBaseManagedTimeIT { dropTable(INDEX_NAME); } } + + @Test + public void testLoadOfSaltTable() throws Exception { + final String TABLE = "TABLE11"; + final String sourceTableddl = "CREATE TABLE " + TABLE + + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, AGE INTEGER, SAL INTEGER) SALT_BUCKETS=2 "; + + conn.createStatement().execute(sourceTableddl); + + //prepare data with 10 rows having age 25 and the other 30. + final String dml = "UPSERT INTO " + TABLE + " VALUES(?,?,?,?)"; + PreparedStatement stmt = conn.prepareStatement(dml); + int rows = 20; + int j = 0, k = 0; + for(int i = 0 ; i < rows; i++) { + stmt.setInt(1, i); + stmt.setString(2, "a"+i); + if(i % 2 == 0) { + stmt.setInt(3, 25); + stmt.setInt(4, 10 * 2 * j++); + } else { + stmt.setInt(3, 30); + stmt.setInt(4, 10 * 3 * k++); + } + + stmt.execute(); + } + conn.commit(); + + final Data data = Storage.resetData(pigServer); + List<Tuple> expectedList = new ArrayList<Tuple>(); + expectedList.add(Storage.tuple(25,10)); + expectedList.add(Storage.tuple(30,10)); + + pigServer.setBatchOn(); + pigServer.registerQuery(String.format( + "A = load 'hbase://table/%s' using " + PhoenixHBaseLoader.class.getName() + "('%s');", TABLE, + zkQuorum)); + + pigServer.registerQuery("B = GROUP A BY AGE;"); + pigServer.registerQuery("C = FOREACH B GENERATE group,COUNT(A);"); + pigServer.registerQuery("STORE C INTO 'out' using mock.Storage();"); + pigServer.executeBatch(); + + List<Tuple> actualList = data.get("out"); + assertEquals(expectedList.size(), actualList.size()); + } @After public void tearDown() throws Exception {
