Pig: fix duplicate schema alias Patch by Alex Liu, reviewed by brandonwilliams for CASSANDRA-6309
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f7efaffa Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f7efaffa Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f7efaffa Branch: refs/heads/trunk Commit: f7efaffadace3e344eeb4a1384fa72c73d8422b0 Parents: 667e3db Author: Brandon Williams <[email protected]> Authored: Thu Dec 5 16:14:43 2013 -0600 Committer: Brandon Williams <[email protected]> Committed: Thu Dec 5 16:14:43 2013 -0600 ---------------------------------------------------------------------- build.xml | 1 + .../apache/cassandra/db/ConsistencyLevel.java | 2 - .../hadoop/pig/AbstractCassandraStorage.java | 21 +--- .../cassandra/pig/CqlTableDataTypeTest.java | 35 ++---- .../org/apache/cassandra/pig/CqlTableTest.java | 9 +- .../pig/ThriftColumnFamilyDataTypeTest.java | 21 ---- .../cassandra/pig/ThriftColumnFamilyTest.java | 121 +++++-------------- 7 files changed, 48 insertions(+), 162 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/f7efaffa/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index 41a7fb0..e21091a 100644 --- a/build.xml +++ b/build.xml @@ -990,6 +990,7 @@ </classpath> <src path="${test.unit.src}"/> <src path="${test.long.src}"/> + <src path="${test.pig.src}"/> </javac> <!-- Non-java resources needed by the test suite --> http://git-wip-us.apache.org/repos/asf/cassandra/blob/f7efaffa/src/java/org/apache/cassandra/db/ConsistencyLevel.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ConsistencyLevel.java b/src/java/org/apache/cassandra/db/ConsistencyLevel.java index cbb4bb1..0f6aba7 100644 --- a/src/java/org/apache/cassandra/db/ConsistencyLevel.java +++ b/src/java/org/apache/cassandra/db/ConsistencyLevel.java @@ -285,9 +285,7 @@ public enum ConsistencyLevel { switch (this) { - case LOCAL_QUORUM: case EACH_QUORUM: - case LOCAL_ONE: requireNetworkTopologyStrategy(keyspaceName); break; case SERIAL: http://git-wip-us.apache.org/repos/asf/cassandra/blob/f7efaffa/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java b/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java index 9e2e301..3fb1c5a 100644 --- a/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java +++ b/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java @@ -614,20 +614,7 @@ public abstract class AbstractCassandraStorage extends LoadFunc implements Store cfDef.default_validation_class = ByteBufferUtil.string(cqlRow.columns.get(3).value); cfDef.key_validation_class = ByteBufferUtil.string(cqlRow.columns.get(4).value); String keyAliases = ByteBufferUtil.string(cqlRow.columns.get(5).value); - List<String> keys = FBUtilities.fromJsonList(keyAliases); - // classis thrift tables - if (keys.size() == 0) - { - CFDefinition cfDefinition = getCfDefinition(keyspace, column_family, client); - for (ColumnIdentifier column : cfDefinition.keys.keySet()) - { - String key = column.toString(); - String type = cfDefinition.keys.get(column).type.toString(); - logger.debug("name: {}, type: {} ", key, type); - keys.add(key); - } - } - else + if (FBUtilities.fromJsonList(keyAliases).size() > 0) cql3Table = true; } cfDef.column_metadata = getColumnMetadata(client); @@ -666,7 +653,8 @@ public abstract class AbstractCassandraStorage extends LoadFunc implements Store { String query = "SELECT column_name, " + " validator, " + - " index_type " + + " index_type, " + + " type " + "FROM system.schema_columns " + "WHERE keyspace_name = '%s' " + " AND columnfamily_name = '%s'"; @@ -717,6 +705,9 @@ public abstract class AbstractCassandraStorage extends LoadFunc implements Store { CqlRow row = iterator.next(); ColumnDef cDef = new ColumnDef(); + String type = ByteBufferUtil.string(row.getColumns().get(3).value); + if (!type.equals("regular")) + continue; cDef.setName(ByteBufferUtil.clone(row.getColumns().get(0).value)); cDef.validation_class = ByteBufferUtil.string(row.getColumns().get(1).value); ByteBuffer indexType = row.getColumns().get(2).value; http://git-wip-us.apache.org/repos/asf/cassandra/blob/f7efaffa/test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java ---------------------------------------------------------------------- diff --git a/test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java b/test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java index 1ae9806..2020b0a 100644 --- a/test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java +++ b/test/pig/org/apache/cassandra/pig/CqlTableDataTypeTest.java @@ -7,37 +7,16 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cassandra.pig; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ +package org.apache.cassandra.pig; import java.io.IOException; import java.nio.charset.CharacterCodingException; http://git-wip-us.apache.org/repos/asf/cassandra/blob/f7efaffa/test/pig/org/apache/cassandra/pig/CqlTableTest.java ---------------------------------------------------------------------- diff --git a/test/pig/org/apache/cassandra/pig/CqlTableTest.java b/test/pig/org/apache/cassandra/pig/CqlTableTest.java index 785d819..e33364f 100644 --- a/test/pig/org/apache/cassandra/pig/CqlTableTest.java +++ b/test/pig/org/apache/cassandra/pig/CqlTableTest.java @@ -50,6 +50,7 @@ public class CqlTableTest extends PigTestBase "INSERT INTO compactcqltable (key1, column1, column2) values ('key1', 100, 10.1)", "CREATE TABLE test (a int PRIMARY KEY, b int);", + "CREATE INDEX test_b on test (b);", "CREATE TABLE moredata (x int PRIMARY KEY, y int);", "INSERT INTO test (a,b) VALUES (1,1);", @@ -158,10 +159,12 @@ public class CqlTableTest extends PigTestBase //(9,10,Ninen,nomatch) pig.registerQuery("result= LOAD 'cql://cql3ks/compotable?" + defaultParameters + "' USING CqlStorage();"); Iterator<Tuple> it = pig.openIterator("result"); - if (it.hasNext()) { - Tuple t = it.next(); - Assert.assertEquals(t.get(3), "match"); + int count = 0; + while (it.hasNext()) { + it.next(); + count ++; } + Assert.assertEquals(count, 9); } @Test http://git-wip-us.apache.org/repos/asf/cassandra/blob/f7efaffa/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java ---------------------------------------------------------------------- diff --git a/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java b/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java index 7bccc23..cc54620 100644 --- a/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java +++ b/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyDataTypeTest.java @@ -17,27 +17,6 @@ * under the License. */ package org.apache.cassandra.pig; -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - import java.io.IOException; import java.nio.charset.CharacterCodingException; http://git-wip-us.apache.org/repos/asf/cassandra/blob/f7efaffa/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java ---------------------------------------------------------------------- diff --git a/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java b/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java index 223cbf4..e114d37 100644 --- a/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java +++ b/test/pig/org/apache/cassandra/pig/ThriftColumnFamilyTest.java @@ -29,7 +29,6 @@ import org.apache.cassandra.thrift.AuthenticationException; import org.apache.cassandra.thrift.AuthorizationException; import org.apache.cassandra.thrift.Cassandra; import org.apache.cassandra.thrift.ColumnOrSuperColumn; -import org.apache.cassandra.thrift.ColumnParent; import org.apache.cassandra.thrift.ColumnPath; import org.apache.cassandra.thrift.ConsistencyLevel; import org.apache.cassandra.thrift.InvalidRequestException; @@ -210,9 +209,8 @@ public class ThriftColumnFamilyTest extends PigTestBase while (it.hasNext()) { count ++; Tuple t = it.next(); - if (count == 1) + if ("bar".equals(t.get(0))) { - Assert.assertEquals(t.get(0), "bar"); Assert.assertEquals(t.get(1), 3.141592653589793d); Assert.assertEquals(t.get(3), "User Bar"); Assert.assertEquals(t.get(4), 35.0f); @@ -220,16 +218,16 @@ public class ThriftColumnFamilyTest extends PigTestBase Assert.assertEquals(t.get(6), 15000L); Assert.assertEquals(t.get(7), "like"); } - else if (count == 2) + else if ("baz".equals(t.get(0))) { - Assert.assertEquals(t.get(0), "baz"); Assert.assertEquals(t.get(1), 1.61803399d); Assert.assertEquals(t.get(3), "User Baz"); Assert.assertEquals(t.get(4), 95.3f); Assert.assertEquals(t.get(5), 3); Assert.assertEquals(t.get(6), 512000L); Assert.assertEquals(t.get(7), "dislike"); - }else if (count == 3) + } + else if ("foo".equals(t.get(0))) { Assert.assertEquals(t.get(0), "foo"); Assert.assertEquals(t.get(1), 2.718281828459045d); @@ -239,7 +237,7 @@ public class ThriftColumnFamilyTest extends PigTestBase Assert.assertEquals(t.get(6), 125000L); Assert.assertEquals(t.get(7), "like"); } - else if (count == 4) + else if ("qux".equals(t.get(0))) { Assert.assertEquals(t.get(0), "qux"); Assert.assertEquals(t.get(1), 0.660161815846869d); @@ -264,18 +262,10 @@ public class ThriftColumnFamilyTest extends PigTestBase while (it.hasNext()) { count ++; Tuple t = it.next(); - if (count == 1) - { - Assert.assertEquals(t.get(0), "chuck"); - Assert.assertEquals(t.get(1), "fist"); + if ("chuck".equals(t.get(0)) && "fist".equals(t.get(1))) Assert.assertEquals(t.get(2), 1L); - } - else if (count == 2) - { - Assert.assertEquals(t.get(0), "chuck"); - Assert.assertEquals(t.get(1), "kick"); + else if ("chuck".equals(t.get(0)) && "kick".equals(t.get(1))) Assert.assertEquals(t.get(2), 3L); - } } Assert.assertEquals(count, 2); @@ -293,34 +283,14 @@ public class ThriftColumnFamilyTest extends PigTestBase while (it.hasNext()) { count ++; Tuple t = it.next(); - if (count == 1) - { - Assert.assertEquals(t.get(0), "kick"); - Assert.assertEquals(t.get(1), "bruce"); - Assert.assertEquals(t.get(2), "bruce"); + if ("kick".equals(t.get(0)) && "bruce".equals(t.get(1)) && "bruce".equals(t.get(2))) Assert.assertEquals(t.get(3), "watch it, mate"); - } - else if (count == 2) - { - Assert.assertEquals(t.get(0), "kick"); - Assert.assertEquals(t.get(1), "bruce"); - Assert.assertEquals(t.get(2), "lee"); + else if ("kick".equals(t.get(0)) && "bruce".equals(t.get(1)) && "lee".equals(t.get(2))) Assert.assertEquals(t.get(3), "oww"); - } - else if (count == 3) - { - Assert.assertEquals(t.get(0), "punch"); - Assert.assertEquals(t.get(1), "bruce"); - Assert.assertEquals(t.get(2), "bruce"); + else if ("punch".equals(t.get(0)) && "bruce".equals(t.get(1)) && "bruce".equals(t.get(2))) Assert.assertEquals(t.get(3), "hunh?"); - } - else if (count == 4) - { - Assert.assertEquals(t.get(0), "punch"); - Assert.assertEquals(t.get(1), "bruce"); - Assert.assertEquals(t.get(2), "lee"); + else if ("punch".equals(t.get(0)) && "bruce".equals(t.get(1)) && "lee".equals(t.get(2))) Assert.assertEquals(t.get(3), "ouch"); - } } Assert.assertEquals(count, 4); } @@ -340,7 +310,6 @@ public class ThriftColumnFamilyTest extends PigTestBase //rating: (name: chararray,value: int),score: (name: chararray,value: long), //vote_type: (name: chararray,value: chararray),columns: {(name: chararray,value: chararray)}} Iterator<Tuple> it = pig.openIterator("rows"); - int count = 0; if (it.hasNext()) { Tuple t = it.next(); String rowKey = t.get(0).toString(); @@ -623,16 +592,12 @@ public class ThriftColumnFamilyTest extends PigTestBase while (it.hasNext()) { count ++; Tuple t = it.next(); - if (count == 1) - Assert.assertEquals(t.get(0), "kick"); - else - Assert.assertEquals(t.get(0), "punch"); Tuple t1 = (Tuple) t.get(1); Assert.assertEquals(t1.get(0), "bruce"); Assert.assertEquals(t1.get(1), "lee"); - if (count == 1) + if ("kick".equals(t.get(0))) Assert.assertEquals(t.get(2), "oww"); - else + else if ("kick".equals(t.get(0))) Assert.assertEquals(t.get(2), "ouch"); } Assert.assertEquals(count, 2); @@ -668,30 +633,14 @@ public class ThriftColumnFamilyTest extends PigTestBase count ++; Tuple t1 = iter.next(); Tuple inner = (Tuple) t1.get(0); - if (count == 1) - { - Assert.assertEquals(inner.get(0), 1L); - Assert.assertEquals(inner.get(1), 0L); + if ((Long) inner.get(0) == 1L && (Long) inner.get(1) == 0L) Assert.assertEquals(t1.get(1), "z"); - } - else if (count == 2) - { - Assert.assertEquals(inner.get(0), 1L); - Assert.assertEquals(inner.get(1), 30L); + else if ((Long) inner.get(0) == 1L && (Long) inner.get(1) == 30L) Assert.assertEquals(t1.get(1), "zzzz"); - } - else if (count == 3) - { - Assert.assertEquals(inner.get(0), 2L); - Assert.assertEquals(inner.get(1), 30L); + else if ((Long) inner.get(0) == 2L && (Long) inner.get(1) == 30L) Assert.assertEquals(t1.get(1), "daddy?"); - } - else if (count == 4) - { - Assert.assertEquals(inner.get(0), 6L); - Assert.assertEquals(inner.get(1), 30L); + else if ((Long) inner.get(0) == 6L && (Long) inner.get(1) == 30L) Assert.assertEquals(t1.get(1), "coffee..."); - } } Assert.assertEquals(count, 4); } @@ -733,11 +682,9 @@ public class ThriftColumnFamilyTest extends PigTestBase while (it.hasNext()) { Tuple t = it.next(); count ++; - if (count == 1) + Tuple key = (Tuple) t.get(0); + if ("clock".equals(key.get(0)) && (Long) key.get(1) == 10L) { - Tuple key = (Tuple) t.get(0); - Assert.assertEquals(key.get(0), "clock"); - Assert.assertEquals(key.get(1), 10L); DataBag columns = (DataBag) t.get(1); Iterator<Tuple> iter = columns.iterator(); if (iter.hasNext()) @@ -747,46 +694,37 @@ public class ThriftColumnFamilyTest extends PigTestBase Assert.assertEquals(t1.get(1), "z"); } } - else if (count == 2) + else if ("clock".equals(key.get(0)) && (Long) key.get(1) == 40L) { - Tuple key = (Tuple) t.get(0); - Assert.assertEquals(key.get(0), "clock"); - Assert.assertEquals(key.get(1), 20L); DataBag columns = (DataBag) t.get(1); Iterator<Tuple> iter = columns.iterator(); if (iter.hasNext()) { Tuple t1 = iter.next(); - Assert.assertEquals(t1.get(0), 1L); - Assert.assertEquals(t1.get(1), "zzzz"); + Assert.assertEquals(t1.get(0), 6L); + Assert.assertEquals(t1.get(1), "coffee..."); } } - else if (count == 3) + else if ("clock".equals(key.get(0)) && (Long) key.get(1) == 20L) { - Tuple key = (Tuple) t.get(0); - Assert.assertEquals(key.get(0), "clock"); - Assert.assertEquals(key.get(1), 30L); DataBag columns = (DataBag) t.get(1); Iterator<Tuple> iter = columns.iterator(); if (iter.hasNext()) { Tuple t1 = iter.next(); - Assert.assertEquals(t1.get(0), 2L); - Assert.assertEquals(t1.get(1), "daddy?"); + Assert.assertEquals(t1.get(0), 1L); + Assert.assertEquals(t1.get(1), "zzzz"); } } - else if (count == 4) + else if ("clock".equals(key.get(0)) && (Long) key.get(1) == 30L) { - Tuple key = (Tuple) t.get(0); - Assert.assertEquals(key.get(0), "clock"); - Assert.assertEquals(key.get(1), 40L); DataBag columns = (DataBag) t.get(1); Iterator<Tuple> iter = columns.iterator(); if (iter.hasNext()) { Tuple t1 = iter.next(); - Assert.assertEquals(t1.get(0), 6L); - Assert.assertEquals(t1.get(1), "coffee..."); + Assert.assertEquals(t1.get(0), 2L); + Assert.assertEquals(t1.get(1), "daddy?"); } } } @@ -800,10 +738,7 @@ public class ThriftColumnFamilyTest extends PigTestBase client.set_keyspace(ks); ByteBuffer key_user_id = ByteBufferUtil.bytes(key); - - long timestamp = System.currentTimeMillis(); ColumnPath cp = new ColumnPath(cf); - ColumnParent par = new ColumnParent(cf); cp.column = ByteBufferUtil.bytes(colName); // read
