Author: yanz Date: Fri Dec 4 22:39:17 2009 New Revision: 887401 URL: http://svn.apache.org/viewvc?rev=887401&view=rev Log: PIG-1119 Support of group as a column name (Gaurav Jain via yanz)
Added: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnNameGroup.java Modified: hadoop/pig/trunk/contrib/zebra/CHANGES.txt hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName2.java hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnGroupName.java hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnSecurity.java hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc1.java Modified: hadoop/pig/trunk/contrib/zebra/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/CHANGES.txt?rev=887401&r1=887400&r2=887401&view=diff ============================================================================== --- hadoop/pig/trunk/contrib/zebra/CHANGES.txt (original) +++ hadoop/pig/trunk/contrib/zebra/CHANGES.txt Fri Dec 4 22:39:17 2009 @@ -11,6 +11,8 @@ IMPROVEMENTS + PIG-1119 Support of "group" as a column name (Gaurav Jain via yanz) + PIG-653 Pig Projection Push Down (Gaurav Jain via yanz) PIG-1111 Multiple Outputs Support (Gaurav Jain via yanz) Modified: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt?rev=887401&r1=887400&r2=887401&view=diff ============================================================================== --- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt (original) +++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt Fri Dec 4 22:39:17 2009 @@ -60,8 +60,8 @@ TOKEN : { <ASC : "ASC"> } TOKEN : { <DESC: "DESC"> } TOKEN : { <SECURE : "secure by"> } -TOKEN : { <USER : "user"> } -TOKEN : { <GROUP : "group"> } +TOKEN : { <USER : "uid"> } +TOKEN : { <GROUP : "gid"> } TOKEN : { <PERM : "perm"> } TOKEN : { <AS : "as"> } @@ -224,7 +224,7 @@ { if(owner != null) { - String msg = "User defined more than once"; + String msg = "Uid defined more than once"; throw new ParseException(msg); } else owner = t1.image; @@ -236,7 +236,7 @@ { if(group != null) { - String msg = "Group defined more than once"; + String msg = "Gid defined more than once"; throw new ParseException(msg); } else group = t1.image; Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName2.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName2.java?rev=887401&r1=887400&r2=887401&view=diff ============================================================================== --- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName2.java (original) +++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName2.java Fri Dec 4 22:39:17 2009 @@ -42,7 +42,7 @@ public class TestColumnGroupName2 { final static String STR_SCHEMA = "s1:bool, s2:int, s3:long, s4:float, s5:string, s6:bytes"; - final static String STR_STORAGE = "[s1, s2] as PI secure by user:chaow perm:777; [s3, s4] as General; [s5, s6] as ULT"; + final static String STR_STORAGE = "[s1, s2] as PI secure by uid:users perm:777; [s3, s4] as General; [s5, s6] as ULT"; private static Configuration conf; private static Path path; private static FileSystem fs; Added: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnNameGroup.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnNameGroup.java?rev=887401&view=auto ============================================================================== --- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnNameGroup.java (added) +++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnNameGroup.java Fri Dec 4 22:39:17 2009 @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.zebra.io; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.StringTokenizer; + +import junit.framework.Assert; +import junit.framework.TestCase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.zebra.io.BasicTable; +import org.apache.hadoop.zebra.io.TableInserter; +import org.apache.hadoop.zebra.io.TableScanner; +import org.apache.hadoop.zebra.io.BasicTable.Reader.RangeSplit; +import org.apache.hadoop.zebra.parser.ParseException; +import org.apache.hadoop.zebra.types.Projection; +import org.apache.hadoop.zebra.schema.Schema; +import org.apache.hadoop.zebra.types.TypesUtils; +import org.apache.pig.backend.executionengine.ExecException; +import org.apache.pig.data.DataBag; +import org.apache.pig.data.DataByteArray; +import org.apache.pig.data.Tuple; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * + * Test projections on complicated column types. + * + */ +public class TestColumnNameGroup { + // final static String STR_SCHEMA = + // "s1:bool, s2:int, s3:long, s4:float, s5:string, s6:bytes, r1:record(f1:int, f2:long), m1:map(string), c:collection(f13:double, f14:float, f15:bytes)"; + // final static String STR_STORAGE = + // "[s1, s2]; [r1.f1]; [s3, s4]; [s5, s6]; [r1.f2]; [c.f13]"; + private static Configuration conf; + private static Path path; + private static FileSystem fs; + + @BeforeClass + public static void setUp() throws IOException { + conf = new Configuration(); + conf.setInt("table.output.tfile.minBlock.size", 64 * 1024); + conf.setInt("table.input.split.minSize", 64 * 1024); + conf.set("table.output.tfile.compression", "none"); + + RawLocalFileSystem rawLFS = new RawLocalFileSystem(); + fs = new LocalFileSystem(rawLFS); + path = new Path(fs.getWorkingDirectory(), "TestColumnNameGroup"); + fs = path.getFileSystem(conf); + // drop any previous tables + BasicTable.drop(path, conf); + } + + /** + * Return the name of the routine that called getCurrentMethodName + * + */ + public String getCurrentMethodName() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + (new Throwable()).printStackTrace(pw); + pw.flush(); + String stackTrace = baos.toString(); + pw.close(); + + StringTokenizer tok = new StringTokenizer(stackTrace, "\n"); + tok.nextToken(); // 'java.lang.Throwable' + tok.nextToken(); // 'at ...getCurrentMethodName' + String l = tok.nextToken(); // 'at ...<caller to getCurrentRoutine>' + // Parse line 3 + tok = new StringTokenizer(l.trim(), " <("); + String t = tok.nextToken(); // 'at' + t = tok.nextToken(); // '...<caller to getCurrentRoutine>' + return t; + } + + @AfterClass + public static void tearDownOnce() throws IOException { + // BasicTable.drop(path, conf); + } + + @Test + public void testSimple() throws IOException, ParseException { + String STR_SCHEMA = "group:string, s6:bytes"; + String STR_STORAGE = "[group]; [s6]"; + + // Build Table and column groups + BasicTable.Writer writer = new BasicTable.Writer(path, STR_SCHEMA, + STR_STORAGE, conf); + writer.finish(); + Schema schema = writer.getSchema(); + Tuple tuple = TypesUtils.createTuple(schema); + BasicTable.Writer writer1 = new BasicTable.Writer(path, conf); + int part = 0; + TableInserter inserter = writer1.getInserter("part" + part, true); + TypesUtils.resetTuple(tuple); + + // insert data in row 1 + int row = 0; + + tuple.set(0, "hello world 1"); // string + tuple.set(1, new DataByteArray("hello byte 1")); // byte + inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1) + .getBytes()), tuple); + + // insert data in row 2 + row++; + + tuple.set(0, "hello world 2"); // string + tuple.set(1, new DataByteArray("hello byte 2")); // byte + inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1) + .getBytes()), tuple); + + // finish building table, closing out the inserter, writer, writer1 + inserter.close(); + writer1.finish(); + writer.close(); + + // Starting read + String projection = new String("group"); + BasicTable.Reader reader = new BasicTable.Reader(path, conf); + reader.setProjection(projection); + List<RangeSplit> splits = reader.rangeSplit(1); + TableScanner scanner = reader.getScanner(splits.get(0), true); + BytesWritable key = new BytesWritable(); + Tuple RowValue = TypesUtils.createTuple(scanner.getSchema()); + + scanner.getKey(key); + Assert.assertEquals(key, new BytesWritable("k11".getBytes())); + scanner.getValue(RowValue); + + Assert.assertEquals("hello world 1", RowValue.get(0)); + + scanner.advance(); + scanner.getKey(key); + Assert.assertEquals(key, new BytesWritable("k12".getBytes())); + scanner.getValue(RowValue); + + Assert.assertEquals("hello world 2", RowValue.get(0)); + + + reader.close(); + } +} Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnGroupName.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnGroupName.java?rev=887401&r1=887400&r2=887401&view=diff ============================================================================== --- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnGroupName.java (original) +++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnGroupName.java Fri Dec 4 22:39:17 2009 @@ -73,7 +73,7 @@ @Test public void testStorageValid1() { try { - String strStorage = "[f1, f2] as PI; [f3, f4] as General secure by user:joe perm:640 COMPRESS BY gz SERIALIZE BY avro; [f5, f6] as ULT"; + String strStorage = "[f1, f2] as PI; [f3, f4] as General secure by uid:joe gid:secure perm:640 COMPRESS BY gz SERIALIZE BY avro; [f5, f6] as ULT"; Partition p = new Partition(schema.toString(), strStorage, null); CGSchema[] cgschemas = p.getCGSchemas(); Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnSecurity.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnSecurity.java?rev=887401&r1=887400&r2=887401&view=diff ============================================================================== --- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnSecurity.java (original) +++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnSecurity.java Fri Dec 4 22:39:17 2009 @@ -127,7 +127,7 @@ */ System.out.println("In test1..."); String schema = "s1:string, s2:string"; - String storage = "[s1, s2 ] secure by user:user1 group:group1 perm:777"; + String storage = "[s1, s2 ] secure by uid:user1 gid:group1 perm:777"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest1"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -176,7 +176,7 @@ */ System.out.println("In test2..."); String schema = "s1:string, s2:string"; - String storage = "[s1, s2] secure by user:user1 group:group2 perm:730"; + String storage = "[s1, s2] secure by uid:user1 gid:group2 perm:730"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest2"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -237,7 +237,7 @@ */ System.out.println("In test3..."); String schema = "s1:string, s2:string"; - String storage = "[s1, s2] secure by user:user1 group:group2 perm:700"; + String storage = "[s1, s2] secure by uid:user1 gid:group2 perm:700"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest3"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -294,7 +294,7 @@ */ System.out.println("In test4..."); String schema = "s1:string, s2:string"; - String storage = "[s1, s2] secure by user:user1 group:group2 perm:740"; + String storage = "[s1, s2] secure by uid:user1 gid:group2 perm:740"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest4"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -352,7 +352,7 @@ */ System.out.println("In test5..."); String schema = "s1:string, s2:string"; - String storage = "[s1, s2] secure by user:user1 group:group1 perm:702"; + String storage = "[s1, s2] secure by uid:user1 gid:group1 perm:702"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest5"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -408,7 +408,7 @@ */ System.out.println("In test6..."); String schema = "s1:string, s2:string"; - String storage = "[s1,s2] secure by user:user1 group:group1 perm:705;"; + String storage = "[s1,s2] secure by uid:user1 gid:group1 perm:705;"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest6"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -466,7 +466,7 @@ */ System.out.println("In test7..."); String schema = "s1:string, s2:string"; - String storage = "[s1, s2] secure by user:user1 group:group1 perm:200"; + String storage = "[s1, s2] secure by uid:user1 gid:group1 perm:200"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest7"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -488,7 +488,7 @@ */ System.out.println("In test8..."); String schema = "s1:string, s2:string"; - String storage = "[s1] secure by user:user1 group:group2 perm:770; [s2]"; + String storage = "[s1] secure by uid:user1 gid:group2 perm:770; [s2]"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest8"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -560,7 +560,7 @@ */ System.out.println("In test9..."); String schema = "s1:string, s2:string"; - String storage = "[s1, s2] secure by user:user1 group:group2 perm:707"; + String storage = "[s1, s2] secure by uid:user1 gid:group2 perm:707"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest9"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -618,7 +618,7 @@ */ System.out.println("In test10..."); String schema = "s1:string, s2:string"; - String storage = "[s1, s2] secure by user:user1 group:group1 perm:400"; + String storage = "[s1, s2] secure by uid:user1 gid:group1 perm:400"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest10"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -640,7 +640,7 @@ */ System.out.println("In test11..."); String schema = "s1:string, s2:string"; - String storage = "[s1] secure by user:user1 group:group2 perm:770;[s2] secure by user:user1 group:group2 perm:770"; + String storage = "[s1] secure by uid:user1 gid:group2 perm:770;[s2] secure by user:user1 gid:group2 perm:770"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest11"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -678,7 +678,7 @@ */ System.out.println("In test12..."); String schema = "s1:string, s2:string"; - String storage = "[s1] secure by user:user1 group:group2 perm:770;[s2] secure by user:user1 group:group2 perm:700"; + String storage = "[s1] secure by uid:user1 gid:group2 perm:770;[s2] secure by user:user1 gid:group2 perm:700"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest12"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -717,7 +717,7 @@ */ System.out.println("In test13..."); String schema = "s1:string, s2:string"; - String storage = "[s1] secure by user:user1 group:group2 perm:700;[s2] secure by user:user1 group:group2 perm:770"; + String storage = "[s1] secure by uid:user1 gid:group2 perm:700;[s2] secure by user:user1 gid:group2 perm:770"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest13"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -755,7 +755,7 @@ */ System.out.println("In test14..."); String schema = "s1:string, s2:string"; - String storage = "[s1] secure by user:user1 group:group2 perm:730;[s2] secure by user:user1 group:group2 perm:730"; + String storage = "[s1] secure by uid:user1 gid:group2 perm:730;[s2] secure by user:user1 gid:group2 perm:730"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest14"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -793,7 +793,7 @@ */ System.out.println("In test15..."); String schema = "s1:string, s2:string"; - String storage = "[s1] secure by user:user1 group:group2 perm:770;[s2] secure by user:user1 group:group2 perm:770"; + String storage = "[s1] secure by uid:user1 gid:group2 perm:770;[s2] secure by user:user1 gid:group2 perm:770"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest15"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -905,7 +905,7 @@ */ System.out.println("In test17..."); String schema = "s1:string, s2:string"; - String storage = "[s1, s2] secure by user:user1 group:group1 perm:000"; + String storage = "[s1, s2] secure by uid:user1 gid:group1 perm:000"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest17"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -927,7 +927,7 @@ */ System.out.println("In test18..."); String schema = "s1:string, s2:string"; - String storage = "[s1, s2] secure by user:user1 group:nonexisting perm:000"; + String storage = "[s1, s2] secure by uid:user1 gid:nonexisting perm:000"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest18"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -949,7 +949,7 @@ */ System.out.println("In test19..."); String schema = "s1:string, s2:string"; - String storage = "[s1, s2] secure by user:user1 group:group2 perm:880"; + String storage = "[s1, s2] secure by uid:user1 gid:group2 perm:880"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest19"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -970,7 +970,7 @@ */ System.out.println("In test120..."); String schema = "s1:string, s2:string"; - String storage = "[s1, s2] secure by user:user1 group:group2 perm:7777"; + String storage = "[s1, s2] secure by uid:user1 gid:group2 perm:7777"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest20"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -992,9 +992,9 @@ */ System.out.println("In test21..."); String schema = "s1:string, s2:string"; - String storage = "[s1] secure by user:" + String storage = "[s1] secure by uid:" + SUPERUSER.getUserName() - + " group:supergroup perm:755;[s2] secure by user:user1 group:group2 perm:755"; + + " gid:supergroup perm:755;[s2] secure by uid:user1 gid:group2 perm:755"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest21"); login(SUPERUSER); fs = path1.getFileSystem(conf); @@ -1065,9 +1065,9 @@ */ System.out.println("In test22..."); String schema = "s1:string, s2:string"; - String storage = "[s1] secure by user:" + String storage = "[s1] secure by uid:" + SUPERUSER.getUserName() - + " group:supergroup perm:775;[s2] secure by user:user1 group:group2 perm:775"; + + " gid:supergroup perm:775;[s2] secure by uid:user1 gid:group2 perm:775"; Path path1 = new Path(path.toString() + "/TestColumnStorageTest22"); login(SUPERUSER); fs = path1.getFileSystem(conf); Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java?rev=887401&r1=887400&r2=887401&view=diff ============================================================================== --- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java (original) +++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java Fri Dec 4 22:39:17 2009 @@ -91,26 +91,26 @@ System.setProperty("group", "users"); } group = System.getProperty("group"); - System.out.println("user:" + user + " group: " + group); - STR_STORAGE = "[s1, s2] COMPRESS BY gz SECURE BY user:" + System.out.println("uid:" + user + " gid: " + group); + STR_STORAGE = "[s1, s2] COMPRESS BY gz SECURE BY uid:" + user - + " group:" + + " gid:" + group - + " perm:777 SERIALIZE BY pig; [m1#{a}] SERIALIZE BY pig COMPRESS BY gz SECURE BY user:" + + " perm:777 SERIALIZE BY pig; [m1#{a}] SERIALIZE BY pig COMPRESS BY gz SECURE BY uid:" + user - + " group:" + + " gid:" + group - + " perm:777 ; [r1.f1] SECURE BY user:" + + " perm:777 ; [r1.f1] SECURE BY uid:" + user - + " group:" + + " gid:" + group - + " perm:777 SERIALIZE BY pig COMPRESS BY gz ; [s3, s4, r2.r3.f3] SERIALIZE BY pig SECURE BY user:" + + " perm:777 SERIALIZE BY pig COMPRESS BY gz ; [s3, s4, r2.r3.f3] SERIALIZE BY pig SECURE BY uid:" + user - + " group: " + + " gid: " + group - + " perm:777 COMPRESS BY gz ; [s5, s6, m2#{x|y}] compREss by gz secURe by user:" + + " perm:777 COMPRESS BY gz ; [s5, s6, m2#{x|y}] compREss by gz secURe by uid:" + user - + " group:" + + " gid:" + group + " perm:777 SerialIZE BY pig; [r1.f2, m1#{b}]; [r2.r3.f4, m2#{z}] SERIALIZE BY avro;[s7,s8] SERIALIZE BY AVRO;[s9,s10] COMPRESS BY gz "; System.out.println("storage: " + STR_STORAGE); @@ -118,7 +118,7 @@ conf.setInt("table.input.split.minSize", 64 * 1024); conf.set("table.output.tfile.compression", "none"); // String STR_STORAGE1 = - // "[s1, s2] COMPRESS BY gz SECURE BY user:"+USER1.getUserName()+" group:"+USER1.getGroupNames()[0]+" perm:777 SERIALIZE BY pig"; + // "[s1, s2] COMPRESS BY gz SECURE BY uid:"+USER1.getUserName()+" gid:"+USER1.getGroupNames()[0]+" perm:777 SERIALIZE BY pig"; RawLocalFileSystem rawLFS = new RawLocalFileSystem(); fs = new LocalFileSystem(rawLFS); path = new Path(fs.getWorkingDirectory(), "TestStorageGrammar"); @@ -326,7 +326,7 @@ public void test1() throws IOException, ParseException { String schema = "s1:string, s2:string"; - String storage = "[s1, s2]COMPRESS BY gz SECURE BY user:user1 group:users perm:744 SERIALIZE BY pig"; + String storage = "[s1, s2]COMPRESS BY gz SECURE BY uid:user1 gid:users perm:744 SERIALIZE BY pig"; RawLocalFileSystem rawLFS = new RawLocalFileSystem(); fs = new LocalFileSystem(rawLFS); Path path1 = new Path(path.toString() + "1"); @@ -350,8 +350,8 @@ @Test public void test2() throws IOException, ParseException { String schema = "m1:map(string),m2:map(map(int))"; - String storage = "[m1#{a}] SERIALIZE BY pig COMPRESS BY gz SECURE BY user:" - + user + " group:" + group + " perm:770 "; + String storage = "[m1#{a}] SERIALIZE BY pig COMPRESS BY gz SECURE BY uid:" + + user + " gid:" + group + " perm:770 "; Path path1 = new Path(path.toString() + "2"); BasicTable.Writer writer = null; try { @@ -373,7 +373,7 @@ @Test public void test3() throws IOException, ParseException { String schema = "r1:record(f1:int, f2:long)"; - String storage = "[r1.f1] SECURE BY user:" + user + " group:" + group + String storage = "[r1.f1] SECURE BY uid:" + user + " gid:" + group + " perm:777 SERIALIZE BY pig COMPRESS BY gz"; Path path1 = new Path(path.toString() + "3"); BasicTable.Writer writer = null; @@ -395,8 +395,8 @@ @Test public void test4() throws IOException, ParseException { String schema = "s3:string, s4:string, r2:record(r3:record(f3:float, f4)), c:collection(f13:double, f14:float, f15:bytes)"; - String storage = "[s3, s4, r2.r3.f3, c] SERIALIZE BY pig SECURE BY user:" - + user + " group:" + group + " perm:777 COMPRESS BY gz"; + String storage = "[s3, s4, r2.r3.f3, c] SERIALIZE BY pig SECURE BY uid:" + + user + " gid:" + group + " perm:777 COMPRESS BY gz"; Path path1 = new Path(path.toString() + "4"); BasicTable.Writer writer = null; try { @@ -417,8 +417,8 @@ @Test public void test5() throws IOException, ParseException { String schema = "s5:string, s6:string, m2:map(map(int))"; - String storage = "[s5, s6, m2#{x|y}] compREss by gz secURe by user:" + user - + " group:" + group + " perm:777 SerialIZE BY pig"; + String storage = "[s5, s6, m2#{x|y}] compREss by gz secURe by uid:" + user + + " gid:" + group + " perm:777 SerialIZE BY pig"; Path path1 = new Path(path.toString() + "5"); BasicTable.Writer writer = null; try { @@ -541,7 +541,7 @@ @Test(expected = IOException.class) public void test12() throws IOException, ParseException { String schema = "some1:string, some2:string"; - String storage = "[some1,some2] COMPRESS BY gz SECURrrrrrE BY user:user1 group:grop1 perm:760 SERIALIZE BY pig"; + String storage = "[some1,some2] COMPRESS BY gz SECURrrrrrE BY uid:user1 gid:grop1 perm:760 SERIALIZE BY pig"; Path path1 = new Path(path.toString() + "12"); BasicTable.Writer writer = null; @@ -553,7 +553,7 @@ @Test(expected = IOException.class) public void test13() throws IOException, ParseException { String schema = "some1:string, some2:string"; - String storage = "[some1,some2] COMPRESS gz SECURE BY user:user1 group:users perm:760 SERIALIZE BY pig"; + String storage = "[some1,some2] COMPRESS gz SECURE BY uid:user1 gid:users perm:760 SERIALIZE BY pig"; Path path1 = new Path(path.toString() + "13"); BasicTable.Writer writer = null; @@ -565,7 +565,7 @@ @Test(expected = IOException.class) public void test14() throws IOException, ParseException { String schema = "some1:string, some2:string"; - String storage = "[some1,some2] COMPRESS BY gz SECURE BY usssser:user1 group:users perm:760 SERIALIZE BY pig"; + String storage = "[some1,some2] COMPRESS BY gz SECURE BY usssser:user1 gid:users perm:760 SERIALIZE BY pig"; Path path1 = new Path(path.toString() + "14"); BasicTable.Writer writer = null; writer = new BasicTable.Writer(path1, schema, storage, conf); @@ -576,7 +576,7 @@ @Test(expected = IOException.class) public void test15() throws IOException, ParseException { String schema = "some1:string, some2:string"; - String storage = "[some1,some2] COMPRESS BY gz SECURE BY user:user1 grouuuup:group1 perm:760 SERIALIZE BY pig"; + String storage = "[some1,some2] COMPRESS BY gz SECURE BY uid:user1 grouuuup:group1 perm:760 SERIALIZE BY pig"; Path path1 = new Path(path.toString() + "15"); BasicTable.Writer writer = null; writer = new BasicTable.Writer(path1, schema, storage, conf); @@ -587,7 +587,7 @@ @Test(expected = IOException.class) public void test16() throws IOException, ParseException { String schema = "some1:string, some2:string"; - String storage = "[some1,some2] COMPRESS BY gz SECURE BY user:user1 group:users perrrrrm:760 SERIALIZE BY pig"; + String storage = "[some1,some2] COMPRESS BY gz SECURE BY uid:user1 gid:users perrrrrm:760 SERIALIZE BY pig"; Path path1 = new Path(path.toString() + "16"); BasicTable.Writer writer = null; writer = new BasicTable.Writer(path1, schema, storage, conf); @@ -658,7 +658,7 @@ */ public void test20() throws IOException, ParseException { String schema = "s9:string, s10:string, s11:string"; - String storage = "[s9,s10] COMPRESS BY gZ; secure by user:user1 group:users perm:755"; + String storage = "[s9,s10] COMPRESS BY gZ; secure by uid:user1 gid:users perm:755"; Path path1 = new Path(path.toString() + "20"); Runtime.getRuntime().exec("rm -rf " + path1.toString()); @@ -681,7 +681,7 @@ @Test(expected = IOException.class) public void test21() throws IOException, ParseException { String schema = "s9:string, s10:string, s11:string"; - String storage = "[s9,s10] COMPRESS BY gZ; secure by user:user1 group:users perm:755;secure by user:user1 group:users perm:755"; + String storage = "[s9,s10] COMPRESS BY gZ; secure by uid:user1 gid:users perm:755;secure by user:user1 gid:users perm:755"; Path path1 = new Path(path.toString() + "16"); BasicTable.Writer writer = null; writer = new BasicTable.Writer(path1, schema, storage, conf); @@ -692,7 +692,7 @@ @Test public void test22() throws IOException, ParseException { String schema = "s9:string, s10:string"; - String storage = "[s9,s10] secure by user:user3"; + String storage = "[s9,s10] secure by uid:user3"; Path path1 = new Path(path.toString() + "22"); Runtime.getRuntime().exec("rm -rf " + path1.toString()); @@ -720,7 +720,7 @@ */ public void test23() throws IOException, ParseException { String schema = "s9:string, s10:string"; - String storage = "[s9,s10] secure by group:users"; + String storage = "[s9,s10] secure by gid:users"; Path path1 = new Path(path.toString() + "23"); Runtime.getRuntime().exec("rm -rf " + path1.toString()); @@ -773,7 +773,7 @@ */ public void test25() throws IOException, ParseException { String schema = "s9:string, s10:string"; - String storage = "[s9,s10] secure by user:user1 group:users"; + String storage = "[s9,s10] secure by uid:user1 gid:users"; Path path1 = new Path(path.toString() + "25"); Runtime.getRuntime().exec("rm -rf " + path1.toString()); @@ -798,7 +798,7 @@ //@Test public void test26() throws IOException, ParseException { String schema = "s9:string, s10:string"; - String storage = "[s9,s10] secure by perm:755 group:users"; + String storage = "[s9,s10] secure by perm:755 gid:users"; Path path1 = new Path(path.toString() + "26"); Runtime.getRuntime().exec("rm -rf " + path1.toString()); Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc1.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc1.java?rev=887401&r1=887400&r2=887401&view=diff ============================================================================== --- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc1.java (original) +++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc1.java Fri Dec 4 22:39:17 2009 @@ -81,13 +81,13 @@ @Test public void testStorageValid1() { try { - String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gz SECURE BY user:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro"; -// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY user:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro"; -// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY user:root group:data perm:0766; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro"; -// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY user:root group:data perm:966; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro"; + String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gz SECURE BY uid:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro"; +// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY uid:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro"; +// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY uid:root group:data perm:0766; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro"; +// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY uid:root group:data perm:966; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro"; // String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro"; -// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY user:ggg SECURE BY group:fff; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro"; -// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY user:root user:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro"; +// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY uid:ggg SECURE BY group:fff; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro"; +// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY uid:root user:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro"; Partition p = new Partition(schema.toString(), strStorage, null); CGSchema[] cgschemas = p.getCGSchemas(); @@ -161,6 +161,7 @@ } } } catch (Exception e) { + System.out.println("Error is [" + e.getMessage() + "]"); Assert.assertTrue(false); } }