Author: yanz
Date: Fri Dec 4 22:39:17 2009
New Revision: 887401
URL: http://svn.apache.org/viewvc?rev=887401&view=rev
Log:
PIG-1119 Support of group as a column name (Gaurav Jain via yanz)
Added:
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnNameGroup.java
Modified:
hadoop/pig/trunk/contrib/zebra/CHANGES.txt
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName2.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnGroupName.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnSecurity.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc1.java
Modified: hadoop/pig/trunk/contrib/zebra/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/CHANGES.txt?rev=887401&r1=887400&r2=887401&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/CHANGES.txt (original)
+++ hadoop/pig/trunk/contrib/zebra/CHANGES.txt Fri Dec 4 22:39:17 2009
@@ -11,6 +11,8 @@
IMPROVEMENTS
+ PIG-1119 Support of "group" as a column name (Gaurav Jain via yanz)
+
PIG-653 Pig Projection Push Down (Gaurav Jain via yanz)
PIG-1111 Multiple Outputs Support (Gaurav Jain via yanz)
Modified:
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt?rev=887401&r1=887400&r2=887401&view=diff
==============================================================================
---
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt
(original)
+++
hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt
Fri Dec 4 22:39:17 2009
@@ -60,8 +60,8 @@
TOKEN : { <ASC : "ASC"> }
TOKEN : { <DESC: "DESC"> }
TOKEN : { <SECURE : "secure by"> }
-TOKEN : { <USER : "user"> }
-TOKEN : { <GROUP : "group"> }
+TOKEN : { <USER : "uid"> }
+TOKEN : { <GROUP : "gid"> }
TOKEN : { <PERM : "perm"> }
TOKEN : { <AS : "as"> }
@@ -224,7 +224,7 @@
{
if(owner != null)
{
- String msg = "User defined more than once";
+ String msg = "Uid defined more than once";
throw new ParseException(msg);
} else
owner = t1.image;
@@ -236,7 +236,7 @@
{
if(group != null)
{
- String msg = "Group defined more than once";
+ String msg = "Gid defined more than once";
throw new ParseException(msg);
} else
group = t1.image;
Modified:
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName2.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName2.java?rev=887401&r1=887400&r2=887401&view=diff
==============================================================================
---
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName2.java
(original)
+++
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName2.java
Fri Dec 4 22:39:17 2009
@@ -42,7 +42,7 @@
public class TestColumnGroupName2 {
final static String STR_SCHEMA = "s1:bool, s2:int, s3:long, s4:float,
s5:string, s6:bytes";
- final static String STR_STORAGE = "[s1, s2] as PI secure by user:chaow
perm:777; [s3, s4] as General; [s5, s6] as ULT";
+ final static String STR_STORAGE = "[s1, s2] as PI secure by uid:users
perm:777; [s3, s4] as General; [s5, s6] as ULT";
private static Configuration conf;
private static Path path;
private static FileSystem fs;
Added:
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnNameGroup.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnNameGroup.java?rev=887401&view=auto
==============================================================================
---
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnNameGroup.java
(added)
+++
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnNameGroup.java
Fri Dec 4 22:39:17 2009
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.hadoop.zebra.io;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.StringTokenizer;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.io.TableScanner;
+import org.apache.hadoop.zebra.io.BasicTable.Reader.RangeSplit;
+import org.apache.hadoop.zebra.parser.ParseException;
+import org.apache.hadoop.zebra.types.Projection;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.Tuple;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ *
+ * Test projections on complicated column types.
+ *
+ */
+public class TestColumnNameGroup {
+ // final static String STR_SCHEMA =
+ // "s1:bool, s2:int, s3:long, s4:float, s5:string, s6:bytes,
r1:record(f1:int, f2:long), m1:map(string), c:collection(f13:double,
f14:float, f15:bytes)";
+ // final static String STR_STORAGE =
+ // "[s1, s2]; [r1.f1]; [s3, s4]; [s5, s6]; [r1.f2]; [c.f13]";
+ private static Configuration conf;
+ private static Path path;
+ private static FileSystem fs;
+
+ @BeforeClass
+ public static void setUp() throws IOException {
+ conf = new Configuration();
+ conf.setInt("table.output.tfile.minBlock.size", 64 * 1024);
+ conf.setInt("table.input.split.minSize", 64 * 1024);
+ conf.set("table.output.tfile.compression", "none");
+
+ RawLocalFileSystem rawLFS = new RawLocalFileSystem();
+ fs = new LocalFileSystem(rawLFS);
+ path = new Path(fs.getWorkingDirectory(), "TestColumnNameGroup");
+ fs = path.getFileSystem(conf);
+ // drop any previous tables
+ BasicTable.drop(path, conf);
+ }
+
+ /**
+ * Return the name of the routine that called getCurrentMethodName
+ *
+ */
+ public String getCurrentMethodName() {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ (new Throwable()).printStackTrace(pw);
+ pw.flush();
+ String stackTrace = baos.toString();
+ pw.close();
+
+ StringTokenizer tok = new StringTokenizer(stackTrace, "\n");
+ tok.nextToken(); // 'java.lang.Throwable'
+ tok.nextToken(); // 'at ...getCurrentMethodName'
+ String l = tok.nextToken(); // 'at ...<caller to getCurrentRoutine>'
+ // Parse line 3
+ tok = new StringTokenizer(l.trim(), " <(");
+ String t = tok.nextToken(); // 'at'
+ t = tok.nextToken(); // '...<caller to getCurrentRoutine>'
+ return t;
+ }
+
+ @AfterClass
+ public static void tearDownOnce() throws IOException {
+ // BasicTable.drop(path, conf);
+ }
+
+ @Test
+ public void testSimple() throws IOException, ParseException {
+ String STR_SCHEMA = "group:string, s6:bytes";
+ String STR_STORAGE = "[group]; [s6]";
+
+ // Build Table and column groups
+ BasicTable.Writer writer = new BasicTable.Writer(path, STR_SCHEMA,
+ STR_STORAGE, conf);
+ writer.finish();
+ Schema schema = writer.getSchema();
+ Tuple tuple = TypesUtils.createTuple(schema);
+ BasicTable.Writer writer1 = new BasicTable.Writer(path, conf);
+ int part = 0;
+ TableInserter inserter = writer1.getInserter("part" + part, true);
+ TypesUtils.resetTuple(tuple);
+
+ // insert data in row 1
+ int row = 0;
+
+ tuple.set(0, "hello world 1"); // string
+ tuple.set(1, new DataByteArray("hello byte 1")); // byte
+ inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
+ .getBytes()), tuple);
+
+ // insert data in row 2
+ row++;
+
+ tuple.set(0, "hello world 2"); // string
+ tuple.set(1, new DataByteArray("hello byte 2")); // byte
+ inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
+ .getBytes()), tuple);
+
+ // finish building table, closing out the inserter, writer, writer1
+ inserter.close();
+ writer1.finish();
+ writer.close();
+
+ // Starting read
+ String projection = new String("group");
+ BasicTable.Reader reader = new BasicTable.Reader(path, conf);
+ reader.setProjection(projection);
+ List<RangeSplit> splits = reader.rangeSplit(1);
+ TableScanner scanner = reader.getScanner(splits.get(0), true);
+ BytesWritable key = new BytesWritable();
+ Tuple RowValue = TypesUtils.createTuple(scanner.getSchema());
+
+ scanner.getKey(key);
+ Assert.assertEquals(key, new BytesWritable("k11".getBytes()));
+ scanner.getValue(RowValue);
+
+ Assert.assertEquals("hello world 1", RowValue.get(0));
+
+ scanner.advance();
+ scanner.getKey(key);
+ Assert.assertEquals(key, new BytesWritable("k12".getBytes()));
+ scanner.getValue(RowValue);
+
+ Assert.assertEquals("hello world 2", RowValue.get(0));
+
+
+ reader.close();
+ }
+}
Modified:
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnGroupName.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnGroupName.java?rev=887401&r1=887400&r2=887401&view=diff
==============================================================================
---
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnGroupName.java
(original)
+++
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnGroupName.java
Fri Dec 4 22:39:17 2009
@@ -73,7 +73,7 @@
@Test
public void testStorageValid1() {
try {
- String strStorage = "[f1, f2] as PI; [f3, f4] as General secure by
user:joe perm:640 COMPRESS BY gz SERIALIZE BY avro; [f5, f6] as ULT";
+ String strStorage = "[f1, f2] as PI; [f3, f4] as General secure by
uid:joe gid:secure perm:640 COMPRESS BY gz SERIALIZE BY avro; [f5, f6] as ULT";
Partition p = new Partition(schema.toString(), strStorage, null);
CGSchema[] cgschemas = p.getCGSchemas();
Modified:
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnSecurity.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnSecurity.java?rev=887401&r1=887400&r2=887401&view=diff
==============================================================================
---
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnSecurity.java
(original)
+++
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnSecurity.java
Fri Dec 4 22:39:17 2009
@@ -127,7 +127,7 @@
*/
System.out.println("In test1...");
String schema = "s1:string, s2:string";
- String storage = "[s1, s2 ] secure by user:user1 group:group1 perm:777";
+ String storage = "[s1, s2 ] secure by uid:user1 gid:group1 perm:777";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest1");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -176,7 +176,7 @@
*/
System.out.println("In test2...");
String schema = "s1:string, s2:string";
- String storage = "[s1, s2] secure by user:user1 group:group2 perm:730";
+ String storage = "[s1, s2] secure by uid:user1 gid:group2 perm:730";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest2");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -237,7 +237,7 @@
*/
System.out.println("In test3...");
String schema = "s1:string, s2:string";
- String storage = "[s1, s2] secure by user:user1 group:group2 perm:700";
+ String storage = "[s1, s2] secure by uid:user1 gid:group2 perm:700";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest3");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -294,7 +294,7 @@
*/
System.out.println("In test4...");
String schema = "s1:string, s2:string";
- String storage = "[s1, s2] secure by user:user1 group:group2 perm:740";
+ String storage = "[s1, s2] secure by uid:user1 gid:group2 perm:740";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest4");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -352,7 +352,7 @@
*/
System.out.println("In test5...");
String schema = "s1:string, s2:string";
- String storage = "[s1, s2] secure by user:user1 group:group1 perm:702";
+ String storage = "[s1, s2] secure by uid:user1 gid:group1 perm:702";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest5");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -408,7 +408,7 @@
*/
System.out.println("In test6...");
String schema = "s1:string, s2:string";
- String storage = "[s1,s2] secure by user:user1 group:group1 perm:705;";
+ String storage = "[s1,s2] secure by uid:user1 gid:group1 perm:705;";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest6");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -466,7 +466,7 @@
*/
System.out.println("In test7...");
String schema = "s1:string, s2:string";
- String storage = "[s1, s2] secure by user:user1 group:group1 perm:200";
+ String storage = "[s1, s2] secure by uid:user1 gid:group1 perm:200";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest7");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -488,7 +488,7 @@
*/
System.out.println("In test8...");
String schema = "s1:string, s2:string";
- String storage = "[s1] secure by user:user1 group:group2 perm:770; [s2]";
+ String storage = "[s1] secure by uid:user1 gid:group2 perm:770; [s2]";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest8");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -560,7 +560,7 @@
*/
System.out.println("In test9...");
String schema = "s1:string, s2:string";
- String storage = "[s1, s2] secure by user:user1 group:group2 perm:707";
+ String storage = "[s1, s2] secure by uid:user1 gid:group2 perm:707";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest9");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -618,7 +618,7 @@
*/
System.out.println("In test10...");
String schema = "s1:string, s2:string";
- String storage = "[s1, s2] secure by user:user1 group:group1 perm:400";
+ String storage = "[s1, s2] secure by uid:user1 gid:group1 perm:400";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest10");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -640,7 +640,7 @@
*/
System.out.println("In test11...");
String schema = "s1:string, s2:string";
- String storage = "[s1] secure by user:user1 group:group2 perm:770;[s2]
secure by user:user1 group:group2 perm:770";
+ String storage = "[s1] secure by uid:user1 gid:group2 perm:770;[s2] secure
by user:user1 gid:group2 perm:770";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest11");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -678,7 +678,7 @@
*/
System.out.println("In test12...");
String schema = "s1:string, s2:string";
- String storage = "[s1] secure by user:user1 group:group2 perm:770;[s2]
secure by user:user1 group:group2 perm:700";
+ String storage = "[s1] secure by uid:user1 gid:group2 perm:770;[s2] secure
by user:user1 gid:group2 perm:700";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest12");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -717,7 +717,7 @@
*/
System.out.println("In test13...");
String schema = "s1:string, s2:string";
- String storage = "[s1] secure by user:user1 group:group2 perm:700;[s2]
secure by user:user1 group:group2 perm:770";
+ String storage = "[s1] secure by uid:user1 gid:group2 perm:700;[s2] secure
by user:user1 gid:group2 perm:770";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest13");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -755,7 +755,7 @@
*/
System.out.println("In test14...");
String schema = "s1:string, s2:string";
- String storage = "[s1] secure by user:user1 group:group2 perm:730;[s2]
secure by user:user1 group:group2 perm:730";
+ String storage = "[s1] secure by uid:user1 gid:group2 perm:730;[s2] secure
by user:user1 gid:group2 perm:730";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest14");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -793,7 +793,7 @@
*/
System.out.println("In test15...");
String schema = "s1:string, s2:string";
- String storage = "[s1] secure by user:user1 group:group2 perm:770;[s2]
secure by user:user1 group:group2 perm:770";
+ String storage = "[s1] secure by uid:user1 gid:group2 perm:770;[s2] secure
by user:user1 gid:group2 perm:770";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest15");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -905,7 +905,7 @@
*/
System.out.println("In test17...");
String schema = "s1:string, s2:string";
- String storage = "[s1, s2] secure by user:user1 group:group1 perm:000";
+ String storage = "[s1, s2] secure by uid:user1 gid:group1 perm:000";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest17");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -927,7 +927,7 @@
*/
System.out.println("In test18...");
String schema = "s1:string, s2:string";
- String storage = "[s1, s2] secure by user:user1 group:nonexisting
perm:000";
+ String storage = "[s1, s2] secure by uid:user1 gid:nonexisting perm:000";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest18");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -949,7 +949,7 @@
*/
System.out.println("In test19...");
String schema = "s1:string, s2:string";
- String storage = "[s1, s2] secure by user:user1 group:group2 perm:880";
+ String storage = "[s1, s2] secure by uid:user1 gid:group2 perm:880";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest19");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -970,7 +970,7 @@
*/
System.out.println("In test120...");
String schema = "s1:string, s2:string";
- String storage = "[s1, s2] secure by user:user1 group:group2 perm:7777";
+ String storage = "[s1, s2] secure by uid:user1 gid:group2 perm:7777";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest20");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -992,9 +992,9 @@
*/
System.out.println("In test21...");
String schema = "s1:string, s2:string";
- String storage = "[s1] secure by user:"
+ String storage = "[s1] secure by uid:"
+ SUPERUSER.getUserName()
- + " group:supergroup perm:755;[s2] secure by user:user1 group:group2
perm:755";
+ + " gid:supergroup perm:755;[s2] secure by uid:user1 gid:group2
perm:755";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest21");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
@@ -1065,9 +1065,9 @@
*/
System.out.println("In test22...");
String schema = "s1:string, s2:string";
- String storage = "[s1] secure by user:"
+ String storage = "[s1] secure by uid:"
+ SUPERUSER.getUserName()
- + " group:supergroup perm:775;[s2] secure by user:user1 group:group2
perm:775";
+ + " gid:supergroup perm:775;[s2] secure by uid:user1 gid:group2
perm:775";
Path path1 = new Path(path.toString() + "/TestColumnStorageTest22");
login(SUPERUSER);
fs = path1.getFileSystem(conf);
Modified:
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java?rev=887401&r1=887400&r2=887401&view=diff
==============================================================================
---
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java
(original)
+++
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java
Fri Dec 4 22:39:17 2009
@@ -91,26 +91,26 @@
System.setProperty("group", "users");
}
group = System.getProperty("group");
- System.out.println("user:" + user + " group: " + group);
- STR_STORAGE = "[s1, s2] COMPRESS BY gz SECURE BY user:"
+ System.out.println("uid:" + user + " gid: " + group);
+ STR_STORAGE = "[s1, s2] COMPRESS BY gz SECURE BY uid:"
+ user
- + " group:"
+ + " gid:"
+ group
- + " perm:777 SERIALIZE BY pig; [m1#{a}] SERIALIZE BY pig COMPRESS BY
gz SECURE BY user:"
+ + " perm:777 SERIALIZE BY pig; [m1#{a}] SERIALIZE BY pig COMPRESS BY
gz SECURE BY uid:"
+ user
- + " group:"
+ + " gid:"
+ group
- + " perm:777 ; [r1.f1] SECURE BY user:"
+ + " perm:777 ; [r1.f1] SECURE BY uid:"
+ user
- + " group:"
+ + " gid:"
+ group
- + " perm:777 SERIALIZE BY pig COMPRESS BY gz ; [s3, s4, r2.r3.f3]
SERIALIZE BY pig SECURE BY user:"
+ + " perm:777 SERIALIZE BY pig COMPRESS BY gz ; [s3, s4, r2.r3.f3]
SERIALIZE BY pig SECURE BY uid:"
+ user
- + " group: "
+ + " gid: "
+ group
- + " perm:777 COMPRESS BY gz ; [s5, s6, m2#{x|y}] compREss by gz secURe
by user:"
+ + " perm:777 COMPRESS BY gz ; [s5, s6, m2#{x|y}] compREss by gz secURe
by uid:"
+ user
- + " group:"
+ + " gid:"
+ group
+ " perm:777 SerialIZE BY pig; [r1.f2, m1#{b}]; [r2.r3.f4, m2#{z}]
SERIALIZE BY avro;[s7,s8] SERIALIZE BY AVRO;[s9,s10] COMPRESS BY gz ";
System.out.println("storage: " + STR_STORAGE);
@@ -118,7 +118,7 @@
conf.setInt("table.input.split.minSize", 64 * 1024);
conf.set("table.output.tfile.compression", "none");
// String STR_STORAGE1 =
- // "[s1, s2] COMPRESS BY gz SECURE BY user:"+USER1.getUserName()+"
group:"+USER1.getGroupNames()[0]+" perm:777 SERIALIZE BY pig";
+ // "[s1, s2] COMPRESS BY gz SECURE BY uid:"+USER1.getUserName()+"
gid:"+USER1.getGroupNames()[0]+" perm:777 SERIALIZE BY pig";
RawLocalFileSystem rawLFS = new RawLocalFileSystem();
fs = new LocalFileSystem(rawLFS);
path = new Path(fs.getWorkingDirectory(), "TestStorageGrammar");
@@ -326,7 +326,7 @@
public void test1() throws IOException, ParseException {
String schema = "s1:string, s2:string";
- String storage = "[s1, s2]COMPRESS BY gz SECURE BY user:user1 group:users
perm:744 SERIALIZE BY pig";
+ String storage = "[s1, s2]COMPRESS BY gz SECURE BY uid:user1 gid:users
perm:744 SERIALIZE BY pig";
RawLocalFileSystem rawLFS = new RawLocalFileSystem();
fs = new LocalFileSystem(rawLFS);
Path path1 = new Path(path.toString() + "1");
@@ -350,8 +350,8 @@
@Test
public void test2() throws IOException, ParseException {
String schema = "m1:map(string),m2:map(map(int))";
- String storage = "[m1#{a}] SERIALIZE BY pig COMPRESS BY gz SECURE BY user:"
- + user + " group:" + group + " perm:770 ";
+ String storage = "[m1#{a}] SERIALIZE BY pig COMPRESS BY gz SECURE BY uid:"
+ + user + " gid:" + group + " perm:770 ";
Path path1 = new Path(path.toString() + "2");
BasicTable.Writer writer = null;
try {
@@ -373,7 +373,7 @@
@Test
public void test3() throws IOException, ParseException {
String schema = "r1:record(f1:int, f2:long)";
- String storage = "[r1.f1] SECURE BY user:" + user + " group:" + group
+ String storage = "[r1.f1] SECURE BY uid:" + user + " gid:" + group
+ " perm:777 SERIALIZE BY pig COMPRESS BY gz";
Path path1 = new Path(path.toString() + "3");
BasicTable.Writer writer = null;
@@ -395,8 +395,8 @@
@Test
public void test4() throws IOException, ParseException {
String schema = "s3:string, s4:string, r2:record(r3:record(f3:float, f4)),
c:collection(f13:double, f14:float, f15:bytes)";
- String storage = "[s3, s4, r2.r3.f3, c] SERIALIZE BY pig SECURE BY user:"
- + user + " group:" + group + " perm:777 COMPRESS BY gz";
+ String storage = "[s3, s4, r2.r3.f3, c] SERIALIZE BY pig SECURE BY uid:"
+ + user + " gid:" + group + " perm:777 COMPRESS BY gz";
Path path1 = new Path(path.toString() + "4");
BasicTable.Writer writer = null;
try {
@@ -417,8 +417,8 @@
@Test
public void test5() throws IOException, ParseException {
String schema = "s5:string, s6:string, m2:map(map(int))";
- String storage = "[s5, s6, m2#{x|y}] compREss by gz secURe by user:" + user
- + " group:" + group + " perm:777 SerialIZE BY pig";
+ String storage = "[s5, s6, m2#{x|y}] compREss by gz secURe by uid:" + user
+ + " gid:" + group + " perm:777 SerialIZE BY pig";
Path path1 = new Path(path.toString() + "5");
BasicTable.Writer writer = null;
try {
@@ -541,7 +541,7 @@
@Test(expected = IOException.class)
public void test12() throws IOException, ParseException {
String schema = "some1:string, some2:string";
- String storage = "[some1,some2] COMPRESS BY gz SECURrrrrrE BY user:user1
group:grop1 perm:760 SERIALIZE BY pig";
+ String storage = "[some1,some2] COMPRESS BY gz SECURrrrrrE BY uid:user1
gid:grop1 perm:760 SERIALIZE BY pig";
Path path1 = new Path(path.toString() + "12");
BasicTable.Writer writer = null;
@@ -553,7 +553,7 @@
@Test(expected = IOException.class)
public void test13() throws IOException, ParseException {
String schema = "some1:string, some2:string";
- String storage = "[some1,some2] COMPRESS gz SECURE BY user:user1
group:users perm:760 SERIALIZE BY pig";
+ String storage = "[some1,some2] COMPRESS gz SECURE BY uid:user1 gid:users
perm:760 SERIALIZE BY pig";
Path path1 = new Path(path.toString() + "13");
BasicTable.Writer writer = null;
@@ -565,7 +565,7 @@
@Test(expected = IOException.class)
public void test14() throws IOException, ParseException {
String schema = "some1:string, some2:string";
- String storage = "[some1,some2] COMPRESS BY gz SECURE BY usssser:user1
group:users perm:760 SERIALIZE BY pig";
+ String storage = "[some1,some2] COMPRESS BY gz SECURE BY usssser:user1
gid:users perm:760 SERIALIZE BY pig";
Path path1 = new Path(path.toString() + "14");
BasicTable.Writer writer = null;
writer = new BasicTable.Writer(path1, schema, storage, conf);
@@ -576,7 +576,7 @@
@Test(expected = IOException.class)
public void test15() throws IOException, ParseException {
String schema = "some1:string, some2:string";
- String storage = "[some1,some2] COMPRESS BY gz SECURE BY user:user1
grouuuup:group1 perm:760 SERIALIZE BY pig";
+ String storage = "[some1,some2] COMPRESS BY gz SECURE BY uid:user1
grouuuup:group1 perm:760 SERIALIZE BY pig";
Path path1 = new Path(path.toString() + "15");
BasicTable.Writer writer = null;
writer = new BasicTable.Writer(path1, schema, storage, conf);
@@ -587,7 +587,7 @@
@Test(expected = IOException.class)
public void test16() throws IOException, ParseException {
String schema = "some1:string, some2:string";
- String storage = "[some1,some2] COMPRESS BY gz SECURE BY user:user1
group:users perrrrrm:760 SERIALIZE BY pig";
+ String storage = "[some1,some2] COMPRESS BY gz SECURE BY uid:user1
gid:users perrrrrm:760 SERIALIZE BY pig";
Path path1 = new Path(path.toString() + "16");
BasicTable.Writer writer = null;
writer = new BasicTable.Writer(path1, schema, storage, conf);
@@ -658,7 +658,7 @@
*/
public void test20() throws IOException, ParseException {
String schema = "s9:string, s10:string, s11:string";
- String storage = "[s9,s10] COMPRESS BY gZ; secure by user:user1
group:users perm:755";
+ String storage = "[s9,s10] COMPRESS BY gZ; secure by uid:user1 gid:users
perm:755";
Path path1 = new Path(path.toString() + "20");
Runtime.getRuntime().exec("rm -rf " + path1.toString());
@@ -681,7 +681,7 @@
@Test(expected = IOException.class)
public void test21() throws IOException, ParseException {
String schema = "s9:string, s10:string, s11:string";
- String storage = "[s9,s10] COMPRESS BY gZ; secure by user:user1
group:users perm:755;secure by user:user1 group:users perm:755";
+ String storage = "[s9,s10] COMPRESS BY gZ; secure by uid:user1 gid:users
perm:755;secure by user:user1 gid:users perm:755";
Path path1 = new Path(path.toString() + "16");
BasicTable.Writer writer = null;
writer = new BasicTable.Writer(path1, schema, storage, conf);
@@ -692,7 +692,7 @@
@Test
public void test22() throws IOException, ParseException {
String schema = "s9:string, s10:string";
- String storage = "[s9,s10] secure by user:user3";
+ String storage = "[s9,s10] secure by uid:user3";
Path path1 = new Path(path.toString() + "22");
Runtime.getRuntime().exec("rm -rf " + path1.toString());
@@ -720,7 +720,7 @@
*/
public void test23() throws IOException, ParseException {
String schema = "s9:string, s10:string";
- String storage = "[s9,s10] secure by group:users";
+ String storage = "[s9,s10] secure by gid:users";
Path path1 = new Path(path.toString() + "23");
Runtime.getRuntime().exec("rm -rf " + path1.toString());
@@ -773,7 +773,7 @@
*/
public void test25() throws IOException, ParseException {
String schema = "s9:string, s10:string";
- String storage = "[s9,s10] secure by user:user1 group:users";
+ String storage = "[s9,s10] secure by uid:user1 gid:users";
Path path1 = new Path(path.toString() + "25");
Runtime.getRuntime().exec("rm -rf " + path1.toString());
@@ -798,7 +798,7 @@
//@Test
public void test26() throws IOException, ParseException {
String schema = "s9:string, s10:string";
- String storage = "[s9,s10] secure by perm:755 group:users";
+ String storage = "[s9,s10] secure by perm:755 gid:users";
Path path1 = new Path(path.toString() + "26");
Runtime.getRuntime().exec("rm -rf " + path1.toString());
Modified:
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc1.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc1.java?rev=887401&r1=887400&r2=887401&view=diff
==============================================================================
---
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc1.java
(original)
+++
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc1.java
Fri Dec 4 22:39:17 2009
@@ -81,13 +81,13 @@
@Test
public void testStorageValid1() {
try {
- String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gz SECURE BY
user:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
-// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY
user:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
-// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY
user:root group:data perm:0766; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE
BY avro";
-// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY
user:root group:data perm:966; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY
avro";
+ String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gz SECURE BY
uid:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
+// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY
uid:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
+// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY
uid:root group:data perm:0766; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY
avro";
+// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY
uid:root group:data perm:966; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY
avro";
// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY;
[r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
-// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY
user:ggg SECURE BY group:fff; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY
avro";
-// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY
user:root user:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
+// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY
uid:ggg SECURE BY group:fff; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY
avro";
+// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY
uid:root user:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
Partition p = new Partition(schema.toString(), strStorage, null);
CGSchema[] cgschemas = p.getCGSchemas();
@@ -161,6 +161,7 @@
}
}
} catch (Exception e) {
+ System.out.println("Error is [" + e.getMessage() + "]");
Assert.assertTrue(false);
}
}
