Added: hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java?rev=882929&view=auto ============================================================================== --- hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java (added) +++ hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java Sat Nov 21 15:36:12 2009 @@ -0,0 +1,833 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.zebra.types; + +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintStream; +import java.util.HashMap; +import java.util.Map; +import javax.security.auth.login.LoginException; + +import junit.framework.Assert; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.zebra.io.BasicTable; +import org.apache.hadoop.zebra.io.TableInserter; +import org.apache.hadoop.zebra.parser.ParseException; +import org.apache.hadoop.zebra.schema.Schema; +import org.apache.hadoop.zebra.types.TypesUtils; +import org.apache.pig.data.DataBag; +import org.apache.pig.data.DataByteArray; +import org.apache.pig.data.Tuple; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * + * Test projections on complicated column types. + * + */ +public class TestStorageGrammar { + + final static String STR_SCHEMA = "s1:bool, s2:int, s3:long, s4:float, s5:string, s6:bytes, r1:record(f1:int, f2:long), r2:record(r3:record(f3:float, f4)), m1:map(string),m2:map(map(int)), c:collection(f13:double, f14:float, f15:bytes),s7:string, s8:string, s9:string, s10:string, s11:string, s12:string, s13:string, s14:string, s15:string, s16:string, s17:string, s18:string, s19:string, s20:string, s21:string, s22:string, s23:string"; + static String STR_STORAGE = null; + final private static Configuration conf = new Configuration(); + private static Path path; + private static FileSystem fs; + private static String user; + private static String group; + private static String defaultUser; + + static{ + try{ + String command = "whoami"; + Process process = new ProcessBuilder(command).start(); + InputStream is = process.getInputStream(); + InputStreamReader isr = new InputStreamReader(is); + BufferedReader br = new BufferedReader(isr); + System.out.printf("Output of running %s is:", + command); + String line; + while ((line = br.readLine()) != null) { + System.out.println("default user0:" +line); + defaultUser = line; + + } + }catch(Exception e){ + e.printStackTrace(); + } + } + + @BeforeClass + public static void setUpOnce() throws IOException, LoginException { + if (System.getProperty("user") == null) { + System.setProperty("user", defaultUser); + } + user = System.getProperty("user"); + if (System.getProperty("group") == null) { + System.setProperty("group", "users"); + } + group = System.getProperty("group"); + System.out.println("user:" + user + " group: " + group); + STR_STORAGE = "[s1, s2] COMPRESS BY gz SECURE BY user:" + + user + + " group:" + + group + + " perm:777 SERIALIZE BY pig; [m1#{a}] SERIALIZE BY pig COMPRESS BY gz SECURE BY user:" + + user + + " group:" + + group + + " perm:777 ; [r1.f1] SECURE BY user:" + + user + + " group:" + + group + + " perm:777 SERIALIZE BY pig COMPRESS BY gz ; [s3, s4, r2.r3.f3] SERIALIZE BY pig SECURE BY user:" + + user + + " group: " + + group + + " perm:777 COMPRESS BY gz ; [s5, s6, m2#{x|y}] compREss by gz secURe by user:" + + user + + " group:" + + group + + " perm:777 SerialIZE BY pig; [r1.f2, m1#{b}]; [r2.r3.f4, m2#{z}] SERIALIZE BY avro;[s7,s8] SERIALIZE BY AVRO;[s9,s10] COMPRESS BY gz "; + System.out.println("storage: " + STR_STORAGE); + conf.setInt("table.output.tfile.minBlock.size", 64 * 1024); + conf.setInt("table.input.split.minSize", 64 * 1024); + conf.set("table.output.tfile.compression", "none"); + // String STR_STORAGE1 = + // "[s1, s2] COMPRESS BY gz SECURE BY user:"+USER1.getUserName()+" group:"+USER1.getGroupNames()[0]+" perm:777 SERIALIZE BY pig"; + RawLocalFileSystem rawLFS = new RawLocalFileSystem(); + fs = new LocalFileSystem(rawLFS); + path = new Path(fs.getWorkingDirectory(), "TestStorageGrammar"); + System.out.println("path: " + path.toString()); + // Process p = Runtime.getRuntime().exec("rm -rf "+path.toString()+"*"); + + fs = path.getFileSystem(conf); + // drop any previous tables + BasicTable.drop(path, conf); + + BasicTable.Writer writer = new BasicTable.Writer(path, STR_SCHEMA, + STR_STORAGE, conf); + + writer.finish(); + + Schema schema = writer.getSchema(); + Tuple tuple = TypesUtils.createTuple(schema); + BasicTable.Writer writer1 = new BasicTable.Writer(path, conf); + int part = 0; + TableInserter inserter = null; + try { + inserter = writer1.getInserter("part" + part, true); + } catch (Exception e) { + e.printStackTrace(); + } + TypesUtils.resetTuple(tuple); + Tuple tupRecord1 = null; + try { + tupRecord1 = TypesUtils.createTuple(schema.getColumnSchema("r1") + .getSchema()); + } catch (ParseException e) { + e.printStackTrace(); + } + + Tuple tupRecord2 = null; + try { + tupRecord2 = TypesUtils.createTuple(schema.getColumnSchema("r2") + .getSchema()); + } catch (ParseException e) { + e.printStackTrace(); + } + + Tuple tupRecord3 = null; + try { + tupRecord3 = TypesUtils.createTuple(new Schema("f3:float, f4")); + } catch (ParseException e) { + e.printStackTrace(); + } + + // row 1 + tuple.set(0, true); // bool + tuple.set(1, 1); // int + tuple.set(2, 1001L); // long + tuple.set(3, 1.1); // float + tuple.set(4, "hello world 1"); // string + tuple.set(5, new DataByteArray("hello byte 1")); // byte + + // r1:record(f1:int, f2:long + tupRecord1.set(0, 1); + tupRecord1.set(1, 1001L); + tuple.set(6, tupRecord1); + + // r2:record(r3:record(f3:float, f4)) + tupRecord2.set(0, tupRecord3); + tupRecord3.set(0, 1.3); + tupRecord3.set(1, new DataByteArray("r3 row 1 byte array ")); + tuple.set(7, tupRecord2); + + // m1:map(string) + Map<String, String> m1 = new HashMap<String, String>(); + m1.put("a", "A"); + m1.put("b", "B"); + m1.put("c", "C"); + tuple.set(8, m1); + + // m2:map(map(int)) + HashMap<String, Map> m2 = new HashMap<String, Map>(); + Map<String, Integer> m3 = new HashMap<String, Integer>(); + m3.put("m311", 311); + m3.put("m321", 321); + m3.put("m331", 331); + Map<String, Integer> m4 = new HashMap<String, Integer>(); + m4.put("m411", 411); + m4.put("m421", 421); + m4.put("m431", 431); + m2.put("x", m3); + m2.put("y", m4); + tuple.set(9, m2); + + // c:collection(f13:double, f14:float, f15:bytes) + DataBag bagColl = TypesUtils.createBag(); + Schema schColl = schema.getColumn(10).getSchema(); + Tuple tupColl1 = TypesUtils.createTuple(schColl); + Tuple tupColl2 = TypesUtils.createTuple(schColl); + byte[] abs1 = new byte[3]; + byte[] abs2 = new byte[4]; + tupColl1.set(0, 3.1415926); + tupColl1.set(1, 1.6); + abs1[0] = 11; + abs1[1] = 12; + abs1[2] = 13; + tupColl1.set(2, new DataByteArray(abs1)); + bagColl.add(tupColl1); + tupColl2.set(0, 123.456789); + tupColl2.set(1, 100); + abs2[0] = 21; + abs2[1] = 22; + abs2[2] = 23; + abs2[3] = 24; + tupColl2.set(2, new DataByteArray(abs2)); + bagColl.add(tupColl2); + tuple.set(10, bagColl); + + // set s7 to s23 + for (int i = 7; i <= 23; i++) { + tuple.set(i + 4, "s" + "i" + ", line1"); + } + + int row = 0; + inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1) + .getBytes()), tuple); + + // row 2 + row++; + TypesUtils.resetTuple(tuple); + TypesUtils.resetTuple(tupRecord1); + TypesUtils.resetTuple(tupRecord2); + TypesUtils.resetTuple(tupRecord3); + m1.clear(); + m2.clear(); + m3.clear(); + m4.clear(); + tuple.set(0, false); + tuple.set(1, 2); // int + tuple.set(2, 1002L); // long + tuple.set(3, 3.1); // float + tuple.set(4, "hello world 2"); // string + tuple.set(5, new DataByteArray("hello byte 2")); // byte + + // r1:record(f1:int, f2:long + tupRecord1.set(0, 2); + tupRecord1.set(1, 1002L); + tuple.set(6, tupRecord1); + + // r2:record(r3:record(f3:float, f4)) + tupRecord2.set(0, tupRecord3); + tupRecord3.set(0, 2.3); + tupRecord3.set(1, new DataByteArray("r3 row2 byte array")); + tuple.set(7, tupRecord2); + + // m1:map(string) + m1.put("a2", "A2"); + m1.put("b2", "B2"); + m1.put("c2", "C2"); + tuple.set(8, m1); + + // m2:map(map(int)) + m3.put("m321", 321); + m3.put("m322", 322); + m3.put("m323", 323); + m2.put("z", m3); + tuple.set(9, m2); + + // c:collection(f13:double, f14:float, f15:bytes) + bagColl.clear(); + TypesUtils.resetTuple(tupColl1); + TypesUtils.resetTuple(tupColl2); + tupColl1.set(0, 7654.321); + tupColl1.set(1, 0.0001); + abs1[0] = 31; + abs1[1] = 32; + abs1[2] = 33; + tupColl1.set(2, new DataByteArray(abs1)); + bagColl.add(tupColl1); + tupColl2.set(0, 0.123456789); + tupColl2.set(1, 0.3333); + abs2[0] = 41; + abs2[1] = 42; + abs2[2] = 43; + abs2[3] = 44; + tupColl2.set(2, new DataByteArray(abs2)); + bagColl.add(tupColl2); + tuple.set(10, bagColl); + // set s7 to s23 + for (int i = 7; i <= 23; i++) { + tuple.set(i + 4, "s" + "i" + ", line2"); + } + inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1) + .getBytes()), tuple); + + inserter.close(); + writer1.finish(); + + writer.close(); + } + + @AfterClass + public static void tearDownOnce() throws IOException { + } + + //@Test + /* + * group1 is user specific + */ + public void test1() throws IOException, ParseException { + + String schema = "s1:string, s2:string"; + String storage = "[s1, s2]COMPRESS BY gz SECURE BY user:user1 group:users perm:744 SERIALIZE BY pig"; + RawLocalFileSystem rawLFS = new RawLocalFileSystem(); + fs = new LocalFileSystem(rawLFS); + Path path1 = new Path(path.toString() + "1"); + Runtime.getRuntime().exec("rm -rf " + path1.toString()); + + fs = path.getFileSystem(conf); + BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage, + conf); + writer.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(bos); + System.out.println("start dumpinfo ==========="); + BasicTable.dumpInfo(path1.toString(), ps, conf); + + Assert.assertEquals(true, bos.toString().contains("Serializer: pig")); + Assert.assertEquals(true, bos.toString().contains("Compressor: gz")); + Assert.assertEquals(true, bos.toString().contains( + "Schema : s1:string,s2:string")); + } + + @Test + public void test2() throws IOException, ParseException { + String schema = "m1:map(string),m2:map(map(int))"; + String storage = "[m1#{a}] SERIALIZE BY pig COMPRESS BY gz SECURE BY user:" + + user + " group:" + group + " perm:770 "; + Path path1 = new Path(path.toString() + "2"); + BasicTable.Writer writer = null; + try { + writer = new BasicTable.Writer(path1, schema, storage, conf); + } catch (Exception e) { + e.printStackTrace(); + } + writer.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(bos); + BasicTable.dumpInfo(path1.toString(), ps, conf); + System.out.println("start dumpinfo ===========\n" + bos.toString()); + Assert.assertEquals(true, bos.toString().contains("Serializer: pig")); + Assert.assertEquals(true, bos.toString().contains("Compressor: gz")); + Assert.assertEquals(true, bos.toString().contains( + "Schema : m1:map(string),m2:map(map(int))")); + } + + @Test + public void test3() throws IOException, ParseException { + String schema = "r1:record(f1:int, f2:long)"; + String storage = "[r1.f1] SECURE BY user:" + user + " group:" + group + + " perm:777 SERIALIZE BY pig COMPRESS BY gz"; + Path path1 = new Path(path.toString() + "3"); + BasicTable.Writer writer = null; + try { + writer = new BasicTable.Writer(path1, schema, storage, conf); + } catch (Exception e) { + e.printStackTrace(); + } + writer.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(bos); + BasicTable.dumpInfo(path1.toString(), ps, conf); + System.out.println("start dumpinfo ===========\n" + bos.toString()); + Assert.assertEquals(true, bos.toString().contains("Serializer: pig")); + Assert.assertEquals(true, bos.toString().contains("Compressor: gz")); + // Assert.assertEquals(true,bos.toString().contains("Schema : r1:record(f1:int, f2:long)")); + } + + @Test + public void test4() throws IOException, ParseException { + String schema = "s3:string, s4:string, r2:record(r3:record(f3:float, f4)), c:collection(f13:double, f14:float, f15:bytes)"; + String storage = "[s3, s4, r2.r3.f3, c] SERIALIZE BY pig SECURE BY user:" + + user + " group:" + group + " perm:777 COMPRESS BY gz"; + Path path1 = new Path(path.toString() + "4"); + BasicTable.Writer writer = null; + try { + writer = new BasicTable.Writer(path1, schema, storage, conf); + } catch (Exception e) { + e.printStackTrace(); + } + writer.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(bos); + BasicTable.dumpInfo(path1.toString(), ps, conf); + System.out.println("start dumpinfo ===========\n" + bos.toString()); + Assert.assertEquals(true, bos.toString().contains("Serializer: pig")); + Assert.assertEquals(true, bos.toString().contains("Compressor: gz")); + // Assert.assertEquals(true,bos.toString().contains("s3:string,s4:string,r2:record(r3:record(f3:float, f4)),c:collection(f13:double,f14:float,f15:bytes)")); + } + + @Test + public void test5() throws IOException, ParseException { + String schema = "s5:string, s6:string, m2:map(map(int))"; + String storage = "[s5, s6, m2#{x|y}] compREss by gz secURe by user:" + user + + " group:" + group + " perm:777 SerialIZE BY pig"; + Path path1 = new Path(path.toString() + "5"); + BasicTable.Writer writer = null; + try { + writer = new BasicTable.Writer(path1, schema, storage, conf); + } catch (Exception e) { + e.printStackTrace(); + } + writer.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(bos); + BasicTable.dumpInfo(path1.toString(), ps, conf); + System.out.println("start dumpinfo ===========\n" + bos.toString()); + Assert.assertEquals(true, bos.toString().contains("Serializer: pig")); + // Assert.assertEquals(true,bos.toString().contains("Compressor: gz")); + Assert.assertEquals(true, bos.toString().contains( + "Schema : s5:string,s6:string,m2:map(map(int))")); + } + + @Test + public void test6() throws IOException, ParseException { + String schema = "r1:record(f1:int, f2:long),m1:map(string)"; + String storage = "[r1.f2, m1#{b}]"; + Path path1 = new Path(path.toString() + "6"); + BasicTable.Writer writer = null; + try { + writer = new BasicTable.Writer(path1, schema, storage, conf); + } catch (Exception e) { + e.printStackTrace(); + } + writer.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(bos); + BasicTable.dumpInfo(path1.toString(), ps, conf); + System.out.println("start dumpinfo ===========\n" + bos.toString()); + Assert.assertEquals(true, bos.toString().contains("Serializer: pig")); + // Assert.assertEquals(true,bos.toString().contains("Compressor: gz")); + // Assert.assertEquals(true,bos.toString().contains("Schema : r1:record(f1:int, f2:long),m1:map(string)")); + } + + @Test + public void test7() throws IOException, ParseException { + System.out.println("========7777"); + String schema = "r2:record(r3:record(f3:float, f4)),m2:map(map(int))"; + String storage = "[r2.r3.f4, m2#{z}] SERIALIZE BY avro"; + Path path1 = new Path(path.toString() + "7"); + BasicTable.Writer writer = null; + try { + writer = new BasicTable.Writer(path1, schema, storage, conf); + } catch (Exception e) { + e.printStackTrace(); + } + writer.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(bos); + BasicTable.dumpInfo(path1.toString(), ps, conf); + System.out.println("start dumpinfo ===========\n" + bos.toString()); + Assert.assertEquals(true, bos.toString().contains("Serializer: avro")); + Assert.assertEquals(true, bos.toString().contains("Compressor: gz")); + // Assert.assertEquals(true,bos.toString().contains("Schema : r2:record(r3:record(f3:float, f4)),m2:map(map(int))")); + } + + @Test + public void test8() throws IOException, ParseException { + String schema = "s7:string, s8:string"; + String storage = "[s7,s8] SERIALIZE BY AvRO"; + Path path1 = new Path(path.toString() + "8"); + BasicTable.Writer writer = null; + try { + writer = new BasicTable.Writer(path1, schema, storage, conf); + } catch (Exception e) { + e.printStackTrace(); + } + writer.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(bos); + BasicTable.dumpInfo(path1.toString(), ps, conf); + System.out.println("start dumpinfo ===========\n" + bos.toString()); + Assert.assertEquals(true, bos.toString().contains("Serializer: AvRO")); + Assert.assertEquals(true, bos.toString().contains("Compressor: gz")); + Assert.assertEquals(true, bos.toString().contains( + "Schema : s7:string,s8:string")); + } + + @Test(expected = IOException.class) + public void test9() throws IOException, ParseException { + String schema = "some1:string, some2:string"; + String storage = "[some1,some2] SERIALIZE BY something"; + Path path1 = new Path(path.toString() + "9"); + BasicTable.Writer writer = null; + + writer = new BasicTable.Writer(path1, schema, storage, conf); + Assert.fail("should throw exception, none defined serializer"); + writer.finish(); + } + + @Test(expected = IOException.class) + public void test10() throws IOException, ParseException { + String schema = "some1:string, some2:string"; + String storage = "[some1,some2] COMPRESS BY gz SERIALLLLIZE BY pig"; + Path path1 = new Path(path.toString() + "10"); + BasicTable.Writer writer = null; + + writer = new BasicTable.Writer(path1, schema, storage, conf); + Assert.fail("should throw exception, none defined serializer"); + writer.finish(); + } + + @Test(expected = IOException.class) + public void test11() throws IOException, ParseException { + String schema = "some1:string, some2:string"; + String storage = "[some1,some2] COMPREEEEESS BY gz SERIALIZE BY pig"; + Path path1 = new Path(path.toString() + "11"); + BasicTable.Writer writer = null; + + writer = new BasicTable.Writer(path1, schema, storage, conf); + Assert.fail("should throw exception, none defined serializer"); + writer.finish(); + } + + @Test(expected = IOException.class) + public void test12() throws IOException, ParseException { + String schema = "some1:string, some2:string"; + String storage = "[some1,some2] COMPRESS BY gz SECURrrrrrE BY user:user1 group:grop1 perm:760 SERIALIZE BY pig"; + Path path1 = new Path(path.toString() + "12"); + BasicTable.Writer writer = null; + + writer = new BasicTable.Writer(path1, schema, storage, conf); + Assert.fail("should throw exception, none defined serializer"); + writer.finish(); + } + + @Test(expected = IOException.class) + public void test13() throws IOException, ParseException { + String schema = "some1:string, some2:string"; + String storage = "[some1,some2] COMPRESS gz SECURE BY user:user1 group:users perm:760 SERIALIZE BY pig"; + Path path1 = new Path(path.toString() + "13"); + BasicTable.Writer writer = null; + + writer = new BasicTable.Writer(path1, schema, storage, conf); + Assert.fail("should throw exception, none defined serializer"); + writer.finish(); + } + + @Test(expected = IOException.class) + public void test14() throws IOException, ParseException { + String schema = "some1:string, some2:string"; + String storage = "[some1,some2] COMPRESS BY gz SECURE BY usssser:user1 group:users perm:760 SERIALIZE BY pig"; + Path path1 = new Path(path.toString() + "14"); + BasicTable.Writer writer = null; + writer = new BasicTable.Writer(path1, schema, storage, conf); + Assert.fail("should throw exception, none defined serializer"); + writer.finish(); + } + + @Test(expected = IOException.class) + public void test15() throws IOException, ParseException { + String schema = "some1:string, some2:string"; + String storage = "[some1,some2] COMPRESS BY gz SECURE BY user:user1 grouuuup:group1 perm:760 SERIALIZE BY pig"; + Path path1 = new Path(path.toString() + "15"); + BasicTable.Writer writer = null; + writer = new BasicTable.Writer(path1, schema, storage, conf); + Assert.fail("should throw exception, none defined serializer"); + writer.finish(); + } + + @Test(expected = IOException.class) + public void test16() throws IOException, ParseException { + String schema = "some1:string, some2:string"; + String storage = "[some1,some2] COMPRESS BY gz SECURE BY user:user1 group:users perrrrrm:760 SERIALIZE BY pig"; + Path path1 = new Path(path.toString() + "16"); + BasicTable.Writer writer = null; + writer = new BasicTable.Writer(path1, schema, storage, conf); + Assert.fail("should throw exception, none defined serializer"); + writer.finish(); + } + + @Test + public void test17() throws IOException, ParseException { + String schema = "s9:string, s10:string"; + String storage = "[s9,s10] COMPRESS BY lzo"; + + Path path1 = new Path(path.toString() + "17"); + Runtime.getRuntime().exec("rm -rf " + path1.toString()); + fs = path.getFileSystem(conf); + BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage, + conf); + writer.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(bos); + System.out.println("start dumpinfo 17 ==========="); + BasicTable.dumpInfo(path1.toString(), ps, conf); + System.out.println("bos: " + bos.toString()); + Assert.assertEquals(true, bos.toString().contains("Serializer: pig")); + Assert.assertEquals(true, bos.toString().contains("Compressor: lzo")); + Assert.assertEquals(true, bos.toString().contains( + "Schema : s9:string,s10:string")); + } + + //@Test + public void test18() throws IOException, ParseException { + String schema = "s9:string, s10:string"; + String storage = "[s9,s10] COMPRESS BY gZ"; + + Path path1 = new Path(path.toString() + "18"); + Runtime.getRuntime().exec("rm -rf " + path1.toString()); + fs = path.getFileSystem(conf); + BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage, + conf); + writer.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(bos); + System.out.println("start dumpinfo 18===========" + bos.toString()); + BasicTable.dumpInfo(path1.toString(), ps, conf); + System.out.println("bos: " + bos.toString()); + + Assert.assertEquals(true, bos.toString().contains("Serializer: pig")); + + Assert.assertEquals(true, bos.toString().contains("Compressor: gz")); + Assert.assertEquals(true, bos.toString().contains( + "Schema : s9:string,s10:string")); + } + + @Test(expected = IOException.class) + public void test19() throws IOException, ParseException { + String schema = "some1:string, some2:string"; + String storage = "[some1,some2] COMPRESS BY something"; + Path path1 = new Path(path.toString() + "19"); + BasicTable.Writer writer = null; + writer = new BasicTable.Writer(path1, schema, storage, conf); + Assert.fail("should throw exception, none defined serializer"); + writer.finish(); + } + + //@Test + /* + * group1 is user specific + */ + public void test20() throws IOException, ParseException { + String schema = "s9:string, s10:string, s11:string"; + String storage = "[s9,s10] COMPRESS BY gZ; secure by user:user1 group:users perm:755"; + + Path path1 = new Path(path.toString() + "20"); + Runtime.getRuntime().exec("rm -rf " + path1.toString()); + fs = path.getFileSystem(conf); + BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage, + conf); + writer.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(bos); + System.out.println("start dumpinfo 20===========" + bos.toString()); + BasicTable.dumpInfo(path1.toString(), ps, conf); + System.out.println("bos: " + bos.toString()); + + Assert.assertEquals(true, bos.toString().contains("Serializer: pig")); + Assert.assertEquals(true, bos.toString().contains("Compressor: gz")); + Assert.assertEquals(true, bos.toString().contains( + "Schema : s9:string,s10:string")); + } + + @Test(expected = IOException.class) + public void test21() throws IOException, ParseException { + String schema = "s9:string, s10:string, s11:string"; + String storage = "[s9,s10] COMPRESS BY gZ; secure by user:user1 group:users perm:755;secure by user:user1 group:users perm:755"; + Path path1 = new Path(path.toString() + "16"); + BasicTable.Writer writer = null; + writer = new BasicTable.Writer(path1, schema, storage, conf); + Assert.fail("should throw exception, none defined serializer"); + writer.finish(); + } + + @Test + public void test22() throws IOException, ParseException { + String schema = "s9:string, s10:string"; + String storage = "[s9,s10] secure by user:user3"; + + Path path1 = new Path(path.toString() + "22"); + Runtime.getRuntime().exec("rm -rf " + path1.toString()); + fs = path.getFileSystem(conf); + BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage, + conf); + writer.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(bos); + System.out.println("start dumpinfo 22===========" + bos.toString()); + BasicTable.dumpInfo(path1.toString(), ps, conf); + System.out.println("bos: " + bos.toString()); + + Assert.assertEquals(true, bos.toString().contains("Serializer: pig")); + Assert.assertEquals(true, bos.toString().contains("Compressor: gz")); + Assert.assertEquals(true, bos.toString().contains( + "Schema : s9:string,s10:string")); + Assert.assertEquals(true, bos.toString().contains("Group: null")); + Assert.assertEquals(true, bos.toString().contains("Perm: -1")); + } + + //@Test + /* + * group1 is user specific + */ + public void test23() throws IOException, ParseException { + String schema = "s9:string, s10:string"; + String storage = "[s9,s10] secure by group:users"; + + Path path1 = new Path(path.toString() + "23"); + Runtime.getRuntime().exec("rm -rf " + path1.toString()); + fs = path.getFileSystem(conf); + BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage, + conf); + writer.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(bos); + System.out.println("start dumpinfo 23===========" + bos.toString()); + BasicTable.dumpInfo(path1.toString(), ps, conf); + System.out.println("bos: " + bos.toString()); + + Assert.assertEquals(true, bos.toString().contains("Serializer: pig")); + Assert.assertEquals(true, bos.toString().contains("Compressor: gz")); + Assert.assertEquals(true, bos.toString().contains( + "Schema : s9:string,s10:string")); + Assert.assertEquals(true, bos.toString().contains("Group: users")); + Assert.assertEquals(true, bos.toString().contains("Perm: -1")); + } + + //@Test + public void test24() throws IOException, ParseException { + String schema = "s9:string, s10:string"; + String storage = "[s9,s10] secure by perm:755"; + + Path path1 = new Path(path.toString() + "24"); + Runtime.getRuntime().exec("rm -rf " + path1.toString()); + fs = path.getFileSystem(conf); + BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage, + conf); + writer.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(bos); + System.out.println("start dumpinfo 24===========" + bos.toString()); + BasicTable.dumpInfo(path1.toString(), ps, conf); + System.out.println("bos: " + bos.toString()); + + Assert.assertEquals(true, bos.toString().contains("Serializer: pig")); + Assert.assertEquals(true, bos.toString().contains("Compressor: gz")); + Assert.assertEquals(true, bos.toString().contains( + "Schema : s9:string,s10:string")); + Assert.assertEquals(true, bos.toString().contains("Group:")); + Assert.assertEquals(true, bos.toString().contains("Perm: 0755")); + } + + //@Test + /* + * group1 is user specific + */ + public void test25() throws IOException, ParseException { + String schema = "s9:string, s10:string"; + String storage = "[s9,s10] secure by user:user1 group:users"; + + Path path1 = new Path(path.toString() + "25"); + Runtime.getRuntime().exec("rm -rf " + path1.toString()); + fs = path.getFileSystem(conf); + BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage, + conf); + writer.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(bos); + System.out.println("start dumpinfo 25===========" + bos.toString()); + BasicTable.dumpInfo(path1.toString(), ps, conf); + System.out.println("bos: " + bos.toString()); + + Assert.assertEquals(true, bos.toString().contains("Serializer: pig")); + Assert.assertEquals(true, bos.toString().contains("Compressor: gz")); + Assert.assertEquals(true, bos.toString().contains( + "Schema : s9:string,s10:string")); + Assert.assertEquals(true, bos.toString().contains("Group: users")); + Assert.assertEquals(true, bos.toString().contains("Perm: -1")); + } + + //@Test + public void test26() throws IOException, ParseException { + String schema = "s9:string, s10:string"; + String storage = "[s9,s10] secure by perm:755 group:users"; + + Path path1 = new Path(path.toString() + "26"); + Runtime.getRuntime().exec("rm -rf " + path1.toString()); + fs = path.getFileSystem(conf); + BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage, + conf); + writer.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(bos); + System.out.println("start dumpinfo 26===========" + bos.toString()); + BasicTable.dumpInfo(path1.toString(), ps, conf); + System.out.println("bos: " + bos.toString()); + + Assert.assertEquals(true, bos.toString().contains("Serializer: pig")); + Assert.assertEquals(true, bos.toString().contains("Compressor: gz")); + Assert.assertEquals(true, bos.toString().contains( + "Schema : s9:string,s10:string")); + Assert.assertEquals(true, bos.toString().contains("Group: users")); + Assert.assertEquals(true, bos.toString().contains("Perm: 755")); + } + + @Test(expected = IOException.class) + public void test27() throws IOException, ParseException { + String schema = "s9:string, s10:string"; + String storage = "[s9,s10] COMPRESS BY gZ; secure by "; + Path path1 = new Path(path.toString() + "27"); + BasicTable.Writer writer = null; + writer = new BasicTable.Writer(path1, schema, storage, conf); + Assert.fail("should throw exception, none defined serializer"); + writer.finish(); + } +}