Author: olga Date: Mon Jul 21 10:50:56 2008 New Revision: 678512 URL: http://svn.apache.org/viewvc?rev=678512&view=rev Log: Missef from PIG-111 merge
Added: incubator/pig/branches/types/conf/ incubator/pig/branches/types/conf/log4j.properties incubator/pig/branches/types/conf/pig.properties incubator/pig/branches/types/test/org/apache/pig/test/TestPigContext.java Added: incubator/pig/branches/types/conf/log4j.properties URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/conf/log4j.properties?rev=678512&view=auto ============================================================================== --- incubator/pig/branches/types/conf/log4j.properties (added) +++ incubator/pig/branches/types/conf/log4j.properties Mon Jul 21 10:50:56 2008 @@ -0,0 +1,8 @@ +# ***** Set root logger level to DEBUG and its only appender to A. +log4j.rootLogger=info, A + +# ***** A is set to be a ConsoleAppender. +log4j.appender.A=org.apache.log4j.ConsoleAppender +# ***** A uses PatternLayout. +log4j.appender.A.layout=org.apache.log4j.PatternLayout +log4j.appender.A.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n Added: incubator/pig/branches/types/conf/pig.properties URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/conf/pig.properties?rev=678512&view=auto ============================================================================== --- incubator/pig/branches/types/conf/pig.properties (added) +++ incubator/pig/branches/types/conf/pig.properties Mon Jul 21 10:50:56 2008 @@ -0,0 +1,62 @@ +# Pig configuration file. All values can be overwritten by command line arguments. +# see bin/pig -help + +# log4jconf log4j configuration file +# log4jconf=./conf/log4j.properties + +# brief logging (no timestamps) +brief=false + +# clustername, name of the hadoop jobtracker. If no port is defined port 50020 will be used. +#cluster + +#debug level, INFO is default +debug=INFO + +# a file that contains pig script +#file= + +# load jarfile, colon separated +#jar= + +#verbose print all log messages to screen (default to print only INFO and above to screen) +verbose=false + +#exectype local|mapreduce, mapreduce is default +#exectype=mapreduce +# hod realted properties +#ssh.gateway +#hod.expect.root +#hod.expect.uselatest +#hod.command +#hod.config.dir +#hod.param + + +#Do not spill temp files smaller than this size (bytes) +pig.spill.size.threshold=5000000 +#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes) +#This should help reduce the number of files being spilled. +pig.spill.gc.activation.size=40000000 + + +###################### +# Everything below this line is Yahoo specific. Note that I've made +# (almost) no changes to the lines above to make merging in from Apache +# easier. Any values I don't want from above I override below. +# +# This file is configured for use with HOD on the production clusters. If you +# want to run pig with a static cluster you will need to remove everything +# below this line and set the cluster value (above) to the +# hostname and port of your job tracker. + +exectype=mapreduce + +hod.config.dir=/export/crawlspace/kryptonite/hod/current/conf +hod.server=local + +cluster.domain=inktomisearch.com + +log.file= + +yinst.cluster=kryptonite Added: incubator/pig/branches/types/test/org/apache/pig/test/TestPigContext.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestPigContext.java?rev=678512&view=auto ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestPigContext.java (added) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestPigContext.java Mon Jul 21 10:50:56 2008 @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.pig.test; + + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import junit.framework.TestCase; + +import org.apache.pig.ExecType; +import org.apache.pig.PigServer; +import org.apache.pig.impl.PigContext; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class TestPigContext extends TestCase { + + private static final String TMP_DIR_PROP = "/tmp/hadoop-hadoop"; + private static final String FS_NAME = "machine:9000"; + private static final String JOB_TRACKER = "machine:9001"; + + private File input; + private PigContext pigContext; + + @Before + @Override + protected void setUp() throws Exception { + pigContext = new PigContext(ExecType.LOCAL, getProperties()); + input = File.createTempFile("PigContextTest-", ".txt"); + } + + /** + * Passing an already configured pigContext in PigServer constructor. + */ + @Test + public void testSetProperties_way_num01() throws Exception { + PigServer pigServer = new PigServer(pigContext); + registerAndStore(pigServer); + + check_asserts(); + } + + /** + * Setting properties through PigServer constructor directly. + */ + @Test + public void testSetProperties_way_num02() throws Exception { + PigServer pigServer = new PigServer(ExecType.LOCAL, getProperties()); + registerAndStore(pigServer); + + check_asserts(); + } + + /** + * using connect() method. + */ + @Test + public void testSetProperties_way_num03() throws Exception { + pigContext.connect(); + PigServer pigServer = new PigServer(pigContext); + registerAndStore(pigServer); + + check_asserts(); + } + + @After + @Override + protected void tearDown() throws Exception { + input.delete(); + } + + private static Properties getProperties() { + Properties props = new Properties(); + props.put("mapred.job.tracker", JOB_TRACKER); + props.put("fs.default.name", FS_NAME); + props.put("hadoop.tmp.dir", TMP_DIR_PROP); + return props; + } + + private List<String> getCommands() { + List<String> commands = new ArrayList<String>(); + commands.add("my_input = LOAD '" + input.getAbsolutePath() + "' USING PigStorage();"); + commands.add("words = FOREACH my_input GENERATE FLATTEN(TOKENIZE(*));"); + commands.add("grouped = GROUP words BY $0;"); + commands.add("counts = FOREACH grouped GENERATE group, COUNT(words);"); + return commands; + } + + private void registerAndStore(PigServer pigServer) throws IOException { + pigServer.debugOn(); + List<String> commands = getCommands(); + for (final String command : commands) { + pigServer.registerQuery(command); + } + pigServer.store("counts", input.getAbsolutePath() + ".out"); + } + + private void check_asserts() { + assertEquals(JOB_TRACKER, pigContext.getProperties().getProperty("mapred.job.tracker")); + assertEquals(FS_NAME, pigContext.getProperties().getProperty("fs.default.name")); + assertEquals(TMP_DIR_PROP, pigContext.getProperties().getProperty("hadoop.tmp.dir")); + } +}