Author: olga
Date: Mon Jul 21 10:50:56 2008
New Revision: 678512
URL: http://svn.apache.org/viewvc?rev=678512&view=rev
Log:
Missef from PIG-111 merge
Added:
incubator/pig/branches/types/conf/
incubator/pig/branches/types/conf/log4j.properties
incubator/pig/branches/types/conf/pig.properties
incubator/pig/branches/types/test/org/apache/pig/test/TestPigContext.java
Added: incubator/pig/branches/types/conf/log4j.properties
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/conf/log4j.properties?rev=678512&view=auto
==============================================================================
--- incubator/pig/branches/types/conf/log4j.properties (added)
+++ incubator/pig/branches/types/conf/log4j.properties Mon Jul 21 10:50:56 2008
@@ -0,0 +1,8 @@
+# ***** Set root logger level to DEBUG and its only appender to A.
+log4j.rootLogger=info, A
+
+# ***** A is set to be a ConsoleAppender.
+log4j.appender.A=org.apache.log4j.ConsoleAppender
+# ***** A uses PatternLayout.
+log4j.appender.A.layout=org.apache.log4j.PatternLayout
+log4j.appender.A.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
Added: incubator/pig/branches/types/conf/pig.properties
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/conf/pig.properties?rev=678512&view=auto
==============================================================================
--- incubator/pig/branches/types/conf/pig.properties (added)
+++ incubator/pig/branches/types/conf/pig.properties Mon Jul 21 10:50:56 2008
@@ -0,0 +1,62 @@
+# Pig configuration file. All values can be overwritten by command line
arguments.
+# see bin/pig -help
+
+# log4jconf log4j configuration file
+# log4jconf=./conf/log4j.properties
+
+# brief logging (no timestamps)
+brief=false
+
+# clustername, name of the hadoop jobtracker. If no port is defined port 50020
will be used.
+#cluster
+
+#debug level, INFO is default
+debug=INFO
+
+# a file that contains pig script
+#file=
+
+# load jarfile, colon separated
+#jar=
+
+#verbose print all log messages to screen (default to print only INFO and
above to screen)
+verbose=false
+
+#exectype local|mapreduce, mapreduce is default
+#exectype=mapreduce
+# hod realted properties
+#ssh.gateway
+#hod.expect.root
+#hod.expect.uselatest
+#hod.command
+#hod.config.dir
+#hod.param
+
+
+#Do not spill temp files smaller than this size (bytes)
+pig.spill.size.threshold=5000000
+#EXPERIMENT: Activate garbage collection when spilling a file bigger than this
size (bytes)
+#This should help reduce the number of files being spilled.
+pig.spill.gc.activation.size=40000000
+
+
+######################
+# Everything below this line is Yahoo specific. Note that I've made
+# (almost) no changes to the lines above to make merging in from Apache
+# easier. Any values I don't want from above I override below.
+#
+# This file is configured for use with HOD on the production clusters. If you
+# want to run pig with a static cluster you will need to remove everything
+# below this line and set the cluster value (above) to the
+# hostname and port of your job tracker.
+
+exectype=mapreduce
+
+hod.config.dir=/export/crawlspace/kryptonite/hod/current/conf
+hod.server=local
+
+cluster.domain=inktomisearch.com
+
+log.file=
+
+yinst.cluster=kryptonite
Added: incubator/pig/branches/types/test/org/apache/pig/test/TestPigContext.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestPigContext.java?rev=678512&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestPigContext.java
(added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestPigContext.java
Mon Jul 21 10:50:56 2008
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test;
+
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import junit.framework.TestCase;
+
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.impl.PigContext;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestPigContext extends TestCase {
+
+ private static final String TMP_DIR_PROP = "/tmp/hadoop-hadoop";
+ private static final String FS_NAME = "machine:9000";
+ private static final String JOB_TRACKER = "machine:9001";
+
+ private File input;
+ private PigContext pigContext;
+
+ @Before
+ @Override
+ protected void setUp() throws Exception {
+ pigContext = new PigContext(ExecType.LOCAL, getProperties());
+ input = File.createTempFile("PigContextTest-", ".txt");
+ }
+
+ /**
+ * Passing an already configured pigContext in PigServer constructor.
+ */
+ @Test
+ public void testSetProperties_way_num01() throws Exception {
+ PigServer pigServer = new PigServer(pigContext);
+ registerAndStore(pigServer);
+
+ check_asserts();
+ }
+
+ /**
+ * Setting properties through PigServer constructor directly.
+ */
+ @Test
+ public void testSetProperties_way_num02() throws Exception {
+ PigServer pigServer = new PigServer(ExecType.LOCAL, getProperties());
+ registerAndStore(pigServer);
+
+ check_asserts();
+ }
+
+ /**
+ * using connect() method.
+ */
+ @Test
+ public void testSetProperties_way_num03() throws Exception {
+ pigContext.connect();
+ PigServer pigServer = new PigServer(pigContext);
+ registerAndStore(pigServer);
+
+ check_asserts();
+ }
+
+ @After
+ @Override
+ protected void tearDown() throws Exception {
+ input.delete();
+ }
+
+ private static Properties getProperties() {
+ Properties props = new Properties();
+ props.put("mapred.job.tracker", JOB_TRACKER);
+ props.put("fs.default.name", FS_NAME);
+ props.put("hadoop.tmp.dir", TMP_DIR_PROP);
+ return props;
+ }
+
+ private List<String> getCommands() {
+ List<String> commands = new ArrayList<String>();
+ commands.add("my_input = LOAD '" + input.getAbsolutePath() + "' USING
PigStorage();");
+ commands.add("words = FOREACH my_input GENERATE
FLATTEN(TOKENIZE(*));");
+ commands.add("grouped = GROUP words BY $0;");
+ commands.add("counts = FOREACH grouped GENERATE group, COUNT(words);");
+ return commands;
+ }
+
+ private void registerAndStore(PigServer pigServer) throws IOException {
+ pigServer.debugOn();
+ List<String> commands = getCommands();
+ for (final String command : commands) {
+ pigServer.registerQuery(command);
+ }
+ pigServer.store("counts", input.getAbsolutePath() + ".out");
+ }
+
+ private void check_asserts() {
+ assertEquals(JOB_TRACKER,
pigContext.getProperties().getProperty("mapred.job.tracker"));
+ assertEquals(FS_NAME,
pigContext.getProperties().getProperty("fs.default.name"));
+ assertEquals(TMP_DIR_PROP,
pigContext.getProperties().getProperty("hadoop.tmp.dir"));
+ }
+}
