svn commit: r829126 - in /hadoop/pig/trunk: CHANGES.txt src/org/apache/pig/tools/pigstats/PigStats.java test/org/apache/pig/test/TestPigStats.java
Author: gates
Date: Fri Oct 23 17:01:24 2009
New Revision: 829126
URL: http://svn.apache.org/viewvc?rev=829126view=rev
Log:
PIG-1027: Number of bytes written are always zero in local mode.
Added:
hadoop/pig/trunk/test/org/apache/pig/test/TestPigStats.java
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=829126r1=829125r2=829126view=diff
==
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Fri Oct 23 17:01:24 2009
@@ -78,6 +78,8 @@
BUG FIXES
+PIG-1027: Number of bytes written are always zero in local mode (zjffdu via
gates).
+
PIG-976: Multi-query optimization throws ClassCastException (rding via
pradeepkth)
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java?rev=829126r1=829125r2=829126view=diff
==
--- hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java Fri Oct 23
17:01:24 2009
@@ -102,11 +102,19 @@
stats.put(op.toString(), jobStats);
POCounter counter = (POCounter) php.getPredecessors(op).get(0);
jobStats.put(PIG_STATS_LOCAL_OUTPUT_RECORDS,
(Long.valueOf(counter.getCount())).toString());
-jobStats.put(PIG_STATS_LOCAL_BYTES_WRITTEN, (Long.valueOf((new
File(((POStore)op).getSFile().getFileName())).length())).toString());
+String
localFilePath=normalizeToLocalFilePath(((POStore)op).getSFile().getFileName());
+jobStats.put(PIG_STATS_LOCAL_BYTES_WRITTEN, (Long.valueOf(new
File(localFilePath).length())).toString());
}
return stats;
}
+private String normalizeToLocalFilePath(String fileName) {
+if (fileName.startsWith(file:)){
+return fileName.substring(5);
+}
+return fileName;
+}
+
private MapString, MapString, String accumulateMRStats() throws
ExecException {
for(Job job : jc.getSuccessfulJobs()) {
Added: hadoop/pig/trunk/test/org/apache/pig/test/TestPigStats.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPigStats.java?rev=829126view=auto
==
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPigStats.java (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPigStats.java Fri Oct 23
17:01:24 2009
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test;
+
+import java.io.File;
+import java.io.IOException;
+
+import junit.framework.TestCase;
+
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.tools.pigstats.PigStats;
+
+public class TestPigStats extends TestCase {
+
+public void testBytesWritten_JIRA_1027() {
+
+File outputFile = null;
+try {
+outputFile = File.createTempFile(JIAR_1027, .out);
+PigServer pig = new PigServer(ExecType.LOCAL);
+pig
+.registerQuery(A = load
'test/org/apache/pig/test/data/passwd';);
+PigStats stats = pig.store(A, outputFile.getAbsolutePath())
+.getStatistics();
+assertEquals(outputFile.length(), stats.getBytesWritten());
+} catch (IOException e) {
+fail(IOException happened);
+} finally {
+if (outputFile != null) {
+outputFile.delete();
+}
+}
+
+}
+}
svn commit: r829156 - /hadoop/pig/trunk/CHANGES.txt
Author: gates Date: Fri Oct 23 18:05:23 2009 New Revision: 829156 URL: http://svn.apache.org/viewvc?rev=829156view=rev Log: Minor syntax cleanup in CHANGES.txt to make it easier to automate contributor counting. Modified: hadoop/pig/trunk/CHANGES.txt Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=829156r1=829155r2=829156view=diff == --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Fri Oct 23 18:05:23 2009 @@ -30,7 +30,7 @@ String(String) constructor (olgan) PIG-984: Add map side grouping for data that is already collected when -it is read into the map (rding via gates). +it is read into the map (rding via gates) PIG-1025: Add ability to set job priority from Pig Latin script (kevinweil via gates) @@ -64,7 +64,7 @@ PIG-975: Need a databag that does not register with SpillableMemoryManager and spill data pro-actively (yinghe via olgan) -PIG-891: Fixing dfs statement for Pig (zjffdu via daijy). +PIG-891: Fixing dfs statement for Pig (zjffdu via daijy PIG-956: 10 minute commit tests (olgan) @@ -78,7 +78,7 @@ BUG FIXES -PIG-1027: Number of bytes written are always zero in local mode (zjffdu via gates). +PIG-1027: Number of bytes written are always zero in local mode (zjffdu via gates) PIG-976: Multi-query optimization throws ClassCastException (rding via pradeepkth) @@ -87,17 +87,17 @@ from physical plan (ashutoshc via gates) PIG-968: Fix findContainingJar to work properly when there is a + in the jar - path (tlipcon via gates). + path (tlipcon via gates) PIG-738: Regexp passed from pigscript fails in UDF (pradeepkth) PIG-942: Maps are not implicitly casted (pradeepkth) PIG-513: Removed unecessary bounds check in DefaultTuple (ashutoshc via - gates). + gates) PIG-951: Set parallelism explicitly to 1 for indexing job in merge join - (ashutoc via gates). + (ashutoshc via gates) PIG-592: schema inferred incorrectly (daijy) @@ -136,7 +136,7 @@ PIG-892: Make COUNT and AVG deal with nulls accordingly with SQL standart (olgan) -PIG-734: Changed maps to only take strings as keys (gates). +PIG-734: Changed maps to only take strings as keys (gates) IMPROVEMENTS @@ -159,7 +159,7 @@ PIG-845: PERFORMANCE: Merge Join (ashutoshc via pradeepkth) -PIG-893: Added string - integer, long, float, and double casts (zjffdu via gates). +PIG-893: Added string - integer, long, float, and double casts (zjffdu via gates) PIG-833: Added Zebra, new columnar storage mechanism for HDFS (rangadi plus many others via gates) @@ -169,15 +169,15 @@ PIG-820: Change RandomSampleLoader to take a LoadFunc instead of extending BinStorage. Added new Samplable interface for loaders to implement - allowing them to be used by RandomSampleLoader (ashutoshc via gates). + allowing them to be used by RandomSampleLoader (ashutoshc via gates) PIG-832: Make import list configurable (daijy) PIG-697: Proposed improvements to pig's optimizer (sms) -PIG-753: Allow UDFs with no parameters (zjffdu via gates). +PIG-753: Allow UDFs with no parameters (zjffdu via gates) -PIG-765: jdiff for pig ( gkesavan ). +PIG-765: jdiff for pig ( gkesavan OPTIMIZATIONS @@ -227,7 +227,7 @@ PIG-695: Pig should not fail when error logs cannot be created (sms) - PIG-878: Pig is returning too many blocks in the input split. (arunc via gates). + PIG-878: Pig is returning too many blocks in the input split. (arunc via gates) PIG-888: Pig do not pass udf to the backend in some situation (daijy) @@ -267,29 +267,30 @@ PIG-817: documentation update (chandec via olgan) -PIG-830: Add RegExLoader and apache log utils to piggybank (dvryaboy via gates). +PIG-830: Add RegExLoader and apache log utils to piggybank (dvryaboy via gates) PIG-831: Turned off reporting of records and bytes written for mutli-store queries as the returned results are confusing and wrong. (gates) PIG-813: documentation updates (chandec via olgan) -PIG-825: PIG_HADOOP_VERSION should be set to 18 (dvryaboy via gates). +PIG-825: PIG_HADOOP_VERSION should be set to 18 (dvryaboy via gates) PIG-795: support for SAMPLE command (ericg via olgan) PIG-619: Create one InputSplit even when the input file is zero length so that hadoop runs maps and creates output for the next -job (gates). +job (gates) PIG-697: Proposed improvements to pig's optimizer (sms) PIG-700: To automate the pig patch test process (gkesavan via sms) PIG-712: Added utility functions to create schemas for tuples and bags (zjffdu -via gates). +via gates) -PIG-652: Adapt changes in store interface to multi-query changes (hagleitn via gates). +PIG-652: Adapt changes in store interface to multi-query
svn commit: r829195 - in /hadoop/pig/trunk/contrib/zebra: CHANGES.txt build-contrib.xml build.xml
Author: gates
Date: Fri Oct 23 19:59:53 2009
New Revision: 829195
URL: http://svn.apache.org/viewvc?rev=829195view=rev
Log:
PIG-996 Add findbugs, checkstyle, and clover to zebra build file.
Modified:
hadoop/pig/trunk/contrib/zebra/CHANGES.txt
hadoop/pig/trunk/contrib/zebra/build-contrib.xml
hadoop/pig/trunk/contrib/zebra/build.xml
Modified: hadoop/pig/trunk/contrib/zebra/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/CHANGES.txt?rev=829195r1=829194r2=829195view=diff
==
--- hadoop/pig/trunk/contrib/zebra/CHANGES.txt (original)
+++ hadoop/pig/trunk/contrib/zebra/CHANGES.txt Fri Oct 23 19:59:53 2009
@@ -6,6 +6,9 @@
IMPROVEMENTS
+ PIG-996 Add findbugs, checkstyle, and clover to zebra build file (chaow
via
+ gates)
+
PIG-993 Ability to drop a column group in a table (yanz and rangadi via
gates)
PIG-992 Separate schema related files into a schema package (yanz via
Modified: hadoop/pig/trunk/contrib/zebra/build-contrib.xml
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/build-contrib.xml?rev=829195r1=829194r2=829195view=diff
==
--- hadoop/pig/trunk/contrib/zebra/build-contrib.xml (original)
+++ hadoop/pig/trunk/contrib/zebra/build-contrib.xml Fri Oct 23 19:59:53 2009
@@ -61,6 +61,18 @@
property name=build.ivy.lib.dir location=${build.ivy.dir}/lib /
property name=ivy.lib.dir location=${build.ivy.lib.dir}/Pig/
+ property name=clover.db.dir location=${build.dir}/test/clover/db/
+property name=clover.report.dir
location=${build.dir}/test/clover/reports/
+property name=clover.jar location=${clover.home}/lib/clover.jar/
+available property=clover.present file=${clover.jar} /
+!-- check if clover reports should be generated --
+condition property=clover.enabled
+ and
+isset property=run.clover/
+isset property=clover.present/
+ /and
+/condition
+
!-- javacc properties --
property name=src.gen.dir value=${basedir}/src-gen/ /
property name=src.gen.zebra.parser.dir
value=${src.gen.dir}/org/apache/hadoop/zebra/parser /
@@ -107,6 +119,7 @@
pathelement location=${conf.dir}/
pathelement location=${pig.root}/build/
pathelement location=${build.examples}/
+ pathelement path=${clover.jar}/
path refid=classpath/
/path
@@ -235,48 +248,6 @@
/fileset
/copy
/target
-
- !-- == --
- !-- Run unit tests --
- !-- == --
- target name=test depends=compile-test, compile if=test.available
-echo message=contrib: ${name}/
-delete dir=${pig.log.dir}/
-mkdir dir=${pig.log.dir}/
-junit
- printsummary=yes showoutput=${test.output}
- haltonfailure=no fork=yes maxmemory=256m
- errorProperty=tests.failed failureProperty=tests.failed
- timeout=${test.timeout}
-
- sysproperty key=test.build.data value=${build.test}/data/
- sysproperty key=build.test value=${build.test}/
- sysproperty key=contrib.name value=${name}/
- sysproperty key=hadoop.log.dir value=${pig.log.dir}/
-
- !-- requires fork=yes for:
-relative File paths to use the specified user.dir
-classpath to use build/contrib/*.jar
- --
- sysproperty key=user.dir value=${build.test}/data/
-
- sysproperty key=fs.default.name value=${fs.default.name}/
- sysproperty key=pig.test.localoutputfile
value=${pig.test.localoutputfile}/
- sysproperty key=pig.log.dir value=${pig.log.dir}/
- classpath refid=test.classpath/
- formatter type=${test.junit.output.format} /
-
- batchtest todir=${build.test} unless=testcase
-fileset dir=${src.test}
- includes=**/Test*.java excludes=**/${test.exclude}.java /
- /batchtest
- batchtest todir=${build.test} if=testcase
-fileset dir=${src.test} includes=**/${testcase}.java/
- /batchtest
-/junit
-fail if=tests.failedTests failed!/fail
-
- /target
!-- == --
!-- Clean. Delete the build files, and their directories --
Modified: hadoop/pig/trunk/contrib/zebra/build.xml
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/build.xml?rev=829195r1=829194r2=829195view=diff
==
--- hadoop/pig/trunk/contrib/zebra/build.xml (original)
+++ hadoop/pig/trunk/contrib/zebra/build.xml Fri Oct 23 19:59:53 2009
@@ -24,10 +24,10 @@
project name=zebra default=jar
import file=build-contrib.xml/
- property name=zebraVersion value=0.1.0 /
+ property name=zebraVersion
svn commit: r829216 - in /hadoop/pig/trunk: ./ lib-src/bzip2/org/apache/tools/bzip2r/ src/org/apache/pig/ src/org/apache/pig/backend/executionengine/util/ src/org/apache/pig/backend/hadoop/executionen
Author: daijy
Date: Fri Oct 23 20:39:39 2009
New Revision: 829216
URL: http://svn.apache.org/viewvc?rev=829216view=rev
Log:
PIG-1041: javac warnings: cast, fallthrough, serial
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CBZip2InputStream.java
hadoop/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CBZip2OutputStream.java
hadoop/pig/trunk/src/org/apache/pig/Main.java
hadoop/pig/trunk/src/org/apache/pig/backend/executionengine/util/ExecTools.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/plans/DotMRPrinter.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/plans/MROperPlan.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogicalToPhysicalTranslatorException.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POBinCond.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POIsNull.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POMapLookUp.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/PONegative.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalExecutionEngine.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalPigLauncher.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/LocalLogToPhyTranslationVisitor.java
hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POCogroup.java
hadoop/pig/trunk/src/org/apache/pig/builtin/FloatMax.java
hadoop/pig/trunk/src/org/apache/pig/builtin/FloatSum.java
hadoop/pig/trunk/src/org/apache/pig/builtin/IntMax.java
hadoop/pig/trunk/src/org/apache/pig/builtin/IntSum.java
hadoop/pig/trunk/src/org/apache/pig/builtin/LongSum.java
hadoop/pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java
hadoop/pig/trunk/src/org/apache/pig/data/DataByteArray.java
hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java
hadoop/pig/trunk/src/org/apache/pig/data/SingleTupleBag.java
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckerException.java
hadoop/pig/trunk/src/org/apache/pig/impl/plan/PlanException.java
hadoop/pig/trunk/src/org/apache/pig/impl/plan/VisitorException.java
hadoop/pig/trunk/src/org/apache/pig/impl/plan/optimizer/OptimizerException.java
hadoop/pig/trunk/src/org/apache/pig/impl/util/Pair.java
hadoop/pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java
hadoop/pig/trunk/src/org/apache/pig/pen/DerivedDataVisitor.java
hadoop/pig/trunk/src/org/apache/pig/pen/LineageTrimmingVisitor.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=829216r1=829215r2=829216view=diff
==
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Fri Oct 23 20:39:39 2009
@@ -76,6 +76,8 @@
PIG-1033: javac warnings: deprecated hadoop APIs (daijy)
+PIG-1041: javac warnings: cast, fallthrough, serial (daijy)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CBZip2InputStream.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CBZip2InputStream.java?rev=829216r1=829215r2=829216view=diff
==
---
hadoop/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CBZip2InputStream.java
(original)
+++
hadoop/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CBZip2InputStream.java
Fri Oct 23 20:39:39 2009
@@ -433,11 +433,11 @@
}
private int bsGetIntVS(int numBits) throws IOException {
-return (int) bsR(numBits);
+return bsR(numBits);
}
private int bsGetInt32() throws IOException {
-return (int) bsGetint();
+return bsGetint();
}
private void hbCreateDecodeTables(int[] limit, int[] base,
@@ -796,7 +796,7 @@
}
for (i = 0; i = last; i++) {
-ch = (char) ll8[i];
+ch = ll8[i];
tt[cftab[ch]] = i;
cftab[ch]++;
}
@@ -830,7 +830,7 @@
}
svn commit: r829277 - /hadoop/pig/branches/load-store-redesign/
Author: pradeepkth Date: Fri Oct 23 23:18:34 2009 New Revision: 829277 URL: http://svn.apache.org/viewvc?rev=829277view=rev Log: branch to develop new load store interfaces as laid out in http://wiki.apache.org/pig/LoadStoreRedesignProposal - this work will be based on the hadoop 20 api changes submitted in PIG-1045 Added: hadoop/pig/branches/load-store-redesign/ (props changed) - copied from r829276, hadoop/pig/trunk/ Propchange: hadoop/pig/branches/load-store-redesign/ -- --- svn:ignore (added) +++ svn:ignore Fri Oct 23 23:18:34 2009 @@ -0,0 +1,4 @@ + +dist +depend +pig.jar Propchange: hadoop/pig/branches/load-store-redesign/ -- svn:mergeinfo = /hadoop/pig/branches/multiquery:741727-770826
