Author: daijy
Date: Wed Nov 9 00:30:57 2016
New Revision: 1768807
URL: http://svn.apache.org/viewvc?rev=1768807&view=rev
Log:
PIG-4939: QueryParserUtils.setHdfsServers(QueryParserUtils.java:104) should not
be called for non-dfs methods
Added:
pig/trunk/src/org/apache/pig/NonFSLoadFunc.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/docs/src/documentation/content/xdocs/udf.xml
pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java
pig/trunk/test/org/apache/pig/parser/TestQueryParserUtils.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1768807&r1=1768806&r2=1768807&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Nov 9 00:30:57 2016
@@ -24,6 +24,9 @@ INCOMPATIBLE CHANGES
Â
IMPROVEMENTS
+PIG-4939: QueryParserUtils.setHdfsServers(QueryParserUtils.java:104) should
not be called for non-dfs
+ methods (szita via daijy)
+
PIG-5034: Remove org.apache.hadoop.hive.serde2.objectinspector.primitive
package (nkollar via daijy)
PIG-5036: Remove biggish from e2e input dataset (daijy)
Modified: pig/trunk/src/docs/src/documentation/content/xdocs/udf.xml
URL:
http://svn.apache.org/viewvc/pig/trunk/src/docs/src/documentation/content/xdocs/udf.xml?rev=1768807&r1=1768806&r2=1768807&view=diff
==============================================================================
--- pig/trunk/src/docs/src/documentation/content/xdocs/udf.xml (original)
+++ pig/trunk/src/docs/src/documentation/content/xdocs/udf.xml Wed Nov 9
00:30:57 2016
@@ -1038,6 +1038,9 @@ has methods to convert byte arrays to sp
<li id="loadpredicatepushdown"><a
href="http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/LoadPredicatePushdown.java?view=markup";>LoadPredicatePushdown</a>
has the methods to push predicates to the loader. It is different than
LoadMetadata.setPartitionFilter in that loader may load records which does not
satisfy the predicates. In other words, predicates is only a hint. Note this
interface is still in development and might change in next version. Currently
only OrcStorage implements this interface.</li>
+
+<li id="nonfsloadfunc"><a
href="http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/NonFSLoadFunc.java?view=markup";>NonFSLoadFunc</a>
+ is a marker interface to indicate that a LoadFunc implementation is not a
filesystem loader. This is useful for LoadFunc classes that for example supply
queries instead of filesystem pathes to the LOAD operator.</li>
</ul>
<p>The LoadFunc abstract class is the main class to extend for implementing a
loader. The methods which need to be overridden are explained below:</p>
Added: pig/trunk/src/org/apache/pig/NonFSLoadFunc.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/NonFSLoadFunc.java?rev=1768807&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/NonFSLoadFunc.java (added)
+++ pig/trunk/src/org/apache/pig/NonFSLoadFunc.java Wed Nov 9 00:30:57 2016
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig;
+
+/**
+ * Marker interface to distinguish LoadFunc implementations that don't use
file system sources.
+ */
+public interface NonFSLoadFunc {
+
+}
Modified: pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java?rev=1768807&r1=1768806&r2=1768807&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java (original)
+++ pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java Wed Nov 9
00:30:57 2016
@@ -34,6 +34,7 @@ import org.antlr.runtime.RecognitionExce
import org.apache.pig.ExecType;
import org.apache.pig.FuncSpec;
import org.apache.pig.LoadFunc;
+import org.apache.pig.NonFSLoadFunc;
import org.apache.pig.PigConfiguration;
import org.apache.pig.StoreFuncInterface;
import org.apache.pig.backend.executionengine.ExecException;
@@ -888,7 +889,7 @@ public class LogicalPlanBuilder {
if (absolutePath == null) {
absolutePath = loFunc.relativeToAbsolutePath( filename,
QueryParserUtils.getCurrentDir( pigContext ) );
- if (absolutePath!=null) {
+ if (absolutePath!=null && !(loFunc instanceof NonFSLoadFunc)) {
QueryParserUtils.setHdfsServers( absolutePath, pigContext
);
}
fileNameMap.put( fileNameKey, absolutePath );
Modified: pig/trunk/test/org/apache/pig/parser/TestQueryParserUtils.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/parser/TestQueryParserUtils.java?rev=1768807&r1=1768806&r2=1768807&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/parser/TestQueryParserUtils.java (original)
+++ pig/trunk/test/org/apache/pig/parser/TestQueryParserUtils.java Wed Nov 9
00:30:57 2016
@@ -19,10 +19,20 @@ package org.apache.pig.parser;
import static org.junit.Assert.assertEquals;
+import java.io.IOException;
import java.util.Properties;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.pig.ExecType;
+import org.apache.pig.LoadFunc;
+import org.apache.pig.NonFSLoadFunc;
+import org.apache.pig.PigServer;
import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRConfiguration;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
+import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
import org.apache.pig.test.Util;
import org.junit.Test;
@@ -107,8 +117,44 @@ public class TestQueryParserUtils {
}
+ }
+ @Test
+ public void testNonFSLoadFunc() throws Exception {
+ PigServer pigServer = new PigServer(Util.getLocalTestMode(), new
Properties());
+ pigServer.registerQuery("A = load 'hbase://query/SELECT ID, NAME,
DATE FROM HIRES WHERE DATE > TO_DATE(\"1990-12-21 05:55:00.000\")' using
org.apache.pig.parser.TestQueryParserUtils$DummyNonFSLoader();");
+ pigServer.shutdown();
}
+ /**
+ * Test class for testNonFSLoadFuncNoSetHdfsServersCall test case
+ */
+ public static class DummyNonFSLoader extends LoadFunc implements
NonFSLoadFunc {
+
+ @Override
+ public void setLocation(String location, Job job) throws IOException {
+ throw new RuntimeException("Should not be called");
+ }
+
+ @Override
+ public InputFormat getInputFormat() throws IOException {
+ throw new RuntimeException("Should not be called");
+ }
+
+ @Override
+ public void prepareToRead(RecordReader reader, PigSplit split) throws
IOException {
+ throw new RuntimeException("Should not be called");
+ }
+
+ @Override
+ public Tuple getNext() throws IOException {
+ throw new RuntimeException("Should not be called");
+ }
+
+ @Override
+ public String relativeToAbsolutePath(String location, Path curDir)
throws IOException {
+ return location;
+ }
+ }
}
