Author: szita
Date: Tue Jan 9 10:03:07 2018
New Revision: 1820631
URL: http://svn.apache.org/viewvc?rev=1820631&view=rev
Log:
PIG-5325: Schema disambiguation can't be turned off for nested schemas (szita)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java
pig/trunk/test/org/apache/pig/test/TestSchema.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1820631&r1=1820630&r2=1820631&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Jan 9 10:03:07 2018
@@ -64,6 +64,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-5325: Schema disambiguation can't be turned off for nested schemas (szita)
+
PIG-5311: POReservoirSample fails for more than Integer.MAX_VALUE records
(rohini)
PIG-3864: ToDate(userstring, format, timezone) computes DateTime with strange
handling of Daylight Saving Time with location based timezones (daijy via
rohini)
Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java?rev=1820631&r1=1820630&r2=1820631&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java
(original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java Tue
Jan 9 10:03:07 2018
@@ -63,16 +63,31 @@ public class LOStore extends LogicalRela
public LogicalSchema getSchema() throws FrontendException {
schema =
((LogicalRelationalOperator)plan.getPredecessors(this).get(0)).getSchema();
- if (!disambiguationEnabled && schema != null && schema.getFields() !=
null) {
+ if (!disambiguationEnabled) {
//If requested try and remove parent alias substring including
colon(s)
+ removeDisambiguation(schema);
+ }
+
+ return schema;
+ }
+
+ /**
+ * Removes schema disambiguation parts (parent alias and :) from field
aliases
+ * @param schema
+ * @return
+ */
+ private static LogicalSchema removeDisambiguation(LogicalSchema schema) {
+ if (schema != null && schema.getFields() != null) {
for (LogicalSchema.LogicalFieldSchema field : schema.getFields()) {
+ if (field.schema != null) {
+ removeDisambiguation(field.schema);
+ }
if (field.alias == null || !field.alias.contains(":")) {
continue;
}
field.alias =
field.alias.substring(field.alias.lastIndexOf(":") + 1);
}
}
-
return schema;
}
Modified: pig/trunk/test/org/apache/pig/test/TestSchema.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestSchema.java?rev=1820631&r1=1820630&r2=1820631&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestSchema.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestSchema.java Tue Jan 9 10:03:07 2018
@@ -990,6 +990,33 @@ public class TestSchema {
}
@Test
+ public void testDisabledDisambiguationContainsNoColonsForNestedSchema()
throws IOException {
+ resetDisambiguationTestPropertyOverride();
+
+ String inputFileName = "testPrepend-nested-input.txt";
+ String[] inputData = new String[]{"apple\t1\tred",
"orange\t2\torange", "kiwi\t3\tgreen", "orange\t4\torange"};
+ Util.createInputFile(cluster, inputFileName, inputData);
+
+ String script = "A = LOAD '" + inputFileName + "' AS (fruit:chararray,
foo:int, color: chararray);" +
+ "B = LOAD '" + inputFileName + "' AS (id:chararray, bar:int);"
+
+ "C = JOIN A by fruit, B by id;" +
+ "D = GROUP C by fruit;" +
+ "E = LOAD '" + inputFileName + "' AS (name:chararray,
qwe:int);" +
+ "F = JOIN E by name, D by group;";
+
+ Util.registerMultiLineQuery(pigServer, script);
+
+ //Prepending should happen with default settings
+ assertEquals("{E::name: chararray,E::qwe: int,D::group:
chararray,D::C: {(A::fruit: chararray,A::foo: int,A::color: chararray,B::id:
chararray,B::bar: int)}}", pigServer.dumpSchema("F").toString());
+
+ //Override prepend property setting (check for flatten, join)
+
pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_STORE_SCHEMA_DISAMBIGUATE,
"false");
+ assertEquals("{name: chararray,qwe: int,group: chararray,C: {(fruit:
chararray,foo: int,color: chararray,id:" +
+ " chararray,bar: int)}}",
pigServer.dumpSchema("F").toString());
+ assertTrue(pigServer.openIterator("F").hasNext());
+ }
+
+ @Test
public void testEnabledDisambiguationPassesForDupeAliases() throws
IOException {
resetDisambiguationTestPropertyOverride();
