Author: kamrul
Date: Tue Nov 20 01:38:25 2012
New Revision: 1411500
URL: http://svn.apache.org/viewvc?rev=1411500&view=rev
Log:
OOZIE-1069 Update dataIn and dataOut EL functions to support partitions
(mohammad)
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/util/HCatURI.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestHCatURI.java
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java?rev=1411500&r1=1411499&r2=1411500&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java
(original)
+++
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java
Tue Nov 20 01:38:25 2012
@@ -491,7 +491,31 @@ public class CoordELFunctions {
if (unresolved != null && unresolved.booleanValue() == true) {
return "${coord:dataIn('" + dataInName + "')}";
}
- return uris;
+ return handlePartitionFilter(uris);
+ }
+
+ private static String handlePartitionFilter(String uris) {
+ String[] uriList = uris.split(DIR_SEPARATOR);
+ // If HCat URI, change its to a filter, otherwise return the original
+ // uris
+ if (uriList.length > 0 && HCatURI.isHcatURI(uriList[0])) {
+ StringBuilder filter = new StringBuilder();
+ for (String uri : uriList) {
+ if (filter.length() > 0) {
+ filter.append(" OR ");
+ }
+ try {
+ filter.append(new HCatURI(uri).toFilter());
+ }
+ catch (URISyntaxException e) {
+ throw new RuntimeException("Parsing exception for HCatURI
" + uri + ". details: " + e);
+ }
+ }
+ return filter.toString();
+ }
+ else {
+ return uris;
+ }
}
/**
@@ -505,7 +529,7 @@ public class CoordELFunctions {
String uris = "";
ELEvaluator eval = ELEvaluator.getCurrent();
uris = (String) eval.getVariable(".dataout." + dataOutName);
- return uris;
+ return handlePartitionFilter(uris);
}
/**
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/util/HCatURI.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/util/HCatURI.java?rev=1411500&r1=1411499&r2=1411500&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/util/HCatURI.java
(original)
+++
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/util/HCatURI.java
Tue Nov 20 01:38:25 2012
@@ -39,6 +39,7 @@ public class HCatURI {
public static final String PARTITION_KEYVAL_SEPARATOR = "=";
public static final String PATH_SEPARATOR = "/";
public static final String PARTITION_PREFIX = "?";
+ public static final String PARTITION_VALUE_QUOTE = "'";
private URI uri;
private String server;
@@ -276,6 +277,27 @@ public class HCatURI {
return equals;
}
+ /**
+ * Convert the partition map to filter string. Each key value pair is
+ * separated by AND
+ *
+ * @return filter string
+ */
+ public String toFilter() {
+ StringBuilder filter = new StringBuilder();
+ for (Map.Entry<String, String> entry : partitions.entrySet()) {
+ if (filter.length() > 0) {
+ filter.append(" AND ");
+ }
+ filter.append(entry.getKey());
+ filter.append("=");
+ filter.append(PARTITION_VALUE_QUOTE);
+ filter.append(entry.getValue());
+ filter.append(PARTITION_VALUE_QUOTE);
+ }
+ return filter.toString();
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
Modified:
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java?rev=1411500&r1=1411499&r2=1411500&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java
(original)
+++
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java
Tue Nov 20 01:38:25 2012
@@ -1015,6 +1015,14 @@ public class TestCoordELFunctions extend
eval.setVariable(".datain.ABC",
"file:///tmp/coord/US/2009/1/30,file:///tmp/coord/US/2009/1/31");
eval.setVariable(".datain.ABC.unresolved", Boolean.TRUE);
assertEquals(expr, CoordELFunctions.evalAndWrap(eval, expr));
+
+ // HCat partition specific
+ eval.setVariable(".datain.ABC",
"hcat://hcat.yahoo.com:5080/mydb/clicks/?datastamp=12®ion=us");
+ eval.setVariable(".datain.ABC.unresolved", Boolean.FALSE);
+ expr = "${coord:dataIn('ABC')}";
+ String res = CoordELFunctions.evalAndWrap(eval, expr);
+ assertTrue(res.equals("datastamp='12' AND region='us'") ||
res.equals("region='us' AND datastamp='12'"));
+
}
public void testDataOut() throws Exception {
@@ -1023,6 +1031,13 @@ public class TestCoordELFunctions extend
String expr = "${coord:dataOut('ABC')}";
assertEquals("file:///tmp/coord/US/2009/1/30,file:///tmp/coord/US/2009/1/31",
CoordELFunctions.evalAndWrap(eval, expr));
+
+ // HCat partition specific
+ eval.setVariable(".dataout.ABC",
"hcat://hcat.yahoo.com:5080/mydb/clicks/?datastamp=20120230®ion=us");
+ eval.setVariable(".dataout.ABC.unresolved", Boolean.FALSE);
+ String res = CoordELFunctions.evalAndWrap(eval, expr);
+ assertTrue(res.equals("datastamp='20120230' AND region='us'")
+ || res.equals("region='us' AND datastamp='20120230'"));
}
public void testActionId() throws Exception {
Modified:
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestHCatURI.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestHCatURI.java?rev=1411500&r1=1411499&r2=1411500&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestHCatURI.java
(original)
+++
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestHCatURI.java
Tue Nov 20 01:38:25 2012
@@ -240,4 +240,18 @@ public class TestHCatURI {
hcatURI =
"hdfs://hcat.yahoo.com:5080/mydb/clicks/?region=us×tamp=1201";
assertFalse(HCatURI.isHcatURI(hcatURI)); // -ve test
}
+
+ @Test
+ public void testToFilter() {
+ String hcatURI =
"hcat://hcat.yahoo.com:5080/mydb/clicks/?datastamp=20120230®ion=us";
+ String filter = "";
+ try {
+ filter = new HCatURI(hcatURI).toFilter();
+ }
+ catch (URISyntaxException e) {
+ fail(e.getMessage());
+ }
+ assertTrue(filter.equals("datastamp='20120230' AND region='us'")
+ || filter.equals("region='us' AND datastamp='20120230'"));
+ }
}
