Author: rohini
Date: Fri Jan 23 05:39:39 2015
New Revision: 1654119
URL: http://svn.apache.org/r1654119
Log:
PIG-4375: ObjectCache should use ProcessorContext.getObjectRegistry() (rohini)
Modified:
pig/branches/branch-0.14/CHANGES.txt
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/ObjectCache.java
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java
Modified: pig/branches/branch-0.14/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.14/CHANGES.txt?rev=1654119&r1=1654118&r2=1654119&view=diff
==============================================================================
--- pig/branches/branch-0.14/CHANGES.txt (original)
+++ pig/branches/branch-0.14/CHANGES.txt Fri Jan 23 05:39:39 2015
@@ -26,6 +26,8 @@ IMPROVEMENTS
BUG FIXES
+PIG-4375: ObjectCache should use ProcessorContext.getObjectRegistry() (rohini)
+
PIG-4334: PigProcessor does not set pig.datetime.default.tz (rohini)
PIG-4342: Pig 0.14 cannot identify the uppercase of DECLARE and DEFAULT (daijy)
Modified:
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java?rev=1654119&r1=1654118&r2=1654119&view=diff
==============================================================================
---
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
(original)
+++
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
Fri Jan 23 05:39:39 2015
@@ -268,6 +268,8 @@ public class PigCombiner {
leaf = null;
pack = null;
pigReporter = null;
+ // Avoid OOM in Tez.
+ PhysicalOperator.setReporter(null);
pigContext = null;
roots = null;
cp = null;
Modified:
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/ObjectCache.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/ObjectCache.java?rev=1654119&r1=1654118&r2=1654119&view=diff
==============================================================================
---
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/ObjectCache.java
(original)
+++
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/ObjectCache.java
Fri Jan 23 05:39:39 2015
@@ -21,15 +21,15 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.tez.runtime.api.ObjectRegistry;
-import org.apache.tez.runtime.common.objectregistry.ObjectRegistryImpl;
[email protected]
[email protected]
public class ObjectCache {
private static final Log LOG = LogFactory.getLog(ObjectCache.class);
- private final ObjectRegistry registry = new ObjectRegistryImpl();
private static ObjectCache cache = new ObjectCache();
+ private ObjectRegistry registry;
+
private ObjectCache() {
}
@@ -37,11 +37,34 @@ public class ObjectCache {
return cache;
}
+ /**
+ * Returns the tez ObjectRegistry which allows caching of objects at the
+ * Session, DAG and Vertex level on container reuse for better performance
+ * and savings
+ */
+ public ObjectRegistry getObjectRegistry() {
+ return registry;
+ }
+
+ /**
+ * For internal use only. This method to be called only by PigProcessor
+ */
+ @InterfaceAudience.Private
+ void setObjectRegistry(ObjectRegistry registry) {
+ this.registry = registry;
+ }
+
+ /**
+ * Convenience method to cache objects in ObjectRegistry for a vertex
+ */
public void cache(String key, Object value) {
LOG.info("Adding " + key + " to cache");
registry.cacheForVertex(key, value);
}
+ /**
+ * Convenience method to retrieve objects cached for the vertex from
ObjectRegistry
+ */
public Object retrieve(String key) {
Object o = registry.get(key);
if (o != null) {
Modified:
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java?rev=1654119&r1=1654118&r2=1654119&view=diff
==============================================================================
---
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java
(original)
+++
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java
Fri Jan 23 05:39:39 2015
@@ -103,6 +103,7 @@ public class PigProcessor extends Abstra
public PigProcessor(ProcessorContext context) {
super(context);
+
ObjectCache.getInstance().setObjectRegistry(context.getObjectRegistry());
}
@SuppressWarnings("unchecked")
