Author: heyongqiang
Date: Fri Dec 2 21:27:37 2011
New Revision: 1209696
URL: http://svn.apache.org/viewvc?rev=1209696&view=rev
Log:
Hive-2607: Add caching to json_tuple (Kevin Wilfong via He Yongqiang)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java?rev=1209696&r1=1209695&r2=1209696&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java
Fri Dec 2 21:27:37 2011
@@ -19,6 +19,8 @@
package org.apache.hadoop.hive.ql.udf.generic;
import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -55,6 +57,28 @@ public class GenericUDTFJSONTuple extend
boolean pathParsed = false;
boolean seenErrors = false;
+ //An LRU cache using a linked hash map
+ static class HashCache<K, V> extends LinkedHashMap<K, V> {
+
+ private static final int CACHE_SIZE = 16;
+ private static final int INIT_SIZE = 32;
+ private static final float LOAD_FACTOR = 0.6f;
+
+ HashCache() {
+ super(INIT_SIZE, LOAD_FACTOR);
+ }
+
+ private static final long serialVersionUID = 1;
+
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
+ return size() > CACHE_SIZE;
+ }
+
+ }
+
+ static Map<String, JSONObject> jsonObjectCache = new HashCache<String,
JSONObject>();
+
@Override
public void close() throws HiveException {
}
@@ -124,7 +148,11 @@ public class GenericUDTFJSONTuple extend
return;
}
try {
- JSONObject jsonObj = new JSONObject(jsonStr);
+ JSONObject jsonObj = jsonObjectCache.get(jsonStr);
+ if (jsonObj == null) {
+ jsonObj = new JSONObject(jsonStr);
+ jsonObjectCache.put(jsonStr, jsonObj);
+ }
for (int i = 0; i < numCols; ++i) {
if (jsonObj.isNull(paths[i])) {
