Used TSONProtocol instead of thrift binary protocol and cleaned up some unused code
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/09fb701c Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/09fb701c Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/09fb701c Branch: refs/heads/apache-blur-0.2 Commit: 09fb701cf4c0a1a74878af2b1c75d68551688a66 Parents: 12e7afe Author: Rahul Challapalli <[email protected]> Authored: Tue Oct 29 23:23:14 2013 -0700 Committer: Rahul Challapalli <[email protected]> Committed: Tue Oct 29 23:23:14 2013 -0700 ---------------------------------------------------------------------- blur-289.patch | 151 +++++++++++++++++++ .../clusterstatus/ZookeeperClusterStatus.java | 77 +--------- .../clusterstatus/ZookeeperPathConstants.java | 4 - 3 files changed, 157 insertions(+), 75 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/09fb701c/blur-289.patch ---------------------------------------------------------------------- diff --git a/blur-289.patch b/blur-289.patch new file mode 100644 index 0000000..7cb04cb --- /dev/null +++ b/blur-289.patch @@ -0,0 +1,151 @@ +From 12e7afe3e630783fbafacd09967d25d89643e1bd Mon Sep 17 00:00:00 2001 +From: Rahul Challapalli <[email protected]> +Date: Sun, 27 Oct 2013 23:41:43 -0700 +Subject: [PATCH] Modified the way in which TableDescriptor object is + serialized and stored in zookeeper + +--- + .../clusterstatus/ZookeeperClusterStatus.java | 90 +++++++++++++++------- + 1 file changed, 64 insertions(+), 26 deletions(-) + +diff --git a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java +index 9a0f420..0ce0d55 100644 +--- a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java ++++ b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java +@@ -37,6 +37,10 @@ import org.apache.blur.BlurConfiguration; + import org.apache.blur.log.Log; + import org.apache.blur.log.LogFactory; + import org.apache.blur.lucene.search.FairSimilarity; ++import org.apache.blur.thirdparty.thrift_0_9_0.TDeserializer; ++import org.apache.blur.thirdparty.thrift_0_9_0.TException; ++import org.apache.blur.thirdparty.thrift_0_9_0.TSerializer; ++import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TBinaryProtocol; + import org.apache.blur.thrift.generated.TableDescriptor; + import org.apache.blur.utils.BlurConstants; + import org.apache.blur.utils.BlurUtil; +@@ -409,6 +413,52 @@ public class ZookeeperClusterStatus extends ClusterStatus { + TableDescriptor tableDescriptor = new TableDescriptor(); + try { + checkIfOpen(); ++ ++ TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory()); ++ String blurTablePath = ZookeeperPathConstants.getTablePath(cluster, table); ++ byte[] bytes = getData(blurTablePath); ++ ++ if (bytes == null || bytes.length == 0) { ++ /* ++ * table descriptor is stored in an older format where we manually ++ * serialized each field into a different zookeeper node ++ * so we fetch it using old code and serialize it again with thrift protocol ++ */ ++ getOldTableDescriptor(useCache, cluster, table, tableDescriptor); ++ ++ // store it using thrift protocol ++ byte[] newFormatBytes = serializeTableDescriptor(tableDescriptor); ++ BlurUtil.createPath(_zk, blurTablePath, newFormatBytes); ++ ++ // remove old child nodes ++ List<String> list = _zk.getChildren(blurTablePath, false); ++ for (String p : list) { ++ BlurUtil.removeAll(_zk, blurTablePath + "/" + p); ++ } ++ } else { ++ deserializer.deserialize(tableDescriptor, bytes); ++ } ++ ++ ++ } catch (TException e) { ++ throw new RuntimeException(e); ++ } catch (KeeperException e) { ++ throw new RuntimeException(e); ++ } catch (InterruptedException e) { ++ throw new RuntimeException(e); ++ } finally { ++ long e = System.nanoTime(); ++ LOG.debug("trace getTableDescriptor took [" + (e - s) / 1000000.0 + " ms]"); ++ } ++ tableDescriptor.cluster = cluster; ++ _tableDescriptorCache.put(table, tableDescriptor); ++ return tableDescriptor; ++ } ++ ++ private TableDescriptor getOldTableDescriptor(boolean useCache, String cluster, String table, TableDescriptor tableDescriptor) { ++ long s = System.nanoTime(); ++ try { ++ + NullPointerException npe = null; + LOOP: for (int i = 0; i < 10; i++) { + npe = null; +@@ -444,13 +494,12 @@ public class ZookeeperClusterStatus extends ClusterStatus { + throw new RuntimeException(e); + } finally { + long e = System.nanoTime(); +- LOG.debug("trace getTableDescriptor took [" + (e - s) / 1000000.0 + " ms]"); ++ LOG.debug("trace getOldTableDescriptor took [" + (e - s) / 1000000.0 + " ms]"); + } +- tableDescriptor.cluster = cluster; +- _tableDescriptorCache.put(table, tableDescriptor); ++ + return tableDescriptor; + } +- ++ + private boolean internalGetReadOnly(String tableReadOnlyPath) throws KeeperException, InterruptedException { + Stat stat = _zk.exists(tableReadOnlyPath, false); + if (stat == null) { +@@ -718,34 +767,14 @@ public class ZookeeperClusterStatus extends ClusterStatus { + String uri = BlurUtil.nullCheck(tableDescriptor.tableUri, "tableDescriptor.tableUri cannot be null."); + int shardCount = BlurUtil.zeroCheck(tableDescriptor.shardCount, + "tableDescriptor.shardCount cannot be less than 1"); +- Similarity similarity = BlurUtil.getInstance(tableDescriptor.similarityClass, Similarity.class); +- boolean blockCaching = tableDescriptor.blockCaching; +- Set<String> blockCachingFileTypes = tableDescriptor.blockCachingFileTypes; + String blurTablePath = ZookeeperPathConstants.getTablePath(cluster, table); + + if (_zk.exists(blurTablePath, false) != null) { + throw new IOException("Table [" + table + "] already exists."); + } + BlurUtil.setupFileSystem(uri, shardCount); +- BlurUtil.createPath(_zk, blurTablePath, null); +- BlurUtil.createPath(_zk, ZookeeperPathConstants.getTableColumnsToPreCache(cluster, table), +- toBytes(tableDescriptor.preCacheCols)); +- BlurUtil.createPath(_zk, ZookeeperPathConstants.getTableUriPath(cluster, table), uri.getBytes()); +- BlurUtil.createPath(_zk, ZookeeperPathConstants.getTableShardCountPath(cluster, table), +- Integer.toString(shardCount).getBytes()); +- BlurUtil.createPath(_zk, ZookeeperPathConstants.getTableSimilarityPath(cluster, table), similarity.getClass() +- .getName().getBytes()); +- BlurUtil.createPath(_zk, ZookeeperPathConstants.getLockPath(cluster, table), null); +- BlurUtil.createPath(_zk, ZookeeperPathConstants.getTableFieldNamesPath(cluster, table), null); +- if (tableDescriptor.readOnly) { +- BlurUtil.createPath(_zk, ZookeeperPathConstants.getTableReadOnlyPath(cluster, table), null); +- } +- if (blockCaching) { +- BlurUtil.createPath(_zk, ZookeeperPathConstants.getTableBlockCachingPath(cluster, table), null); +- } +- BlurUtil.createPath(_zk, ZookeeperPathConstants.getTableBlockCachingFileTypesPath(cluster, table), +- toBytes(blockCachingFileTypes)); +- BlurUtil.createPath(_zk, ZookeeperPathConstants.getTablePropertiesPath(cluster, table), toBytes(tableDescriptor.getTableProperties())); ++ byte[] bytes = serializeTableDescriptor(tableDescriptor); ++ BlurUtil.createPath(_zk, blurTablePath, bytes); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (KeeperException e) { +@@ -758,6 +787,15 @@ public class ZookeeperClusterStatus extends ClusterStatus { + } + } + ++ private byte[] serializeTableDescriptor(TableDescriptor td) { ++ try{ ++ TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory()); ++ return serializer.serialize(td); ++ } catch (TException e) { ++ throw new RuntimeException(e); ++ } ++ } ++ + private void assignTableUri(TableDescriptor tableDescriptor) { + if (tableDescriptor.getTableUri() != null) { + return; +-- +1.7.12.4 (Apple Git-37) + http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/09fb701c/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java index 0ce0d55..3cf948a 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java +++ b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -40,7 +39,7 @@ import org.apache.blur.lucene.search.FairSimilarity; import org.apache.blur.thirdparty.thrift_0_9_0.TDeserializer; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thirdparty.thrift_0_9_0.TSerializer; -import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TBinaryProtocol; +import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TJSONProtocol; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.utils.BlurConstants; import org.apache.blur.utils.BlurUtil; @@ -50,7 +49,6 @@ import org.apache.blur.zookeeper.WatchNodeData; import org.apache.blur.zookeeper.WatchNodeExistance; import org.apache.blur.zookeeper.ZkUtils; import org.apache.blur.zookeeper.ZooKeeperLockManager; -import org.apache.lucene.search.similarities.Similarity; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; @@ -414,7 +412,6 @@ public class ZookeeperClusterStatus extends ClusterStatus { try { checkIfOpen(); - TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory()); String blurTablePath = ZookeeperPathConstants.getTablePath(cluster, table); byte[] bytes = getData(blurTablePath); @@ -424,22 +421,19 @@ public class ZookeeperClusterStatus extends ClusterStatus { * serialized each field into a different zookeeper node * so we fetch it using old code and serialize it again with thrift protocol */ + LOG.info("The schema of Table [{0}] was stored in an older format. Now converting it to the new format", table); getOldTableDescriptor(useCache, cluster, table, tableDescriptor); + BlurUtil.removeAll(_zk, blurTablePath); + // store it using thrift protocol byte[] newFormatBytes = serializeTableDescriptor(tableDescriptor); BlurUtil.createPath(_zk, blurTablePath, newFormatBytes); - // remove old child nodes - List<String> list = _zk.getChildren(blurTablePath, false); - for (String p : list) { - BlurUtil.removeAll(_zk, blurTablePath + "/" + p); - } } else { + TDeserializer deserializer = new TDeserializer(new TJSONProtocol.Factory()); deserializer.deserialize(tableDescriptor, bytes); } - - } catch (TException e) { throw new RuntimeException(e); } catch (KeeperException e) { @@ -472,7 +466,6 @@ public class ZookeeperClusterStatus extends ClusterStatus { tableDescriptor.readOnly = internalGetReadOnly(ZookeeperPathConstants.getTableReadOnlyPath(cluster, table)); tableDescriptor.preCacheCols = toList(getData(ZookeeperPathConstants .getTableColumnsToPreCache(cluster, table))); - tableDescriptor.tableProperties = toMap(getData(ZookeeperPathConstants.getTablePropertiesPath(cluster, table))); byte[] data = getData(ZookeeperPathConstants.getTableSimilarityPath(cluster, table)); if (data != null) { tableDescriptor.similarityClass = new String(data); @@ -521,20 +514,6 @@ public class ZookeeperClusterStatus extends ClusterStatus { return list; } - private static byte[] toBytes(List<String> list) { - if (list == null) { - return null; - } - StringBuilder builder = new StringBuilder(); - for (String s : list) { - if (builder.length() != 0) { - builder.append(','); - } - builder.append(s); - } - return builder.toString().getBytes(); - } - private void updateEnabled(boolean useCache, TableDescriptor tableDescriptor, String cluster, String table) { if (tableDescriptor != null) { tableDescriptor.setEnabled(isEnabled(useCache, cluster, table)); @@ -768,7 +747,6 @@ public class ZookeeperClusterStatus extends ClusterStatus { int shardCount = BlurUtil.zeroCheck(tableDescriptor.shardCount, "tableDescriptor.shardCount cannot be less than 1"); String blurTablePath = ZookeeperPathConstants.getTablePath(cluster, table); - if (_zk.exists(blurTablePath, false) != null) { throw new IOException("Table [" + table + "] already exists."); } @@ -789,7 +767,7 @@ public class ZookeeperClusterStatus extends ClusterStatus { private byte[] serializeTableDescriptor(TableDescriptor td) { try{ - TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory()); + TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); return serializer.serialize(td); } catch (TException e) { throw new RuntimeException(e); @@ -891,49 +869,6 @@ public class ZookeeperClusterStatus extends ClusterStatus { } } - private static byte[] toBytes(Set<String> blockCachingFileTypes) { - if (blockCachingFileTypes == null || blockCachingFileTypes.isEmpty()) { - return null; - } - StringBuilder builder = new StringBuilder(); - for (String type : blockCachingFileTypes) { - builder.append(type).append(','); - } - return builder.substring(0, builder.length() - 1).getBytes(); - } - - private static byte[] toBytes(Map<String,String> properties) { - if (properties == null || properties.isEmpty()) { - return null; - } - StringBuilder builder = new StringBuilder(); - Set<Entry<String, String>> entrySet = properties.entrySet(); - for (Entry<String, String> entry : entrySet) { - String key = entry.getKey(); - String value = entry.getValue(); - String keyValue = key+"->"+value; - builder.append(keyValue).append(','); - } - - return builder.substring(0, builder.length() - 1).getBytes(); - } - - private static Map<String,String> toMap(byte[] mapBytes) { - Map<String,String> properties = new HashMap<String, String>(); - - if (mapBytes == null) { - return null; - } - String str = new String(mapBytes); - String[] keyValuePairs = str.split(","); - for (String keyValue : keyValuePairs) { - String[] split = keyValue.split("->"); - properties.put(split[0].trim(), split[1].trim()); - } - - return properties; - } - @Override public boolean isOpen() { return _running.get(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/09fb701c/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java index 11ab6e5..5278aca 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java +++ b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java @@ -107,8 +107,4 @@ public class ZookeeperPathConstants { return getClusterPath(cluster) + "/layout"; } - public static String getTablePropertiesPath(String cluster, String table) { - return getTablePath(cluster, table) + "/tableproperties"; - } - }
