Repository: incubator-blur Updated Branches: refs/heads/master 317245f4c -> 8b7eb8f09
Changing csd to register memory used with cloudera manager. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/8b7eb8f0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/8b7eb8f0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/8b7eb8f0 Branch: refs/heads/master Commit: 8b7eb8f096ef260629f06e890c68f747f5c7b6f6 Parents: 317245f Author: Aaron McCurry <[email protected]> Authored: Thu Oct 15 08:49:50 2015 -0400 Committer: Aaron McCurry <[email protected]> Committed: Thu Oct 15 08:49:50 2015 -0400 ---------------------------------------------------------------------- .../resources/descriptor/service.sdl.template | 50 ++++++++++++++++++-- .../apache/blur/doc/CreateCSDDescriptor.java | 29 +++++++++++- 2 files changed, 74 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8b7eb8f0/cdh-csd/src/main/resources/descriptor/service.sdl.template ---------------------------------------------------------------------- diff --git a/cdh-csd/src/main/resources/descriptor/service.sdl.template b/cdh-csd/src/main/resources/descriptor/service.sdl.template index 2f66d28..17b2068 100644 --- a/cdh-csd/src/main/resources/descriptor/service.sdl.template +++ b/cdh-csd/src/main/resources/descriptor/service.sdl.template @@ -72,7 +72,7 @@ "BLUR_DEFAULT_TABLE_PATH": "/user/${user}/${blur_cluster_name}/tables", "BLUR_COMMAND_PATH":"/user/${user}/${blur_cluster_name}/commands", "BLUR_CLUSTER_NAME":"${blur_cluster_name}", - "BLUR_CONTROLLER_JVM_OPTIONS":"${blur_controller_jvm_options}" + "BLUR_CONTROLLER_JVM_OPTIONS":"-Xmx${blur_controller_jvm_heap} -Xms${blur_controller_jvm_heap} ${blur_controller_jvm_options}" } }, "externalLink" : { @@ -102,9 +102,23 @@ "configName":"blur.controller.jvm.options", "required":"false", "type":"string", - "default":"-Xmx1024m -Djava.net.preferIPv4Stack=true", + "default":"-Djava.net.preferIPv4Stack=true", "configurableInWizard":true }, + { + "name" : "blur_controller_jvm_heap", + "configName": "blur.controller.jvm.heap", + "label" : "Total Java Heap Sizes of Controller in Bytes", + "description" : "Memory available to the Controller. Passed to Java -Xmx. Measured in bytes.", + "required" : "true", + "type" : "memory", + "unit" : "bytes", + "min" : 2147483648, + "default" : 8589934592, + "scaleFactor" : 1.3, + "autoConfigShare" : 100 + } + , |||Controller-Server-Properties||| ] }, @@ -120,7 +134,7 @@ "BLUR_DEFAULT_TABLE_PATH": "/user/${user}/${blur_cluster_name}/tables", "BLUR_COMMAND_PATH":"/user/${user}/${blur_cluster_name}/commands", "BLUR_CLUSTER_NAME":"${blur_cluster_name}", - "BLUR_SHARD_JVM_OPTIONS":"${blur_shard_jvm_options}" + "BLUR_SHARD_JVM_OPTIONS":"-Xmx${blur_shard_jvm_heap} -Xms${blur_shard_jvm_heap} ${blur_shard_jvm_options}" } }, "additionalExternalLinks" : [{ @@ -150,9 +164,37 @@ "configName":"blur.shard.jvm.options", "required":"false", "type":"string", - "default":"-Xmx1024m -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=256m", + "default":"-Djava.net.preferIPv4Stack=true", "configurableInWizard":true }, + { + "name" : "blur_shard_jvm_heap", + "configName": "blur.shard.jvm.heap", + "label" : "Total Java Heap Sizes of Shard in Bytes", + "description" : "Memory available to the Shard. Passed to Java -Xmx. Measured in bytes.", + "required" : "true", + "type" : "memory", + "unit" : "bytes", + "min" : 2147483648, + "default" : 8589934592, + "scaleFactor" : 1.3, + "autoConfigShare" : 100 + }, + + { + "name" : "blur_shard_block_cache_total_size", + "configName": "blur.shard.block.cache.total.size", + "label" : "Total Block Cache Sizes of Shard in Bytes", + "description" : "Block Cache Memory available to the Shard. Measured in bytes.", + "required" : "true", + "type" : "memory", + "unit" : "bytes", + "min" : 134217728, + "default" : 8589934592, + "scaleFactor" : 1.3, + "autoConfigShare" : 100 + }, + |||Shard-Server-Properties||| ] } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8b7eb8f0/cdh-csd/src/test/java/org/apache/blur/doc/CreateCSDDescriptor.java ---------------------------------------------------------------------- diff --git a/cdh-csd/src/test/java/org/apache/blur/doc/CreateCSDDescriptor.java b/cdh-csd/src/test/java/org/apache/blur/doc/CreateCSDDescriptor.java index eacc7dc..321631d 100644 --- a/cdh-csd/src/test/java/org/apache/blur/doc/CreateCSDDescriptor.java +++ b/cdh-csd/src/test/java/org/apache/blur/doc/CreateCSDDescriptor.java @@ -22,19 +22,25 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintWriter; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import org.apache.blur.doc.BlurPropertyParser.BlurProp; +import org.apache.commons.io.IOUtils; +import org.json.JSONException; +import org.json.JSONObject; public class CreateCSDDescriptor { private static final String BLUR_VERSION = "|||BLUR-VERSION|||"; - public static void main(String[] args) throws IOException { + public static void main(String[] args) throws IOException, JSONException { BlurPropertyParser parser = new BlurPropertyParser(); Map<String, List<BlurProp>> props = parser.parse(); + removeProps("blur.shard.block.cache.total.size", props); Map<String, StringBuffer> map = new HashMap<String, StringBuffer>(); JsonPropertyFormatter formatter = new JsonPropertyFormatter(); @@ -62,6 +68,27 @@ public class CreateCSDDescriptor { String blurVersion = args[2]; replaceValuesInFile(source, dest, map, formatVersion(blurVersion)); + + FileInputStream input = new FileInputStream(dest); + try { + // Check if the csd is a good json file. + new JSONObject(IOUtils.toString(input)); + } finally { + input.close(); + } + } + + private static void removeProps(String propertyName, Map<String, List<BlurProp>> props) { + for (Entry<String, List<BlurProp>> e : props.entrySet()) { + List<BlurProp> value = e.getValue(); + List<BlurProp> remove = new ArrayList<BlurProp>(); + for (BlurProp blurProp : value) { + if (propertyName.equals(blurProp.getName())) { + remove.add(blurProp); + } + } + value.removeAll(remove); + } } public static String formatVersion(String blurVersion) {
