Fixing logging issues with cdh and removing duplicate deps from parcel.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/3499d679 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/3499d679 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/3499d679 Branch: refs/heads/master Commit: 3499d679a94fb71d43ceaf00c7880bdd565c7b1d Parents: 610e242 Author: Aaron McCurry <amccu...@gmail.com> Authored: Tue Jul 7 15:00:17 2015 -0400 Committer: Aaron McCurry <amccu...@gmail.com> Committed: Tue Jul 7 15:00:17 2015 -0400 ---------------------------------------------------------------------- .../org/apache/blur/doc/BlurPropertyParser.java | 28 +++++++++++++++++--- .../cdh/csd/descriptor/service.sdl.template | 11 +++----- .../src/assemble/cdh/csd/scripts/control.sh | 2 +- distribution/src/assemble/parcel-hadoop2.xml | 6 +---- .../bin/start-supervised-controller-server.sh | 8 +++--- .../bin/start-supervised-shard-server.sh | 8 +++--- 6 files changed, 38 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3499d679/blur-util/src/main/java/org/apache/blur/doc/BlurPropertyParser.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/doc/BlurPropertyParser.java b/blur-util/src/main/java/org/apache/blur/doc/BlurPropertyParser.java index f386c2c..752f266 100644 --- a/blur-util/src/main/java/org/apache/blur/doc/BlurPropertyParser.java +++ b/blur-util/src/main/java/org/apache/blur/doc/BlurPropertyParser.java @@ -21,8 +21,11 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Properties; +import java.util.Set; import com.google.common.collect.Lists; @@ -36,6 +39,8 @@ public class BlurPropertyParser { String key = "|||General-Server-Properties|||"; + Set<String> requiredProperties = getRequiredProperties(); + Map<String, List<BlurProp>> map = new HashMap<String, List<BlurProp>>(); while ((line = reader.readLine()) != null) { line = line.trim(); @@ -59,6 +64,7 @@ public class BlurPropertyParser { p.setDefaultVal(value); p.setDescription(desc); p.setType(type); // infer type... + p.setRequired(requiredProperties.contains(name)); props.add(p); } prevLine = line; @@ -66,6 +72,18 @@ public class BlurPropertyParser { return map; } + private Set<String> getRequiredProperties() throws IOException { + InputStream inputStream = getClass().getResourceAsStream("/blur-site.properties"); + Properties properties = new Properties(); + properties.load(inputStream); + inputStream.close(); + Set<String> result = new HashSet<String>(); + for (Object o : properties.keySet()) { + result.add(o.toString()); + } + return result; + } + String getType(String value) { if (value == null || value.isEmpty()) { return "string"; @@ -135,7 +153,7 @@ public class BlurPropertyParser { private String description; private String defaultVal; private String type; - private boolean isRequired; + private boolean required; public String getName() { return name; @@ -169,10 +187,14 @@ public class BlurPropertyParser { this.type = type; } - // We'll have a default if it is. public boolean isRequired() { - return ((defaultVal != null) && (!defaultVal.isEmpty())); + return required; } + + public void setRequired(boolean required) { + this.required = required; + } + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3499d679/distribution/src/assemble/cdh/csd/descriptor/service.sdl.template ---------------------------------------------------------------------- diff --git a/distribution/src/assemble/cdh/csd/descriptor/service.sdl.template b/distribution/src/assemble/cdh/csd/descriptor/service.sdl.template index a8f6b01..d3e3d46 100644 --- a/distribution/src/assemble/cdh/csd/descriptor/service.sdl.template +++ b/distribution/src/assemble/cdh/csd/descriptor/service.sdl.template @@ -82,9 +82,7 @@ }, "logging": { "dir":"/var/log/blur", - "filename": "blur--controller-server-${host}-0_main.log", - "configName" :"blur.log.dir", - "isModifiable":true, + "filename": "blur-controller-server-${host}.log", "loggingType":"log4j" }, "configWriter" : { @@ -132,9 +130,7 @@ }, "logging": { "dir":"/var/log/blur", - "filename": "blur--shard-server-${host}-0_main.log", - "configName" :"blur.log.dir", - "isModifiable":true, + "filename": "blur-shard-server-${host}.log", "loggingType":"log4j" }, "configWriter" : { @@ -167,8 +163,7 @@ }, "serviceDependencies" : [ {"name":"HDFS", "required":"true"}, - {"name":"ZOOKEEPER", "required":"true"}, - {"name":"YARN", "required":"true"} + {"name":"ZOOKEEPER", "required":"true"} ] http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3499d679/distribution/src/assemble/cdh/csd/scripts/control.sh ---------------------------------------------------------------------- diff --git a/distribution/src/assemble/cdh/csd/scripts/control.sh b/distribution/src/assemble/cdh/csd/scripts/control.sh index b4f5ab9..b5c18e6 100644 --- a/distribution/src/assemble/cdh/csd/scripts/control.sh +++ b/distribution/src/assemble/cdh/csd/scripts/control.sh @@ -69,7 +69,7 @@ setup_environment() { BLUR_SITE=${blur_conf_dir}/blur-site.properties - cp ${BLUR_HOME}/conf/log* ${blur_conf_dir}/ +# cp ${BLUR_HOME}/conf/log* ${blur_conf_dir}/ if [ -z "$BLUR_ZK_ROOT" ]; then echo "Rooting zookeeper at [${BLUR_ZK_ROOT}]" http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3499d679/distribution/src/assemble/parcel-hadoop2.xml ---------------------------------------------------------------------- diff --git a/distribution/src/assemble/parcel-hadoop2.xml b/distribution/src/assemble/parcel-hadoop2.xml index 276af73..50a290b 100644 --- a/distribution/src/assemble/parcel-hadoop2.xml +++ b/distribution/src/assemble/parcel-hadoop2.xml @@ -33,11 +33,7 @@ under the License. <includes> <include>org.apache.blur:*</include> - <include>org.apache.zookeeper:zookeeper</include> - <include>org.slf4j:slf4j-api</include> - <include>org.slf4j:slf4j-log4j12</include> <include>org.json:json</include> - <include>log4j:log4j</include> <include>com.yammer.metrics:*</include> <include>com.google.guava:guava</include> <include>org.apache.httpcomponents:*</include> @@ -48,7 +44,6 @@ under the License. <include>com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru</include> <include>jline:jline</include> <include>com.fasterxml.jackson.core:*</include> - <include>com.github.amccurry:lucene-document-security</include> <include>javax.servlet:*</include> </includes> </dependencySet> @@ -60,6 +55,7 @@ under the License. <outputDirectory>blur-${project.version}/conf</outputDirectory> <excludes> <exclude>**/.empty</exclude> + <exclude>**/log4j*.xml</exclude> </excludes> </fileSet> <fileSet> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3499d679/distribution/src/main/scripts/bin/start-supervised-controller-server.sh ---------------------------------------------------------------------- diff --git a/distribution/src/main/scripts/bin/start-supervised-controller-server.sh b/distribution/src/main/scripts/bin/start-supervised-controller-server.sh index a8e6fa5..7afac4c 100755 --- a/distribution/src/main/scripts/bin/start-supervised-controller-server.sh +++ b/distribution/src/main/scripts/bin/start-supervised-controller-server.sh @@ -21,7 +21,7 @@ bin=`cd "$bin"; pwd` echo "Sourcing configs.." . $BLUR_HOME/bin/blur-supervised-config.sh - PROC_NAME=controller-server-$HOSTNAME-0 - echo "Launching controller [$PROC_NAME] now..." - echo "Using BLUR_CLASSPATH: ${BLUR_CLASSPATH}" - exec "$JAVA_HOME"/bin/java -Dblur.name=$PROC_NAME -Djava.library.path=$JAVA_LIBRARY_PATH -Dblur-controller-$INSTANCE $BLUR_CONTROLLER_JVM_OPTIONS -Dblur.logs.dir=$BLUR_LOGS -Dblur.log.file=blur-$PROC_NAME -cp "$BLUR_CLASSPATH" org.apache.blur.thrift.ThriftBlurControllerServer -s 0 +PROC_NAME=controller-server-$HOSTNAME-0 +echo "Launching controller [$PROC_NAME] now..." +echo "Using BLUR_CLASSPATH: ${BLUR_CLASSPATH}" +exec "$JAVA_HOME"/bin/java -Dblur.name=$PROC_NAME -Dblur-controller -Dlog4j.configuration=log4j.properties $BLUR_CONTROLLER_JVM_OPTIONS -Djava.library.path=$JAVA_LIBRARY_PATH -cp "$BLUR_CLASSPATH" org.apache.blur.thrift.ThriftBlurControllerServer -s 0 http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3499d679/distribution/src/main/scripts/bin/start-supervised-shard-server.sh ---------------------------------------------------------------------- diff --git a/distribution/src/main/scripts/bin/start-supervised-shard-server.sh b/distribution/src/main/scripts/bin/start-supervised-shard-server.sh index 89ab0e8..88e5352 100755 --- a/distribution/src/main/scripts/bin/start-supervised-shard-server.sh +++ b/distribution/src/main/scripts/bin/start-supervised-shard-server.sh @@ -21,7 +21,7 @@ bin=`cd "$bin"; pwd` echo "Sourcing configs.." . $BLUR_HOME/bin/blur-supervised-config.sh - PROC_NAME=shard-server-$HOSTNAME-0 - echo "Launching shard [$PROC_NAME] now..." - echo "Using BLUR_CLASSPATH: ${BLUR_CLASSPATH}" - exec "$JAVA_HOME"/bin/java -Dblur.name=$PROC_NAME -Djava.library.path=$JAVA_LIBRARY_PATH -Dblur-shard-$INSTANCE $BLUR_SHARD_JVM_OPTIONS -Dblur.logs.dir=$BLUR_LOGS -Dblur.log.file=blur-$USER-$PROC_NAME -cp "$BLUR_CLASSPATH" org.apache.blur.thrift.ThriftBlurShardServer -s 0 +PROC_NAME=shard-server-$HOSTNAME-0 +echo "Launching shard [$PROC_NAME] now..." +echo "Using BLUR_CLASSPATH: ${BLUR_CLASSPATH}" +exec "$JAVA_HOME"/bin/java -Dblur.name=$PROC_NAME -Dblur-shard -Dlog4j.configuration=log4j.properties $BLUR_SHARD_JVM_OPTIONS -Djava.library.path=$JAVA_LIBRARY_PATH -cp "$BLUR_CLASSPATH" org.apache.blur.thrift.ThriftBlurShardServer -s 0
