Repository: incubator-blur Updated Branches: refs/heads/blur-0.2.4-parcel 0358fe057 -> fea0b3b89 refs/heads/master c9d5e24f3 -> fea0b3b89
First cdh parcel commit. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/053589d4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/053589d4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/053589d4 Branch: refs/heads/master Commit: 053589d49d75129c665a63e93a4b8c680a14bf6e Parents: 2225c8e Author: Aaron McCurry <amccu...@gmail.com> Authored: Thu Jun 18 21:50:04 2015 -0400 Committer: Aaron McCurry <amccu...@gmail.com> Committed: Thu Jun 18 21:50:04 2015 -0400 ---------------------------------------------------------------------- .../mapreduce/lib/BlurOutputFormatTest.java | 2 +- blur-thrift/pom.xml | 18 +- .../org/apache/blur/doc/BlurPropertyParser.java | 181 ++++++++++++++++++ .../apache/blur/doc/CreateBlurApiHtmlPage.java | 4 +- .../blur/doc/CreateBlurServerSetupHtmlPage.java | 69 ++----- .../apache/blur/doc/CreateCSDDescriptor.java | 89 +++++++++ .../apache/blur/doc/JsonPropertyFormatter.java | 43 +++++ .../apache/blur/doc/BlurPropertyParserTest.java | 56 ++++++ .../doc/CreateBlurServerSetupHtmlPageTest.java | 91 +++++++++ .../blur/doc/CreateCSDDescriptorTest.java | 89 +++++++++ blur-util/src/test/resources/prop.doc.base.html | 5 + .../src/test/resources/service.sdl.template | 191 +++++++++++++++++++ distribution/pom.xml | 56 +++++- distribution/src/assemble/cdh/README.txt | 37 ++++ .../cdh/csd/descriptor/service.sdl.template | 191 +++++++++++++++++++ .../src/assemble/cdh/csd/scripts/control.sh | 128 +++++++++++++ .../assemble/cdh/parcel/meta/blur_parcel_env.sh | 28 +++ .../src/assemble/cdh/parcel/meta/parcel.json | 58 ++++++ distribution/src/assemble/csd-hadoop2.xml | 55 ++++++ distribution/src/assemble/parcel-hadoop2.xml | 101 ++++++++++ .../main/scripts/bin/blur-supervised-config.sh | 97 ++++++++++ .../bin/start-blocking-controller-server.sh | 41 ++++ .../bin/start-supervised-controller-server.sh | 27 +++ .../bin/start-supervised-shard-server.sh | 27 +++ pom.xml | 2 + 25 files changed, 1615 insertions(+), 71 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java index f4e7074..d197b90 100644 --- a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java +++ b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java @@ -320,7 +320,7 @@ public class BlurOutputFormatTest { for (int i = 0; i < tableDescriptor.getShardCount(); i++) { Path path = new Path(output, ShardUtil.getShardName(i)); Collection<Path> commitedTasks = getCommitedTasks(path); - assertTrue(multiple >= commitedTasks.size()); + assertTrue(commitedTasks.size() >= multiple); for (Path p : commitedTasks) { DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(_conf, p)); total += reader.numDocs(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/blur-thrift/pom.xml ---------------------------------------------------------------------- diff --git a/blur-thrift/pom.xml b/blur-thrift/pom.xml index 748f6c0..49e7137 100644 --- a/blur-thrift/pom.xml +++ b/blur-thrift/pom.xml @@ -51,7 +51,7 @@ <groupId>javax.servlet</groupId> <artifactId>javax.servlet-api</artifactId> <version>${servlet-api.version}</version> - <scope>provided</scope> + <scope>compile</scope> </dependency> </dependencies> @@ -149,14 +149,6 @@ <name>hadoop2-mr1</name> </property> </activation> - <dependencies> - <dependency> - <groupId>javax.servlet</groupId> - <artifactId>javax.servlet-api</artifactId> - <version>${servlet-api.version}</version> - <scope>provided</scope> - </dependency> - </dependencies> </profile> <profile> <id>hadoop2</id> @@ -165,14 +157,6 @@ <name>hadoop2</name> </property> </activation> - <dependencies> - <dependency> - <groupId>javax.servlet</groupId> - <artifactId>javax.servlet-api</artifactId> - <version>${servlet-api.version}</version> - <scope>provided</scope> - </dependency> - </dependencies> </profile> </profiles> </project> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/blur-util/src/main/java/org/apache/blur/doc/BlurPropertyParser.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/doc/BlurPropertyParser.java b/blur-util/src/main/java/org/apache/blur/doc/BlurPropertyParser.java new file mode 100644 index 0000000..6973aea --- /dev/null +++ b/blur-util/src/main/java/org/apache/blur/doc/BlurPropertyParser.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.doc; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.google.common.collect.Lists; + +public class BlurPropertyParser { + + public Map<String, List<BlurProp>> parse() throws IOException { + InputStream inputStream = BlurPropertyParser.class.getResourceAsStream("/blur-default.properties"); + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); + String line; + String prevLine = null; + + String key = "|||General-Server-Properties|||"; + + Map<String, List<BlurProp>> map = new HashMap<String, List<BlurProp>>(); + while ((line = reader.readLine()) != null) { + line = line.trim(); + if (line.equals("### Shard Server Configuration")) { + key = "|||Shard-Server-Properties|||"; + } else if (line.equals("### Controller Server Configuration")) { + key = "|||Controller-Server-Properties|||"; + } + if (!line.startsWith("#") && !line.isEmpty()) { + String desc = getDesc(prevLine); + String name = getName(line); + String value = getValue(line); + String type = getType(value); + List<BlurProp> props = map.get(key); + if (props == null) { + props = Lists.newArrayList(); + map.put(key, props); + } + BlurProp p = new BlurProp(); + p.setName(name); + p.setDefaultVal(value); + p.setDescription(desc); + p.setType(type); // infer type... + props.add(p); + } + prevLine = line; + } + return map; + } + + String getType(String value) { + if (value == null || value.isEmpty()) { + return "string"; + } + + if (isNumeric(value)) { + return "long"; + } + + if ("true".equals(value) || "false".equals(value)) { + return "boolean"; + } + + if (isDouble(value)) { + return "double"; + } + return "string"; + } + + private boolean isDouble(String value) { + try { + Double.parseDouble(value); + return true; + } catch (NumberFormatException e) { + return false; + } + } + + private boolean isNumeric(String value) { + byte[] chars = value.getBytes(); + + int start = 0; + if (value.charAt(0) == '-') { + start = 1; + } + + for (int i = start; i < chars.length; i++) { + if (!Character.isDigit(chars[i])) { + return false; + } + } + return true; + } + + private static String getValue(String line) { + int index = line.indexOf('='); + if (index < 0) { + throw new RuntimeException(); + } + return line.substring(index + 1); + } + + private static String getName(String line) { + int index = line.indexOf('='); + if (index < 0) { + throw new RuntimeException(); + } + return line.substring(0, index); + } + + private static String getDesc(String prevLine) { + return prevLine.substring(1).trim(); + } + + public static class BlurProp { + private String name; + private String description; + private String defaultVal; + private String type; + private boolean isRequired; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getDefaultVal() { + return defaultVal; + } + + public void setDefaultVal(String defaultVal) { + this.defaultVal = defaultVal; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + // We'll have a default if it is. + public boolean isRequired() { + return ((defaultVal != null) && (!defaultVal.isEmpty())); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/blur-util/src/main/java/org/apache/blur/doc/CreateBlurApiHtmlPage.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/doc/CreateBlurApiHtmlPage.java b/blur-util/src/main/java/org/apache/blur/doc/CreateBlurApiHtmlPage.java index d867f31..7b860f4 100644 --- a/blur-util/src/main/java/org/apache/blur/doc/CreateBlurApiHtmlPage.java +++ b/blur-util/src/main/java/org/apache/blur/doc/CreateBlurApiHtmlPage.java @@ -1,5 +1,3 @@ -package org.apache.blur.doc; - /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -16,6 +14,8 @@ package org.apache.blur.doc; * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.blur.doc; + import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/blur-util/src/main/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPage.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPage.java b/blur-util/src/main/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPage.java index bcba22b..fb81a00 100644 --- a/blur-util/src/main/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPage.java +++ b/blur-util/src/main/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPage.java @@ -24,45 +24,36 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintWriter; import java.util.HashMap; +import java.util.List; import java.util.Map; +import org.apache.blur.doc.BlurPropertyParser.BlurProp; + public class CreateBlurServerSetupHtmlPage { public static void main(String[] args) throws IOException { - InputStream inputStream = CreateBlurServerSetupHtmlPage.class.getResourceAsStream("/blur-default.properties"); - BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); - String line; - String prevLine = null; - - String key = "|||General-Server-Properties|||"; + BlurPropertyParser parser = new BlurPropertyParser(); + Map<String, List<BlurProp>> props = parser.parse(); Map<String, StringBuffer> map = new HashMap<String, StringBuffer>(); - while ((line = reader.readLine()) != null) { - line = line.trim(); - if (line.equals("### Shard Server Configuration")) { - key = "|||Shard-Server-Properties|||"; - } else if (line.equals("### Controller Server Configuration")) { - key = "|||Controller-Server-Properties|||"; + + for (Map.Entry<String, List<BlurProp>> prop : props.entrySet()) { + + StringBuffer buffer = map.get(prop.getKey()); + if (buffer == null) { + buffer = new StringBuffer(); + map.put(prop.getKey(), buffer); } - if (!line.startsWith("#") && !line.isEmpty()) { - System.out.println(prevLine); - System.out.println(line); - String desc = getDesc(prevLine); - String name = getName(line); - String value = getValue(line); - StringBuffer buffer = map.get(key); - if (buffer == null) { - buffer = new StringBuffer(); - map.put(key, buffer); - } - buffer.append("<tr><td>").append(name); - if (!value.trim().isEmpty()) { - buffer.append(" (").append(value).append(")"); + for (BlurProp p : prop.getValue()) { + + buffer.append("<tr><td>").append(p.getName()); + if (!p.getDefaultVal().trim().isEmpty()) { + buffer.append(" (").append(p.getDefaultVal()).append(")"); } - buffer.append("</td><td>").append(desc).append("</td></tr>"); + buffer.append("</td><td>").append(p.getDescription()).append("</td></tr>"); } - prevLine = line; + } - reader.close(); + String source = args[0]; String dest = args[1]; replaceValuesInFile(source, dest, map); @@ -91,24 +82,4 @@ public class CreateBlurServerSetupHtmlPage { } - private static String getValue(String line) { - int index = line.indexOf('='); - if (index < 0) { - throw new RuntimeException(); - } - return line.substring(index + 1); - } - - private static String getName(String line) { - int index = line.indexOf('='); - if (index < 0) { - throw new RuntimeException(); - } - return line.substring(0, index); - } - - private static String getDesc(String prevLine) { - return prevLine.substring(1).trim(); - } - } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/blur-util/src/main/java/org/apache/blur/doc/CreateCSDDescriptor.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/doc/CreateCSDDescriptor.java b/blur-util/src/main/java/org/apache/blur/doc/CreateCSDDescriptor.java new file mode 100644 index 0000000..2804ed8 --- /dev/null +++ b/blur-util/src/main/java/org/apache/blur/doc/CreateCSDDescriptor.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.doc; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.blur.doc.BlurPropertyParser.BlurProp; + +public class CreateCSDDescriptor { + + public static void main(String[] args) throws IOException { + BlurPropertyParser parser = new BlurPropertyParser(); + Map<String, List<BlurProp>> props = parser.parse(); + Map<String, StringBuffer> map = new HashMap<String, StringBuffer>(); + + JsonPropertyFormatter formatter = new JsonPropertyFormatter(); + + for (Map.Entry<String, List<BlurProp>> prop : props.entrySet()) { + + StringBuffer buffer = map.get(prop.getKey()); + if (buffer == null) { + buffer = new StringBuffer(); + map.put(prop.getKey(), buffer); + } + boolean first = true; + + for (BlurProp p : prop.getValue()) { + if (!first) { + buffer.append(formatter.separator()); + } + buffer.append(formatter.format(p)); + first = false; + } + + } + + String source = args[0]; + String dest = args[1]; + + replaceValuesInFile(source, dest, map); + } + + private static void replaceValuesInFile(String s, String o, Map<String, StringBuffer> replacements) + throws IOException { + + File source = new File(s); + File output = new File(o); + System.out.println("Source[" + source.getAbsolutePath() + "]"); + System.out.println("Output[" + output.getAbsolutePath() + "]"); + PrintWriter writer = new PrintWriter(output); + + BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(source))); + String line; + while ((line = reader.readLine()) != null) { + StringBuffer newData = replacements.get(line.trim()); + + if (newData != null) { + writer.println(newData.toString()); + } else { + writer.println(line); + } + } + writer.close(); + reader.close(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/blur-util/src/main/java/org/apache/blur/doc/JsonPropertyFormatter.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/doc/JsonPropertyFormatter.java b/blur-util/src/main/java/org/apache/blur/doc/JsonPropertyFormatter.java new file mode 100644 index 0000000..395bc65 --- /dev/null +++ b/blur-util/src/main/java/org/apache/blur/doc/JsonPropertyFormatter.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.doc; + +import org.apache.blur.doc.BlurPropertyParser.BlurProp; + +public class JsonPropertyFormatter { + + public String separator() { + return ","; + } + + public String format(BlurProp prop) { + return " {\n" + + " \"name\":\"" + prop.getName().replace(".", "_") + "\",\n" + + " \"label\":\"" + prop.getName().replace(".", " ") + "\",\n" + + " \"description\": \"" + escape(prop.getDescription()) + "\",\n" + + " \"configName\":\"" + prop.getName() + "\",\n" + + " \"required\":\"" + prop.isRequired() + "\",\n" + + " \"type\":\"" + prop.getType() + "\",\n" + + " \"default\":\"" + prop.getDefaultVal() + "\",\n" + + " \"configurableInWizard\":true\n" + + " }\n"; + } + + private String escape(String value) { + return value.replace("\"", "'"); + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/blur-util/src/test/java/org/apache/blur/doc/BlurPropertyParserTest.java ---------------------------------------------------------------------- diff --git a/blur-util/src/test/java/org/apache/blur/doc/BlurPropertyParserTest.java b/blur-util/src/test/java/org/apache/blur/doc/BlurPropertyParserTest.java new file mode 100644 index 0000000..e8ecc89 --- /dev/null +++ b/blur-util/src/test/java/org/apache/blur/doc/BlurPropertyParserTest.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.doc; + +import static org.junit.Assert.assertEquals; + +import org.junit.Before; +import org.junit.Test; + +public class BlurPropertyParserTest { + private BlurPropertyParser parser; + + @Before + public void setUp() throws Exception { + parser = new BlurPropertyParser(); + } + + @Test + public void testDefaultValNull() { + assertEquals("string", parser.getType("")); + assertEquals("string", parser.getType(null)); + } + + @Test + public void testDefaultValInt() { + assertEquals("long", parser.getType(Integer.toString(Integer.MIN_VALUE))); + assertEquals("long", parser.getType(Integer.toString(Integer.MAX_VALUE))); + } + + @Test + public void testDefaultValBoolean() { + assertEquals("boolean", parser.getType("true")); + assertEquals("boolean", parser.getType("false")); + } + + @Test + public void testDefaultValDouble() { + assertEquals("double", parser.getType("0.75")); + assertEquals("double", parser.getType("-0.75")); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/blur-util/src/test/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPageTest.java ---------------------------------------------------------------------- diff --git a/blur-util/src/test/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPageTest.java b/blur-util/src/test/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPageTest.java new file mode 100644 index 0000000..9015cbb --- /dev/null +++ b/blur-util/src/test/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPageTest.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.doc; + +import static org.junit.Assert.assertEquals; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Properties; + +import org.junit.Before; +import org.junit.Test; + +public class CreateBlurServerSetupHtmlPageTest { + private File source; + private File dest = new File("target/gen.prop.doc.html"); + + @Before + public void before() { + source = new File(CreateBlurServerSetupHtmlPageTest.class.getResource("/prop.doc.base.html").getFile()); + } + + // Not a great test but makes sure i get it mostly put back together. + @Test + public void testGeneration() throws IOException { + CreateBlurServerSetupHtmlPage.main(new String[] { source.getAbsolutePath(), dest.getAbsolutePath() }); + + Properties defaultProperties = new Properties(); + defaultProperties.load(CreateBlurServerSetupHtmlPageTest.class.getResourceAsStream("/blur-default.properties")); + int numProps = defaultProperties.size(); + int numDocumentedProps = getDocumentedProps(); + + assertEquals(numProps, numDocumentedProps); + } + + private int getDocumentedProps() throws IOException { + int count = 0; + String docs = readGeneratedDocs(); + String[] props = docs.split("<tr>"); + + // System.out.println("PROPERTIES:"); + + for (String p : props) { + if (!p.isEmpty()) { + count++; + // System.out.println("PROP: " + p); + } + } + + return count; + } + + private String readGeneratedDocs() throws IOException { + StringBuffer buffer = new StringBuffer(); + FileInputStream fis = new FileInputStream(dest); + BufferedReader reader = null; + try { + reader = new BufferedReader(new InputStreamReader(fis)); + String line; + while ((line = reader.readLine()) != null) { + buffer.append(line); + } + } finally { + if (fis != null) { + fis.close(); + } + if (reader != null) { + reader.close(); + } + } + return buffer.toString(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/blur-util/src/test/java/org/apache/blur/doc/CreateCSDDescriptorTest.java ---------------------------------------------------------------------- diff --git a/blur-util/src/test/java/org/apache/blur/doc/CreateCSDDescriptorTest.java b/blur-util/src/test/java/org/apache/blur/doc/CreateCSDDescriptorTest.java new file mode 100644 index 0000000..cc9217e --- /dev/null +++ b/blur-util/src/test/java/org/apache/blur/doc/CreateCSDDescriptorTest.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.doc; + +import static org.junit.Assert.assertEquals; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Properties; + +import org.junit.Before; +import org.junit.Test; + +public class CreateCSDDescriptorTest { + private File source; + private File dest = new File("target/service.sdl"); + + @Before + public void before() { + source = new File(CreateCSDDescriptorTest.class.getResource("/service.sdl.template").getFile()); + } + + // Not a great test but makes sure i get it mostly put back together. + @Test + public void testGeneration() throws IOException { + Properties defaultProperties = new Properties(); + defaultProperties.load(CreateCSDDescriptorTest.class.getResourceAsStream("/blur-default.properties")); + int numProps = defaultProperties.size(); + int numDocumentedProps = getDocumentedProps(); + + assertEquals(numProps, numDocumentedProps); + } + + private int getDocumentedProps() throws IOException { + int count = 0; + String docs = readGeneratedDocs(); + String[] props = docs.split("configurableInWizard"); + + // System.out.println("PROPERTIES:"); + + for (String p : props) { + if (!p.isEmpty()) { + count++; + // System.out.println("PROP: " + p); + } + } + + return count; + } + + private String readGeneratedDocs() throws IOException { + StringBuffer buffer = new StringBuffer(); + FileInputStream fis = new FileInputStream(dest); + BufferedReader reader = null; + try { + reader = new BufferedReader(new InputStreamReader(fis)); + String line; + while ((line = reader.readLine()) != null) { + buffer.append(line); + } + } finally { + if (fis != null) { + fis.close(); + } + if (reader != null) { + reader.close(); + } + } + return buffer.toString(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/blur-util/src/test/resources/prop.doc.base.html ---------------------------------------------------------------------- diff --git a/blur-util/src/test/resources/prop.doc.base.html b/blur-util/src/test/resources/prop.doc.base.html new file mode 100644 index 0000000..2e85566 --- /dev/null +++ b/blur-util/src/test/resources/prop.doc.base.html @@ -0,0 +1,5 @@ +|||General-Server-Properties||| + +|||Shard-Server-Properties||| + +|||Controller-Server-Properties||| \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/blur-util/src/test/resources/service.sdl.template ---------------------------------------------------------------------- diff --git a/blur-util/src/test/resources/service.sdl.template b/blur-util/src/test/resources/service.sdl.template new file mode 100644 index 0000000..3f9f6dc --- /dev/null +++ b/blur-util/src/test/resources/service.sdl.template @@ -0,0 +1,191 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +{ + "name" : "BLUR", + "label" : "Blur", + "description": "Apache Blur Search Service", + "version" : "0.2.4", + "runAs" : { + "user" : "blur", + "group" : "blur" + }, + "rolesWithExternalLinks":["BLUR_CONTROLLER","BLUR_SHARD"], + "hdfsDirs" : [ + { + "name" : "CreateBlurTablesDir", + "label": "Create Blur Tables Directory", + "description" : "Creates the Blur Tables Directory", + "directoryDescription": "Location of Blur tables.", + "path":"/user/${user}/${blur_cluster_name}/tables", + "permissions":"0755" + }, + { + "name" : "CreateBlurCommandsDir", + "label": "Create Blur Commands Directory", + "description" : "Creates the Blur Commands Directory", + "directoryDescription": "Path that Blur will look for installed commands.", + "path":"/user/${user}/${blur_cluster_name}/commands", + "permissions":"0755" + } + ], + "serviceInit": { + "preStartSteps": [ + {"commandName":"CreateBlurTablesDir"}, + {"commandName":"CreateBlurCommandsDir"} + ] + }, + + "parameters": [ + + { + "name":"blur_cluster_name", + "label":"Blur Cluster Name", + "description": "Name of Blur Cluster", + "configName":"blur.cluster.name", + "required":"true", + "type":"string", + "default":"default", + "configurableInWizard":true + }, + { + "name":"blur_zookeeper_root", + "label":"Zookeeper Root", + "description": "Advanced: Root ZK connection to alternate path.", + "configName":"blur.zookeeper.root", + "required":"false", + "type":"string", + "default":"", + "configurableInWizard":true + }, + +|||General-Server-Properties||| + ], + + "roles" : [ + { + "name" : "BLUR_CONTROLLER", + "label" : "Blur Controller Server", + "pluralLabel" : "Blur Controller Servers", + "startRunner" : { + "program" : "scripts/control.sh", + "args" : [ "start-controller" ], + "environmentVariables" : { + "BLUR_ZK_ROOT": "${blur_zookeeper_root}", + "BLUR_DEFAULT_TABLE_PATH": "/user/${user}/${blur_cluster_name}/tables", + "BLUR_COMMAND_PATH":"/user/${user}/${blur_cluster_name}/commands", + "BLUR_CLUSTER_NAME":"${blur_cluster_name}", + "BLUR_CONTROLLER_JVM_OPTIONS":"${blur_controller_jvm_options}" + } + }, + "externalLink" : { + "name":"blur_controller_ui", + "label":"Blur Controller Status UI", + "url":"http://${host}:${blur_gui_controller_port}"; + }, + "logging": { + "dir":"/var/log/blur", + "filename": "blur--controller-server-${host}-0_main.log", + "configName" :"blur.log.dir", + "isModifiable":true, + "loggingType":"log4j" + }, + "configWriter" : { + "generators" : [ + { + "filename" : "blur-site.properties", + "configFormat" : "properties", + "excludedParams": ["blur_controller_jvm_options"] + } + ] + }, + "parameters": [ + { + "name":"blur_controller_jvm_options", + "label":"JVM Options for Controllers", + "description": "JAVA JVM OPTIONS for the controller servers, jvm tuning parameters are placed here.", + "configName":"blur.controller.jvm.options", + "required":"false", + "type":"string", + "default":"-Xmx1024m -Djava.net.preferIPv4Stack=true", + "configurableInWizard":true + }, +|||Controller-Server-Properties||| + ] + }, + { + "name" : "BLUR_SHARD", + "label" : "Blur Shard Server", + "pluralLabel" : "Blur Shard Servers", + "startRunner" : { + "program" : "scripts/control.sh", + "args" : [ "start-shard" ], + "environmentVariables" : { + "BLUR_ZK_ROOT": "${blur_zookeeper_root}", + "BLUR_DEFAULT_TABLE_PATH": "/user/${user}/${blur_cluster_name}/tables", + "BLUR_COMMAND_PATH":"/user/${user}/${blur_cluster_name}/commands", + "BLUR_CLUSTER_NAME":"${blur_cluster_name}", + "BLUR_SHARD_JVM_OPTIONS":"${blur_shard_jvm_options}" + } + }, + "externalLink" : { + "name":"blur_shard_ui", + "label":"Blur Shard Status UI", + "url":"http://${host}:${blur_gui_shard_port}"; + }, + "logging": { + "dir":"/var/log/blur", + "filename": "blur--shard-server-${host}-0_main.log", + "configName" :"blur.log.dir", + "isModifiable":true, + "loggingType":"log4j" + }, + "configWriter" : { + "generators" : [ + { + "filename" : "blur-site.properties", + "configFormat" : "properties", + "excludedParams": ["blur_shard_jvm_options"] + } + ] + }, + "parameters": [ + { + "name":"blur_shard_jvm_options", + "label":"JVM OPTIONS for Shards", + "description": "JAVA JVM OPTIONS for the shard servers, jvm tuning parameters are placed here.", + "configName":"blur.shard.jvm.options", + "required":"false", + "type":"string", + "default":"-Xmx1024m -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=256m", + "configurableInWizard":true + }, +|||Shard-Server-Properties||| + ] + } + ], + "parcel" : { + "requiredTags": ["blurcontroller"], + "optionalTags": ["blurcontroller", "blurshard", "blurconsole"] + }, + "serviceDependencies" : [ + {"name":"HDFS", "required":"true"}, + {"name":"ZOOKEEPER", "required":"true"}, + {"name":"YARN", "required":"true"} + ] + + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/distribution/pom.xml ---------------------------------------------------------------------- diff --git a/distribution/pom.xml b/distribution/pom.xml index 97707df..0bb9e5a 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -58,8 +58,28 @@ under the License. </dependencies> <build> - <plugins> - <plugin> + <plugins> <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.2.1</version> + <executions> + <execution> + <id>create-service-descriptor</id> + <phase>package</phase> + <goals> + <goal>java</goal> + </goals> + </execution> + </executions> + <configuration> + <mainClass>org.apache.blur.doc.CreateCSDDescriptor</mainClass> + <arguments> + <argument>${project.build.scriptSourceDirectory}/../../assemble/cdh/csd/descriptor/service.sdl.template</argument> + <argument>${project.build.directory}/service.sdl</argument> + </arguments> + </configuration> + </plugin> + <plugin> <artifactId>maven-assembly-plugin</artifactId> <executions> <execution> @@ -87,6 +107,35 @@ under the License. </descriptors> </configuration> </execution> + <execution> + <id>distro-assembly-parcel</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <descriptors> + <descriptor>${parcel.assembly.file}</descriptor> + </descriptors> + <finalName>blur-${project.version}-el6.parcel</finalName> + <appendAssemblyId>false</appendAssemblyId> + <ignoreDirFormatExtensions>false</ignoreDirFormatExtensions> + </configuration> + </execution> + <execution> + <id>distro-assembly-csd</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <descriptors> + <descriptor>${csd.assembly.file}</descriptor> + </descriptors> + <finalName>BLUR-0.2.4</finalName> + <appendAssemblyId>false</appendAssemblyId> + </configuration> + </execution> </executions> </plugin> </plugins> @@ -124,6 +173,9 @@ under the License. </activation> <properties> <bin.assembly.file>src/assemble/bin-hadoop2.xml</bin.assembly.file> + + <csd.assembly.file>src/assemble/csd-hadoop2.xml</csd.assembly.file> + <parcel.assembly.file>src/assemble/parcel-hadoop2.xml</parcel.assembly.file> </properties> </profile> </profiles> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/distribution/src/assemble/cdh/README.txt ---------------------------------------------------------------------- diff --git a/distribution/src/assemble/cdh/README.txt b/distribution/src/assemble/cdh/README.txt new file mode 100644 index 0000000..588cbfa --- /dev/null +++ b/distribution/src/assemble/cdh/README.txt @@ -0,0 +1,37 @@ +== Overview == +This provides some info on testing the CSD/Parcel packaging for Apache Blur. + +== CSD == + sudo cp distribution/target/BLUR-0.2.4.jar /opt/cloudera/csd/BLUR-0.2.4.jar + sudo service cloudera-scm-server restart + +== Parcel == +Note that the '.tar.gz' extension is ripped off in the copy +sudo cp distribution/target/blur-hadoop2-2.5.0-cdh5.2.0-0.2.4-incubating-SNAPSHOT-el6.parcel.tar.gz /opt/cloudera/parcel-repo/blur-hadoop2-2.5.0-cdh5.2.0-0.2.4-incubating-SNAPSHOT-el6.parcel + +Then, in /opt/cloudera/parcel-repo: +sha1sum blur-hadoop2-2.5.0-cdh5.2.0-0.2.4-incubating-SNAPSHOT-el6.parcel | awk '{print $1}' > blur-hadoop2-2.5.0-cdh5.2.0-0.2.4-incubating-SNAPSHOT-el6.parcel.sha && cat blur-hadoop2-2.5.0-cdh5.2.0-0.2.4-incubating-SNAPSHOT-el6.parcel.sha + +... which will write the sha file and print it to the screen. You can then copy that into a manifest in the + same directory [/opt/cloudera/parcel-repo/manifest.json] and substitute it in the hash value, which will look + like this: + + +{ + "lastUpdated":14286139390000, + "parcels": [ + { + "parcelName":"blur-hadoop2-2.5.0-cdh5.2.0-0.2.4-incubating-SNAPSHOT-el6.parcel", + "components": [ + { + "name" : "blur", + "version" : "0.2.4", + "pkg_version": "0.2.4" + } + ], + "hash":"632bd8d320f27ba68b9595f39ca2d99a203dd43c" + } + ] +} + +Now you should be able to click the "Check for new parcels" in CM and begin activating it. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/distribution/src/assemble/cdh/csd/descriptor/service.sdl.template ---------------------------------------------------------------------- diff --git a/distribution/src/assemble/cdh/csd/descriptor/service.sdl.template b/distribution/src/assemble/cdh/csd/descriptor/service.sdl.template new file mode 100644 index 0000000..3f9f6dc --- /dev/null +++ b/distribution/src/assemble/cdh/csd/descriptor/service.sdl.template @@ -0,0 +1,191 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +{ + "name" : "BLUR", + "label" : "Blur", + "description": "Apache Blur Search Service", + "version" : "0.2.4", + "runAs" : { + "user" : "blur", + "group" : "blur" + }, + "rolesWithExternalLinks":["BLUR_CONTROLLER","BLUR_SHARD"], + "hdfsDirs" : [ + { + "name" : "CreateBlurTablesDir", + "label": "Create Blur Tables Directory", + "description" : "Creates the Blur Tables Directory", + "directoryDescription": "Location of Blur tables.", + "path":"/user/${user}/${blur_cluster_name}/tables", + "permissions":"0755" + }, + { + "name" : "CreateBlurCommandsDir", + "label": "Create Blur Commands Directory", + "description" : "Creates the Blur Commands Directory", + "directoryDescription": "Path that Blur will look for installed commands.", + "path":"/user/${user}/${blur_cluster_name}/commands", + "permissions":"0755" + } + ], + "serviceInit": { + "preStartSteps": [ + {"commandName":"CreateBlurTablesDir"}, + {"commandName":"CreateBlurCommandsDir"} + ] + }, + + "parameters": [ + + { + "name":"blur_cluster_name", + "label":"Blur Cluster Name", + "description": "Name of Blur Cluster", + "configName":"blur.cluster.name", + "required":"true", + "type":"string", + "default":"default", + "configurableInWizard":true + }, + { + "name":"blur_zookeeper_root", + "label":"Zookeeper Root", + "description": "Advanced: Root ZK connection to alternate path.", + "configName":"blur.zookeeper.root", + "required":"false", + "type":"string", + "default":"", + "configurableInWizard":true + }, + +|||General-Server-Properties||| + ], + + "roles" : [ + { + "name" : "BLUR_CONTROLLER", + "label" : "Blur Controller Server", + "pluralLabel" : "Blur Controller Servers", + "startRunner" : { + "program" : "scripts/control.sh", + "args" : [ "start-controller" ], + "environmentVariables" : { + "BLUR_ZK_ROOT": "${blur_zookeeper_root}", + "BLUR_DEFAULT_TABLE_PATH": "/user/${user}/${blur_cluster_name}/tables", + "BLUR_COMMAND_PATH":"/user/${user}/${blur_cluster_name}/commands", + "BLUR_CLUSTER_NAME":"${blur_cluster_name}", + "BLUR_CONTROLLER_JVM_OPTIONS":"${blur_controller_jvm_options}" + } + }, + "externalLink" : { + "name":"blur_controller_ui", + "label":"Blur Controller Status UI", + "url":"http://${host}:${blur_gui_controller_port}"; + }, + "logging": { + "dir":"/var/log/blur", + "filename": "blur--controller-server-${host}-0_main.log", + "configName" :"blur.log.dir", + "isModifiable":true, + "loggingType":"log4j" + }, + "configWriter" : { + "generators" : [ + { + "filename" : "blur-site.properties", + "configFormat" : "properties", + "excludedParams": ["blur_controller_jvm_options"] + } + ] + }, + "parameters": [ + { + "name":"blur_controller_jvm_options", + "label":"JVM Options for Controllers", + "description": "JAVA JVM OPTIONS for the controller servers, jvm tuning parameters are placed here.", + "configName":"blur.controller.jvm.options", + "required":"false", + "type":"string", + "default":"-Xmx1024m -Djava.net.preferIPv4Stack=true", + "configurableInWizard":true + }, +|||Controller-Server-Properties||| + ] + }, + { + "name" : "BLUR_SHARD", + "label" : "Blur Shard Server", + "pluralLabel" : "Blur Shard Servers", + "startRunner" : { + "program" : "scripts/control.sh", + "args" : [ "start-shard" ], + "environmentVariables" : { + "BLUR_ZK_ROOT": "${blur_zookeeper_root}", + "BLUR_DEFAULT_TABLE_PATH": "/user/${user}/${blur_cluster_name}/tables", + "BLUR_COMMAND_PATH":"/user/${user}/${blur_cluster_name}/commands", + "BLUR_CLUSTER_NAME":"${blur_cluster_name}", + "BLUR_SHARD_JVM_OPTIONS":"${blur_shard_jvm_options}" + } + }, + "externalLink" : { + "name":"blur_shard_ui", + "label":"Blur Shard Status UI", + "url":"http://${host}:${blur_gui_shard_port}"; + }, + "logging": { + "dir":"/var/log/blur", + "filename": "blur--shard-server-${host}-0_main.log", + "configName" :"blur.log.dir", + "isModifiable":true, + "loggingType":"log4j" + }, + "configWriter" : { + "generators" : [ + { + "filename" : "blur-site.properties", + "configFormat" : "properties", + "excludedParams": ["blur_shard_jvm_options"] + } + ] + }, + "parameters": [ + { + "name":"blur_shard_jvm_options", + "label":"JVM OPTIONS for Shards", + "description": "JAVA JVM OPTIONS for the shard servers, jvm tuning parameters are placed here.", + "configName":"blur.shard.jvm.options", + "required":"false", + "type":"string", + "default":"-Xmx1024m -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=256m", + "configurableInWizard":true + }, +|||Shard-Server-Properties||| + ] + } + ], + "parcel" : { + "requiredTags": ["blurcontroller"], + "optionalTags": ["blurcontroller", "blurshard", "blurconsole"] + }, + "serviceDependencies" : [ + {"name":"HDFS", "required":"true"}, + {"name":"ZOOKEEPER", "required":"true"}, + {"name":"YARN", "required":"true"} + ] + + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/distribution/src/assemble/cdh/csd/scripts/control.sh ---------------------------------------------------------------------- diff --git a/distribution/src/assemble/cdh/csd/scripts/control.sh b/distribution/src/assemble/cdh/csd/scripts/control.sh new file mode 100644 index 0000000..b4f5ab9 --- /dev/null +++ b/distribution/src/assemble/cdh/csd/scripts/control.sh @@ -0,0 +1,128 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +: <<DOCUMENTATION + -------------------------------------------------------------------------- +| control.sh + -------------------------------------------------------------------------- +Script used by CM to control services of Blur. This should mostly be +bridge + -------------------------------------------------------------------------- +DOCUMENTATION + +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "$this")" && pwd -P) +script="$(basename -- "$this")" +this="$bin/$script" + +set -e +set -u +set -o pipefail + +IFS=$'\n\t' + +scratch=$(mktemp -d -t scratch.tmp.XXXXXXXXXX) + +function finish { + rm -rf "${scratch}" + # Your cleanup code here +} +trap finish EXIT + +#__args +declare -A OPT_ARGS +usage() { echo "Usage: $0 <action>" 1>&2; exit 1; } + +#__functions + +write_to_blur_site_props() { + prop="$1=$2" + echo -n "Adding property[$prop] to blur-site.properties..." + #For now, rely on last one in + echo $prop >> $BLUR_SITE + echo " done." +} + +setup_environment() { + blur_tmp_path="${CONF_DIR}/tmp" + blur_conf_dir="${CONF_DIR}" + + mkdir -p $blur_tmp_path + mkdir -p $blur_conf_dir + + #Wacky, yo! + chown blur:blur $blur_tmp_path + + BLUR_SITE=${blur_conf_dir}/blur-site.properties + + cp ${BLUR_HOME}/conf/log* ${blur_conf_dir}/ + + if [ -z "$BLUR_ZK_ROOT" ]; then + echo "Rooting zookeeper at [${BLUR_ZK_ROOT}]" + fi + + write_to_blur_site_props blur.zookeeper.connection "$ZK_QUORUM/$BLUR_ZK_ROOT" + + DFS_PATH=$(hdfs getconf -confKey fs.defaultFS) + + #TODO: Should allow BLUR_DEFAULT_TABLE_PATH to be absolute to another cluster too. + write_to_blur_site_props "blur.cluster.${BLUR_CLUSTER_NAME}.table.uri" "$DFS_PATH/$BLUR_DEFAULT_TABLE_PATH" + write_to_blur_site_props blur.command.lib.path "$DFS_PATH/$BLUR_COMMAND_PATH" + + export BLUR_CONF_DIR=$blur_conf_dir + export BLUR_LOGS="/var/log/blur" + export HADOOP_CLASSPATH=$(hadoop classpath) +} + +start_controller() { + setup_environment + exec $BLUR_HOME/bin/start-supervised-controller-server.sh +} + +start_shard() { + setup_environment + exec $BLUR_HOME/bin/start-supervised-shard-server.sh +} + +#__main +action="$1" + +if [ "${action}" == "" ] ;then + usage +fi + +echo "Executing [$action] with BLUR_HOME [$BLUR_HOME]" + +export BLUR_LOGS=${CONF_DIR}/logs + +case ${action} in + (start-controller) + start_controller + ;; + (start-shard) + start_shard + ;; + (*) + echo "Unknown command[${action}]" + ;; +esac + + + + + + http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/distribution/src/assemble/cdh/parcel/meta/blur_parcel_env.sh ---------------------------------------------------------------------- diff --git a/distribution/src/assemble/cdh/parcel/meta/blur_parcel_env.sh b/distribution/src/assemble/cdh/parcel/meta/blur_parcel_env.sh new file mode 100644 index 0000000..ff839e7 --- /dev/null +++ b/distribution/src/assemble/cdh/parcel/meta/blur_parcel_env.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export HADOOP_HOME=$CDH_HADOOP_HOME +export BLUR_HOME=$PARCELS_ROOT/$PARCEL_DIRNAME + +source $COMMON_SCRIPT +set_hadoop_classpath +HCP=$(hadoop classpath) +export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$HCP +echo "***** PREFIX:" +env +echo "**** DONE." + http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/distribution/src/assemble/cdh/parcel/meta/parcel.json ---------------------------------------------------------------------- diff --git a/distribution/src/assemble/cdh/parcel/meta/parcel.json b/distribution/src/assemble/cdh/parcel/meta/parcel.json new file mode 100644 index 0000000..f0145e7 --- /dev/null +++ b/distribution/src/assemble/cdh/parcel/meta/parcel.json @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +{ + "schema_version": 1, + "name": "blur", + "version" : "hadoop2-2.5.0-cdh5.2.0-0.2.4-incubating-SNAPSHOT", + "setActiveSymlink": false, + + "depends": "", + "replaces":"blur", + "conflicts":"", + + "provides": [ + "blurcontroller", + "blurshard", + "blurconsole" + ], + + "scripts": { + "defines":"blur_parcel_env.sh" + }, + + "components": [ + { + "name" : "blur", + "version" : "0.2.4", + "pkg_version": "0.2.4" + } + ], + + "packages" : [], + + "users": { + "blur": { + "longname" : "Blur", + "home" : "/var/lib/blur", + "shell" : "/bin/bash", + "extra_groups": [] + + } + }, + "groups": [] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/distribution/src/assemble/csd-hadoop2.xml ---------------------------------------------------------------------- diff --git a/distribution/src/assemble/csd-hadoop2.xml b/distribution/src/assemble/csd-hadoop2.xml new file mode 100644 index 0000000..13e638e --- /dev/null +++ b/distribution/src/assemble/csd-hadoop2.xml @@ -0,0 +1,55 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd";> + <id>csd</id> + <formats> + <format>jar</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + + <fileSets> + <fileSet> + <directory>${project.build.scriptSourceDirectory}/../../assemble/cdh/csd/scripts</directory> + <outputDirectory>scripts</outputDirectory> + <excludes> + <exclude>**/.empty</exclude> + </excludes> + </fileSet> + </fileSets> + + <files> + <file> + <source>${project.build.directory}/service.sdl</source> + <outputDirectory>descriptor</outputDirectory> + <destName>service.sdl</destName> + </file> + <file> + <source>${project.build.scriptSourceDirectory}/../resources-hadoop2/NOTICE-bin.txt</source> + <outputDirectory></outputDirectory> + <destName>NOTICE</destName> + </file> + <file> + <source>${project.build.scriptSourceDirectory}/../resources-hadoop2/LICENSE-bin.txt</source> + <outputDirectory></outputDirectory> + <destName>LICENSE</destName> + </file> + </files> +</assembly> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/distribution/src/assemble/parcel-hadoop2.xml ---------------------------------------------------------------------- diff --git a/distribution/src/assemble/parcel-hadoop2.xml b/distribution/src/assemble/parcel-hadoop2.xml new file mode 100644 index 0000000..4eeacb4 --- /dev/null +++ b/distribution/src/assemble/parcel-hadoop2.xml @@ -0,0 +1,101 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd";> + <id>parcel</id> + <formats> + <format>tar.gz</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + + <dependencySets> + <dependencySet> + <useProjectArtifact>false</useProjectArtifact> + <outputDirectory>blur-${project.version}/lib</outputDirectory> + <unpack>false</unpack> + <includes> + <include>org.apache.blur:*</include> + + <include>org.apache.zookeeper:zookeeper</include> + <include>org.slf4j:slf4j-api</include> + <include>org.slf4j:slf4j-log4j12</include> + <include>org.json:json</include> + <include>log4j:log4j</include> + <include>com.yammer.metrics:*</include> + <include>com.google.guava:guava</include> + <include>org.apache.httpcomponents:*</include> + <include>org.apache.lucene:*</include> + <include>com.spatial4j:spatial4j</include> + <include>commons-cli:commons-cli</include> + <include>org.eclipse.jetty:*</include> + <include>com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru</include> + <include>jline:jline</include> + <include>com.fasterxml.jackson.core:*</include> + <include>com.github.amccurry:lucene-document-security</include> + <include>javax.servlet:*</include> + </includes> + </dependencySet> + </dependencySets> + + <fileSets> + <fileSet> + <directory>${project.build.scriptSourceDirectory}/conf</directory> + <outputDirectory>blur-${project.version}/conf</outputDirectory> + <excludes> + <exclude>**/.empty</exclude> + </excludes> + </fileSet> + <fileSet> + <directory>${project.build.scriptSourceDirectory}/bin</directory> + <outputDirectory>blur-${project.version}/bin</outputDirectory> + <excludes> + <exclude>**/.empty</exclude> + </excludes> + </fileSet> + <fileSet> + <directory>${project.build.scriptSourceDirectory}/../../assemble/cdh/parcel/meta</directory> + <outputDirectory>blur-${project.version}/meta</outputDirectory> + <excludes> + <exclude>**/.empty</exclude> + </excludes> + </fileSet> + <fileSet> + <directory>${project.build.scriptSourceDirectory}/../../../../</directory> + <outputDirectory>blur-${project.version}/</outputDirectory> + <includes> + <include>DISCLAIMER</include> + <include>README</include> + </includes> + </fileSet> + </fileSets> + + <files> + <file> + <source>${project.build.scriptSourceDirectory}/../resources-hadoop2/NOTICE-bin.txt</source> + <outputDirectory>blur-${project.version}</outputDirectory> + <destName>NOTICE</destName> + </file> + <file> + <source>${project.build.scriptSourceDirectory}/../resources-hadoop2/LICENSE-bin.txt</source> + <outputDirectory>blur-${project.version}</outputDirectory> + <destName>LICENSE</destName> + </file> + </files> +</assembly> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/distribution/src/main/scripts/bin/blur-supervised-config.sh ---------------------------------------------------------------------- diff --git a/distribution/src/main/scripts/bin/blur-supervised-config.sh b/distribution/src/main/scripts/bin/blur-supervised-config.sh new file mode 100755 index 0000000..16605d1 --- /dev/null +++ b/distribution/src/main/scripts/bin/blur-supervised-config.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +export BLUR_HOME="$bin"/.. +export BLUR_HOME_CONF=$BLUR_HOME/conf + +#. $BLUR_HOME/conf/blur-env.sh +if [ -z "$JAVA_HOME" ]; then + if which java >/dev/null 2>&1 ; then + export JAVA_HOME=`java -cp $bin/../lib/blur-util-*.jar org.apache.blur.FindJavaHome` + fi +fi +if [ -z "$JAVA_HOME" ]; then + cat 1>&2 <<EOF ++======================================================================+ +| Error: JAVA_HOME is not set and Java could not be found | ++----------------------------------------------------------------------+ +| Please download the latest Sun JDK from the Sun Java web site | +| > http://java.sun.com/javase/downloads/ < | +| | +| Hadoop and Blur requires Java 1.6 or later. | +| NOTE: This script will find Sun Java whether you install using the | +| binary or the RPM based installer. | ++======================================================================+ +EOF + exit 1 +fi + +export JAVA=$JAVA_HOME/bin/java + +if [ -z "$BLUR_CONF_DIR" ]; then + BLUR_CONF_DIR=$BLUR_HOME/conf +fi + +BLUR_CLASSPATH=$BLUR_CONF_DIR + +for f in $BLUR_HOME/lib/*.jar; do + BLUR_CLASSPATH=${BLUR_CLASSPATH}:$f; +done + +for f in $BLUR_HOME/lib/*.war; do + BLUR_CLASSPATH=${BLUR_CLASSPATH}:$f; +done + +BLUR_CORE_FILE=`ls -d1 $BLUR_HOME/lib/blur-core-*.jar | head -1` + + BLUR_CLASSPATH=${BLUR_CLASSPATH}:$HADOOP_HOME/etc/hadoop + BLUR_CLASSPATH=${BLUR_CLASSPATH}:$HADOOP_CLASSPATH + + for f in $HADOOP_HOME/*.jar; do + BLUR_CLASSPATH=${BLUR_CLASSPATH}:$f; + done + + for f in $HADOOP_HOME/lib/*.jar; do + BLUR_CLASSPATH=${BLUR_CLASSPATH}:$f; + done + + for f in $HADOOP_HOME/share/hadoop/yarn/*.jar; do + BLUR_CLASSPATH=${BLUR_CLASSPATH}:$f; + done + + for f in $HADOOP_HOME/share/hadoop/yarn/lib/*.jar; do + BLUR_CLASSPATH=${BLUR_CLASSPATH}:$f; + done + + for f in $HADOOP_HOME/share/hadoop/common/*.jar; do + BLUR_CLASSPATH=${BLUR_CLASSPATH}:$f; + done + + for f in $HADOOP_HOME/share/hadoop/common/lib/*.jar; do + BLUR_CLASSPATH=${BLUR_CLASSPATH}:$f; + done + + + +export BLUR_CLASSPATH + +echo "BLUR_CLASSPATH: [$BLUR_CLASSPATH]" + +HOSTNAME=`hostname` http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/distribution/src/main/scripts/bin/start-blocking-controller-server.sh ---------------------------------------------------------------------- diff --git a/distribution/src/main/scripts/bin/start-blocking-controller-server.sh b/distribution/src/main/scripts/bin/start-blocking-controller-server.sh new file mode 100755 index 0000000..f3df832 --- /dev/null +++ b/distribution/src/main/scripts/bin/start-blocking-controller-server.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/blur-config.sh + +INSTANCE=0 +while [ $INSTANCE -lt $BLUR_NUMBER_OF_CONTROLLER_SERVER_INSTANCES_PER_MACHINE ]; do + PID_FILE=$BLUR_HOME/pids/controller-$INSTANCE.pid + + if [ -f $PID_FILE ]; then + if kill -0 `cat $PID_FILE` > /dev/null 2>&1; then + echo Controller server already running as process `cat $PID_FILE`. Stop it first. + let INSTANCE=INSTANCE+1 + continue + fi + fi + + PROC_NAME=controller-server-$HOSTNAME-$INSTANCE + "$JAVA_HOME"/bin/java -Dblur.name=$PROC_NAME -Djava.library.path=$JAVA_LIBRARY_PATH -Dblur-controller-$INSTANCE $BLUR_CONTROLLER_JVM_OPTIONS -Dblur.logs.dir=$BLUR_LOGS -Dblur.log.file=blur-$USER-$PROC_NAME -cp $BLUR_CLASSPATH org.apache.blur.thrift.ThriftBlurControllerServer -s $INSTANCE > "$BLUR_LOGS/blur-$USER-$PROC_NAME.out" 2>&1 < /dev/null + echo $! > $PID_FILE + echo Controller [$INSTANCE] starting as process `cat $PID_FILE`. + + let INSTANCE=INSTANCE+1 +done \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/distribution/src/main/scripts/bin/start-supervised-controller-server.sh ---------------------------------------------------------------------- diff --git a/distribution/src/main/scripts/bin/start-supervised-controller-server.sh b/distribution/src/main/scripts/bin/start-supervised-controller-server.sh new file mode 100755 index 0000000..a8e6fa5 --- /dev/null +++ b/distribution/src/main/scripts/bin/start-supervised-controller-server.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +echo "Sourcing configs.." +. $BLUR_HOME/bin/blur-supervised-config.sh + + PROC_NAME=controller-server-$HOSTNAME-0 + echo "Launching controller [$PROC_NAME] now..." + echo "Using BLUR_CLASSPATH: ${BLUR_CLASSPATH}" + exec "$JAVA_HOME"/bin/java -Dblur.name=$PROC_NAME -Djava.library.path=$JAVA_LIBRARY_PATH -Dblur-controller-$INSTANCE $BLUR_CONTROLLER_JVM_OPTIONS -Dblur.logs.dir=$BLUR_LOGS -Dblur.log.file=blur-$PROC_NAME -cp "$BLUR_CLASSPATH" org.apache.blur.thrift.ThriftBlurControllerServer -s 0 http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/distribution/src/main/scripts/bin/start-supervised-shard-server.sh ---------------------------------------------------------------------- diff --git a/distribution/src/main/scripts/bin/start-supervised-shard-server.sh b/distribution/src/main/scripts/bin/start-supervised-shard-server.sh new file mode 100755 index 0000000..89ab0e8 --- /dev/null +++ b/distribution/src/main/scripts/bin/start-supervised-shard-server.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +echo "Sourcing configs.." +. $BLUR_HOME/bin/blur-supervised-config.sh + + PROC_NAME=shard-server-$HOSTNAME-0 + echo "Launching shard [$PROC_NAME] now..." + echo "Using BLUR_CLASSPATH: ${BLUR_CLASSPATH}" + exec "$JAVA_HOME"/bin/java -Dblur.name=$PROC_NAME -Djava.library.path=$JAVA_LIBRARY_PATH -Dblur-shard-$INSTANCE $BLUR_SHARD_JVM_OPTIONS -Dblur.logs.dir=$BLUR_LOGS -Dblur.log.file=blur-$USER-$PROC_NAME -cp "$BLUR_CLASSPATH" org.apache.blur.thrift.ThriftBlurShardServer -s 0 http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/053589d4/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index b9602a6..622f873 100644 --- a/pom.xml +++ b/pom.xml @@ -389,6 +389,8 @@ under the License. <!-- Used to create empty directories in git --> <exclude>**/.empty</exclude> + + <exclude>**/test/**/prop.doc.base.html</exclude> </excludes> </configuration> <executions>
