Repository: bigtop Updated Branches: refs/heads/master 1f47e2773 -> 0154e3c87
BIGTOP-1865. circular dependency between bigtop-groovy and bigtop-utils Signed-off-by: Evans Ye <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/0154e3c8 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/0154e3c8 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/0154e3c8 Branch: refs/heads/master Commit: 0154e3c872b3b6e55d5350b54450c57f0b8f81c6 Parents: 1f47e27 Author: Konstantin Boudnik <[email protected]> Authored: Sun May 10 16:41:37 2015 -0700 Committer: Evans Ye <[email protected]> Committed: Wed May 13 16:43:19 2015 +0000 ---------------------------------------------------------------------- .../src/common/bigtop-utils/init-hcfs.groovy | 343 ------------------- .../src/common/hadoop/init-hcfs.groovy | 343 +++++++++++++++++++ bigtop-packages/src/common/hadoop/init-hdfs.sh | 2 +- .../src/common/hadoop/install_hadoop.sh | 1 + bigtop-packages/src/deb/bigtop-utils/control | 2 +- bigtop-packages/src/deb/bigtop-utils/rules | 1 - bigtop-packages/src/deb/hadoop/control | 2 +- .../src/deb/hadoop/hadoop-hdfs.install | 1 + .../rpm/bigtop-utils/SPECS/bigtop-utils.spec | 5 +- .../src/rpm/hadoop/SPECS/hadoop.spec | 3 +- 10 files changed, 351 insertions(+), 352 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/0154e3c8/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy b/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy deleted file mode 100644 index d07ef01..0000000 --- a/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy +++ /dev/null @@ -1,343 +0,0 @@ -#!/usr/bin/env /usr/lib/bigtop-groovy/bin/groovy -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import groovy.json.JsonSlurper; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.permission.FsPermission; - -def final LOG = LogFactory.getLog(this.getClass()); -def final jsonParser = new JsonSlurper(); - -def final USAGE = """\ - ********************************************************************* - USAGE: - - This script provisions the skeleton of a hadoop file system. - It takes a single argument: The json schema (a list of lists), - of 4 element tuples. For an example , see the bigtop init-hcfs.json - file. The main elements of the JSON file are: - - A copy of init-hcfs.json ships with bigtop distributions. - - dir: list of dirs to create with permissions. - user: list of users to setup home dirs with permissions. - root_user: The root owner of distributed FS, to run shell commands. - - To run this script, you will want to setup your environment using - init-hcfs.json, - which defines the properties above, and then invoke this script. - - Details below. - - SETUP YOUR CLUSTER ENVIRONMENT - - As mentinoed above, the init-hcfs.json file is what guides the - directories/users to setup. - So first you will want to edit that file as you need to. Some common - modifications: - - - - Usually the "root_user" on HDFS is just hdfs. For other file systems - the root user might be "root". - - The default hadoop users you may find in the init-hcfs.json template - you follow "tom"/"alice"/etc.. aren't necessarily on all clusters. - - HOW TO INVOKE: - - 1) Simple groovy based method: Just manually construct a hadoop classpath: - - groovy -classpath /usr/lib/hadoop/hadoop-common-2.0.6-alpha.jar - :/usr/lib/hadoop/lib/guava-11.0.2.jar - :/etc/hadoop/conf/:/usr/lib/hadoop/hadoop-common-2.0.6-alpha.jar - :/usr/lib/hadoop/lib/commons-configuration-1.6.jar - :/usr/lib/hadoop/lib/commons-lang-2.5.jar:/usr/lib/hadoop/hadoop-auth.jar - :/usr/lib/hadoop/lib/slf4j-api-1.6.1.jar - :/usr/lib/hadoop-hdfs/hadoop-hdfs.jar - :/usr/lib/hadoop/lib/protobuf-java-2.4.0a.jar /vagrant/init-hcfs.groovy - /vagrant/init-hcfs.json - - 2) Another method: Follow the instructions on groovy.codehaus.org/Running - for setting up groovy runtime environment with - CLASSPATH and/or append those libraries to the shebang command as - necessary, and then simply do: - - chmod +x init-hcfs.groovy - ./init-hcfs.groovy init-hcfs.json - - ********************************************************************* -""" - -/** - * The HCFS generic provisioning process: - * - * 1) Create a file system skeleton. - * 2) Create users with home dirs in /user. - * 3) Copy jars and libs into the DFS for oozie. - * - * In the future maybe we will add more optional steps (i.e. adding libs to - * the distribtued cache, mounting FUSE over HDFS, etc...). - **/ - -def errors = [ - ("0: No init-hcfs.json input file provided !"): { - LOG.info("Checking argument length: " + args.length + " " + args); - return args.length == 1 - }, - ("1: init-hcfs json not found."): { - LOG.info("Checking for file : " + args[0]); - return new File(args[0]).exists() - }]; - -errors.each { error_message, passed -> - if (!passed.call()) { - System.err.println("ERROR:" + error_message); - System.err.println(USAGE); - System.exit(1); - } -} - -def final json = args[0]; -def final parsedData = jsonParser.parse(new FileReader(json)); - -/** - * Groovy is smart enough to convert JSON - * fields to objects for us automagically. - * */ -def dirs = parsedData.dir as List; -def users = parsedData.user as List; -def hcfs_super_user = parsedData.root_user; - -def final Configuration conf = new Configuration(); - -LOG.info("Provisioning file system for file system from Configuration: " + - conf.get("fs.defaultFS")); - -/** - * We create a single FileSystem instance to use for all the file system calls. - * This script makes anywhere from 20-100 file system operations so it's - * important to cache and create this only once. - * */ -def final FileSystem fs = FileSystem.get(conf); - -LOG.info("PROVISIONING WITH FILE SYSTEM : " + fs.getClass()); - -// Longest back off time to check whether the file system is ready for write -def final int maxBackOff = 64; - -/** - * Make a directory. Note when providing input to this functino that if - * nulls are given, the commands will work but behaviour varies depending on - * the HCFS implementation ACLs, etc. - * @param fs The HCFS implementation to create the Directory on. - * @param dname Required. - * @param mode can be null. - * @param user can be null. - * @param group can be null, - */ -def mkdir = { FileSystem fsys, Path dname, FsPermission mode, String user, String group -> - boolean success = false; - for(i = 1; i <= maxBackOff; i*=2) { - try { - success = fsys.mkdirs(dname) - break; - } catch(Exception e) { - LOG.info("Failed to create directory " + dname + "... Retry after " + i + " second(s)"); - Thread.sleep(i*1000); - } - } - if (!success) { - LOG.info("Can not create directory " + dname + " on " + fsys.getClass()); - } - if (user != null) { - fsys.setOwner(dname, user, group); - } - if (mode != null) { - fsys.setPermission(dname, mode); - FsPermission result = fsys.getFileStatus(dname).getPermission(); - /** Confirm that permission took properly. - * important to do this since while we work on better - * docs for modifying and maintaining this new approach - * to HCFS provisioning.*/ - if (!fsys.getFileStatus(dname).getPermission().equals(mode)) { - throw new RuntimeException("Failed at setting permission to " + mode + - "... target directory permission is incorrect: " + result); - } - } -} - -/** - * Create a perm from raw string representing an octal perm. - * @param mode The stringified octal mode (i.e. "1777") - * */ -private FsPermission readPerm(String mode) { - Short permValue = Short.decode("0" + mode); - //This constructor will decode the octal perm bits - //out of the short. - return new FsPermission(permValue); -} - -int dirs_created = 0; -/** - * Provisioning the directories on the file system. This is the - * most important task of this script, as a basic directory skeleton - * is needed even for basic yarn/mapreduce apps before startup. - * */ -dirs.each() { - def (dname, mode, user, group) = it; - - dname = new Path(dname); - - //We encode permissions as strings, since they are octal. - //JSON doesn't support octal natively. - if (mode != null) - mode = readPerm(mode) as FsPermission; - - if (user?.equals("HCFS_SUPER_USER")) - user = hcfs_super_user; - - LOG.info("mkdirs " + dname + " " + user + " " + mode + " " + group); - mkdir(fs, dname, mode, user, group); - - dirs_created++; -} - -LOG.info("Succesfully created " + dirs_created + " directories in the DFS."); - -/** - * Now, for most clusters we will generally start out with at least one - * user. You should modify your init-hcfs.json file accordingly if you - * have a set of users you want to setup for using hadoop. - * - * For each user we do initial setup, create a home directory, etc... - * You may also need to do special tasks if running LinuxTaskControllers, - * etc, which aren't (yet) handled by this provisioner. - * */ -users.each() { - def (user, permission, group) = it; - LOG.info("current user: " + user); - Path homedir = new Path("/user/" + user); - - //perms should be ALL, RX,RX ^^ - fs.mkdirs(homedir); - fs.setOwner(homedir, user, group); - FsPermission perm = readPerm(permission); - fs.setPermission(homedir, perm); -} - - -/** - * Copys jar files from a destination into the distributed FS. - * Build specifically for the common task of getting jars into - * oozies classpath so that oozie can run pig/hive/etc based - * applications. Directories and broken symlinks will be skipped. - * - * @param fs An instance of an HCFS FileSystem . - * - * @param input The LOCAL DIRECTORY containing jar files. - * - * @param jarstr A jar file name filter used to reject/accept jar names. - * See the script below for example of how it's used. Jars matching this - * string will be copied into the specified path on the "target" directory. - * - * @param target The path on the DISTRIBUTED FS where jars should be copied - * to. - * - * @return The total number of jars copied into the DFS. - */ -def copyJars = { FileSystem fsys, File input, String jarstr, Path target -> - int copied = 0; - input.listFiles(new FileFilter() { - public boolean accept(File f) { - String filename = f.getName(); - boolean validJar = filename.endsWith("jar") && f.isFile(); - return validJar && filename.contains(jarstr) - } - }).each({ jar_file -> - boolean success = false; - for(i = 1; i <= maxBackOff; i*=2) { - try { - fsys.copyFromLocalFile(new Path(jar_file.getAbsolutePath()), target) - copied++; - success = true; - break; - } catch(Exception e) { - LOG.info("Failed to upload " + jar_file.getAbsolutePath() + " to " + target + "... Retry after " + i + " second(s)"); - Thread.sleep(i*1000); - } - if (!success) { - LOG.info("Can not upload " + jar_file.getAbsolutePath() + " to " + target + " on " + fsys.getClass()); - } - } - }); - return copied; -} - -/** - * Copy shared libraries into oozie. - * Particular applications might want to modify this for example - * if one wanted to add a custom file system or always available - * custom library to be used in oozie workflows. - * */ -total_jars = 0; - -LOG.info("Now copying Jars into the DFS for oozie "); -LOG.info("This might take a few seconds..."); - -def final OOZIE_SHARE = "/user/oozie/share/lib/"; -def final MAPREDUCE = "/usr/lib/hadoop-mapreduce/"; -def final PIG_HOME = "/usr/lib/pig/"; -def final HIVE_HOME = "/usr/lib/hive/"; - -total_jars += copyJars(fs, - new File(HIVE_HOME, "lib"), "", - new Path(OOZIE_SHARE, "hive/")) - -total_jars += copyJars(fs, - new File(MAPREDUCE), "hadoop-streaming", - new Path(OOZIE_SHARE, "mapreduce-streaming/")) - -total_jars += copyJars(fs, - new File(MAPREDUCE), "hadoop-distcp", - new Path(OOZIE_SHARE, "distcp")) - -total_jars += copyJars(fs, - new File(PIG_HOME, "lib/"), "", - new Path(OOZIE_SHARE, "pig")) - -total_jars += copyJars(fs, - new File(PIG_HOME), "", - new Path(OOZIE_SHARE, "pig")) - -LOG.info("Now copying Jars into the DFS for tez "); -LOG.info("This might take a few seconds..."); - -def final TEZ_APPS = "/apps"; -def final TEZ_HOME = "/usr/lib/tez/"; - -total_jars += copyJars(fs, - new File(TEZ_HOME, "lib/"), "", - new Path(TEZ_APPS, "tez/lib")) - -total_jars += copyJars(fs, - new File(TEZ_HOME), "", - new Path(TEZ_APPS, "tez")) - -LOG.info("Total jars copied into the DFS : " + total_jars); http://git-wip-us.apache.org/repos/asf/bigtop/blob/0154e3c8/bigtop-packages/src/common/hadoop/init-hcfs.groovy ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hadoop/init-hcfs.groovy b/bigtop-packages/src/common/hadoop/init-hcfs.groovy new file mode 100644 index 0000000..d07ef01 --- /dev/null +++ b/bigtop-packages/src/common/hadoop/init-hcfs.groovy @@ -0,0 +1,343 @@ +#!/usr/bin/env /usr/lib/bigtop-groovy/bin/groovy +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import groovy.json.JsonSlurper; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.permission.FsPermission; + +def final LOG = LogFactory.getLog(this.getClass()); +def final jsonParser = new JsonSlurper(); + +def final USAGE = """\ + ********************************************************************* + USAGE: + + This script provisions the skeleton of a hadoop file system. + It takes a single argument: The json schema (a list of lists), + of 4 element tuples. For an example , see the bigtop init-hcfs.json + file. The main elements of the JSON file are: + + A copy of init-hcfs.json ships with bigtop distributions. + + dir: list of dirs to create with permissions. + user: list of users to setup home dirs with permissions. + root_user: The root owner of distributed FS, to run shell commands. + + To run this script, you will want to setup your environment using + init-hcfs.json, + which defines the properties above, and then invoke this script. + + Details below. + + SETUP YOUR CLUSTER ENVIRONMENT + + As mentinoed above, the init-hcfs.json file is what guides the + directories/users to setup. + So first you will want to edit that file as you need to. Some common + modifications: + + + - Usually the "root_user" on HDFS is just hdfs. For other file systems + the root user might be "root". + - The default hadoop users you may find in the init-hcfs.json template + you follow "tom"/"alice"/etc.. aren't necessarily on all clusters. + + HOW TO INVOKE: + + 1) Simple groovy based method: Just manually construct a hadoop classpath: + + groovy -classpath /usr/lib/hadoop/hadoop-common-2.0.6-alpha.jar + :/usr/lib/hadoop/lib/guava-11.0.2.jar + :/etc/hadoop/conf/:/usr/lib/hadoop/hadoop-common-2.0.6-alpha.jar + :/usr/lib/hadoop/lib/commons-configuration-1.6.jar + :/usr/lib/hadoop/lib/commons-lang-2.5.jar:/usr/lib/hadoop/hadoop-auth.jar + :/usr/lib/hadoop/lib/slf4j-api-1.6.1.jar + :/usr/lib/hadoop-hdfs/hadoop-hdfs.jar + :/usr/lib/hadoop/lib/protobuf-java-2.4.0a.jar /vagrant/init-hcfs.groovy + /vagrant/init-hcfs.json + + 2) Another method: Follow the instructions on groovy.codehaus.org/Running + for setting up groovy runtime environment with + CLASSPATH and/or append those libraries to the shebang command as + necessary, and then simply do: + + chmod +x init-hcfs.groovy + ./init-hcfs.groovy init-hcfs.json + + ********************************************************************* +""" + +/** + * The HCFS generic provisioning process: + * + * 1) Create a file system skeleton. + * 2) Create users with home dirs in /user. + * 3) Copy jars and libs into the DFS for oozie. + * + * In the future maybe we will add more optional steps (i.e. adding libs to + * the distribtued cache, mounting FUSE over HDFS, etc...). + **/ + +def errors = [ + ("0: No init-hcfs.json input file provided !"): { + LOG.info("Checking argument length: " + args.length + " " + args); + return args.length == 1 + }, + ("1: init-hcfs json not found."): { + LOG.info("Checking for file : " + args[0]); + return new File(args[0]).exists() + }]; + +errors.each { error_message, passed -> + if (!passed.call()) { + System.err.println("ERROR:" + error_message); + System.err.println(USAGE); + System.exit(1); + } +} + +def final json = args[0]; +def final parsedData = jsonParser.parse(new FileReader(json)); + +/** + * Groovy is smart enough to convert JSON + * fields to objects for us automagically. + * */ +def dirs = parsedData.dir as List; +def users = parsedData.user as List; +def hcfs_super_user = parsedData.root_user; + +def final Configuration conf = new Configuration(); + +LOG.info("Provisioning file system for file system from Configuration: " + + conf.get("fs.defaultFS")); + +/** + * We create a single FileSystem instance to use for all the file system calls. + * This script makes anywhere from 20-100 file system operations so it's + * important to cache and create this only once. + * */ +def final FileSystem fs = FileSystem.get(conf); + +LOG.info("PROVISIONING WITH FILE SYSTEM : " + fs.getClass()); + +// Longest back off time to check whether the file system is ready for write +def final int maxBackOff = 64; + +/** + * Make a directory. Note when providing input to this functino that if + * nulls are given, the commands will work but behaviour varies depending on + * the HCFS implementation ACLs, etc. + * @param fs The HCFS implementation to create the Directory on. + * @param dname Required. + * @param mode can be null. + * @param user can be null. + * @param group can be null, + */ +def mkdir = { FileSystem fsys, Path dname, FsPermission mode, String user, String group -> + boolean success = false; + for(i = 1; i <= maxBackOff; i*=2) { + try { + success = fsys.mkdirs(dname) + break; + } catch(Exception e) { + LOG.info("Failed to create directory " + dname + "... Retry after " + i + " second(s)"); + Thread.sleep(i*1000); + } + } + if (!success) { + LOG.info("Can not create directory " + dname + " on " + fsys.getClass()); + } + if (user != null) { + fsys.setOwner(dname, user, group); + } + if (mode != null) { + fsys.setPermission(dname, mode); + FsPermission result = fsys.getFileStatus(dname).getPermission(); + /** Confirm that permission took properly. + * important to do this since while we work on better + * docs for modifying and maintaining this new approach + * to HCFS provisioning.*/ + if (!fsys.getFileStatus(dname).getPermission().equals(mode)) { + throw new RuntimeException("Failed at setting permission to " + mode + + "... target directory permission is incorrect: " + result); + } + } +} + +/** + * Create a perm from raw string representing an octal perm. + * @param mode The stringified octal mode (i.e. "1777") + * */ +private FsPermission readPerm(String mode) { + Short permValue = Short.decode("0" + mode); + //This constructor will decode the octal perm bits + //out of the short. + return new FsPermission(permValue); +} + +int dirs_created = 0; +/** + * Provisioning the directories on the file system. This is the + * most important task of this script, as a basic directory skeleton + * is needed even for basic yarn/mapreduce apps before startup. + * */ +dirs.each() { + def (dname, mode, user, group) = it; + + dname = new Path(dname); + + //We encode permissions as strings, since they are octal. + //JSON doesn't support octal natively. + if (mode != null) + mode = readPerm(mode) as FsPermission; + + if (user?.equals("HCFS_SUPER_USER")) + user = hcfs_super_user; + + LOG.info("mkdirs " + dname + " " + user + " " + mode + " " + group); + mkdir(fs, dname, mode, user, group); + + dirs_created++; +} + +LOG.info("Succesfully created " + dirs_created + " directories in the DFS."); + +/** + * Now, for most clusters we will generally start out with at least one + * user. You should modify your init-hcfs.json file accordingly if you + * have a set of users you want to setup for using hadoop. + * + * For each user we do initial setup, create a home directory, etc... + * You may also need to do special tasks if running LinuxTaskControllers, + * etc, which aren't (yet) handled by this provisioner. + * */ +users.each() { + def (user, permission, group) = it; + LOG.info("current user: " + user); + Path homedir = new Path("/user/" + user); + + //perms should be ALL, RX,RX ^^ + fs.mkdirs(homedir); + fs.setOwner(homedir, user, group); + FsPermission perm = readPerm(permission); + fs.setPermission(homedir, perm); +} + + +/** + * Copys jar files from a destination into the distributed FS. + * Build specifically for the common task of getting jars into + * oozies classpath so that oozie can run pig/hive/etc based + * applications. Directories and broken symlinks will be skipped. + * + * @param fs An instance of an HCFS FileSystem . + * + * @param input The LOCAL DIRECTORY containing jar files. + * + * @param jarstr A jar file name filter used to reject/accept jar names. + * See the script below for example of how it's used. Jars matching this + * string will be copied into the specified path on the "target" directory. + * + * @param target The path on the DISTRIBUTED FS where jars should be copied + * to. + * + * @return The total number of jars copied into the DFS. + */ +def copyJars = { FileSystem fsys, File input, String jarstr, Path target -> + int copied = 0; + input.listFiles(new FileFilter() { + public boolean accept(File f) { + String filename = f.getName(); + boolean validJar = filename.endsWith("jar") && f.isFile(); + return validJar && filename.contains(jarstr) + } + }).each({ jar_file -> + boolean success = false; + for(i = 1; i <= maxBackOff; i*=2) { + try { + fsys.copyFromLocalFile(new Path(jar_file.getAbsolutePath()), target) + copied++; + success = true; + break; + } catch(Exception e) { + LOG.info("Failed to upload " + jar_file.getAbsolutePath() + " to " + target + "... Retry after " + i + " second(s)"); + Thread.sleep(i*1000); + } + if (!success) { + LOG.info("Can not upload " + jar_file.getAbsolutePath() + " to " + target + " on " + fsys.getClass()); + } + } + }); + return copied; +} + +/** + * Copy shared libraries into oozie. + * Particular applications might want to modify this for example + * if one wanted to add a custom file system or always available + * custom library to be used in oozie workflows. + * */ +total_jars = 0; + +LOG.info("Now copying Jars into the DFS for oozie "); +LOG.info("This might take a few seconds..."); + +def final OOZIE_SHARE = "/user/oozie/share/lib/"; +def final MAPREDUCE = "/usr/lib/hadoop-mapreduce/"; +def final PIG_HOME = "/usr/lib/pig/"; +def final HIVE_HOME = "/usr/lib/hive/"; + +total_jars += copyJars(fs, + new File(HIVE_HOME, "lib"), "", + new Path(OOZIE_SHARE, "hive/")) + +total_jars += copyJars(fs, + new File(MAPREDUCE), "hadoop-streaming", + new Path(OOZIE_SHARE, "mapreduce-streaming/")) + +total_jars += copyJars(fs, + new File(MAPREDUCE), "hadoop-distcp", + new Path(OOZIE_SHARE, "distcp")) + +total_jars += copyJars(fs, + new File(PIG_HOME, "lib/"), "", + new Path(OOZIE_SHARE, "pig")) + +total_jars += copyJars(fs, + new File(PIG_HOME), "", + new Path(OOZIE_SHARE, "pig")) + +LOG.info("Now copying Jars into the DFS for tez "); +LOG.info("This might take a few seconds..."); + +def final TEZ_APPS = "/apps"; +def final TEZ_HOME = "/usr/lib/tez/"; + +total_jars += copyJars(fs, + new File(TEZ_HOME, "lib/"), "", + new Path(TEZ_APPS, "tez/lib")) + +total_jars += copyJars(fs, + new File(TEZ_HOME), "", + new Path(TEZ_APPS, "tez")) + +LOG.info("Total jars copied into the DFS : " + total_jars); http://git-wip-us.apache.org/repos/asf/bigtop/blob/0154e3c8/bigtop-packages/src/common/hadoop/init-hdfs.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hadoop/init-hdfs.sh b/bigtop-packages/src/common/hadoop/init-hdfs.sh index 9b9271b..1f111a3 100755 --- a/bigtop-packages/src/common/hadoop/init-hdfs.sh +++ b/bigtop-packages/src/common/hadoop/init-hdfs.sh @@ -43,4 +43,4 @@ cd $HADOOP_LIB_DIR for d in $HADOOP_DEPENDENCIES; do CLASSPATH=$CLASSPATH:$HADOOP_LIB_DIR/$d; done for d in $HDFS_DEPENDENCIES; do CLASSPATH=$CLASSPATH:$HDFS_LIB_DIR/$d; done popd -su -s /bin/bash hdfs -c "/usr/lib/bigtop-groovy/bin/groovy -classpath $CLASSPATH /usr/lib/bigtop-utils/init-hcfs.groovy /usr/lib/hadoop/libexec/init-hcfs.json" +su -s /bin/bash hdfs -c "/usr/lib/bigtop-groovy/bin/groovy -classpath $CLASSPATH /usr/lib/hadoop/libexec/init-hcfs.groovy /usr/lib/hadoop/libexec/init-hcfs.json" http://git-wip-us.apache.org/repos/asf/bigtop/blob/0154e3c8/bigtop-packages/src/common/hadoop/install_hadoop.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hadoop/install_hadoop.sh b/bigtop-packages/src/common/hadoop/install_hadoop.sh index 9736e5a..f8aef9b 100755 --- a/bigtop-packages/src/common/hadoop/install_hadoop.sh +++ b/bigtop-packages/src/common/hadoop/install_hadoop.sh @@ -185,6 +185,7 @@ cp ${BUILD_DIR}/libexec/* ${SYSTEM_LIBEXEC_DIR}/ cp ${DISTRO_DIR}/hadoop-layout.sh ${SYSTEM_LIBEXEC_DIR}/ install -m 0755 ${DISTRO_DIR}/init-hdfs.sh ${SYSTEM_LIBEXEC_DIR}/ install -m 0755 ${DISTRO_DIR}/init-hcfs.json ${SYSTEM_LIBEXEC_DIR}/ +install -m 0755 ${DISTRO_DIR}/init-hcfs.groovy ${SYSTEM_LIBEXEC_DIR}/ rm -rf ${SYSTEM_LIBEXEC_DIR}/*.cmd # hadoop jar http://git-wip-us.apache.org/repos/asf/bigtop/blob/0154e3c8/bigtop-packages/src/deb/bigtop-utils/control ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/bigtop-utils/control b/bigtop-packages/src/deb/bigtop-utils/control index e3b4d9a..7618882 100644 --- a/bigtop-packages/src/deb/bigtop-utils/control +++ b/bigtop-packages/src/deb/bigtop-utils/control @@ -22,6 +22,6 @@ Homepage: http://bigtop.apache.org/ Package: bigtop-utils Architecture: all -Depends: ${misc:Depends}, bigtop-groovy +Depends: ${misc:Depends} Description: Collection of useful tools for Bigtop This includes a collection of useful tools and files for Bigtop http://git-wip-us.apache.org/repos/asf/bigtop/blob/0154e3c8/bigtop-packages/src/deb/bigtop-utils/rules ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/bigtop-utils/rules b/bigtop-packages/src/deb/bigtop-utils/rules index 4efcf01..e55e985 100644 --- a/bigtop-packages/src/deb/bigtop-utils/rules +++ b/bigtop-packages/src/deb/bigtop-utils/rules @@ -32,7 +32,6 @@ override_dh_auto_install: # Copy repo file install -d -p -m 755 debian/bigtop-utils/var/lib/bigtop/ install -d -p -m 755 debian/bigtop-utils/usr/lib/bigtop-utils/ - install -p -m 755 debian/init-hcfs.groovy debian/bigtop-utils/usr/lib/bigtop-utils/ install -p -m 755 debian/bigtop-detect-javahome debian/bigtop-utils/usr/lib/bigtop-utils/ install -p -m 755 debian/bigtop-detect-javalibs debian/bigtop-utils/usr/lib/bigtop-utils/ install -p -m 755 debian/bigtop-detect-classpath debian/bigtop-utils/usr/lib/bigtop-utils/ http://git-wip-us.apache.org/repos/asf/bigtop/blob/0154e3c8/bigtop-packages/src/deb/hadoop/control ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/hadoop/control b/bigtop-packages/src/deb/hadoop/control index c6655bd..87370db 100644 --- a/bigtop-packages/src/deb/hadoop/control +++ b/bigtop-packages/src/deb/hadoop/control @@ -47,7 +47,7 @@ Description: Hadoop is a software platform for processing vast amounts of data Package: hadoop-hdfs Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends}, adduser, bigtop-utils (>= 0.7), hadoop (= ${binary:Version}), bigtop-jsvc +Depends: ${shlibs:Depends}, ${misc:Depends}, adduser, bigtop-utils (>= 0.7), bigtop-groovy, hadoop (= ${binary:Version}), bigtop-jsvc Description: The Hadoop Distributed File System Hadoop Distributed File System (HDFS) is the primary storage system used by Hadoop applications. HDFS creates multiple replicas of data blocks and distributes http://git-wip-us.apache.org/repos/asf/bigtop/blob/0154e3c8/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install b/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install index 3aece60..1851ea9 100644 --- a/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install +++ b/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install @@ -4,6 +4,7 @@ /usr/lib/hadoop/libexec/hdfs-config.sh /usr/lib/hadoop/libexec/init-hdfs.sh /usr/lib/hadoop/libexec/init-hcfs.json +/usr/lib/hadoop/libexec/init-hcfs.groovy /usr/bin/hdfs /var/lib/hadoop-hdfs /var/log/hadoop-hdfs http://git-wip-us.apache.org/repos/asf/bigtop/blob/0154e3c8/bigtop-packages/src/rpm/bigtop-utils/SPECS/bigtop-utils.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/bigtop-utils/SPECS/bigtop-utils.spec b/bigtop-packages/src/rpm/bigtop-utils/SPECS/bigtop-utils.spec index 9186540..9a89091 100644 --- a/bigtop-packages/src/rpm/bigtop-utils/SPECS/bigtop-utils.spec +++ b/bigtop-packages/src/rpm/bigtop-utils/SPECS/bigtop-utils.spec @@ -32,8 +32,7 @@ Source2: bigtop-utils.default Source3: bigtop-detect-javalibs Source4: bigtop-detect-classpath Source5: bigtop-monitor-service -Source6: init-hcfs.groovy -Requires: bash, bigtop-groovy +Requires: bash # "which" command is needed for a lot of projects. # It is part of the package "util-linux" on suse and "which" everywhere else @@ -55,7 +54,6 @@ install -p -m 644 %{SOURCE2} . install -p -m 644 %{SOURCE3} . install -p -m 644 %{SOURCE4} . install -p -m 644 %{SOURCE5} . -install -p -m 755 %{SOURCE6} . %build @@ -69,7 +67,6 @@ install -p -m 755 %{SOURCE3} $RPM_BUILD_ROOT%{lib_dir}/ install -p -m 755 %{SOURCE4} $RPM_BUILD_ROOT%{lib_dir}/ install -p -m 755 %{SOURCE5} $RPM_BUILD_ROOT%{lib_dir}/ install -p -m 644 %{SOURCE2} $RPM_BUILD_ROOT/etc/default/bigtop-utils -install -p -m 644 %{SOURCE6} $RPM_BUILD_ROOT%{lib_dir}/ %clean rm -rf $RPM_BUILD_ROOT http://git-wip-us.apache.org/repos/asf/bigtop/blob/0154e3c8/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec index abe2dd6..4a6cfaf 100644 --- a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec +++ b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec @@ -222,7 +222,7 @@ located. %package hdfs Summary: The Hadoop Distributed File System Group: System/Daemons -Requires: %{name} = %{version}-%{release}, bigtop-jsvc +Requires: %{name} = %{version}-%{release}, bigtop-groovy, bigtop-jsvc %description hdfs Hadoop Distributed File System (HDFS) is the primary storage system used by @@ -620,6 +620,7 @@ fi %attr(1777,hdfs,hadoop) %{state_hdfs}/cache %{lib_hadoop}/libexec/init-hdfs.sh %{lib_hadoop}/libexec/init-hcfs.json +%{lib_hadoop}/libexec/init-hcfs.groovy %files mapreduce %defattr(-,root,root)
