Added: incubator/hcatalog/trunk/webhcat/svr/src/main/bin/templeton_server.sh
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/bin/templeton_server.sh?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/bin/templeton_server.sh
(added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/bin/templeton_server.sh Wed
Jul 25 20:29:44 2012
@@ -0,0 +1,237 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Support functions
+#
+
+# Follow symlinks on Linux and Darwin
+function real_script_name() {
+ local base=$1
+ local real
+ if readlink -f $base >/dev/null 2>&1; then
+ # Darwin/Mac OS X
+ real=`readlink -f $base`
+ fi
+ if [[ "$?" != "0" || -z "$real" ]]; then
+ # Linux
+ local bin=$(cd -P -- "$(dirname -- "$base")">/dev/null && pwd
-P)
+ local script="$(basename -- "$base")"
+ real="$bin/$script"
+ fi
+ echo "$real"
+}
+
+function usage() {
+ echo "usage: $0 [start|stop|foreground]"
+ echo " start Start the Templeton Server"
+ echo " stop Stop the Templeton Server"
+ echo " foreground Run the Templeton Server in the foreground"
+ exit 1
+}
+
+# Print an error message and exit
+function die() {
+ echo "templeton: $@" 1>&2
+ exit 1
+}
+
+# Print an message
+function log() {
+ echo "templeton: $@"
+}
+
+# Find the templeton jar
+function find_jar_path() {
+ for dir in "." "build/templeton" "share/templeton/"; do
+ local jar="$base_dir/$dir/$TEMPLETON_JAR"
+ if [[ -f $jar ]]; then
+ echo $jar
+ break
+ fi
+ done
+}
+
+# Find the templeton classpath
+function find_classpath() {
+ local classpath=""
+ for dir in "share/templeton/lib" "build/ivy/lib/templeton" "conf" ; do
+ local path="$base_dir/$dir"
+
+ if [[ -d $path ]]; then
+ for jar_or_conf in $path/*; do
+ if [[ -z "$classpath" ]]; then
+ classpath="$jar_or_conf"
+ else
+ classpath="$classpath:$jar_or_conf"
+ fi
+ done
+ fi
+ done
+
+ if [[ -n "$TEMPLETON_CONF_DIR" ]]; then
+ if [[ -z "$classpath" ]]; then
+ classpath="$TEMPLETON_CONF_DIR"
+ else
+ classpath="$classpath:$TEMPLETON_CONF_DIR"
+ fi
+ fi
+
+ # Append hcat classpath
+ local hcat_classpath
+ hcat_classpath=`hcat -classpath`
+ if [[ "$?" != "0" ]]; then
+ die "Unable to get the hcatalog classpath"
+ fi
+ echo "$classpath:$hcat_classpath"
+}
+
+# Check if the pid is running
+function check_pid() {
+ local pid=$1
+ if ps -p $pid > /dev/null; then
+ return 0
+ else
+ return 1
+ fi
+}
+
+# Start the templeton server in the foreground
+function foreground_templeton() {
+ $start_cmd
+}
+
+# Start the templeton server in the background. Record the PID for
+# later use.
+function start_templeton() {
+ if [[ -f $PID_FILE ]]; then
+ # Check if there is a server running
+ local pid=`cat $PID_FILE`
+ if check_pid $pid; then
+ die "already running on process $pid"
+ fi
+ fi
+
+ log "starting ..."
+ log "$start_cmd"
+ nohup $start_cmd >>$CONSOLE_LOG 2>>$ERROR_LOG &
+ local pid=$!
+
+ if [[ -z "${pid}" ]] ; then # we failed right off
+ die "failed to start. Check logs in " `dirname $ERROR_LOG`
+ fi
+
+ sleep $SLEEP_TIME_AFTER_START
+
+ if check_pid $pid; then
+ echo $pid > $PID_FILE
+ log "starting ... started."
+ else
+ die "failed to start. Check logs in " `dirname $ERROR_LOG`
+ fi
+}
+
+# Stop a running server
+function stop_templeton() {
+ local pid
+ if [[ -f $PID_FILE ]]; then
+ # Check if there is a server running
+ local check=`cat $PID_FILE`
+ if check_pid $check; then
+ pid=$check
+ fi
+ fi
+
+ if [[ -z "$pid" ]]; then
+ log "no running server found"
+ else
+ log "stopping ..."
+ kill $pid
+ sleep $SLEEP_TIME_AFTER_START
+ if check_pid $pid; then
+ die "failed to stop"
+ else
+ log "stopping ... stopped"
+ fi
+ fi
+}
+
+#
+# Build command line and run
+#
+
+this=`real_script_name "${BASH_SOURCE-$0}"`
+this_bin=`dirname $this`
+base_dir="$this_bin/.."
+
+if [[ -f "$base_dir/libexec/templeton_config.sh" ]]; then
+ . "$base_dir/libexec/templeton_config.sh"
+else
+ . "$this_bin/templeton_config.sh"
+fi
+
+JAR=`find_jar_path`
+if [[ -z "$JAR" ]]; then
+ die "No templeton jar found"
+fi
+
+CLASSPATH=`find_classpath`
+if [[ -z "$CLASSPATH" ]]; then
+ die "No classpath or jars found"
+fi
+CLASSPATH="$JAR:$CLASSPATH"
+
+if [[ -z "$HADOOP_CLASSPATH" ]]; then
+ export HADOOP_CLASSPATH="$CLASSPATH"
+else
+ export HADOOP_CLASSPATH="$CLASSPATH:$HADOOP_CLASSPATH"
+fi
+
+if [[ -z "$TEMPLETON_LOG4J" ]]; then
+ if [[ -f "$base_dir/conf/templeton-log4j.properties" ]]; then
+ TEMPLETON_LOG4J="$base_dir/conf/templeton-log4j.properties";
+ elif [[ -f "$base_dir/conf/templeton-log4j.properties" ]]; then
+ TEMPLETON_LOG4J="$base_dir/conf/templeton-log4j.properties";
+ else
+ TEMPLETON_LOG4J="templeton-log4j.properties";
+ fi
+fi
+
+export HADOOP_USER_CLASSPATH_FIRST=true
+export HADOOP_OPTS="-Dtempleton.log.dir=$TEMPLETON_LOG_DIR
-Dlog4j.configuration=$TEMPLETON_LOG4J"
+
+start_cmd="$HADOOP_PREFIX/bin/hadoop jar $JAR
org.apache.hcatalog.templeton.Main "
+
+
+cmd=$1
+case $cmd in
+ start)
+ start_templeton
+ ;;
+ stop)
+ stop_templeton
+ ;;
+ foreground)
+ foreground_templeton
+ ;;
+ *)
+ usage
+ ;;
+esac
+
+log "done"
Propchange:
incubator/hcatalog/trunk/webhcat/svr/src/main/bin/templeton_server.sh
------------------------------------------------------------------------------
svn:executable = *
Added:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java?rev=1365722&view=auto
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java
(added)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java
Wed Jul 25 20:29:44 2012
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/*
+ * Communicate with the JobTracker as a specific user.
+ */
+public class TempletonJobTracker {
+ private JobSubmissionProtocol cnx;
+
+ /**
+ * Create a connection to the Job Tracker.
+ */
+ public TempletonJobTracker(UserGroupInformation ugi,
+ InetSocketAddress addr,
+ Configuration conf)
+ throws IOException
+ {
+ cnx = (JobSubmissionProtocol)
+ RPC.getProxy(JobSubmissionProtocol.class,
+ JobSubmissionProtocol.versionID,
+ addr,
+ ugi,
+ conf,
+ NetUtils.getSocketFactory(conf,
+
JobSubmissionProtocol.class));
+ }
+
+ /**
+ * Grab a handle to a job that is already known to the JobTracker.
+ *
+ * @return Profile of the job, or null if not found.
+ */
+ public JobProfile getJobProfile(JobID jobid)
+ throws IOException
+ {
+ return cnx.getJobProfile(jobid);
+ }
+
+ /**
+ * Grab a handle to a job that is already known to the JobTracker.
+ *
+ * @return Status of the job, or null if not found.
+ */
+ public JobStatus getJobStatus(JobID jobid)
+ throws IOException
+ {
+ return cnx.getJobStatus(jobid);
+ }
+
+
+ /**
+ * Kill a job.
+ */
+ public void killJob(JobID jobid)
+ throws IOException
+ {
+ cnx.killJob(jobid);
+ }
+
+ /**
+ * Get all the jobs submitted.
+ */
+ public JobStatus[] getAllJobs()
+ throws IOException
+ {
+ return cnx.getAllJobs();
+ }
+
+ /**
+ * Close the connection to the Job Tracker.
+ */
+ public void close() {
+ RPC.stopProxy(cnx);
+ }
+}
Added:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java?rev=1365722&view=auto
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
(added)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
Wed Jul 25 20:29:44 2012
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.io.File;
+import java.net.URL;
+import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hcatalog.templeton.tool.JobState;
+import org.apache.hcatalog.templeton.tool.ZooKeeperCleanup;
+import org.apache.hcatalog.templeton.tool.ZooKeeperStorage;
+
+/**
+ * The configuration for Templeton. This merges the normal Hadoop
+ * configuration with the Templeton specific variables.
+ *
+ * The Templeton configuration variables are described in
+ * templeton-default.xml
+ *
+ * The Templeton specific configuration is split into two layers
+ *
+ * 1. templeton-default.xml - All the configuration variables that
+ * Templeton needs. These are the defaults that ship with the app
+ * and should only be changed be the app developers.
+ *
+ * 2. templeton-site.xml - The (possibly empty) configuration that the
+ * system administrator can set variables for their Hadoop cluster.
+ *
+ * The configuration files are loaded in this order with later files
+ * overriding earlier ones.
+ *
+ * To find the configuration files, we first attempt to load a file
+ * from the CLASSPATH and then look in the directory specified in the
+ * TEMPLETON_HOME environment variable.
+ *
+ * In addition the configuration files may access the special env
+ * variable env for all environment variables. For example, the
+ * hadoop executable could be specified using:
+ *<pre>
+ * ${env.HADOOP_PREFIX}/bin/hadoop
+ *</pre>
+ */
+public class AppConfig extends Configuration {
+ public static final String[] HADOOP_CONF_FILENAMES = {
+ "core-default.xml", "core-site.xml", "mapred-default.xml",
"mapred-site.xml"
+ };
+
+ public static final String[] HADOOP_PREFIX_VARS = {
+ "HADOOP_PREFIX", "HADOOP_HOME"
+ };
+
+ public static final String TEMPLETON_HOME_VAR = "TEMPLETON_HOME";
+
+ public static final String[] TEMPLETON_CONF_FILENAMES = {
+ "templeton-default.xml",
+ "templeton-site.xml"
+ };
+
+ public static final String PORT = "templeton.port";
+ public static final String EXEC_ENCODING_NAME = "templeton.exec.encoding";
+ public static final String EXEC_ENVS_NAME = "templeton.exec.envs";
+ public static final String EXEC_MAX_BYTES_NAME =
"templeton.exec.max-output-bytes";
+ public static final String EXEC_MAX_PROCS_NAME =
"templeton.exec.max-procs";
+ public static final String EXEC_TIMEOUT_NAME = "templeton.exec.timeout";
+ public static final String HADOOP_NAME = "templeton.hadoop";
+ public static final String HADOOP_CONF_DIR =
"templeton.hadoop.conf.dir";
+ public static final String HCAT_NAME = "templeton.hcat";
+ public static final String HIVE_ARCHIVE_NAME = "templeton.hive.archive";
+ public static final String HIVE_PATH_NAME = "templeton.hive.path";
+ public static final String HIVE_PROPS_NAME =
"templeton.hive.properties";
+ public static final String LIB_JARS_NAME = "templeton.libjars";
+ public static final String PIG_ARCHIVE_NAME = "templeton.pig.archive";
+ public static final String PIG_PATH_NAME = "templeton.pig.path";
+ public static final String STREAMING_JAR_NAME = "templeton.streaming.jar";
+ public static final String TEMPLETON_JAR_NAME = "templeton.jar";
+ public static final String OVERRIDE_JARS_NAME = "templeton.override.jars";
+ public static final String OVERRIDE_JARS_ENABLED =
"templeton.override.enabled";
+ public static final String KERBEROS_SECRET =
"templeton.kerberos.secret";
+ public static final String KERBEROS_PRINCIPAL =
"templeton.kerberos.principal";
+ public static final String KERBEROS_KEYTAB =
"templeton.kerberos.keytab";
+
+ public static final String CALLBACK_INTERVAL_NAME
+ = "templeton.callback.retry.interval";
+ public static final String CALLBACK_RETRY_NAME
+ = "templeton.callback.retry.attempts";
+ public static final String HADOOP_END_INTERVAL_NAME =
"job.end.retry.interval";
+ public static final String HADOOP_END_RETRY_NAME =
"job.end.retry.attempts";
+ public static final String HADOOP_END_URL_NAME =
"job.end.notification.url";
+ public static final String HADOOP_SPECULATIVE_NAME
+ = "mapred.map.tasks.speculative.execution";
+
+ private static final Log LOG = LogFactory.getLog(AppConfig.class);
+
+ public AppConfig() {
+ init();
+ LOG.info("Using Hadoop version " + VersionInfo.getVersion());
+ }
+
+ private void init() {
+ for (Map.Entry<String, String> e : System.getenv().entrySet())
+ set("env." + e.getKey(), e.getValue());
+
+ String templetonDir = getTempletonDir();
+ for (String fname : TEMPLETON_CONF_FILENAMES)
+ if (! loadOneClasspathConfig(fname))
+ loadOneFileConfig(templetonDir, fname);
+
+ String hadoopConfDir = getHadoopConfDir();
+ for (String fname : HADOOP_CONF_FILENAMES)
+ loadOneFileConfig(hadoopConfDir, fname);
+ }
+
+ public void startCleanup() {
+ JobState.getStorageInstance(this).startCleanup(this);
+ }
+
+ public String getHadoopConfDir() {
+ return get(HADOOP_CONF_DIR);
+ }
+
+ public static String getTempletonDir() {
+ return System.getenv(TEMPLETON_HOME_VAR);
+ }
+
+ private boolean loadOneFileConfig(String dir, String fname) {
+ if (dir != null) {
+ File f = new File(dir, fname);
+ if (f.exists()) {
+ addResource(new Path(f.getAbsolutePath()));
+ LOG.debug("loaded config file " + f.getAbsolutePath());
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean loadOneClasspathConfig(String fname) {
+ URL x = getResource(fname);
+ if (x != null) {
+ addResource(x);
+ LOG.debug("loaded config from classpath " + x);
+ return true;
+ }
+
+ return false;
+ }
+
+ public String templetonJar() { return get(TEMPLETON_JAR_NAME); }
+ public String libJars() { return get(LIB_JARS_NAME); }
+ public String clusterHadoop() { return get(HADOOP_NAME); }
+ public String clusterHcat() { return get(HCAT_NAME); }
+ public String pigPath() { return get(PIG_PATH_NAME); }
+ public String pigArchive() { return get(PIG_ARCHIVE_NAME); }
+ public String hivePath() { return get(HIVE_PATH_NAME); }
+ public String hiveArchive() { return get(HIVE_ARCHIVE_NAME); }
+ public String streamingJar() { return get(STREAMING_JAR_NAME); }
+ public String kerberosSecret() { return get(KERBEROS_SECRET); }
+ public String kerberosPrincipal(){ return get(KERBEROS_PRINCIPAL); }
+ public String kerberosKeytab() { return get(KERBEROS_KEYTAB); }
+
+ public String[] overrideJars() {
+ if (getBoolean(OVERRIDE_JARS_ENABLED, true))
+ return getStrings(OVERRIDE_JARS_NAME);
+ else
+ return null;
+ }
+ public String overrideJarsString() {
+ if (getBoolean(OVERRIDE_JARS_ENABLED, true))
+ return get(OVERRIDE_JARS_NAME);
+ else
+ return null;
+ }
+
+ public long zkCleanupInterval() {
+ return getLong(ZooKeeperCleanup.ZK_CLEANUP_INTERVAL,
+ (1000L * 60L * 60L * 12L)); }
+ public long zkMaxAge() {
+ return getLong(ZooKeeperCleanup.ZK_CLEANUP_MAX_AGE,
+ (1000L * 60L * 60L * 24L * 7L)); }
+ public String zkHosts() { return get(ZooKeeperStorage.ZK_HOSTS); }
+ public int zkSessionTimeout() { return
getInt(ZooKeeperStorage.ZK_SESSION_TIMEOUT,
+ 30000); }
+}
Added:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java?rev=1365722&view=auto
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java
(added)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java
Wed Jul 25 20:29:44 2012
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+/**
+ * Missing required or badly configured paramater.
+ */
+public class BadParam extends SimpleWebException {
+ public BadParam(String msg) {
+ super(400, msg);
+ }
+}
Added:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java?rev=1365722&view=auto
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java
(added)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java
Wed Jul 25 20:29:44 2012
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+/**
+ * Simple "we are busy, try again" exception.
+ */
+public class BusyException extends SimpleWebException {
+ public BusyException() {
+ super(503, "Busy, please retry");
+ }
+}
Added:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java?rev=1365722&view=auto
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java
(added)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java
Wed Jul 25 20:29:44 2012
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+/**
+ * The callback failed when it tried to reach the callback URL.
+ */
+public class CallbackFailedException extends SimpleWebException {
+ public CallbackFailedException(String msg) {
+ super(400, msg);
+ }
+}
Added:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java?rev=1365722&view=auto
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java
(added)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java
Wed Jul 25 20:29:44 2012
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Map all exceptions to the Jersey response. This lets us have nice
+ * results in the error body.
+ */
+@Provider
+public class CatchallExceptionMapper
+ implements ExceptionMapper<Exception>
+{
+ private static final Log LOG =
LogFactory.getLog(CatchallExceptionMapper.class);
+
+ public Response toResponse(Exception e) {
+ LOG.error(e.getMessage(), e);
+ return SimpleWebException.buildMessage(500, null, e.getMessage());
+ }
+}
Added:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ColumnDesc.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ColumnDesc.java?rev=1365722&view=auto
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ColumnDesc.java
(added)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ColumnDesc.java
Wed Jul 25 20:29:44 2012
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * A description of the column to create.
+ */
+@XmlRootElement
+public class ColumnDesc extends GroupPermissionsDesc {
+ public String name;
+ public String type;
+ public String comment;
+
+ public ColumnDesc() {}
+
+ /**
+ * Create a new ColumnDesc
+ */
+ public ColumnDesc(String name, String type, String comment) {
+ this.name = name;
+ this.type = type;
+ this.comment = comment;
+ }
+
+ public String toString() {
+ return String.format("ColumnDesc(name=%s, type=%s, comment=%s)",
+ name, type, comment);
+ }
+
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (! (o instanceof ColumnDesc))
+ return false;
+ ColumnDesc that = (ColumnDesc) o;
+ return xequals(this.name, that.name)
+ && xequals(this.type, that.type)
+ && xequals(this.comment, that.comment)
+ && super.equals(that)
+ ;
+ }
+}
Added:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteBean.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteBean.java?rev=1365722&view=auto
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteBean.java
(added)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteBean.java
Wed Jul 25 20:29:44 2012
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+/**
+ * CompleteBean - The results of an CompleteDelegator run.
+ */
+public class CompleteBean {
+ public String status;
+
+ public CompleteBean() {}
+
+ /**
+ * Create a new CompleteBean
+ *
+ * @param status run status
+ */
+ public CompleteBean(String status) {
+ this.status = status;
+ }
+}
Added:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteDelegator.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteDelegator.java?rev=1365722&view=auto
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteDelegator.java
(added)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteDelegator.java
Wed Jul 25 20:29:44 2012
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.io.IOException;
+import java.net.URL;
+import java.net.MalformedURLException;
+import java.util.Date;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.JobID;
+import org.apache.hadoop.mapred.JobProfile;
+import org.apache.hadoop.mapred.JobStatus;
+import org.apache.hadoop.mapred.JobTracker;
+import org.apache.hadoop.mapred.TempletonJobTracker;
+import org.apache.hcatalog.templeton.tool.JobState;
+import org.apache.hcatalog.templeton.tool.TempletonUtils;
+
+/**
+ * Complete a job. This will run the callback if
+ *
+ * - the job is done
+ * - there is a callback
+ * - the callback has not yet been called
+ *
+ * There is a small chance for a race condition if two callers run
+ * this at the same time. That should never happen.
+ *
+ * We use a Hadoop config var to notify this class on the completion
+ * of a job. Hadoop will call use multiple times in the event of
+ * failure. Even if the failure is that the client callback failed.
+ *
+ * See LauncherDelegator for the HADOOP_END_RETRY* vars that are set.
+ */
+public class CompleteDelegator extends TempletonDelegator {
+ private static final Log LOG = LogFactory.getLog(CompleteDelegator.class);
+
+ public CompleteDelegator(AppConfig appConf) {
+ super(appConf);
+ }
+
+ public CompleteBean run(String id)
+ throws CallbackFailedException, IOException
+ {
+ if (id == null)
+ acceptWithError("No jobid given");
+
+ JobState state = null;
+ try {
+ state = new JobState(id, Main.getAppConfigInstance());
+ if (state.getCompleteStatus() == null)
+ failed("Job not yet complete", null);
+
+ Long notified = state.getNotifiedTime();
+ if (notified != null)
+ return acceptWithError("Callback already run on "
+ + new Date(notified.longValue()));
+
+ String callback = state.getCallback();
+ if (callback == null)
+ return new CompleteBean("No callback registered");
+
+ try {
+ doCallback(state.getId(), callback);
+ } catch (Exception e) {
+ failed("Callback failed " + callback + " for " + id, e);
+ }
+
+ state.setNotifiedTime(System.currentTimeMillis());
+ return new CompleteBean("Callback sent");
+ } finally {
+ if (state != null)
+ state.close();
+ }
+ }
+
+ /**
+ * Call the callback url with the jobid to let them know it's
+ * finished. If the url has the string $jobId in it, it will be
+ * replaced with the completed jobid.
+ */
+ public static void doCallback(String jobid, String url)
+ throws MalformedURLException, IOException
+ {
+ if (url.contains("$jobId"))
+ url = url.replace("$jobId", jobid);
+ TempletonUtils.fetchUrl(new URL(url));
+ }
+
+ private void failed(String msg, Exception e)
+ throws CallbackFailedException
+ {
+ if (e != null)
+ LOG.error(msg, e);
+ else
+ LOG.error(msg);
+ throw new CallbackFailedException(msg);
+ }
+
+ private CompleteBean acceptWithError(String msg) {
+ LOG.error(msg);
+ return new CompleteBean(msg);
+ }
+}
Added:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DatabaseDesc.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DatabaseDesc.java?rev=1365722&view=auto
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DatabaseDesc.java
(added)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DatabaseDesc.java
Wed Jul 25 20:29:44 2012
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.util.Map;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * A description of the database to create.
+ */
+@XmlRootElement
+public class DatabaseDesc extends GroupPermissionsDesc {
+ public boolean ifNotExists;
+ public String database;
+ public String comment;
+ public String location;
+ public Map<String, String> properties;
+
+ public DatabaseDesc() {}
+
+ public String toString() {
+ return String.format("DatabaseDesc(database=%s, comment=%s,
location=%s, " +
+ "properties=%s)", database, comment, location,
properties);
+ }
+}
Added:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java?rev=1365722&view=auto
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java
(added)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java
Wed Jul 25 20:29:44 2012
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.io.IOException;
+import org.apache.hadoop.mapred.JobID;
+import org.apache.hadoop.mapred.JobProfile;
+import org.apache.hadoop.mapred.JobStatus;
+import org.apache.hadoop.mapred.JobTracker;
+import org.apache.hadoop.mapred.TempletonJobTracker;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hcatalog.templeton.tool.JobState;
+
+/**
+ * Delete a job
+ */
+public class DeleteDelegator extends TempletonDelegator {
+ public DeleteDelegator(AppConfig appConf) {
+ super(appConf);
+ }
+
+ public QueueStatusBean run(String user, String id)
+ throws NotAuthorizedException, BadParam, IOException
+ {
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
+ TempletonJobTracker tracker = null;
+ JobState state = null;
+ try {
+ tracker = new TempletonJobTracker(ugi,
+ JobTracker.getAddress(appConf),
+ appConf);
+ JobID jobid = StatusDelegator.StringToJobID(id);
+ if (jobid == null)
+ throw new BadParam("Invalid jobid: " + id);
+ tracker.killJob(jobid);
+ state = new JobState(id, Main.getAppConfigInstance());
+ String childid = state.getChildId();
+ if (childid != null)
+ tracker.killJob(StatusDelegator.StringToJobID(childid));
+ return StatusDelegator.makeStatus(tracker, jobid, state);
+ } catch (IllegalStateException e) {
+ throw new BadParam(e.getMessage());
+ } finally {
+ if (tracker != null)
+ tracker.close();
+ if (state != null)
+ state.close();
+ }
+ }
+}
Added:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/EnqueueBean.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/EnqueueBean.java?rev=1365722&view=auto
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/EnqueueBean.java
(added)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/EnqueueBean.java
Wed Jul 25 20:29:44 2012
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+/**
+ * EnqueueBean - The results of a call that enqueues a Hadoop job.
+ */
+public class EnqueueBean {
+ public String id;
+
+ public EnqueueBean() {}
+
+ /**
+ * Create a new EnqueueBean.
+ *
+ * @param id job id
+ */
+ public EnqueueBean(String id) {
+ this.id = id;
+ }
+}
Added:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecBean.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecBean.java?rev=1365722&view=auto
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecBean.java
(added)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecBean.java
Wed Jul 25 20:29:44 2012
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+/**
+ * ExecBean - The results of an exec call.
+ */
+public class ExecBean {
+ public String stdout;
+ public String stderr;
+ public int exitcode;
+
+ public ExecBean() {}
+
+ /**
+ * Create a new ExecBean.
+ *
+ * @param stdout standard output of the the program.
+ * @param stderr error output of the the program.
+ * @param exitcode exit code of the program.
+ */
+ public ExecBean(String stdout, String stderr, int exitcode) {
+ this.stdout = stdout;
+ this.stderr = stderr;
+ this.exitcode = exitcode;
+ }
+
+ public String toString() {
+ return String.format("ExecBean(stdout=%s, stderr=%s, exitcode=%s)",
+ stdout, stderr, exitcode);
+ }
+}
Added:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecService.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecService.java?rev=1365722&view=auto
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecService.java
(added)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecService.java
Wed Jul 25 20:29:44 2012
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.exec.ExecuteException;
+
+public interface ExecService {
+ public ExecBean run(String program, List<String> args,
+ Map<String, String> env)
+ throws NotAuthorizedException, BusyException, ExecuteException,
IOException;
+
+ public ExecBean runUnlimited(String program, List<String> args,
+ Map<String, String> env)
+ throws NotAuthorizedException, ExecuteException, IOException;
+}
Added:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java?rev=1365722&view=auto
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java
(added)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java
Wed Jul 25 20:29:44 2012
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Semaphore;
+import org.apache.commons.exec.CommandLine;
+import org.apache.commons.exec.DefaultExecutor;
+import org.apache.commons.exec.ExecuteException;
+import org.apache.commons.exec.ExecuteWatchdog;
+import org.apache.commons.exec.PumpStreamHandler;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Execute a local program. This is a singleton service that will
+ * execute programs as non-privileged users on the local box. See
+ * ExecService.run and ExecService.runUnlimited for details.
+ */
+public class ExecServiceImpl implements ExecService {
+ private static final Log LOG = LogFactory.getLog(ExecServiceImpl.class);
+ private static AppConfig appConf = Main.getAppConfigInstance();
+
+ private static volatile ExecServiceImpl theSingleton;
+
+ /**
+ * Retrieve the singleton.
+ */
+ public static synchronized ExecServiceImpl getInstance() {
+ if (theSingleton == null) {
+ theSingleton = new ExecServiceImpl();
+ }
+ return theSingleton;
+ }
+
+ private Semaphore avail;
+
+ private ExecServiceImpl() {
+ avail = new Semaphore(appConf.getInt(AppConfig.EXEC_MAX_PROCS_NAME,
16));
+ }
+
+ /**
+ * Run the program synchronously as the given user. We rate limit
+ * the number of processes that can simultaneously created for
+ * this instance.
+ *
+ * @param program The program to run
+ * @param args Arguments to pass to the program
+ * @param env Any extra environment variables to set
+ * @return The result of the run.
+ */
+ public ExecBean run(String program, List<String> args,
+ Map<String, String> env)
+ throws NotAuthorizedException, BusyException, ExecuteException,
IOException
+ {
+ boolean aquired = false;
+ try {
+ aquired = avail.tryAcquire();
+ if (aquired) {
+ return runUnlimited(program, args, env);
+ } else {
+ throw new BusyException();
+ }
+ } finally {
+ if (aquired) {
+ avail.release();
+ }
+ }
+ }
+
+ /**
+ * Run the program synchronously as the given user. Warning:
+ * CommandLine will trim the argument strings.
+ *
+ * @param program The program to run.
+ * @param args Arguments to pass to the program
+ * @param env Any extra environment variables to set
+ * @return The result of the run.
+ */
+ public ExecBean runUnlimited(String program, List<String> args,
+ Map<String, String> env)
+ throws NotAuthorizedException, ExecuteException, IOException
+ {
+ try {
+ return auxRun(program, args, env);
+ } catch (IOException e) {
+ File cwd = new java.io.File(".");
+ if (cwd.canRead() && cwd.canWrite())
+ throw e;
+ else
+ throw new IOException("Invalid permissions on Templeton
directory: "
+ + cwd.getCanonicalPath());
+ }
+ }
+
+ private ExecBean auxRun(String program, List<String> args, Map<String,
String> env)
+ throws NotAuthorizedException, ExecuteException, IOException
+ {
+ DefaultExecutor executor = new DefaultExecutor();
+ executor.setExitValues(null);
+
+ // Setup stdout and stderr
+ int nbytes = appConf.getInt(AppConfig.EXEC_MAX_BYTES_NAME, -1);
+ ByteArrayOutputStream outStream = new MaxByteArrayOutputStream(nbytes);
+ ByteArrayOutputStream errStream = new MaxByteArrayOutputStream(nbytes);
+ executor.setStreamHandler(new PumpStreamHandler(outStream, errStream));
+
+ // Only run for N milliseconds
+ int timeout = appConf.getInt(AppConfig.EXEC_TIMEOUT_NAME, 0);
+ ExecuteWatchdog watchdog = new ExecuteWatchdog(timeout);
+ executor.setWatchdog(watchdog);
+
+ CommandLine cmd = makeCommandLine(program, args);
+
+ LOG.info("Running: " + cmd);
+ ExecBean res = new ExecBean();
+ res.exitcode = executor.execute(cmd, execEnv(env));
+ String enc = appConf.get(AppConfig.EXEC_ENCODING_NAME);
+ res.stdout = outStream.toString(enc);
+ res.stderr = errStream.toString(enc);
+
+ return res;
+ }
+
+ private CommandLine makeCommandLine(String program,
+ List<String> args)
+ throws NotAuthorizedException, IOException
+ {
+ String path = validateProgram(program);
+ CommandLine cmd = new CommandLine(path);
+ if (args != null)
+ for (String arg : args)
+ cmd.addArgument(arg, false);
+
+ return cmd;
+ }
+
+ /**
+ * Build the environment used for all exec calls.
+ *
+ * @return The environment variables.
+ */
+ public Map<String, String> execEnv(Map<String, String> env) {
+ HashMap<String, String> res = new HashMap<String, String>();
+
+ for (String key : appConf.getStrings(AppConfig.EXEC_ENVS_NAME)) {
+ String val = System.getenv(key);
+ if (val != null) {
+ res.put(key, val);
+ }
+ }
+ if (env != null)
+ res.putAll(env);
+ for(Map.Entry<String, String> envs : res.entrySet()){
+ LOG.info("Env " + envs.getKey() + "=" + envs.getValue());
+ }
+ return res;
+ }
+
+ /**
+ * Given a program name, lookup the fully qualified path. Throws
+ * an exception if the program is missing or not authorized.
+ *
+ * @param path The path of the program.
+ * @return The path of the validated program.
+ */
+ public String validateProgram(String path)
+ throws NotAuthorizedException, IOException
+ {
+ File f = new File(path);
+ if (f.canExecute()) {
+ return f.getCanonicalPath();
+ } else {
+ throw new NotAuthorizedException("Unable to access program: " +
path);
+ }
+ }
+}
Added:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/GroupPermissionsDesc.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/GroupPermissionsDesc.java?rev=1365722&view=auto
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/GroupPermissionsDesc.java
(added)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/GroupPermissionsDesc.java
Wed Jul 25 20:29:44 2012
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * The base create permissions for ddl objects.
+ */
+public abstract class GroupPermissionsDesc {
+ public String group;
+ public String permissions;
+
+ public GroupPermissionsDesc() {}
+
+ protected static boolean xequals(Object a, Object b) {
+ if (a == null) {
+ if (b == null)
+ return true;
+ else
+ return false;
+ }
+
+ return a.equals(b);
+ }
+
+ protected static boolean xequals(boolean a, boolean b) { return a == b; }
+ protected static boolean xequals(int a, int b) { return a == b; }
+ protected static boolean xequals(char a, char b) { return a == b; }
+
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (! (o instanceof GroupPermissionsDesc))
+ return false;
+ GroupPermissionsDesc that = (GroupPermissionsDesc) o;
+ return xequals(this.group, that.group)
+ && xequals(this.permissions, that.permissions)
+ ;
+ }
+}
