I think we have two
different conversations about the same topic. I reply here.
Thats not an issue of a env script in windows, because we haven't one so
far. You may need to cp (Classpath) the flume - libs as Inder says.
As I wrote in my howto:
As example, starting from inside of your flume-installation (the
directory where you unzip'ed the build), lets assume D:\apache-flume:
d:\apache flume > "PATH\to\java.exe" -Xmx20m
-Dlog4j.configuration=file:///%CD%\conf\log4j.properties
-cp "d:\apache-flume\flume-1.3.0-SNAPSHOT\lib\*"
org.apache.flume.node.Application -f d:\flume\flume-1.3.0-SNAPSHOT\conf\YOUR_CONFIG.conf -n
agent
Please be sure, that the lib-directory from flume is in place as well in
the CLASSPATH (cp).
- Alex
Thanks Inder,
when I am
running the $echo $CLASSPATH, I am just getting . (dot) and nothing
more.
Please see below the flume-ng files. Can you just pointout
where I should put echo$CLASSPATH.
Thanks
CLEAN_FLAG=1
################################
# functions
################################
info()
{
if [ ${CLEAN_FLAG} -ne 0 ]; then
local msg=$1
echo
"Info: $msg" >&2
fi
}
warn() {
if [ ${CLEAN_FLAG} -ne 0 ]; then
local msg=$1
echo "Warning: $msg" >&2
fi
}
error()
{
local msg=$1
local exit_code=$2
echo "Error: $msg"
>&2
if [ -n "$exit_code" ] ; then
exit $exit_code
fi
}
#
If avail, add Hadoop paths to the FLUME_CLASSPATH and to the
#
FLUME_JAVA_LIBRARY_PATH env vars.
# Requires Flume jars to already be
on FLUME_CLASSPATH.
add_hadoop_paths() {
local
HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" \
which hadoop 2>/dev/null)
if [ -f "${HADOOP_IN_PATH}" ];
then
info "Including Hadoop libraries found via ($HADOOP_IN_PATH)
for HDFS access"
# determine hadoop java.library.path and use that for flume
local HADOOP_CLASSPATH=""
local
HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$FLUME_CLASSPATH" \
${HADOOP_IN_PATH} org.apache.flume.tools.GetJavaProperty \
java.library.path 2>/dev/null)
# look for the line
that has the desired property value
# (considering extraneous
output from some GC options that write to stdout)
# IFS =
InternalFieldSeparator (set to recognize only newline char as delimiter)
IFS=$'\n'
for line in $HADOOP_JAVA_LIBRARY_PATH; do
if [[ $line =~ ^java\.library\.path=(.*)$ ]]; then
HADOOP_JAVA_LIBRARY_PATH=${BASH_REMATCH[1]}
break
fi
done
unset IFS
if [ -n "${HADOOP_JAVA_LIBRARY_PATH}" ]; then
FLUME_JAVA_LIBRARY_PATH="$FLUME_JAVA_LIBRARY_PATH:$HADOOP_JAVA_LIBRARY_PATH"
fi
# determine hadoop classpath
HADOOP_CLASSPATH=$($HADOOP_IN_PATH classpath 2>/dev/null)
# hack up and filter hadoop classpath
local ELEMENTS=$(sed -e
's/:/ /g' <<<${HADOOP_CLASSPATH})
local ELEMENT
for ELEMENT in $ELEMENTS; do
local PIECE
for PIECE in $(echo $ELEMENT); do
if [[ $PIECE =~ slf4j-(api|log4j12).*\.jar ]]; then
info
"Excluding $PIECE from classpath"
continue
else
FLUME_CLASSPATH="$FLUME_CLASSPATH:$PIECE"
fi
done
done
fi
}
add_HBASE_paths() {
local
HBASE_IN_PATH=$(PATH="${HBASE_HOME}/bin:$PATH" \
which hbase
2>/dev/null)
if [ -f "${HBASE_IN_PATH}" ]; then
info "Including HBASE
libraries found via ($HBASE_IN_PATH) for HBASE access"
#
determine HBASE java.library.path and use that for flume
local
HBASE_CLASSPATH=""
local HBASE_JAVA_LIBRARY_PATH=$(HBASE_CLASSPATH="$FLUME_CLASSPATH" \
${HBASE_IN_PATH} org.apache.flume.tools.GetJavaProperty \
java.library.path 2>/dev/null)
# look for the line that
has the desired property value
# (considering extraneous output from some GC options that write to
stdout)
# IFS = InternalFieldSeparator (set to recognize only
newline char as delimiter)
IFS=$'\n'
for line in
$HBASE_JAVA_LIBRARY_PATH; do
if [[ $line =~ ^java\.library\.path=(.*)$ ]]; then
HBASE_JAVA_LIBRARY_PATH=${BASH_REMATCH[1]}
break
fi
done
unset IFS
if [ -n "${HBASE_JAVA_LIBRARY_PATH}"
]; then
FLUME_JAVA_LIBRARY_PATH="$FLUME_JAVA_LIBRARY_PATH:$HBASE_JAVA_LIBRARY_PATH"
fi
# determine HBASE classpath
HBASE_CLASSPATH=$($HBASE_IN_PATH classpath 2>/dev/null)
#
hack up and filter HBASE classpath
local ELEMENTS=$(sed -e 's/:/ /g' <<<${HBASE_CLASSPATH})
local ELEMENT
for ELEMENT in $ELEMENTS; do
local PIECE
for PIECE in $(echo $ELEMENT); do
if [[ $PIECE =~
slf4j-(api|log4j12).*\.jar ]]; then
info "Excluding $PIECE from classpath"
continue
else
FLUME_CLASSPATH="$FLUME_CLASSPATH:$PIECE"
fi
done
done
FLUME_CLASSPATH="$FLUME_CLASSPATH:$HBASE_HOME/conf"
fi
}
display_help() {
cat <<EOF
Usage: $0
<command> [options]...
commands:
help
display this help text
agent run a Flume agent
avro-client run an avro Flume client
version show Flume version info
global options:
--conf,-c <conf> use configs in <conf> directory
--classpath,-C <cp> append to the classpath
--dryrun,-d do not actually start Flume, just print the
command
-Dproperty=value sets a JDK system property value
agent
options:
--conf-file,-f <file> specify a config file
(required)
--name,-n <name> the name of this agent
(required)
--help,-h display help text
avro-client options:
--host,-H <host> hostname to
which events will be sent (required)
--port,-p <port>
port of the avro source (required)
--filename,-F <file> text
file to stream to avro source [default: std input]
--headerFile,-R <file> headerFile containing headers as
key/value pairs on each new line
--help,-h display help
text
Note that if <conf> directory is specified, then it
is always included first
in the classpath.
EOF
}
run_flume() {
local
FLUME_APPLICATION_CLASS
if [ "$#" -gt 0 ]; then
FLUME_APPLICATION_CLASS=$1
shift
else
error "Must
specify flume application class" 1
fi
if [ ${CLEAN_FLAG} -ne 0 ]; then
set -x
fi
$EXEC $JAVA_HOME/bin/java $JAVA_OPTS -cp "$FLUME_CLASSPATH" \
-Djava.library.path=$FLUME_JAVA_LIBRARY_PATH "$FLUME_APPLICATION_CLASS"
$*
}
################################
# main
################################
#
set default params
FLUME_CLASSPATH=""
FLUME_JAVA_LIBRARY_PATH=""
JAVA_OPTS="-Xmx20m"
opt_conf=""
opt_classpath=""
opt_java_props=""
opt_dryrun=""
mode=$1
shift
case
"$mode" in
help)
display_help
exit 0
;;
agent)
opt_agent=1
;;
node)
opt_agent=1
warn "The \"node\" command is deprecated. Please use \"agent\"
instead."
;;
avro-client)
opt_avro_client=1
;;
version)
opt_version=1
CLEAN_FLAG=0
;;
*)
error "Unknown or unspecified command '$mode'"
echo
display_help
exit 1
;;
esac
while [ -n "$*" ] ;
do
arg=$1
shift
case "$arg" in
--conf|-c)
[
-n "$1" ] || error "Option --conf requires an argument" 1
opt_conf=$1
shift
;;
--classpath|-C)
[ -n "$1" ] || error "Option --classpath requires an argument" 1
opt_classpath=$1
shift
;;
--dryrun|-d)
opt_dryrun="1"
;;
-D*)
opt_java_props="$opt_java_props $arg"
;;
*)
args="$args $arg"
;;
esac
done
# make opt_conf
absolute
if [[ -n "$opt_conf" && -d "$opt_conf" ]]; then
opt_conf=$(cd $opt_conf; pwd)
fi
# allow users to override
the default env vars via conf/flume-env.sh
if [ -z "$opt_conf" ];
then
warn "No configuration directory set! Use --conf <dir>
to override."
elif [ -f "$opt_conf/flume-env.sh" ]; then
info "Sourcing
environment configuration script $opt_conf/flume-env.sh"
source
"$opt_conf/flume-env.sh"
fi
# append command-line java options
to stock or env script JAVA_OPTS
if [ -n "${opt_java_props}" ]; then
JAVA_OPTS="${JAVA_OPTS}
${opt_java_props}"
fi
# prepend command-line classpath to env
script classpath
if [ -n "${opt_classpath}" ]; then
if [ -n "${FLUME_CLASSPATH}" ]; then
FLUME_CLASSPATH="${opt_classpath}:${FLUME_CLASSPATH}"
else
FLUME_CLASSPATH="${opt_classpath}"
fi
fi
if [ -z
"${FLUME_HOME}" ]; then
FLUME_HOME=$(cd $(dirname $0)/..; pwd)
fi
# prepend
$FLUME_HOME/lib jars to the specified classpath (if any)
if [ -n
"${FLUME_CLASSPATH}" ] ; then
FLUME_CLASSPATH="${FLUME_HOME}/lib/*:$FLUME_CLASSPATH"
else
FLUME_CLASSPATH="${FLUME_HOME}/lib/*"
fi
# find java
if
[ -z "${JAVA_HOME}" ] ; then
warn "JAVA_HOME is not set!"
#
Try to use Bigtop to autodetect JAVA_HOME if it's available
if [ -e /usr/libexec/bigtop-detect-javahome ] ; then
.
/usr/libexec/bigtop-detect-javahome
elif [ -e
/usr/lib/bigtop-utils/bigtop-detect-javahome ] ; then
.
/usr/lib/bigtop-utils/bigtop-detect-javahome
fi
# Using java from path if bigtop is not installed or
couldn't find it
if [ -z "${JAVA_HOME}" ] ; then
JAVA_DEFAULT=$(type -p java)
[ -n "$JAVA_DEFAULT" ] || error
"Unable to find java executable. Is it in your PATH?" 1
JAVA_HOME=$(cd $(dirname $JAVA_DEFAULT)/..; pwd)
fi
fi
#
look for hadoop libs
add_hadoop_paths
add_HBASE_paths
#
prepend conf dir to classpath
if [ -n "$opt_conf" ]; then
FLUME_CLASSPATH="$opt_conf:$FLUME_CLASSPATH"
fi
# allow dryrun
EXEC="exec"
if [ -n "${opt_dryrun}" ];
then
warn "Dryrun mode enabled (will not actually initiate
startup)"
EXEC="echo"
fi
# finally, invoke the
appropriate command
if [ -n "$opt_agent" ] ; then
run_flume $FLUME_AGENT_CLASS $args
elif
[ -n "$opt_avro_client" ] ; then
run_flume
$FLUME_AVRO_CLIENT_CLASS $args
elif [ -n "${opt_version}" ] ; then
run_flume $FLUME_VERSION_CLASS $args
else
error "This message
should never appear" 1
fi
exit 0
