Thanks Inder,
when I am running the $echo $CLASSPATH, I am just getting . (dot) and
nothing more.
Please see below the flume-ng files. Can you just pointout where I should
put echo$CLASSPATH.
Thanks
CLEAN_FLAG=1
################################
# functions
################################
info() {
if [ ${CLEAN_FLAG} -ne 0 ]; then
local msg=$1
echo "Info: $msg" >&2
fi
}
warn() {
if [ ${CLEAN_FLAG} -ne 0 ]; then
local msg=$1
echo "Warning: $msg" >&2
fi
}
error() {
local msg=$1
local exit_code=$2
echo "Error: $msg" >&2
if [ -n "$exit_code" ] ; then
exit $exit_code
fi
}
# If avail, add Hadoop paths to the FLUME_CLASSPATH and to the
# FLUME_JAVA_LIBRARY_PATH env vars.
# Requires Flume jars to already be on FLUME_CLASSPATH.
add_hadoop_paths() {
local HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" \
which hadoop 2>/dev/null)
if [ -f "${HADOOP_IN_PATH}" ]; then
info "Including Hadoop libraries found via ($HADOOP_IN_PATH) for HDFS
access"
# determine hadoop java.library.path and use that for flume
local HADOOP_CLASSPATH=""
local HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$FLUME_CLASSPATH" \
${HADOOP_IN_PATH} org.apache.flume.tools.GetJavaProperty \
java.library.path 2>/dev/null)
# look for the line that has the desired property value
# (considering extraneous output from some GC options that write to
stdout)
# IFS = InternalFieldSeparator (set to recognize only newline char as
delimiter)
IFS=$'\n'
for line in $HADOOP_JAVA_LIBRARY_PATH; do
if [[ $line =~ ^java\.library\.path=(.*)$ ]]; then
HADOOP_JAVA_LIBRARY_PATH=${BASH_REMATCH[1]}
break
fi
done
unset IFS
if [ -n "${HADOOP_JAVA_LIBRARY_PATH}" ]; then
FLUME_JAVA_LIBRARY_PATH="$FLUME_JAVA_LIBRARY_PATH:$HADOOP_JAVA_LIBRARY_PATH"
fi
# determine hadoop classpath
HADOOP_CLASSPATH=$($HADOOP_IN_PATH classpath 2>/dev/null)
# hack up and filter hadoop classpath
local ELEMENTS=$(sed -e 's/:/ /g' <<<${HADOOP_CLASSPATH})
local ELEMENT
for ELEMENT in $ELEMENTS; do
local PIECE
for PIECE in $(echo $ELEMENT); do
if [[ $PIECE =~ slf4j-(api|log4j12).*\.jar ]]; then
info "Excluding $PIECE from classpath"
continue
else
FLUME_CLASSPATH="$FLUME_CLASSPATH:$PIECE"
fi
done
done
fi
}
add_HBASE_paths() {
local HBASE_IN_PATH=$(PATH="${HBASE_HOME}/bin:$PATH" \
which hbase 2>/dev/null)
if [ -f "${HBASE_IN_PATH}" ]; then
info "Including HBASE libraries found via ($HBASE_IN_PATH) for HBASE
access"
# determine HBASE java.library.path and use that for flume
local HBASE_CLASSPATH=""
local HBASE_JAVA_LIBRARY_PATH=$(HBASE_CLASSPATH="$FLUME_CLASSPATH" \
${HBASE_IN_PATH} org.apache.flume.tools.GetJavaProperty \
java.library.path 2>/dev/null)
# look for the line that has the desired property value
# (considering extraneous output from some GC options that write to
stdout)
# IFS = InternalFieldSeparator (set to recognize only newline char as
delimiter)
IFS=$'\n'
for line in $HBASE_JAVA_LIBRARY_PATH; do
if [[ $line =~ ^java\.library\.path=(.*)$ ]]; then
HBASE_JAVA_LIBRARY_PATH=${BASH_REMATCH[1]}
break
fi
done
unset IFS
if [ -n "${HBASE_JAVA_LIBRARY_PATH}" ]; then
FLUME_JAVA_LIBRARY_PATH="$FLUME_JAVA_LIBRARY_PATH:$HBASE_JAVA_LIBRARY_PATH"
fi
# determine HBASE classpath
HBASE_CLASSPATH=$($HBASE_IN_PATH classpath 2>/dev/null)
# hack up and filter HBASE classpath
local ELEMENTS=$(sed -e 's/:/ /g' <<<${HBASE_CLASSPATH})
local ELEMENT
for ELEMENT in $ELEMENTS; do
local PIECE
for PIECE in $(echo $ELEMENT); do
if [[ $PIECE =~ slf4j-(api|log4j12).*\.jar ]]; then
info "Excluding $PIECE from classpath"
continue
else
FLUME_CLASSPATH="$FLUME_CLASSPATH:$PIECE"
fi
done
done
FLUME_CLASSPATH="$FLUME_CLASSPATH:$HBASE_HOME/conf"
fi
}
display_help() {
cat <<EOF
Usage: $0 <command> [options]...
commands:
help display this help text
agent run a Flume agent
avro-client run an avro Flume client
version show Flume version info
global options:
--conf,-c <conf> use configs in <conf> directory
--classpath,-C <cp> append to the classpath
--dryrun,-d do not actually start Flume, just print the command
-Dproperty=value sets a JDK system property value
agent options:
--conf-file,-f <file> specify a config file (required)
--name,-n <name> the name of this agent (required)
--help,-h display help text
avro-client options:
--host,-H <host> hostname to which events will be sent (required)
--port,-p <port> port of the avro source (required)
--filename,-F <file> text file to stream to avro source [default: std
input]
--headerFile,-R <file> headerFile containing headers as key/value pairs
on each new line
--help,-h display help text
Note that if <conf> directory is specified, then it is always included first
in the classpath.
EOF
}
run_flume() {
local FLUME_APPLICATION_CLASS
if [ "$#" -gt 0 ]; then
FLUME_APPLICATION_CLASS=$1
shift
else
error "Must specify flume application class" 1
fi
if [ ${CLEAN_FLAG} -ne 0 ]; then
set -x
fi
$EXEC $JAVA_HOME/bin/java $JAVA_OPTS -cp "$FLUME_CLASSPATH" \
-Djava.library.path=$FLUME_JAVA_LIBRARY_PATH
"$FLUME_APPLICATION_CLASS" $*
}
################################
# main
################################
# set default params
FLUME_CLASSPATH=""
FLUME_JAVA_LIBRARY_PATH=""
JAVA_OPTS="-Xmx20m"
opt_conf=""
opt_classpath=""
opt_java_props=""
opt_dryrun=""
mode=$1
shift
case "$mode" in
help)
display_help
exit 0
;;
agent)
opt_agent=1
;;
node)
opt_agent=1
warn "The \"node\" command is deprecated. Please use \"agent\" instead."
;;
avro-client)
opt_avro_client=1
;;
version)
opt_version=1
CLEAN_FLAG=0
;;
*)
error "Unknown or unspecified command '$mode'"
echo
display_help
exit 1
;;
esac
while [ -n "$*" ] ; do
arg=$1
shift
case "$arg" in
--conf|-c)
[ -n "$1" ] || error "Option --conf requires an argument" 1
opt_conf=$1
shift
;;
--classpath|-C)
[ -n "$1" ] || error "Option --classpath requires an argument" 1
opt_classpath=$1
shift
;;
--dryrun|-d)
opt_dryrun="1"
;;
-D*)
opt_java_props="$opt_java_props $arg"
;;
*)
args="$args $arg"
;;
esac
done
# make opt_conf absolute
if [[ -n "$opt_conf" && -d "$opt_conf" ]]; then
opt_conf=$(cd $opt_conf; pwd)
fi
# allow users to override the default env vars via conf/flume-env.sh
if [ -z "$opt_conf" ]; then
warn "No configuration directory set! Use --conf <dir> to override."
elif [ -f "$opt_conf/flume-env.sh" ]; then
info "Sourcing environment configuration script $opt_conf/flume-env.sh"
source "$opt_conf/flume-env.sh"
fi
# append command-line java options to stock or env script JAVA_OPTS
if [ -n "${opt_java_props}" ]; then
JAVA_OPTS="${JAVA_OPTS} ${opt_java_props}"
fi
# prepend command-line classpath to env script classpath
if [ -n "${opt_classpath}" ]; then
if [ -n "${FLUME_CLASSPATH}" ]; then
FLUME_CLASSPATH="${opt_classpath}:${FLUME_CLASSPATH}"
else
FLUME_CLASSPATH="${opt_classpath}"
fi
fi
if [ -z "${FLUME_HOME}" ]; then
FLUME_HOME=$(cd $(dirname $0)/..; pwd)
fi
# prepend $FLUME_HOME/lib jars to the specified classpath (if any)
if [ -n "${FLUME_CLASSPATH}" ] ; then
FLUME_CLASSPATH="${FLUME_HOME}/lib/*:$FLUME_CLASSPATH"
else
FLUME_CLASSPATH="${FLUME_HOME}/lib/*"
fi
# find java
if [ -z "${JAVA_HOME}" ] ; then
warn "JAVA_HOME is not set!"
# Try to use Bigtop to autodetect JAVA_HOME if it's available
if [ -e /usr/libexec/bigtop-detect-javahome ] ; then
. /usr/libexec/bigtop-detect-javahome
elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ] ; then
. /usr/lib/bigtop-utils/bigtop-detect-javahome
fi
# Using java from path if bigtop is not installed or couldn't find it
if [ -z "${JAVA_HOME}" ] ; then
JAVA_DEFAULT=$(type -p java)
[ -n "$JAVA_DEFAULT" ] || error "Unable to find java executable. Is it
in your PATH?" 1
JAVA_HOME=$(cd $(dirname $JAVA_DEFAULT)/..; pwd)
fi
fi
# look for hadoop libs
add_hadoop_paths
add_HBASE_paths
# prepend conf dir to classpath
if [ -n "$opt_conf" ]; then
FLUME_CLASSPATH="$opt_conf:$FLUME_CLASSPATH"
fi
# allow dryrun
EXEC="exec"
if [ -n "${opt_dryrun}" ]; then
warn "Dryrun mode enabled (will not actually initiate startup)"
EXEC="echo"
fi
# finally, invoke the appropriate command
if [ -n "$opt_agent" ] ; then
run_flume $FLUME_AGENT_CLASS $args
elif [ -n "$opt_avro_client" ] ; then
run_flume $FLUME_AVRO_CLIENT_CLASS $args
elif [ -n "${opt_version}" ] ; then
run_flume $FLUME_VERSION_CLASS $args
else
error "This message should never appear" 1
fi
exit 0
On Mon, Jul 30, 2012 at 2:40 PM, Inder Pall <[email protected]> wrote:
> Hey if you edit flume-ng script and just before java is invoked put an
>
> echo $CLASSPATH that should help in figuring what all dir's are there in
> CLASSPATH.
> I am sorry but i am a big fan of windows hence don't run that O.S. to try
> it out.
>
> Thanks,
> - inder
>
> On Mon, Jul 30, 2012 at 7:00 PM, mardan Khan <[email protected]> wrote:
>
>> HI Inder,
>>
>> Sorry for the previous email. Actually I past the snapshot directly into
>> editor which convert into some noisy text. Please see the snapshot as
>> attachment .
>>
>> Please let me know for any solution
>>
>> Thanks
>>
>>
>>
>>
>> On Mon, Jul 30, 2012 at 1:20 PM, Inder Pall <[email protected]> wrote:
>>
>>> can you echo your CLASSPATH in the flume startup script.
>>>
>>> - inder
>>>
>>>
>>> On Mon, Jul 30, 2012 at 4:25 PM, mardan Khan <[email protected]>wrote:
>>>
>>>> Hi Brock
>>>>
>>>> Thanks for reply. I have solved the problem of cygwin. Now i am getting
>>>> another error message: Could not find or load main class of
>>>> org.apache.flume.node.Application.
>>>>
>>>> I am running the following command:
>>>>
>>>> bin/flume-ng agent -n agent -c conf -f
>>>> conf/flume-conf.properties.template
>>>>
>>>> Thanks
>>>>
>>>>
>>>>
>>>> On Sun, Jul 29, 2012 at 6:03 PM, Brock Noland <[email protected]>wrote:
>>>>
>>>>> Hi,
>>>>>
>>>>> Unfortunately I am not familiar with cygwin and don't have a windows
>>>>> computer to test on. Hopefully someone else on the list has an insight.
>>>>>
>>>>> Brock
>>>>>
>>>>>
>>>>> On Sun, Jul 29, 2012 at 11:27 AM, mardan Khan <[email protected]>wrote:
>>>>>
>>>>>> Thanks Brock.
>>>>>>
>>>>>>
>>>>>> Now when i am run the command: flume$ bin/flume-ng agent -n agent -f
>>>>>> conf/flume-conf.properties.
>>>>>> template.
>>>>>>
>>>>>>
>>>>>> Cygwin Warning: MS-DOS style path detected
>>>>>> /usr/local/bin/c:\program......................................................................
>>>>>> bin/flume-ng line:210 exec c:\program not found.
>>>>>>
>>>>>> How to solve this problem.
>>>>>>
>>>>>>
>>>>>>
>>>>>>
>>>>>>
>>>>>>
>>>>>>
>>>>>>
>>>>>> On Sun, Jul 29, 2012 at 4:41 PM, Brock Noland <[email protected]>wrote:
>>>>>>
>>>>>>> Hi,
>>>>>>>
>>>>>>> Inline
>>>>>>>
>>>>>>> On Sat, Jul 28, 2012 at 11:54 PM, mardan Khan
>>>>>>> <[email protected]>wrote:
>>>>>>>
>>>>>>>> Hi Brock.
>>>>>>>>
>>>>>>>> Thanks for reply. I want to use the full flume as i have to upload
>>>>>>>> the data into hdfs. I have installed the cygwin and hadoop.
>>>>>>>>
>>>>>>>
>>>>>>> OK this is a great start.
>>>>>>>
>>>>>>>
>>>>>>>> I have download the flume binary from the mentioned link. Now you
>>>>>>>> please let me explain the follow:
>>>>>>>>
>>>>>>>> 1). I need just to decopress the downloaded folder?
>>>>>>>>
>>>>>>>
>>>>>>> Yes you download a tar.gz which is like a zip. In cygwin you can use
>>>>>>> the command tar -zxvf to untar it.
>>>>>>>
>>>>>>>
>>>>>>>> 2). where i should decompress the flume binary. Should i decompress
>>>>>>>> c:\cygwin\home\user-name
>>>>>>>>
>>>>>>>
>>>>>>> Whereever you want, however a common place for "tarball" software is
>>>>>>> /usr/local or /opt
>>>>>>>
>>>>>>>
>>>>>>>> 3). How I will run the flume-ng.
>>>>>>>>
>>>>>>>
>>>>>>> You configure flume and then run the flume-ng command in the bin
>>>>>>> directory
>>>>>>>
>>>>>>>
>>>>>>>> 4) The flume.conf file will configure same as configuring using
>>>>>>>> Linux?
>>>>>>>>
>>>>>>>>
>>>>>>> Yes configure flume will be the same as on Linux in cygwin.
>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>> Many thanks
>>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>> On Sun, Jul 29, 2012 at 4:20 AM, Brock Noland
>>>>>>>> <[email protected]>wrote:
>>>>>>>>
>>>>>>>>> Hi,
>>>>>>>>>
>>>>>>>>> If you just want run a flume agent on Windows you just download
>>>>>>>>> the binary here: http://flume.apache.org/download.html
>>>>>>>>>
>>>>>>>>> Note that the flume startup script is written in bash so unless
>>>>>>>>> you use cygwin you'll need to start the JVM and setup the classpath
>>>>>>>>> yourself.
>>>>>>>>>
>>>>>>>>> Brock
>>>>>>>>>
>>>>>>>>>
>>>>>>>>> On Sat, Jul 28, 2012 at 8:01 PM, mardan Khan <[email protected]
>>>>>>>>> > wrote:
>>>>>>>>>
>>>>>>>>>> Hi,
>>>>>>>>>>
>>>>>>>>>> I am struggling from couple of weeks to install and configure the
>>>>>>>>>> flume1.x on windows but could not success. Could you please someone
>>>>>>>>>> show me
>>>>>>>>>> step-by-step how to install flume on window. I have read a couple of
>>>>>>>>>> posts
>>>>>>>>>> which mentioned the git and Maven but i dont understand how can use
>>>>>>>>>> it to
>>>>>>>>>> install the flume.
>>>>>>>>>>
>>>>>>>>>> Please help me as i am total stuck and running out of time.
>>>>>>>>>>
>>>>>>>>>> Thanks
>>>>>>>>>>
>>>>>>>>>
>>>>>>>>>
>>>>>>>>>
>>>>>>>>> --
>>>>>>>>> Apache MRUnit - Unit testing MapReduce -
>>>>>>>>> http://incubator.apache.org/mrunit/
>>>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>> --
>>>>>>> Apache MRUnit - Unit testing MapReduce -
>>>>>>> http://incubator.apache.org/mrunit/
>>>>>>>
>>>>>>
>>>>>>
>>>>>
>>>>>
>>>>> --
>>>>> Apache MRUnit - Unit testing MapReduce -
>>>>> http://incubator.apache.org/mrunit/
>>>>>
>>>>
>>>>
>>>
>>>
>>> --
>>> Thanks,
>>> - Inder
>>> Tech Platforms @Inmobi
>>> Linkedin - http://goo.gl/eR4Ub
>>>
>>
>>
>
>
> --
> Thanks,
> - Inder
> Tech Platforms @Inmobi
> Linkedin - http://goo.gl/eR4Ub
>
