Repository: flume Updated Branches: refs/heads/trunk 1d49ef704 -> 62a4cad22
FLUME-1334. Windows startup scripts. (Roshan Naik via Hari) Project: http://git-wip-us.apache.org/repos/asf/flume/repo Commit: http://git-wip-us.apache.org/repos/asf/flume/commit/62a4cad2 Tree: http://git-wip-us.apache.org/repos/asf/flume/tree/62a4cad2 Diff: http://git-wip-us.apache.org/repos/asf/flume/diff/62a4cad2 Branch: refs/heads/trunk Commit: 62a4cad223aebfa0aa0f9d20c1c37f04a8b918a9 Parents: 1d49ef7 Author: Hari Shreedharan <[email protected]> Authored: Fri Jan 30 16:55:47 2015 -0800 Committer: Hari Shreedharan <[email protected]> Committed: Fri Jan 30 16:55:47 2015 -0800 ---------------------------------------------------------------------- bin/flume-ng.cmd | 19 +++ bin/flume-ng.ps1 | 403 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 422 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flume/blob/62a4cad2/bin/flume-ng.cmd ---------------------------------------------------------------------- diff --git a/bin/flume-ng.cmd b/bin/flume-ng.cmd new file mode 100644 index 0000000..b1b97a9 --- /dev/null +++ b/bin/flume-ng.cmd @@ -0,0 +1,19 @@ +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. + +powershell.exe -NoProfile -InputFormat none -ExecutionPolicy unrestricted -File %~dp0flume-ng.ps1 %* + http://git-wip-us.apache.org/repos/asf/flume/blob/62a4cad2/bin/flume-ng.ps1 ---------------------------------------------------------------------- diff --git a/bin/flume-ng.ps1 b/bin/flume-ng.ps1 new file mode 100755 index 0000000..1638bf1 --- /dev/null +++ b/bin/flume-ng.ps1 @@ -0,0 +1,403 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +param( + [Parameter(Position=0, Mandatory=$true)] + [ValidateSet('help','agent','avro-client','version')] $command, + [Parameter()] [string] $conf, + [Parameter()] [Alias('C')] [string] $classPath, + [Parameter()] [Alias('property')] [string] $javaProperties, + [Parameter()] [Alias('d')] [switch] $dryrun, + [Parameter()] [Alias('plugins-path')] [string] $pluginsPath, + + [Parameter()] [Alias('f',"conf-file")] [string] $confFile, + [Parameter()] [Alias('n')] [string] $name, + + [Parameter()] [string] $rpcProps, + [Parameter()] [Alias('H',"host")] [string] $avroHost, + [Parameter()] [Alias('p',"port")] [string] $avroPort, + [Parameter()] [Alias("dirname")] [string] $avroDirName, + [Parameter()] [Alias("filename")] [string] $avroFileName, + [Parameter()] [Alias('R',"headerFile")] [string] $avroHeaderFile +) + +# when invoked from cmd shell, array arguments are treated as array with one string value +# so we accept classpath and javaProperty as string argument and split explicitly into array + +if ($classPath -ne "") { + $classPathArr = $classPath.Split(";") +} else { + $classPathArr = @() +} + +if($javaProperties -ne "") { + $javaPropertiesArr = $javaProperties.Split(";") # properties may have embedded comma, so using ; as delim +} else { + $javaPropertiesArr = @() +} + +# FUNCTIONS +Function PrintUsage { + Write-Host " +Usage: flume-ng <command> [options]... + +commands: + help display this help text + agent run a Flume agent + avro-client run an avro Flume client + version show Flume version info + +global options: + -conf <conf> use configs in <conf> directory + -classpath,-C ""value1;value2; .."" append to the classpath + -property ""name1=val;name2=val; .."" sets a JDK system property value + -dryrun,-d do not actually start Flume (test) + -plugins-path ""dir1;dir2"" semi-colon-separated list of plugins.d directories. See the + plugins.d section in the user guide for more details. + Default: \$FLUME_HOME/plugins.d + +agent options: + -conf-file,-f <file> specify a config file (required) + -name,-n <name> the name of this agent (required) + +avro-client options: + -rpcProps <file> RPC client properties file with server connection params + -host,-H <host> hostname to which events will be sent (required) + -port,-p <port> port of the avro source (required) + -dirname <dir> directory to stream to avro source + -filename <file> text file to stream to avro source [default: std input] + -headerFile,-R <file> headerFile containing headers as key/value pairs on each new line + + Either --rpcProps or both --host and --port must be specified. + +Note that if <conf> directory is specified, then it is always included first +in the classpath." +} + +Function EnumerateJars($path) { + return Get-ChildItem $path -Filter *.jar | ? { "$_" -notmatch "slf4j-(api|log4j12).*jar" } | % { $_.FullName} +} + +Function GetHadoopHome { + if($env:HADOOP_PREFIX) { + $hadoopBin = "$env:HADOOP_PREFIX\bin;" + } elseif ($env:HADOOP_HOME) { + $hadoopBin = "$env:HADOOP_HOME\bin;" + } + + #Searches for hadoop.cmd in the HADOOP_HOME, current directory and path + [String[]] $hadoopPaths = ("$hadoopBin;.;$env:PATH").Split(";") | + ? { "$_" -ne "" -and (Test-Path $_ )} | + ? { Test-Path ( Join-Path $_ "hadoop.cmd" ) } + if($hadoopPaths -ne $Null ) { + $binDir = $hadoopPaths[0] + "\.." + return Resolve-Path $binDir + } + Write-Host "WARN: HADOOP_PREFIX or HADOOP_HOME not found" + return $Null +} + +Function GetHbaseHome() { + if($env:HBASE_HOME) { + return $env:HBASE_HOME + } + + #Searches for hbase.cmd in the HBASE_HOME, current directory and path + [String[]] $hbasePaths = (".;$env:PATH").Split(";") | + ? { "$_" -ne "" -and (Test-Path $_) } | + ? { Test-Path (Join-Path $_ "hbase.cmd") } + + if($hbasePaths -ne $Null) { + return $hbasePaths[0] + } + Write-Host "WARN: HBASE_HOME not found" + return $Null +} + +Function GetHiveHome() { + if($env:HIVE_HOME) { + return $env:HIVE_HOME + } + + #Searches for hive.cmd in the HIVE_HOME, current directory and path + [String[]] $hivePaths = (".;$env:PATH").Split(";") | + ? { "$_" -ne "" -and (Test-Path $_) } | + ? { Test-Path (Join-Path $_ "hive.cmd") } + + if($hivePaths -ne $Null) { + return $hivePaths[0] + } + Write-Host "WARN: HIVE_HOME not found" + return $Null +} + +Function GetJavaLibraryPath ($cmd, $flumeLibDir) { + $flumeCoreJar = EnumerateJars( $flumeLibDir ) | ? { $_ -match "flume-ng-core.*jar" } + $output = & "$cmd" -classpath $flumeCoreJar org.apache.flume.tools.GetJavaProperty java.library.path + + # look for the line that has the desired property value + if( $output -match "(?m)^java\.library\.path=(.*)$" ) { + return $Matches[1].split(";") | % { $_ -replace "\\$" , ""} # trim trailing \ char + } + return @(); +} + + +Function GetClassPath ($cmd) { + $output = & "$cmd" "classpath" + return $output; +} + +Function GetJavaPath { + if ($env:JAVA_HOME) { + return "$env:JAVA_HOME\bin\java.exe" }ss + Write-Host "WARN: JAVA_HOME not set" + return '"' + (Resolve-Path "java.exe").Path + '"' +} + + +function runFlume($javaClassPath, $javaLibraryPath, $javaOptions, $class, $javaProcessArgumentList) { + [string]$javaPath = GetJavaPath + [string]$fullJavaCommand = "-classpath $javaClassPath -Djava.library.path=$javaLibraryPath $javaOptions $class $javaProcessArgumentList" + if ($dryrun) { + Write-Host 'Dry run mode enabled (will not actually initiate startup)' + Write-Host "$javaPath $fullJavaCommand" + } else { + Write-Host " + Running FLUME $command : + class: $class + arguments: $javaProcessArgumentList + " + + $ErrorActionPreference = "Continue" + $x = Start-Process $javaPath -ArgumentList "$fullJavaCommand" -Wait -NoNewWindow + } +} + + +# The script will terminate if any steps fail +$ErrorActionPreference = "Stop" + +#SWITCH FOR THE DIFFERENT COMMANDS +[string] $javaProcessArgumentList +switch ($command) +{ + 'help' { + PrintUsage + return } + + 'agent' { + if (!$Name) { + PrintUsage + Write-Host "ERROR: Name parameter missing" + return + } + if (!$ConfFile) { + PrintUsage + Write-Host "ERROR: ConfFile parameter missing" + return + } + $class='org.apache.flume.node.Application' + $confFile = '"' + (Resolve-Path $confFile).Path + '"' + $javaProcessArgumentList = "-n $name -f $confFile" } + + 'avro-client' { + $class='org.apache.flume.client.avro.AvroCLIClient' + if("$rpcProps" -eq "") { + if (!$AvroHost) { + PrintUsage + Write-Host "ERROR: Avro Host parameter missing" + return + } + if (!$AvroPort) { + PrintUsage + Write-Host "ERROR: Avro Port parameter missing" + return + } + $javaProcessArgumentList = " -H $AvroHost -p $AvroPort" + } else { + $javaProcessArgumentList += " --rpcProps ""$rpcProps""" + } + + if ($avroHeaderFile) { + $avroHeaderFile = (Resolve-Path $avroHeaderFile).Path + $javaProcessArgumentList += " -R $avroHeaderFile" } + if ($avroFileName) { + $avroFileName = (Resolve-Path $avroFileName).Path + $javaProcessArgumentList += " -F $avroFileName" } + if($avroDirName) { + $avroDirName = (Resolve-Path $avroDirName).Path + $javaProcessArgumentList += " --dirname ""$avroDirName""" } } + + 'version' { + $class='org.apache.flume.tools.VersionInfo' + $javaProcessArgumentList = "" } + + default { + PrintUsage + Write-Host "ERROR: Invalid command '$command'" + return } +} + +$FlumeHome = $null +if($env:FLUME_HOME) { + $FlumeHome = $env:FLUME_HOME +} else { + $ScriptPath = Split-Path -Parent $MyInvocation.MyCommand.Path + $FlumeHome = Split-Path -Parent $ScriptPath +} + +########### Source flume-env.ps1 ############## + +# allow users to override the default env vars via conf\flume-env.ps1 +if( "$conf" -eq "" ) { + if( Test-path ("$FlumeHome\conf") ) { + $conf = "$FlumeHome\conf" + Write-Host "WARN: Config directory not set. Defaulting to $conf" + } +} +if ( "$conf" -ne "" ) { + Write-Host "Sourcing environment configuration script $conf\flume-env.ps1" + if ( Test-path "$conf\flume-env.ps1" ) { + . "$conf\flume-env.ps1" + } else { + Write-Host "WARN: Did not find $conf\flume-env.ps1" + } +} else { + Write-Host "WARN: No configuration directory found! Use --conf <dir> to set." +} + +########### Setup JAVA_OPTS ############## + +[string]$javaOptions="$JAVA_OPTS" +foreach ($opt in $javaPropertiesArr) { + $javaOptions = "$javaOptions -D$opt" +} + +########### Setup Classpath ############### +# flume\conf ; flume_home\lib\* ; cmdline ; env.ps1 ; plugins.d ; hadoop.cpath ; hbase.cpath ; hive.cpath + +[string]$javaClassPath='"' + $conf + '"' +[string]$flumeLibJars="" +[string]$flumeLibDir = Resolve-Path "$FlumeHome\lib" + +# Add FlumeHome\lib\* to class path +$javaClassPath = "$javaClassPath;""$flumeLibDir\*""" +$flumeLibJars = "$flumeLibDir\*""" + +# Add classpath from cmd line & FLUME_CLASSPATH in flume-env.ps1 +if ( $FLUME_CLASSPATH ) { + $classPathArr = $FLUME_CLASSPATH.Split(";") +} +foreach ($path in $classPathArr) { + $fullPath = (Resolve-Path $path).Path + $javaClassPath = "$javaClassPath;""$fullPath""" +} + +$javaLibraryPath = "" + +# Add plugins.d into classpath and libpath +if ("$pluginsPath" -eq "") { + $pluginsPath = "$FlumeHome\plugins.d" +} + +foreach($plugin in $pluginsPath.Split(";") ) { + if ( Test-path "$plugin" ) { + $pluginTmp = + ( (Get-ChildItem "$plugin\*\lib") + (Get-ChildItem "$plugin\*\libext") ) -join "\*"";""" + if( "$pluginTmp" -ne "" ) { + $javaClassPath="$javaClassPath;""" + $pluginTmp + "\*"";" + } + + $javaLibraryPathTmp = (@(Get-ChildItem "$plugin\*\native") -join "\*"";""") + if( "$javaLibraryPathTmp" -ne "" ) { + $javaLibraryPath= "$javaLibraryPath""" + "$javaLibraryPathTmp" + "\*"";" + } + } +} + +# Add Hadoop classpath & java.library.path +$hadoopHome = GetHadoopHome +if("$hadoopHome" -ne "") { + $hadoopCmd = "$hadoopHome\bin\hadoop.cmd" + if( Test-Path $hadoopCmd ) { + Write-Host "Including Hadoop libraries found in ($hadoopHome) for DFS access" + $javaClassPath = "$javaClassPath;""$hadoopHome\conf"""; + foreach ($path in GetClassPath $hadoopCmd) { + $javaClassPath = "$javaClassPath;""$path""" + } + + foreach ( $path in GetJavaLibraryPath $hadoopCmd $flumeLibDir ) { + $javaLibraryPath = "$javaLibraryPath""$path"";" + } + } else { + Write-Host "WARN: $hadoopCmd not be found. Unable to include Hadoop's classpath & java.library.path" + } +} else { + Write-Host "WARN: HADOOP_PREFIX not set. Unable to include Hadoop's classpath & java.library.path" +} + +# Add HBase classpath & java.library.path +$hbaseHome = GetHbaseHome +if( "$hbaseHome" -ne "" ) { + $hbaseCmd = "$hbaseHome\bin\hbase.cmd" + if( Test-Path $hbaseCmd ) { + Write-Host "Including HBase libraries found via ($hbaseHome) for HBase access" + foreach ( $path in GetClassPath $hbaseCmd ) { + $javaClassPath = "$javaClassPath;""$path""" + } + $javaClassPath = "$javaClassPath;""$hbaseHome\conf""" + + foreach ( $path in GetJavaLibraryPath $hbaseCmd $flumeLibDir ) { + $javaLibraryPath = "$javaLibraryPath""$path"";" + } + } else { + Write-Host "WARN: $hbaseCmd not be found. Unable to include HBase classpath and java.library.path" + } +} + +# Add Hive classpath +$hiveHome = GetHiveHome +if( "$hiveHome" -ne "" ) { + $hiveLib = "$hiveHome\lib" + if( Test-Path $hiveLib ) { + Write-Host "Including Hive libraries found via ($hiveHome) for Hive access" + $javaClassPath = "$javaClassPath;""$hiveLib\*""" + } else { + Write-Host "WARN: $hiveLib not found. Unable to include Hive into classpath" + } + + $hiveConf = "$hiveHome\conf" + if( Test-Path $hiveConf ) { + Write-Host "Including Hive conf dir ($hiveConf) in classpath for Hive access" + $javaClassPath = "$javaClassPath;""$hiveConf""" + } else { + Write-Host "WARN: $hiveConf not found. Unable to include it into classpath" + } + + $hcatLib = "$hiveHome\hcatalog\share\hcatalog" + if( Test-Path $hcatLib ) { + Write-Host "Including HCatalog libraries ($hcatLib) for Hive access" + $javaClassPath = "$javaClassPath;""$hcatLib\*""" + } else { + Write-Host "WARN: $hcatLib not found. Unable to include HCatalog into classpath" + } +} + +runFlume $javaClassPath $javaLibraryPath $javaOptions $class $javaProcessArgumentList
