Updated Branches: refs/heads/master 9d10075a6 -> 34904ff57
OOZIE-1552 Bring Windows shell script functionality and structure in line with trunk (omaliuvanchuk via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/34904ff5 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/34904ff5 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/34904ff5 Branch: refs/heads/master Commit: 34904ff5784da7d0ab9f223dc2445ada9e7ded23 Parents: 9d10075 Author: Robert Kanter <[email protected]> Authored: Mon Feb 10 16:37:31 2014 -0800 Committer: Robert Kanter <[email protected]> Committed: Mon Feb 10 16:37:31 2014 -0800 ---------------------------------------------------------------------- core/src/main/conf/oozie-env.cmd | 8 +- distro/src/main/bin/oozie-setup.ps1 | 262 +++++++++++++++++-------------- distro/src/main/bin/oozie-sys.cmd | 27 ++++ distro/src/main/bin/oozied.cmd | 17 +- release-log.txt | 1 + 5 files changed, 190 insertions(+), 125 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/34904ff5/core/src/main/conf/oozie-env.cmd ---------------------------------------------------------------------- diff --git a/core/src/main/conf/oozie-env.cmd b/core/src/main/conf/oozie-env.cmd index 3a173ec..85e1da5 100644 --- a/core/src/main/conf/oozie-env.cmd +++ b/core/src/main/conf/oozie-env.cmd @@ -56,8 +56,12 @@ set CATALINA_OPTS=%CATALINA_OPTS% -Xmx1024m @rem The location of the keystore for the Oozie server if using SSL (HTTPS) @rem -@rem export OOZIE_HTTPS_KEYSTORE_FILE=${HOME}\.keystore +@rem set OOZIE_HTTPS_KEYSTORE_FILE=%HOME%/.keystore @rem The password of the keystore for the Oozie server if using SSL (HTTPS) @rem -@rem export OOZIE_HTTPS_KEYSTORE_PASS=password \ No newline at end of file +@rem set OOZIE_HTTPS_KEYSTORE_PASS=password + +@rem The Oozie Instance ID +@rem +@rem set OOZIE_INSTANCE_ID=%OOZIE_HTTP_HOSTNAME% \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/34904ff5/distro/src/main/bin/oozie-setup.ps1 ---------------------------------------------------------------------- diff --git a/distro/src/main/bin/oozie-setup.ps1 b/distro/src/main/bin/oozie-setup.ps1 index 6afde9c..2699a07 100644 --- a/distro/src/main/bin/oozie-setup.ps1 +++ b/distro/src/main/bin/oozie-setup.ps1 @@ -14,60 +14,43 @@ ### limitations under the License. param( -[Parameter(Mandatory=$true)] [string] $InputWar, -[Parameter(Mandatory=$true)] [string] $OutputWar, -[string[]] $Jars, -[string] $HadoopVersion, -[string] $HadoopHome, -[string] $ExtJS) +[Parameter(Mandatory=$true,Position = 0)] [string] $Command="", +[string] $D="", +[switch] $Secure, +[Parameter(ValueFromRemainingArguments = $true)][string[]]$args) # FUNCTIONS -Function GetHadoopJars { param ( - [Parameter(Mandatory=$true)] [string] $HadoopVersion, - [Parameter(Mandatory=$true)] [string] $HadoopHome) - if ($HadoopVersion -ieq "0.20.1") - { - $jarsList = "$hadoopHome\hadoop*core*.jar" - } - elseif ($HadoopVersion -ieq "0.20.2") - { - $jarsList = "$hadoopHome\hadoop*core*.jar" - } - elseif ($HadoopVersion -ieq "0.20.104") - { - $jarsList = - "$hadoopHome\hadoop*core*.jar", - "$hadoopHome\lib\jackson-core-asl*.jar", - "$hadoopHome\lib\jackson-mapper-asl-*.jar" - } - else - { - $jarsList = - "$hadoopHome\hadoop-core-$HadoopVersion.jar", - "$hadoopHome\lib\jackson-core-asl*.jar", - "$hadoopHome\lib\jackson-mapper-asl-*.jar", - "$hadoopHome\lib\commons-configuration-*.jar" - } - $hadoopJars = @() - $jarsList | % { if ((Test-Path $_)) { - $hadoopJars += $_ - } - else { - Write-Output "Unable to find Hadoop Jar '$_'" - }} - $hadoopJars -} - Function PrintUsage { param () - Write-Output "Usage : oozie-setup.ps1 [OPTIONS]" - Write-Output " [-ExtJS EXTJS_PATH] (expanded or ZIP, to enable the Oozie webconsole)" - Write-Output " [-HadoopHome HADOOP_PATH -HadoopVersion HADOOP_VERSION] (Hadoop version [1.2.0-SNAPSHOT])" - Write-Output " (Hadoop version [1.2.0-SNAPSHOT])" - Write-Output " [-jars [JAR_PATH, ... ] ] " + Write-Output "Usage : oozie-setup.ps1 COMMAND [OPTIONS]" + Write-Output " prepare-war [-d directory] [-secure] (-d identifies an alternative directory for processing jars" + Write-Output " -secure will configure the war file to use HTTPS (SSL))" + Write-Output " sharelib create -fs FS_URI [-locallib SHARED_LIBRARY] (create sharelib for oozie," + Write-Output " FS_URI is the fs.default.name" + Write-Output " for hdfs uri; SHARED_LIBRARY, path to the" + Write-Output " Oozie sharelib to install, it can be a tarball" + Write-Output " or an expanded version of it. If ommited," + Write-Output " the Oozie sharelib tarball from the Oozie" + Write-Output " installation directory will be used)" + Write-Output " (action failes if sharelib is already installed" + Write-Output " in HDFS)" + Write-Output " sharelib upgrade -fs FS_URI [-locallib SHARED_LIBRARY] ([deprecated]" + Write-Output " [use create command to create new version]" + Write-Output " upgrade existing sharelib, fails if there" + Write-Output " is no existing sharelib installed in HDFS)" + Write-Output " db create|upgrade|postupgrade -run [-sqlfile <FILE>] (create, upgrade or postupgrade oozie db with an" + Write-Output " optional sql File)" Write-Output " EXTJS can be downloaded from http://www.extjs.com/learn/Ext_Version_Archives"; } +function Expand-ZIPFile($file, $destination){ + $shell = new-object -com shell.application + $zip = $shell.NameSpace($file) + foreach($item in $zip.items()){ + $shell.Namespace($destination).copyhere($item) + } +} + # MAIN() # The script will terminate if any steps fail @@ -75,98 +58,141 @@ $ErrorActionPreference = "Stop" # Constants $EXT_SUBDIR = "ext-2.2" -$HADOOP_DEFAULT_VERSION = "1.2.0-SNAPSHOT" +$OOZIE_HOME = (Split-Path $MyInvocation.MyCommand.Path) + "\.." +$OOZIE_HOME = Resolve-Path $OOZIE_HOME -# Finds JAR.EXE +$CATALINA_BASE = "" +if ($env:CATALINA_BASE){ + $CATALINA_BASE = "$env:CATALINA_BASE" +}else{ + $CATALINA_BASE = "$OOZIE_HOME\oozie-server" +} +# Finds JAR.EXE and Java $JAR_EXE="" +$JAVA_BIN="" if ($env:JAVA_HOME) { $JAR_EXE = "$env:JAVA_HOME\bin\jar.exe" + $JAVA_BIN = "$env:JAVA_HOME\bin\java.exe" } else { - Write-Output "WARN: JAVA_HOME not defined. oozie-setup.ps1 will rely on the PATH environment variable to use JAR.exe" + Write-Output "WARN: JAVA_HOME not defined. oozie-setup.ps1 will relay on the PATH environment variable to use JAR.exe" $JAR_EXE = "jar.exe" + $JAVA_BIN = "java.exe" } -# Validates that both/neither $HadoopHome and $HadoopVersion are provided -if ($HadoopHome -and !$HadoopVersion) { - PrintUsage - throw "Need to specify -HadoopVersion for -HadoopHome='$HadoopHome'" -} -if ($HadoopVersion -and !$HadoopHome) { - PrintUsage - throw "Need to specify -HadoopHome for -HadoopVersion='$HadoopHome'" -} +if (($Command -eq "sharelib") -Or ($Command -eq "db")) { + $OOZIE_OPTS="-Doozie.home.dir=$OOZIE_HOME"; + $OOZIE_OPTS="$OOZIE_OPTS -Doozie.config.dir=$OOZIE_HOME\conf"; + $OOZIE_OPTS="$OOZIE_OPTS -Doozie.log.dir=$OOZIE_HOME\log"; + $OOZIE_OPTS="$OOZIE_OPTS -Doozie.data.dir=$OOZIE_HOME\data"; + $OOZIE_OPTS="$OOZIE_OPTS -Dderby.stream.error.file=$OOZIE_HOME\log\derby.log" + + $OOZIECPPATH="" + $OOZIECPPATH="$OOZIE_HOME\libtools\*" + + $COMMAND_OPTS=[string]$args + + if ($Command -eq "sharelib") { + cmd /c $JAVA_BIN $OOZIE_OPTS -cp $OOZIECPPATH org.apache.oozie.tools.OozieSharelibCLI $COMMAND_OPTS + } elseif ($Command -eq "db") { + cmd /c $JAVA_BIN $OOZIE_OPTS -cp $OOZIECPPATH org.apache.oozie.tools.OozieDBCLI $COMMAND_OPTS + } + exit 0 +}elseif ($Command -eq "prepare-war"){ + + $InputWar = "$OOZIE_HOME\oozie.war" + $OutputWar = "$OOZIE_HOME\oozie-server\webapps\oozie.war" + $SecureConfigsDir="$CATALINA_BASE\conf\ssl" + $ExtraLibs = Resolve-Path "$OOZIE_HOME\..\extra_libs" + $EXTJS = "$ExtraLibs\ext-2.2.zip" + + # Validates the input\output wars + if (!(Test-Path $InputWar)){ + PrintUsage + throw "Path '$InputWar' doesn't exist" + } + if (!$InputWar.ToLower().EndsWith(".war")){ + PrintUsage + throw "Invalid input war file '$InputWar'" + } + if (!$OutputWar.ToLower().EndsWith(".war")){ + PrintUsage + throw "Invalid input war file '$OutputWar'" + } + if ($OutputWar -ieq $InputWar){ + PrintUsage + throw "Invalid output\input war file. Both parameters cannot be the same file" + } + # Deletes previous output wars. + if (Test-Path $OutputWar){ + Write-Output "Deleting existing output .war '$OutputWar'" + Remove-Item -Force -Path $OutputWar + } -# Validates the input\output wars -if (!(Test-Path $InputWar)){ - PrintUsage - throw "Path '$InputWar' doesn't exist" -} -if (!$InputWar.ToLower().EndsWith(".war")){ - PrintUsage - throw "Invalid input war file '$InputWar'" -} -if (!$OutputWar.ToLower().EndsWith(".war")){ - PrintUsage - throw "Invalid input war file '$OutputWar'" -} -if ($OutputWar -ieq $InputWar){ - PrintUsage - throw "Invalid output\input war file. Both parameters cannot be the same file" -} -# Deletes previous output wars. -if (Test-Path $OutputWar){ - Write-Output "Deleting existing output .war '$OutputWar'" - Remove-Item -Force -Path $OutputWar -} + # Selects\Creates the temp directory + $OOZIE_TEMP = "$OOZIE_HOME\temp" + $OOZIE_WEB_INF_LIB = "$OOZIE_TEMP\WEB-INF\lib" -# Selects\Creates the temp directory -$OOZIE_HOME = (Split-Path $MyInvocation.MyCommand.Path) + "\.." -$OOZIE_HOME = Resolve-Path $OOZIE_HOME -$OOZIE_TEMP = "$OOZIE_HOME\temp" -$OOZIE_WEB_INF_LIB = "$OOZIE_TEMP\WEB-INF\lib" - -Write-Output "Creating OOZIE_TEMP directory '$OOZIE_TEMP'" -if (Test-Path "$OOZIE_TEMP") { Remove-Item "$OOZIE_TEMP" -Force -Recurse } -$x = New-Item "$OOZIE_WEB_INF_LIB" -type directory - -# Creates the new OutputWar -$x = Copy-Item $InputWar $OutputWar - -# Copy hadoop files -if ($HadoopVersion -or $HadoopHome) { - Write-Output "Extracting files from path '$HadoopHome' from version '$HadoopVersion'" - if (!(Test-Path $HadoopHome)) { throw "Unable to find Hadoop Home '$HadoopHome'" } - $HadoopFiles = GetHadoopJars -HadoopVersion $HadoopVersion -HadoopHome $HadoopHome - $HadoopFiles | % { - Write-Output " Adding HadoopFiles: $_" - Copy-Item $_ $OOZIE_WEB_INF_LIB -force - } -} + Write-Output "Creating OOZIE_TEMP directory '$OOZIE_TEMP'" + if (Test-Path "$OOZIE_TEMP") { Remove-Item "$OOZIE_TEMP" -Force -Recurse } + $x = New-Item "$OOZIE_TEMP" -type directory -# Copy EXT_JS files -if ($ExtJS) { - Write-Output "ExtJS not currently supported!" -} + # Extract the InputWar + pushd $OOZIE_TEMP + cmd /c $JAR_EXE xvf $InputWar + if ($LASTEXITCODE -ne 0) { + throw "Unable to execute 'jar xvf'. Error ($LASTEXITCODE)" + } + popd + # Copy EXT_JS files + if ((Test-Path $EXTJS)) { + $EXTJS_HOME = "$ExtraLibs\ext-2.2" + + if (Test-Path "$EXTJS_HOME") { Remove-Item "$EXTJS_HOME" -Force -Recurse } + $x = New-Item "$EXTJS_HOME" -type directory + + Expand-ZIPFile -File $EXTJS -Destination $EXTJS_HOME + cp -r "$EXTJS_HOME\ext-2.2" "$OOZIE_TEMP" + }else{ + Write-Output "INFO: Oozie webconsole disabled, ExtJS library not specified" + } -# Copy additional Jars -$Jars | % { - Write-Output " Adding JarFiles: $_" - Copy-Item $_ $OOZIE_WEB_INF_LIB -force -} + # Copy additional Jars + if ($D -ne "") { + $ExtraLibs = $D + } + Write-Output " Adding JarFiles: $ExtraLibs\*.jar" + cp -r $ExtraLibs\*.jar $OOZIE_WEB_INF_LIB -$counter = (Get-ChildItem $OOZIE_WEB_INF_LIB).Length -IF ($counter -gt 0) { - Write-Output "Adding files to the war file '$OutputWar'..." + if ($Secure) { + #Use the SSL version of server.xml in oozie-server + if (Test-Path $SecureConfigsDir\ssl-server.xml){ + cp $SecureConfigsDir\ssl-server.xml $CATALINA_BASE\conf\server.xml + } - "$JAR_EXE" + #Inject the SSL version of web.xml in oozie war + if (Test-Path $SecureConfigsDir\ssl-web.xml){ + cp $SecureConfigsDir\ssl-web.xml $OOZIE_TEMP\WEB-INF\web.xml + } - cmd /c $JAR_EXE uvf $OutputWar -C $OOZIE_TEMP WEB-INF\lib + Write-Output "INFO: Using secure server.xml and secure web.xml" + }else{ + #Use the regular version of server.xml in oozie-server + if (Test-Path $SecureConfigsDir\server.xml){ + cp $SecureConfigsDir\server.xml $CATALINA_BASE\conf\server.xml + } + #No need to restore web.xml because its already in the original WAR file + } + Write-Output "Recreating the new war file '$OutputWar'..." + cmd /c $JAR_EXE cvf $OutputWar -C $OOZIE_TEMP '.' if ($LASTEXITCODE -ne 0) { throw "Unable to execute 'jar uvf'. Error ($LASTEXITCODE)" } Write-Output "Done! $counter files added" +} else { + PrintUsage + exit -1 } http://git-wip-us.apache.org/repos/asf/oozie/blob/34904ff5/distro/src/main/bin/oozie-sys.cmd ---------------------------------------------------------------------- diff --git a/distro/src/main/bin/oozie-sys.cmd b/distro/src/main/bin/oozie-sys.cmd index 61cdd17..e9c8100 100644 --- a/distro/src/main/bin/oozie-sys.cmd +++ b/distro/src/main/bin/oozie-sys.cmd @@ -111,6 +111,13 @@ if not defined OOZIE_ADMIN_PORT ( echo Using OOZIE_ADMIN_PORT: '%OOZIE_ADMIN_PORT%' ) +if not defined OOZIE_HTTPS_PORT ( + SET OOZIE_HTTPS_PORT=11443 + echo Setting OOZIE_HTTPS_PORT: '%OOZIE_HTTPS_PORT%' +) else ( + echo Using OOZIE_HTTPS_PORT: '%OOZIE_HTTPS_PORT%' +) + if not defined OOZIE_BASE_URL ( set OOZIE_BASE_URL=http://%OOZIE_HTTP_HOSTNAME%:%OOZIE_HTTP_PORT%/oozie echo Setting OOZIE_BASE_URL: '%OOZIE_BASE_URL%' @@ -125,6 +132,26 @@ if not defined CATALINA_BASE ( echo Using CATALINA_BASE: '%CATALINA_BASE%' ) +if not defined OOZIE_HTTPS_KEYSTORE_FILE ( + set OOZIE_HTTPS_KEYSTORE_FILE=%HOME%\.keystore + echo Setting OOZIE_HTTPS_KEYSTORE_FILE: '%OOZIE_HTTPS_KEYSTORE_FILE%' +) else ( + echo Using OOZIE_HTTPS_KEYSTORE_FILE: '%OOZIE_HTTPS_KEYSTORE_FILE%' +) + +if not defined OOZIE_HTTPS_KEYSTORE_PASS ( + set OOZIE_HTTPS_KEYSTORE_PASS=password + echo Setting OOZIE_HTTPS_KEYSTORE_PASS: '%OOZIE_HTTPS_KEYSTORE_PASS%' +) else ( + echo Using OOZIE_HTTPS_KEYSTORE_PASS: '%OOZIE_HTTPS_KEYSTORE_PASS%' +) + +if not defined OOZIE_INSTANCE_ID ( + set OOZIE_INSTANCE_ID=%OOZIE_HTTP_HOSTNAME% + echo Setting OOZIE_INSTANCE_ID: '%OOZIE_INSTANCE_ID%' +) else ( + echo Using OOZIE_INSTANCE_ID: '%OOZIE_INSTANCE_ID%' +) if not defined CATALINA_HOME ( set CATALINA_HOME=%OOZIE_HOME%\oozie-server http://git-wip-us.apache.org/repos/asf/oozie/blob/34904ff5/distro/src/main/bin/oozied.cmd ---------------------------------------------------------------------- diff --git a/distro/src/main/bin/oozied.cmd b/distro/src/main/bin/oozied.cmd index 6ae2b87..6b8d45b 100644 --- a/distro/src/main/bin/oozied.cmd +++ b/distro/src/main/bin/oozied.cmd @@ -27,10 +27,11 @@ if %argC% == 0 ( set actionCmd=%1 set BASEDIR=%~dp0 -call %BASEDIR%\oozie-sys.cmd +set BASEDIR=%BASEDIR%\.. +call %BASEDIR%\bin\oozie-sys.cmd -set CATALINA=%BASEDIR%\..\oozie-server\bin\catalina.bat +set CATALINA=%BASEDIR%\oozie-server\bin\catalina.bat if "%actionCmd%" == "start" goto setup_catalina_opts if "%actionCmd%" == "run" goto setup_catalina_opts @@ -46,8 +47,8 @@ goto exec_catalina goto :EOF :setup_catalina_opts - @REM The Java System property 'oozie.http.port' it is not used by Oozie, - @REM it is used in Tomcat's server.xml configuration file + @REM The Java System properties 'oozie.http.port' and 'oozie.https.port' are not + @REM used by Oozie, they are used in Tomcat's server.xml configuration file echo "Using CATALINA_OPTS: %CATALINA_OPTS%" echo "OOZIE_HOME" %OOZIE_HOME% @@ -55,17 +56,23 @@ goto exec_catalina set catalina_opts_tmp=%catalina_opts_tmp% -Doozie.config.dir=%OOZIE_CONFIG% set catalina_opts_tmp=%catalina_opts_tmp% -Doozie.log.dir=%OOZIE_LOG% set catalina_opts_tmp=%catalina_opts_tmp% -Doozie.data.dir=%OOZIE_DATA% + set catalina_opts_tmp=%catalina_opts_tmp% -Doozie.instance.id=%OOZIE_INSTANCE_ID% set catalina_opts_tmp=%catalina_opts_tmp% -Doozie.config.file=%OOZIE_CONFIG_FILE% set catalina_opts_tmp=%catalina_opts_tmp% -Doozie.log4j.file=%OOZIE_LOG4J_FILE% set catalina_opts_tmp=%catalina_opts_tmp% -Doozie.log4j.reload=%OOZIE_LOG4J_RELOAD% - set catalina_opts_tmp=%catalina_opts_tmp% -Doozie.http.=%OOZIE_HTTP_HOSTNAME% + set catalina_opts_tmp=%catalina_opts_tmp% -Doozie.http.hostname=%OOZIE_HTTP_HOSTNAME% set catalina_opts_tmp=%catalina_opts_tmp% -Doozie.http.port=%OOZIE_HTTP_PORT% + set catalina_opts_tmp=%catalina_opts_tmp% -Doozie.https.port=%OOZIE_HTTPS_PORT% set catalina_opts_tmp=%catalina_opts_tmp% -Doozie.base.url=%OOZIE_BASE_URL% + set catalina_opts_tmp=%catalina_opts_tmp% -Doozie.https.keystore.file=%OOZIE_HTTPS_KEYSTORE_FILE% + set catalina_opts_tmp=%catalina_opts_tmp% -Doozie.https.keystore.pass=%OOZIE_HTTPS_KEYSTORE_PASS% + # add required native libraries such as compression codecs + set catalina_opts_tmp=%catalina_opts_tmp% -Djava.library.path=%JAVA_LIBRARY_PATH% echo "Adding to CATALINA_OPTS: %catalina_opts_tmp%" set CATALINA_OPTS=%CATALINA_OPTS% %catalina_opts_tmp% echo CATALINA_OPTS: %CATALINA_OPTS% http://git-wip-us.apache.org/repos/asf/oozie/blob/34904ff5/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index c685ace..c5cdf6f 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.1.0 release (trunk - unreleased) +OOZIE-1552 Bring Windows shell script functionality and structure in line with trunk (omaliuvanchuk via rkanter) OOZIE-1608 Update Curator to 2.4.0 when its available to fix security hole (rkanter) OOZIE-1687 Bundle can still be in RUNNINGWITHERROR status after bundle kill (rohini) OOZIE-1684 DB upgrade from 3.3.0 to trunk fails on Oracle (rkanter)
