AMBARI-8373 Refactor the OS-dependent Ambari Server Windows components Refactored the Ambari Server setup+reset+update. +Pervasively adopted the OS-independent factory-based object model +Unwired the setup-related code from ambari-server.py into separate files. +Fixed the unit tests
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2914d681 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2914d681 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2914d681 Branch: refs/heads/trunk Commit: 2914d681dacc675a0a4a379fd20066cfd415e8b8 Parents: bee9379 Author: Florian Barca <fbarca@HW10923.local> Authored: Fri Dec 19 11:41:49 2014 -0800 Committer: Florian Barca <fbarca@HW10923.local> Committed: Fri Dec 19 11:41:49 2014 -0800 ---------------------------------------------------------------------- .../src/main/python/ambari_agent/HostInfo.py | 42 + .../src/main/python/ambari_commons/firewall.py | 93 +- .../main/python/ambari_commons/inet_utils.py | 17 +- .../python/ambari_commons/os_family_impl.py | 35 +- .../src/main/python/ambari_commons/os_linux.py | 13 +- .../src/main/python/ambari_commons/os_utils.py | 20 +- .../main/python/ambari_commons/os_windows.py | 147 +- .../core/providers/windows/system.py | 4 +- .../functions/get_unique_id_and_date.py | 33 +- ambari-server/conf/unix/ambari.properties | 17 +- ambari-server/conf/windows/ambari.properties | 15 +- .../src/main/python/ambari-server-windows.py | 78 +- ambari-server/src/main/python/ambari-server.py | 1272 +------- .../python/ambari_server/dbConfiguration.py | 318 +- .../ambari_server/dbConfiguration_linux.py | 1189 +++---- .../ambari_server/dbConfiguration_windows.py | 242 +- .../src/main/python/ambari_server/properties.py | 15 +- .../python/ambari_server/resourceFilesKeeper.py | 4 +- .../python/ambari_server/serverConfiguration.py | 277 +- .../ambari_server/serverConfiguration_linux.py | 67 - .../serverConfiguration_windows.py | 98 - .../main/python/ambari_server/serverSetup.py | 1038 ++++-- .../python/ambari_server/serverSetup_linux.py | 795 ----- .../python/ambari_server/serverSetup_windows.py | 313 -- .../main/python/ambari_server/setupSecurity.py | 111 +- .../src/main/python/ambari_server/userInput.py | 2 +- .../custom_actions/scripts/check_host.py | 24 +- .../src/test/python/TestAmbariServer.py | 3066 ++++++++++-------- .../test/python/custom_actions/TestCheckHost.py | 14 +- 29 files changed, 4149 insertions(+), 5210 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-agent/src/main/python/ambari_agent/HostInfo.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/HostInfo.py b/ambari-agent/src/main/python/ambari_agent/HostInfo.py index a99a85d..5789160 100644 --- a/ambari-agent/src/main/python/ambari_agent/HostInfo.py +++ b/ambari-agent/src/main/python/ambari_agent/HostInfo.py @@ -113,6 +113,48 @@ class HostInfo(object): pass return False + def getTransparentHugePage(self): + """ + Base declaration. The derived classes are supposed to override it and provide OS-specific implementations. + :return: + """ + pass + + def checkUsers(self, users, results): + """ + Base declaration. The derived classes are supposed to override it and provide OS-specific implementations. + :return: + """ + pass + + def checkIptables(self): + """ + Base declaration. The derived classes are supposed to override it and provide OS-specific implementations. + :return: + """ + pass + + def javaProcs(self, list): + """ + Base declaration. The derived classes are supposed to override it and provide OS-specific implementations. + :return: + """ + pass + + def getServiceStatus(self, serivce_name): + """ + Base declaration. The derived classes are supposed to override it and provide OS-specific implementations. + :return: + """ + pass + + def register(self, dict, componentsMapped=True, commandsInProgress=True): + """ + Base declaration. The derived classes are supposed to override it and provide OS-specific implementations. + :return: + """ + pass + @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) class HostInfoLinux(HostInfo): http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-common/src/main/python/ambari_commons/firewall.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/firewall.py b/ambari-common/src/main/python/ambari_commons/firewall.py index b73cc0c..448d24e 100644 --- a/ambari-common/src/main/python/ambari_commons/firewall.py +++ b/ambari-common/src/main/python/ambari_commons/firewall.py @@ -21,6 +21,8 @@ limitations under the License. import subprocess import shlex from ambari_commons import OSCheck, OSConst +from ambari_commons.logging_utils import print_warning_msg +from ambari_commons.os_family_impl import OsFamilyImpl class Firewall(object): @@ -31,6 +33,16 @@ class Firewall(object): self.OS_FAMILY = OSCheck.get_os_family() def getFirewallObject(self): + pass + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class FirewallWindows(Firewall): + def getFirewallObject(self): + return WindowsFirewallChecks() + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class FirewallLinux(Firewall): + def getFirewallObject(self): if self.OS_TYPE == OSConst.OS_UBUNTU: return UbuntuFirewallChecks() elif self.OS_TYPE == OSConst.OS_FEDORA and int(self.OS_VERSION) >= 18: @@ -49,23 +61,31 @@ class FirewallChecks(object): self.returncode = None self.stdoutdata = None self.stderrdata = None + # stdout message + self.MESSAGE_CHECK_FIREWALL = 'Checking iptables...' def get_command(self): return "%s %s %s" % (self.SERVICE_CMD, self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD) - def check_result(self, retcode, out, err): + def check_result(self): result = False - if retcode == 3: + if self.returncode == 3: result = False - elif retcode == 0: - if "Table: filter" in out: + elif self.returncode == 0: + if "Table: filter" in self.stdoutdata: result = True return result + def run_command(self): + retcode, out, err = self.run_os_command(self.get_command()) + self.returncode = retcode + self.stdoutdata = out + self.stderrdata = err + def check_iptables(self): try: - retcode, out, err = self.run_os_command(self.get_command()) - return self.check_result(retcode, out, err) + self.run_command() + return self.check_result() except OSError: return False @@ -76,11 +96,7 @@ class FirewallChecks(object): process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) (stdoutdata, stderrdata) = process.communicate() - self.returncode = process.returncode - self.stdoutdata = stdoutdata - self.stderrdata = stderrdata - return self.returncode, self.stdoutdata, self.stderrdata - + return process.returncode, stdoutdata, stderrdata class UbuntuFirewallChecks(FirewallChecks): @@ -91,13 +107,13 @@ class UbuntuFirewallChecks(FirewallChecks): def get_command(self): return "%s %s" % (self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD) - def check_result(self, retcode, out, err): + def check_result(self): # On ubuntu, the status command returns 0 whether running or not result = False - if retcode == 0: - if "Status: inactive" in out: + if self.returncode == 0: + if "Status: inactive" in self.stdoutdata: result = False - elif "Status: active" in out: + elif "Status: active" in self.stdoutdata: result = True return result @@ -108,10 +124,10 @@ class Fedora18FirewallChecks(FirewallChecks): def get_command(self): return "systemctl is-active %s" % (self.FIREWALL_SERVICE_NAME) - def check_result(self, retcode, out, err): + def check_result(self): result = False - if retcode == 0: - if "active" in out: + if self.returncode == 0: + if "active" in self.stdoutdata: result = True return result @@ -123,11 +139,44 @@ class SuseFirewallChecks(FirewallChecks): def get_command(self): return "%s %s" % (self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD) - def check_result(self, retcode, out, err): + def check_result(self): result = False - if retcode == 0: - if "SuSEfirewall2 not active" in out: + if self.returncode == 0: + if "SuSEfirewall2 not active" in self.stdoutdata: result = False - elif "### iptables" in out: + elif "### iptables" in self.stdoutdata: result = True return result + + +class WindowsFirewallChecks(FirewallChecks): + def __init__(self): + super(WindowsFirewallChecks, self).__init__() + self.MESSAGE_CHECK_FIREWALL = 'Checking firewall status...' + + def run_command(self): + from ambari_commons.os_windows import run_powershell_script, CHECK_FIREWALL_SCRIPT + + retcode, out, err = run_powershell_script(CHECK_FIREWALL_SCRIPT) + self.returncode = retcode + self.stdoutdata = out + self.stderrdata = err + + def check_result(self): + if self.returncode != 0: + print_warning_msg("Unable to check firewall status:{0}".format(self.stderrdata)) + return False + profiles_status = [i for i in self.stdoutdata.split("\n") if not i == ""] + if "1" in profiles_status: + enabled_profiles = [] + if profiles_status[0] == "1": + enabled_profiles.append("DomainProfile") + if profiles_status[1] == "1": + enabled_profiles.append("StandardProfile") + if profiles_status[2] == "1": + enabled_profiles.append("PublicProfile") + print_warning_msg( + "Following firewall profiles are enabled:{0}. Make sure that the firewall is properly configured.".format( + ",".join(enabled_profiles))) + return False + return True http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-common/src/main/python/ambari_commons/inet_utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/inet_utils.py b/ambari-common/src/main/python/ambari_commons/inet_utils.py index 2a54cb6..804c63f 100644 --- a/ambari-common/src/main/python/ambari_commons/inet_utils.py +++ b/ambari-common/src/main/python/ambari_commons/inet_utils.py @@ -22,8 +22,9 @@ import os import sys import urllib2 -from exceptions import * -from logging_utils import * +from ambari_commons.exceptions import FatalException +from ambari_commons.logging_utils import print_info_msg, print_warning_msg + def download_file(link, destination, chunk_size=16 * 1024): print_info_msg("Downloading {0} to {1}".format(link, destination)) @@ -34,6 +35,10 @@ def download_file(link, destination, chunk_size=16 * 1024): force_download_file(link, destination, chunk_size) +def download_progress_stub(file_name, downloaded_size, blockSize, totalSize): + #Facilitates mocking. Normally, the callback used by force_download_file eludes the mocks. + download_progress(file_name, downloaded_size, blockSize, totalSize) + def download_progress(file_name, downloaded_size, blockSize, totalSize): percent = int(downloaded_size * 100 / totalSize) status = "\r" + file_name @@ -56,7 +61,7 @@ def find_range_components(meta): if len(range_comp1) > 1: range_comp2 = range_comp1[0].split(' ') #split away the "bytes" prefix if len(range_comp2) == 0: - raise FatalException(12, 'Malformed Content-Range response header: "{}".' % hdr_range) + raise FatalException(12, 'Malformed Content-Range response header: "{0}".' % hdr_range) range_comp3 = range_comp2[1].split('-') seek_pos = int(range_comp3[0]) if range_comp1[1] != '*': #'*' == unknown length @@ -72,12 +77,12 @@ def find_range_components(meta): return (file_size, seek_pos) -def force_download_file(link, destination, chunk_size = 16 * 1024, progress_func = download_progress): +def force_download_file(link, destination, chunk_size = 16 * 1024, progress_func = download_progress_stub): request = urllib2.Request(link) if os.path.exists(destination) and not os.path.isfile(destination): #Directory specified as target? Must be a mistake. Bail out, don't assume anything. - err = 'Download target {} is a directory.' % destination + err = 'Download target {0} is a directory.' % destination raise FatalException(1, err) (dest_path, file_name) = os.path.split(destination) @@ -138,7 +143,7 @@ def force_download_file(link, destination, chunk_size = 16 * 1024, progress_func downloaded_size = os.stat(temp_dest).st_size if downloaded_size != file_size: - err = 'Size of downloaded file {} is {} bytes, it is probably damaged or incomplete' % (destination, downloaded_size) + err = 'Size of downloaded file {0} is {0} bytes, it is probably damaged or incomplete' % (destination, downloaded_size) raise FatalException(1, err) # when download is complete -> mv temp_dest destination http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-common/src/main/python/ambari_commons/os_family_impl.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/os_family_impl.py b/ambari-common/src/main/python/ambari_commons/os_family_impl.py index f8a3379..d70d942 100644 --- a/ambari-common/src/main/python/ambari_commons/os_family_impl.py +++ b/ambari-common/src/main/python/ambari_commons/os_family_impl.py @@ -17,15 +17,15 @@ limitations under the License. ''' import types -from os_check import OSCheck +from ambari_commons import OSCheck class OsFamilyImpl(object): """ - Base class for os depended factory. Usage:: + Base class for os dependent factory. Usage:: class BaseFoo(object): pass - @Factory("windows") + @OsFamilyImpl("windows") class OsFoo(object):pass print BaseFoo()# OsFoo @@ -62,3 +62,32 @@ class OsFamilyImpl(object): base_cls.__new__ = types.MethodType(new, base_cls) return cls + +class OsFamilyFuncImpl(object): + """ + Base class for os dependent function. Usage:: + + @OSFamilyFuncImpl("windows") + def os_foo(...):pass + + """ + _func_impls = {} + + def _createFunctionInstance(self, func): + self._func_impls[func.__module__ + "." + func.__name__ + "." + self.os_const] = func + + def thunk(*args, **kwargs): + fn_id_base = func.__module__ + "." + func.__name__ + fn_id = fn_id_base + "." + OSCheck.get_os_family() + if fn_id not in self._func_impls: + fn_id = fn_id_base + "." + OsFamilyImpl.DEFAULT + + fn = self._func_impls[fn_id] + return fn(*args, **kwargs) + return thunk + + def __init__(self, os_family): + self.os_const = os_family + + def __call__(self, func): + return self._createFunctionInstance(func) http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-common/src/main/python/ambari_commons/os_linux.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/os_linux.py b/ambari-common/src/main/python/ambari_commons/os_linux.py index 38f3fb9..eca3c5d 100644 --- a/ambari-common/src/main/python/ambari_commons/os_linux.py +++ b/ambari-common/src/main/python/ambari_commons/os_linux.py @@ -24,7 +24,7 @@ import pwd import shlex import subprocess -from logging_utils import * +from ambari_commons.logging_utils import print_info_msg, print_warning_msg NR_CHMOD_CMD = 'chmod {0} {1} {2}' @@ -33,14 +33,15 @@ NR_CHOWN_CMD = 'chown {0} {1} {2}' ULIMIT_CMD = "ulimit -n" -def run_os_command(cmd): +def os_run_os_command(cmd, env=None): print_info_msg('about to run command: ' + str(cmd)) if type(cmd) == str: cmd = shlex.split(cmd) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.PIPE + stderr=subprocess.PIPE, + env=env ) (stdoutdata, stderrdata) = process.communicate() return process.returncode, stdoutdata, stderrdata @@ -64,17 +65,17 @@ def os_set_file_permissions(file, mod, recursive, user): else: params = "" command = NR_CHMOD_CMD.format(params, mod, file) - retcode, out, err = run_os_command(command) + retcode, out, err = os_run_os_command(command) if retcode != 0: print_warning_msg(WARN_MSG.format(command, file, err)) command = NR_CHOWN_CMD.format(params, user, file) - retcode, out, err = run_os_command(command) + retcode, out, err = os_run_os_command(command) if retcode != 0: print_warning_msg(WARN_MSG.format(command, file, err)) def os_set_open_files_limit(maxOpenFiles): command = "%s %s" % (ULIMIT_CMD, str(maxOpenFiles)) - run_os_command(command) + os_run_os_command(command) def os_getpass(prompt): http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-common/src/main/python/ambari_commons/os_utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/os_utils.py b/ambari-common/src/main/python/ambari_commons/os_utils.py index 3f4819d..6e2c1d2 100644 --- a/ambari-common/src/main/python/ambari_commons/os_utils.py +++ b/ambari-common/src/main/python/ambari_commons/os_utils.py @@ -18,20 +18,23 @@ See the License for the specific language governing permissions and limitations under the License. ''' +import os import shutil import string -import os -from os_check import * +from ambari_commons import OSCheck if OSCheck.is_windows_family(): - from os_windows import * + from ambari_commons.os_windows import os_change_owner, os_getpass, os_is_root, os_run_os_command, \ + os_set_open_files_limit, os_set_file_permissions else: # MacOS not supported - from os_linux import * + from ambari_commons.os_linux import os_change_owner, os_getpass, os_is_root, os_run_os_command, \ + os_set_open_files_limit, os_set_file_permissions + pass -from logging_utils import * -from exceptions import FatalException +from ambari_commons.exceptions import FatalException +from ambari_commons.logging_utils import print_info_msg, print_warning_msg def is_valid_filepath(filepath): @@ -88,6 +91,9 @@ def set_file_permissions(file, mod, user, recursive): else: print_info_msg("File %s does not exist" % file) +def run_os_command(cmd, env=None): + return os_run_os_command(cmd, env) + def is_root(): return os_is_root() @@ -108,4 +114,4 @@ def find_in_path(file): full_path = os.path.join(dir, file) if os.path.exists(full_path): return full_path - raise Exception("File {} not found in PATH".format(file)) \ No newline at end of file + raise Exception("File {} not found in PATH".format(file)) http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-common/src/main/python/ambari_commons/os_windows.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/os_windows.py b/ambari-common/src/main/python/ambari_commons/os_windows.py index 2fb98e4..0cc4ea2 100644 --- a/ambari-common/src/main/python/ambari_commons/os_windows.py +++ b/ambari-common/src/main/python/ambari_commons/os_windows.py @@ -17,41 +17,35 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ''' -import os import getpass +import os +import random import shlex import subprocess import sys +import tempfile import time -import win32api -import win32event -import win32service -import win32con -import win32serviceutil -import wmi -import random import string import ctypes -from win32security import * -from win32api import * -from winerror import ERROR_INVALID_HANDLE -from win32process import GetExitCodeProcess, STARTF_USESTDHANDLES, STARTUPINFO, CreateProcessAsUser -from win32event import WaitForSingleObject, INFINITE import msvcrt -import tempfile -from win32event import * -from win32api import CloseHandle - -from ambari_commons.exceptions import * -from logging_utils import * -from win32security import LsaOpenPolicy, POLICY_CREATE_ACCOUNT, POLICY_LOOKUP_NAMES, LookupAccountName, \ - LsaAddAccountRights, LsaRemoveAccountRights, SE_SERVICE_LOGON_NAME -from win32net import NetUserAdd -from win32netcon import USER_PRIV_USER, UF_NORMAL_ACCOUNT, UF_SCRIPT import pywintypes +import winerror +import win32api +import win32con +import win32event +import win32net +import win32netcon +import win32process +import win32security +import win32service +import win32serviceutil +import wmi + +from ambari_commons.exceptions import FatalException +from ambari_commons.logging_utils import print_info_msg, print_warning_msg SERVICE_STATUS_UNKNOWN = "unknown" SERVICE_STATUS_STARTING = "starting" @@ -77,17 +71,17 @@ class OSVERSIONINFOEXW(ctypes.Structure): ('wReserved', ctypes.c_byte)] def get_windows_version(): - """ - Get's the OS major and minor versions. Returns a tuple of - (OS_MAJOR, OS_MINOR). - """ - os_version = OSVERSIONINFOEXW() - os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version) - retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version)) - if retcode != 0: - raise Exception("Failed to get OS version") - - return os_version.dwMajorVersion, os_version.dwMinorVersion, os_version.dwBuildNumber + """ + Gets the OS major and minor versions. Returns a tuple of + (OS_MAJOR, OS_MINOR). + """ + os_version = OSVERSIONINFOEXW() + os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version) + retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version)) + if retcode != 0: + raise Exception("Failed to get OS version") + + return os_version.dwMajorVersion, os_version.dwMinorVersion, os_version.dwBuildNumber CHECK_FIREWALL_SCRIPT = """[string]$CName = $env:computername $reg = [Microsoft.Win32.RegistryKey]::OpenRemoteBaseKey("LocalMachine",$computer) @@ -116,25 +110,25 @@ def _get_files_output(out, err): def _safe_duplicate_handle(h): try: - h = DuplicateHandle(GetCurrentProcess(), - h, - GetCurrentProcess(), - 0, - True, - win32con.DUPLICATE_SAME_ACCESS) + h = win32api.DuplicateHandle(win32process.GetCurrentProcess(), + h, + win32process.GetCurrentProcess(), + 0, + True, + win32con.DUPLICATE_SAME_ACCESS) return True, h except Exception as exc: - if exc.winerror == ERROR_INVALID_HANDLE: + if exc.winerror == winerror.ERROR_INVALID_HANDLE: return True, None return False, None def run_os_command_impersonated(cmd, user, password, domain='.'): - si = STARTUPINFO() + si = win32process.STARTUPINFO() out_handle, err_handle, out_file, err_file = _create_tmp_files() - ok, si.hStdInput = _safe_duplicate_handle(GetStdHandle(STD_INPUT_HANDLE)) + ok, si.hStdInput = _safe_duplicate_handle(win32api.GetStdHandle(win32api.STD_INPUT_HANDLE)) if not ok: raise Exception("Unable to create StdInput for child process") @@ -145,27 +139,29 @@ def run_os_command_impersonated(cmd, user, password, domain='.'): if not ok: raise Exception("Unable to create StdErr for child process") - si.dwFlags = STARTF_USESTDHANDLES + si.dwFlags = win32process.STARTF_USESTDHANDLES si.lpDesktop = "" - user_token = LogonUser(user, domain, password, win32con.LOGON32_LOGON_SERVICE, win32con.LOGON32_PROVIDER_DEFAULT) - primary_token = DuplicateTokenEx(user_token, SecurityImpersonation, 0, TokenPrimary) - info = CreateProcessAsUser(primary_token, None, cmd, None, None, 1, 0, None, None, si) + user_token = win32security.LogonUser(user, domain, password, + win32con.LOGON32_LOGON_SERVICE, win32con.LOGON32_PROVIDER_DEFAULT) + primary_token = win32security.DuplicateTokenEx(user_token, + win32security.SecurityImpersonation, 0, win32security.TokenPrimary) + info = win32process.CreateProcessAsUser(primary_token, None, cmd, None, None, 1, 0, None, None, si) hProcess, hThread, dwProcessId, dwThreadId = info hThread.Close() try: - WaitForSingleObject(hProcess, INFINITE) + win32event.WaitForSingleObject(hProcess, win32event.INFINITE) except KeyboardInterrupt: pass out, err = _get_files_output(out_file, err_file) - exitcode = GetExitCodeProcess(hProcess) + exitcode = win32process.GetExitCodeProcess(hProcess) return exitcode, out, err -def run_os_command(cmd, env=None): +def os_run_os_command(cmd, env=None): if isinstance(cmd,basestring): cmd = cmd.replace("\\", "\\\\") cmd = shlex.split(cmd) @@ -186,13 +182,13 @@ def run_powershell_script(script_content): script_file = open(os.path.join(tmp_dir,random_filename+".ps1"),"w") script_file.write(script_content) script_file.close() - result = run_os_command("powershell -ExecutionPolicy unrestricted -File {0}".format(script_file.name)) + result = os_run_os_command("powershell -ExecutionPolicy unrestricted -File {0}".format(script_file.name)) os.remove(script_file.name) return result def os_change_owner(filePath, user): cmd = ['icacls', filePath, '/setowner', user] - retcode, outdata, errdata = run_os_command(cmd) + retcode, outdata, errdata = os_run_os_command(cmd) return retcode def os_is_root(): @@ -200,9 +196,10 @@ def os_is_root(): Checks whether the current user is a member of the Administrators group Returns True if yes, otherwise False ''' - retcode, out, err = run_os_command(WHOAMI_GROUPS) + retcode, out, err = os_run_os_command(WHOAMI_GROUPS) if retcode != 0: - err_msg = "Unable to check the current user's group memberships. Command {0} returned exit code {1} with message: {2}".format(WHOAMI_GROUPS, retcode, err) + err_msg = "Unable to check the current user's group memberships. " \ + "Command {0} returned exit code {1} with message: {2}".format(WHOAMI_GROUPS, retcode, err) print_warning_msg(err_msg) raise FatalException(retcode, err_msg) @@ -221,20 +218,20 @@ def os_set_file_permissions(file, mod, recursive, user): #else: # params = "" #command = NR_CHMOD_CMD.format(params, mod, file) - #retcode, out, err = run_os_command(command) + #retcode, out, err = os_run_os_command(command) #if retcode != 0: # print_warning_msg(WARN_MSG.format(command, file, err)) #command = NR_CHOWN_CMD.format(params, user, file) - #retcode, out, err = run_os_command(command) + #retcode, out, err = os_run_os_command(command) #if retcode != 0: # print_warning_msg(WARN_MSG.format(command, file, err)) # rights = mod # acls_remove_cmd = "icacls {0} /remove {1}".format(file, user) - # retcode, out, err = run_os_command(acls_remove_cmd) + # retcode, out, err = os_run_os_command(acls_remove_cmd) # if retcode == 0: # acls_modify_cmd = "icacls {0} /grant {1}:{2}".format(file, user, rights) - # retcode, out, err = run_os_command(acls_modify_cmd) + # retcode, out, err = os_run_os_command(acls_modify_cmd) return retcode @@ -248,8 +245,6 @@ def os_getpass(prompt, stream=None): if sys.stdin is not sys.__stdin__: return getpass.fallback_getpass(prompt, stream) - import msvcrt - for c in prompt: msvcrt.putch(c) @@ -487,7 +482,8 @@ class WinService(win32serviceutil.ServiceFramework): def _StopOrWaitForChildProcessToFinish(self, childProcess): #Wait for the child process to finish or for the stop event to be signaled - if(win32event.WAIT_OBJECT_0 == win32event.WaitForMultipleObjects([self._heventSvcStop, childProcess._handle], False, win32event.INFINITE)): + if(win32event.WAIT_OBJECT_0 == win32event.WaitForMultipleObjects([self._heventSvcStop, childProcess._handle], + False, win32event.INFINITE)): # The OS only detaches the child process when the master process exits. # We must kill it manually. try: @@ -503,24 +499,24 @@ class WinService(win32serviceutil.ServiceFramework): class SystemWideLock(object): def __init__(self, name): - self._mutex = CreateMutex(None, 0, name) + self._mutex = win32event.CreateMutex(None, 0, name) def lock(self, timeout=0): - result = WaitForSingleObject(self._mutex, timeout) - if result in [WAIT_TIMEOUT, WAIT_ABANDONED, WAIT_FAILED]: + result = win32event.WaitForSingleObject(self._mutex, timeout) + if result in [win32event.WAIT_TIMEOUT, win32event.WAIT_ABANDONED, win32event.WAIT_FAILED]: return False - elif result == WAIT_OBJECT_0: + elif result == win32event.WAIT_OBJECT_0: return True def unlock(self): try: - ReleaseMutex(self._mutex) + win32event.ReleaseMutex(self._mutex) return True except: return False def __del__(self): - CloseHandle(self._mutex) + win32api.CloseHandle(self._mutex) class UserHelper(object): ACTION_OK = 0 @@ -528,17 +524,18 @@ class UserHelper(object): ACTION_FAILED = -1 def __init__(self): - self._policy = LsaOpenPolicy(None, POLICY_CREATE_ACCOUNT | POLICY_LOOKUP_NAMES) + self._policy = win32security.LsaOpenPolicy(None, + win32security.POLICY_CREATE_ACCOUNT | win32security.POLICY_LOOKUP_NAMES) def create_user(self, name, password, comment="Ambari user"): user_info = {} user_info['name'] = name user_info['password'] = password - user_info['priv'] = USER_PRIV_USER + user_info['priv'] = win32netcon.USER_PRIV_USER user_info['comment'] = comment - user_info['flags'] = UF_NORMAL_ACCOUNT | UF_SCRIPT + user_info['flags'] = win32netcon.UF_NORMAL_ACCOUNT | win32netcon.UF_SCRIPT try: - NetUserAdd(None, 1, user_info) + win32net.NetUserAdd(None, 1, user_info) except pywintypes.error as e: if e.winerror == 2224: return UserHelper.USER_EXISTS, e.strerror @@ -548,16 +545,16 @@ class UserHelper(object): def add_user_privilege(self, name, privilege): try: - acc_sid = LookupAccountName(None, name)[0] - LsaAddAccountRights(self._policy, acc_sid, (privilege,)) + acc_sid = win32security.LookupAccountName(None, name)[0] + win32security.LsaAddAccountRights(self._policy, acc_sid, (privilege,)) except pywintypes.error as e: return UserHelper.ACTION_FAILED, e.strerror return UserHelper.ACTION_OK, "Privilege added." def remove_user_privilege(self, name, privilege): try: - acc_sid = LookupAccountName(None, name)[0] - LsaRemoveAccountRights(self._policy, acc_sid, 0, (privilege,)) + acc_sid = win32security.LookupAccountName(None, name)[0] + win32security.LsaRemoveAccountRights(self._policy, acc_sid, 0, (privilege,)) except pywintypes.error as e: return UserHelper.ACTION_FAILED, e.strerror return UserHelper.ACTION_OK, "Privilege removed." http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-common/src/main/python/resource_management/core/providers/windows/system.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/windows/system.py b/ambari-common/src/main/python/resource_management/core/providers/windows/system.py index e7a98fc..dac44ec 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/windows/system.py +++ b/ambari-common/src/main/python/resource_management/core/providers/windows/system.py @@ -140,7 +140,7 @@ def QueryPrivilegeState(hToken, priv): for (id, attr) in privList: if id == privId: privState = attr - Logger.debug('Privilege state: {}={} ({}) Enabled={}'.format(privId, priv, LookupPrivilegeDisplayName(None, priv), privState)) + Logger.debug('Privilege state: {0}={1} ({2}) Enabled={3}'.format(privId, priv, LookupPrivilegeDisplayName(None, priv), privState)) return privState # Execute command. As windows hdp stack heavily relies on proper environment it is better to reload fresh environment @@ -182,7 +182,7 @@ def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish= if not ok: raise Exception("Unable to create StdErr for child process") - Logger.debug("Redirecting stdout to '{}', stderr to '{}'".format(out_file.name, err_file.name)) + Logger.debug("Redirecting stdout to '{0}', stderr to '{1}'".format(out_file.name, err_file.name)) si.dwFlags = win32con.STARTF_USESTDHANDLES si.lpDesktop = "" http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py index afc82bb..3f8e983 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py @@ -23,17 +23,30 @@ Ambari Agent __all__ = ["get_unique_id_and_date"] import datetime from resource_management.core import shell -from ambari_commons import os_check +from ambari_commons import os_check, OSConst +from ambari_commons.os_family_impl import OsFamilyImpl, OsFamilyFuncImpl + + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def get_volume_serial_number(): + from ambari_commons.os_windows import os_run_os_command + + id = "" + code, out, err = os_run_os_command("cmd /c vol C:") + for line in out.splitlines(): + if line.startswith(" Volume Serial Number is"): + id = line[25:] + + return id + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def get_volume_serial_number(): + out = shell.checked_call("hostid")[1].split('\n')[-1] # bugfix: take the lastline (stdin is not tty part cut) + id = out.strip() + return id + def get_unique_id_and_date(): - if os_check.OSCheck.is_windows_family(): - from ambari_commons.os_windows import run_os_command - code, out, err = run_os_command("cmd /c vol C:") - for line in out.splitlines(): - if line.startswith(" Volume Serial Number is"): - id = line[25:] - else: - out = shell.checked_call("hostid")[1].split('\n')[-1] # bugfix: take the lastline (stdin is not tty part cut) - id = out.strip() + id = get_volume_serial_number() now = datetime.datetime.now() date = now.strftime("%M%d%y") http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/conf/unix/ambari.properties ---------------------------------------------------------------------- diff --git a/ambari-server/conf/unix/ambari.properties b/ambari-server/conf/unix/ambari.properties index e29a6e2..84c96a3 100644 --- a/ambari-server/conf/unix/ambari.properties +++ b/ambari-server/conf/unix/ambari.properties @@ -22,10 +22,23 @@ security.server.keys_dir = /var/lib/ambari-server/keys resources.dir = /var/lib/ambari-server/resources shared.resources.dir = /usr/lib/ambari-server/lib/ambari_commons/resources custom.action.definitions = /var/lib/ambari-server/resources/custom_action_definitions + +java.releases=jdk1.6,jdk1.7 +jdk1.6.desc=Oracle JDK 1.6 jdk1.6.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-6u31-linux-x64.bin -jce_policy1.6.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jce_policy-6.zip +jdk1.6.dest-file=jdk-6u31-linux-x64.bin +jdk1.6.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/jce_policy-6.zip +jdk1.6.jcpol-file=jce_policy-6.zip +jdk1.6.home=/usr/jdk64/ +jdk1.6.re=Creating (jdk.*)/jre +jdk1.7.desc=Oracle JDK 1.7 jdk1.7.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-linux-x64.tar.gz -jce_policy1.7.url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip +jdk1.7.dest-file=jdk-7u67-linux-x64.tar.gz +jdk1.7.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip +jdk1.7.jcpol-file=UnlimitedJCEPolicyJDK7.zip +jdk1.7.home=/usr/jdk64/ +jdk1.7.re=(jdk.*)/jre + metadata.path=/var/lib/ambari-server/resources/stacks common.services.path=/var/lib/ambari-server/resources/common-services server.version.file=/var/lib/ambari-server/resources/version http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/conf/windows/ambari.properties ---------------------------------------------------------------------- diff --git a/ambari-server/conf/windows/ambari.properties b/ambari-server/conf/windows/ambari.properties index 3982bb9..434e42a 100644 --- a/ambari-server/conf/windows/ambari.properties +++ b/ambari-server/conf/windows/ambari.properties @@ -23,13 +23,14 @@ custom.action.definitions=resources\\custom_action_definitions #Comma-separated list of JDK versions #java.releases=jdk1.8.20,jdk1.6.31 -java.releases=jdk1.7.67 -jdk1.7.67.desc=Oracle JDK 1.7.67 -jdk1.7.67.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-windows-x64.exe -jdk1.7.67.dest-file=jdk-7u67-windows-x64.exe -jdk1.7.67.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip -jdk1.7.67.jcpol-file=UnlimitedJCEPolicyJDK7.zip -jdk1.7.67.home=C:\\jdk1.7.0_67 +java.releases=jdk1.7 +jdk1.7.desc=Oracle JDK 1.7 +jdk1.7.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-windows-x64.exe +jdk1.7.dest-file=jdk-7u67-windows-x64.exe +jdk1.7.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip +jdk1.7.jcpol-file=UnlimitedJCEPolicyJDK7.zip +jdk1.7.home=C:\\jdk1.7.0_67 +jdk1.7.re=(jdk.*)/jre metadata.path=resources\\stacks common.services.path=resources\\common-services http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari-server-windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server-windows.py b/ambari-server/src/main/python/ambari-server-windows.py index 6c4f894..8940972 100644 --- a/ambari-server/src/main/python/ambari-server-windows.py +++ b/ambari-server/src/main/python/ambari-server-windows.py @@ -19,20 +19,27 @@ limitations under the License. ''' import optparse - -from ambari_commons.ambari_service import AmbariService, ENV_PYTHON_PATH -from ambari_commons.logging_utils import * -from ambari_commons.os_utils import remove_file +import os +import subprocess +import sys + +from ambari_commons.ambari_service import AmbariService +from ambari_commons.exceptions import NonFatalException, FatalException +from ambari_commons.logging_utils import print_error_msg, print_info_msg, print_warning_msg +from ambari_commons.os_utils import remove_file, set_open_files_limit from ambari_commons.os_windows import SvcStatusCallback -from ambari_server import utils -from ambari_server.dbConfiguration import DBMSConfig +from ambari_server.dbConfiguration import DBMSConfigFactory from ambari_server.resourceFilesKeeper import ResourceFilesKeeper, KeeperException -from ambari_server.serverConfiguration import * -from ambari_server.serverSetup import setup, reset, is_server_running, upgrade -from ambari_server.setupActions import * -from ambari_server.setupSecurity import * -from ambari_server.serverSetup_windows import SERVICE_PASSWORD_KEY, SERVICE_USERNAME_KEY +from ambari_server.serverConfiguration import find_jdk, get_ambari_properties, get_value_from_properties, \ + get_full_ambari_classpath, configDefaults, VERBOSE_OUTPUT_KEY, DEBUG_MODE_KEY, SUSPEND_START_MODE_KEY, \ + SERVER_OUT_FILE_KEY, RESOURCES_DIR_PROPERTY, RESOURCES_DIR_DEFAULT, STACK_LOCATION_KEY, STACK_LOCATION_DEFAULT +from ambari_server.serverSetup import setup, reset, is_server_running, upgrade, SERVICE_PASSWORD_KEY, SERVICE_USERNAME_KEY +from ambari_server.setupActions import SETUP_ACTION, START_ACTION, PSTART_ACTION, STOP_ACTION, RESET_ACTION, \ + STATUS_ACTION, UPGRADE_ACTION, UPGRADE_STACK_ACTION, LDAP_SETUP_ACTION, SETUP_SECURITY_ACTION, ACTION_REQUIRE_RESTART +from ambari_server.setupSecurity import setup_ambari_krb5_jaas, setup_https, setup_ldap, setup_master_key +from ambari_server.userInput import get_validated_string_input +from ambari_server.utils import check_reverse_lookup, save_pid # debug settings SERVER_START_DEBUG = False @@ -42,11 +49,6 @@ SUSPEND_START_MODE = False ambari_provider_module_option = "" ambari_provider_module = os.environ.get('AMBARI_PROVIDER_MODULE') -#Common setup or upgrade message -SETUP_OR_UPGRADE_MSG = "- If this is a new setup, then run the \"ambari-server setup\" command to create the user\n" \ -"- If this is an upgrade of an existing setup, run the \"ambari-server upgrade\" command.\n" \ -"Refer to the Ambari documentation for more information on setup and upgrade." - AMBARI_SERVER_DIE_MSG = "Ambari Server java process died with exitcode {0}. Check {1} for more information." if ambari_provider_module is not None: @@ -110,7 +112,7 @@ class AmbariServerService(AmbariService): if not self._StopOrWaitForChildProcessToFinish(childProc): return - pid_file_path = PID_DIR + os.sep + PID_NAME + pid_file_path = os.path.join(configDefaults.PID_DIR, PID_NAME) remove_file(pid_file_path) pass @@ -122,7 +124,7 @@ class AmbariServerService(AmbariService): outFilePath = properties[SERVER_OUT_FILE_KEY] if (outFilePath is None or outFilePath == ""): - outFilePath = SERVER_OUT_FILE + outFilePath = configDefaults.SERVER_OUT_FILE self._RedirectOutputStreamsToFile(outFilePath) pass @@ -157,7 +159,7 @@ def start(options): childProc.wait() - pid_file_path = PID_DIR + os.sep + PID_NAME + pid_file_path = os.path.join(configDefaults.PID_DIR, PID_NAME) remove_file(pid_file_path) # @@ -199,7 +201,7 @@ def server_process_main(options, scmStatus=None): except AttributeError: pass - if not utils.check_reverse_lookup(): + if not check_reverse_lookup(): print_warning_msg("The hostname was not found in the reverse DNS lookup. " "This may result in incorrect behavior. " "Please check the DNS setup and fix the issue.") @@ -208,12 +210,11 @@ def server_process_main(options, scmStatus=None): print_info_msg("Ambari Server is not running...") - conf_dir = get_conf_dir() jdk_path = find_jdk() if jdk_path is None: err = "No JDK found, please run the \"ambari-server setup\" " \ "command to install a JDK automatically or install any " \ - "JDK manually to " + JDK_INSTALL_DIR + "JDK manually to " + configDefaults.JDK_INSTALL_DIR raise FatalException(1, err) # Preparations @@ -236,17 +237,15 @@ def server_process_main(options, scmStatus=None): if scmStatus is not None: scmStatus.reportStartPending() - conf_dir = os.path.abspath(conf_dir) + os.pathsep + get_ambari_classpath() - if conf_dir.find(' ') != -1: - conf_dir = '"' + conf_dir + '"' + conf_dir = get_full_ambari_classpath() - java_exe = jdk_path + os.sep + JAVA_EXE_SUBPATH - pidfile = PID_DIR + os.sep + PID_NAME + java_exe = os.path.join(jdk_path, configDefaults.JAVA_EXE_SUBPATH) + pidfile = os.path.join(configDefaults.PID_DIR, PID_NAME) command_base = SERVER_START_CMD_DEBUG if (DEBUG_MODE or SERVER_START_DEBUG) else SERVER_START_CMD suspend_mode = 'y' if SUSPEND_START_MODE else 'n' command = command_base.format(conf_dir, suspend_mode) - if not os.path.exists(PID_DIR): - os.makedirs(PID_DIR, 0755) + if not os.path.exists(configDefaults.PID_DIR): + os.makedirs(configDefaults.PID_DIR, 0755) set_open_files_limit(get_ulimit_open_files()); @@ -264,18 +263,18 @@ def server_process_main(options, scmStatus=None): if pidJava <= 0: procJava.terminate() exitcode = procJava.returncode - exitfile = os.path.join(PID_DIR, EXITCODE_NAME) - utils.save_pid(exitcode, exitfile) + exitfile = os.path.join(configDefaults.PID_DIR, EXITCODE_NAME) + save_pid(exitcode, exitfile) if scmStatus is not None: scmStatus.reportStopPending() - raise FatalException(-1, AMBARI_SERVER_DIE_MSG.format(exitcode, SERVER_OUT_FILE)) + raise FatalException(-1, AMBARI_SERVER_DIE_MSG.format(exitcode, configDefaults.SERVER_OUT_FILE)) else: - utils.save_pid(pidJava, pidfile) - print "Server PID at: "+pidfile - print "Server out at: "+SERVER_OUT_FILE - print "Server log at: "+SERVER_LOG_FILE + save_pid(pidJava, pidfile) + print "Server PID at: "+ pidfile + print "Server out at: "+ configDefaults.SERVER_OUT_FILE + print "Server log at: "+ configDefaults.SERVER_LOG_FILE if scmStatus is not None: scmStatus.reportStarted() @@ -289,13 +288,15 @@ def server_process_main(options, scmStatus=None): #Wait until the status is 'started' or a configured timeout elapses #If the timeout has been reached, bail out with exception def ensure_dbms_is_running(options, properties, scmStatus): - dbms = DBMSConfig.create(options, properties, "Ambari") + factory = DBMSConfigFactory() + + dbms = factory.create(options, properties, "Ambari") if not dbms._is_jdbc_driver_installed(properties): raise FatalException(-1, "JDBC driver is not installed. Run ambari-server setup and try again.") dbms.ensure_dbms_is_running(options, properties, scmStatus) - dbms2 = DBMSConfig.create(options, properties, "Metrics") + dbms2 = factory.create(options, properties, "Metrics") if dbms2.database_host.lower() != dbms.database_host.lower(): dbms2.ensure_dbms_is_running(options, properties, scmStatus) pass @@ -499,6 +500,7 @@ def main(): #perform checks options.warnings = [] + options.exit_message = None options.must_set_database_options = False if are_cmd_line_db_args_blank(options):
