Revert "AMBARI-8373 Refactor the OS-dependent Ambari Server Windows components"
This reverts commit 2914d681dacc675a0a4a379fd20066cfd415e8b8. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fad56746 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fad56746 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fad56746 Branch: refs/heads/trunk Commit: fad56746f7a6181c883fa0b0ba8e58f137dfde52 Parents: 613bdd7 Author: Yusaku Sako <[email protected]> Authored: Fri Dec 19 13:51:22 2014 -0800 Committer: Yusaku Sako <[email protected]> Committed: Fri Dec 19 13:51:22 2014 -0800 ---------------------------------------------------------------------- .../src/main/python/ambari_agent/HostInfo.py | 42 - .../src/main/python/ambari_commons/firewall.py | 93 +- .../main/python/ambari_commons/inet_utils.py | 17 +- .../python/ambari_commons/os_family_impl.py | 35 +- .../src/main/python/ambari_commons/os_linux.py | 13 +- .../src/main/python/ambari_commons/os_utils.py | 20 +- .../main/python/ambari_commons/os_windows.py | 147 +- .../core/providers/windows/system.py | 4 +- .../functions/get_unique_id_and_date.py | 33 +- ambari-server/conf/unix/ambari.properties | 17 +- ambari-server/conf/windows/ambari.properties | 15 +- .../src/main/python/ambari-server-windows.py | 78 +- ambari-server/src/main/python/ambari-server.py | 1272 +++++++- .../python/ambari_server/dbConfiguration.py | 318 +- .../ambari_server/dbConfiguration_linux.py | 1189 ++++--- .../ambari_server/dbConfiguration_windows.py | 242 +- .../src/main/python/ambari_server/properties.py | 15 +- .../python/ambari_server/resourceFilesKeeper.py | 4 +- .../python/ambari_server/serverConfiguration.py | 277 +- .../ambari_server/serverConfiguration_linux.py | 67 + .../serverConfiguration_windows.py | 98 + .../main/python/ambari_server/serverSetup.py | 1038 ++---- .../python/ambari_server/serverSetup_linux.py | 795 +++++ .../python/ambari_server/serverSetup_windows.py | 313 ++ .../main/python/ambari_server/setupSecurity.py | 111 +- .../src/main/python/ambari_server/userInput.py | 2 +- .../custom_actions/scripts/check_host.py | 24 +- .../src/test/python/TestAmbariServer.py | 3066 ++++++++---------- .../test/python/custom_actions/TestCheckHost.py | 14 +- 29 files changed, 5210 insertions(+), 4149 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/fad56746/ambari-agent/src/main/python/ambari_agent/HostInfo.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/HostInfo.py b/ambari-agent/src/main/python/ambari_agent/HostInfo.py index 5789160..a99a85d 100644 --- a/ambari-agent/src/main/python/ambari_agent/HostInfo.py +++ b/ambari-agent/src/main/python/ambari_agent/HostInfo.py @@ -113,48 +113,6 @@ class HostInfo(object): pass return False - def getTransparentHugePage(self): - """ - Base declaration. The derived classes are supposed to override it and provide OS-specific implementations. - :return: - """ - pass - - def checkUsers(self, users, results): - """ - Base declaration. The derived classes are supposed to override it and provide OS-specific implementations. - :return: - """ - pass - - def checkIptables(self): - """ - Base declaration. The derived classes are supposed to override it and provide OS-specific implementations. - :return: - """ - pass - - def javaProcs(self, list): - """ - Base declaration. The derived classes are supposed to override it and provide OS-specific implementations. - :return: - """ - pass - - def getServiceStatus(self, serivce_name): - """ - Base declaration. The derived classes are supposed to override it and provide OS-specific implementations. - :return: - """ - pass - - def register(self, dict, componentsMapped=True, commandsInProgress=True): - """ - Base declaration. The derived classes are supposed to override it and provide OS-specific implementations. - :return: - """ - pass - @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) class HostInfoLinux(HostInfo): http://git-wip-us.apache.org/repos/asf/ambari/blob/fad56746/ambari-common/src/main/python/ambari_commons/firewall.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/firewall.py b/ambari-common/src/main/python/ambari_commons/firewall.py index 448d24e..b73cc0c 100644 --- a/ambari-common/src/main/python/ambari_commons/firewall.py +++ b/ambari-common/src/main/python/ambari_commons/firewall.py @@ -21,8 +21,6 @@ limitations under the License. import subprocess import shlex from ambari_commons import OSCheck, OSConst -from ambari_commons.logging_utils import print_warning_msg -from ambari_commons.os_family_impl import OsFamilyImpl class Firewall(object): @@ -33,16 +31,6 @@ class Firewall(object): self.OS_FAMILY = OSCheck.get_os_family() def getFirewallObject(self): - pass - -@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) -class FirewallWindows(Firewall): - def getFirewallObject(self): - return WindowsFirewallChecks() - -@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) -class FirewallLinux(Firewall): - def getFirewallObject(self): if self.OS_TYPE == OSConst.OS_UBUNTU: return UbuntuFirewallChecks() elif self.OS_TYPE == OSConst.OS_FEDORA and int(self.OS_VERSION) >= 18: @@ -61,31 +49,23 @@ class FirewallChecks(object): self.returncode = None self.stdoutdata = None self.stderrdata = None - # stdout message - self.MESSAGE_CHECK_FIREWALL = 'Checking iptables...' def get_command(self): return "%s %s %s" % (self.SERVICE_CMD, self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD) - def check_result(self): + def check_result(self, retcode, out, err): result = False - if self.returncode == 3: + if retcode == 3: result = False - elif self.returncode == 0: - if "Table: filter" in self.stdoutdata: + elif retcode == 0: + if "Table: filter" in out: result = True return result - def run_command(self): - retcode, out, err = self.run_os_command(self.get_command()) - self.returncode = retcode - self.stdoutdata = out - self.stderrdata = err - def check_iptables(self): try: - self.run_command() - return self.check_result() + retcode, out, err = self.run_os_command(self.get_command()) + return self.check_result(retcode, out, err) except OSError: return False @@ -96,7 +76,11 @@ class FirewallChecks(object): process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) (stdoutdata, stderrdata) = process.communicate() - return process.returncode, stdoutdata, stderrdata + self.returncode = process.returncode + self.stdoutdata = stdoutdata + self.stderrdata = stderrdata + return self.returncode, self.stdoutdata, self.stderrdata + class UbuntuFirewallChecks(FirewallChecks): @@ -107,13 +91,13 @@ class UbuntuFirewallChecks(FirewallChecks): def get_command(self): return "%s %s" % (self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD) - def check_result(self): + def check_result(self, retcode, out, err): # On ubuntu, the status command returns 0 whether running or not result = False - if self.returncode == 0: - if "Status: inactive" in self.stdoutdata: + if retcode == 0: + if "Status: inactive" in out: result = False - elif "Status: active" in self.stdoutdata: + elif "Status: active" in out: result = True return result @@ -124,10 +108,10 @@ class Fedora18FirewallChecks(FirewallChecks): def get_command(self): return "systemctl is-active %s" % (self.FIREWALL_SERVICE_NAME) - def check_result(self): + def check_result(self, retcode, out, err): result = False - if self.returncode == 0: - if "active" in self.stdoutdata: + if retcode == 0: + if "active" in out: result = True return result @@ -139,44 +123,11 @@ class SuseFirewallChecks(FirewallChecks): def get_command(self): return "%s %s" % (self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD) - def check_result(self): + def check_result(self, retcode, out, err): result = False - if self.returncode == 0: - if "SuSEfirewall2 not active" in self.stdoutdata: + if retcode == 0: + if "SuSEfirewall2 not active" in out: result = False - elif "### iptables" in self.stdoutdata: + elif "### iptables" in out: result = True return result - - -class WindowsFirewallChecks(FirewallChecks): - def __init__(self): - super(WindowsFirewallChecks, self).__init__() - self.MESSAGE_CHECK_FIREWALL = 'Checking firewall status...' - - def run_command(self): - from ambari_commons.os_windows import run_powershell_script, CHECK_FIREWALL_SCRIPT - - retcode, out, err = run_powershell_script(CHECK_FIREWALL_SCRIPT) - self.returncode = retcode - self.stdoutdata = out - self.stderrdata = err - - def check_result(self): - if self.returncode != 0: - print_warning_msg("Unable to check firewall status:{0}".format(self.stderrdata)) - return False - profiles_status = [i for i in self.stdoutdata.split("\n") if not i == ""] - if "1" in profiles_status: - enabled_profiles = [] - if profiles_status[0] == "1": - enabled_profiles.append("DomainProfile") - if profiles_status[1] == "1": - enabled_profiles.append("StandardProfile") - if profiles_status[2] == "1": - enabled_profiles.append("PublicProfile") - print_warning_msg( - "Following firewall profiles are enabled:{0}. Make sure that the firewall is properly configured.".format( - ",".join(enabled_profiles))) - return False - return True http://git-wip-us.apache.org/repos/asf/ambari/blob/fad56746/ambari-common/src/main/python/ambari_commons/inet_utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/inet_utils.py b/ambari-common/src/main/python/ambari_commons/inet_utils.py index 804c63f..2a54cb6 100644 --- a/ambari-common/src/main/python/ambari_commons/inet_utils.py +++ b/ambari-common/src/main/python/ambari_commons/inet_utils.py @@ -22,9 +22,8 @@ import os import sys import urllib2 -from ambari_commons.exceptions import FatalException -from ambari_commons.logging_utils import print_info_msg, print_warning_msg - +from exceptions import * +from logging_utils import * def download_file(link, destination, chunk_size=16 * 1024): print_info_msg("Downloading {0} to {1}".format(link, destination)) @@ -35,10 +34,6 @@ def download_file(link, destination, chunk_size=16 * 1024): force_download_file(link, destination, chunk_size) -def download_progress_stub(file_name, downloaded_size, blockSize, totalSize): - #Facilitates mocking. Normally, the callback used by force_download_file eludes the mocks. - download_progress(file_name, downloaded_size, blockSize, totalSize) - def download_progress(file_name, downloaded_size, blockSize, totalSize): percent = int(downloaded_size * 100 / totalSize) status = "\r" + file_name @@ -61,7 +56,7 @@ def find_range_components(meta): if len(range_comp1) > 1: range_comp2 = range_comp1[0].split(' ') #split away the "bytes" prefix if len(range_comp2) == 0: - raise FatalException(12, 'Malformed Content-Range response header: "{0}".' % hdr_range) + raise FatalException(12, 'Malformed Content-Range response header: "{}".' % hdr_range) range_comp3 = range_comp2[1].split('-') seek_pos = int(range_comp3[0]) if range_comp1[1] != '*': #'*' == unknown length @@ -77,12 +72,12 @@ def find_range_components(meta): return (file_size, seek_pos) -def force_download_file(link, destination, chunk_size = 16 * 1024, progress_func = download_progress_stub): +def force_download_file(link, destination, chunk_size = 16 * 1024, progress_func = download_progress): request = urllib2.Request(link) if os.path.exists(destination) and not os.path.isfile(destination): #Directory specified as target? Must be a mistake. Bail out, don't assume anything. - err = 'Download target {0} is a directory.' % destination + err = 'Download target {} is a directory.' % destination raise FatalException(1, err) (dest_path, file_name) = os.path.split(destination) @@ -143,7 +138,7 @@ def force_download_file(link, destination, chunk_size = 16 * 1024, progress_func downloaded_size = os.stat(temp_dest).st_size if downloaded_size != file_size: - err = 'Size of downloaded file {0} is {0} bytes, it is probably damaged or incomplete' % (destination, downloaded_size) + err = 'Size of downloaded file {} is {} bytes, it is probably damaged or incomplete' % (destination, downloaded_size) raise FatalException(1, err) # when download is complete -> mv temp_dest destination http://git-wip-us.apache.org/repos/asf/ambari/blob/fad56746/ambari-common/src/main/python/ambari_commons/os_family_impl.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/os_family_impl.py b/ambari-common/src/main/python/ambari_commons/os_family_impl.py index d70d942..f8a3379 100644 --- a/ambari-common/src/main/python/ambari_commons/os_family_impl.py +++ b/ambari-common/src/main/python/ambari_commons/os_family_impl.py @@ -17,15 +17,15 @@ limitations under the License. ''' import types -from ambari_commons import OSCheck +from os_check import OSCheck class OsFamilyImpl(object): """ - Base class for os dependent factory. Usage:: + Base class for os depended factory. Usage:: class BaseFoo(object): pass - @OsFamilyImpl("windows") + @Factory("windows") class OsFoo(object):pass print BaseFoo()# OsFoo @@ -62,32 +62,3 @@ class OsFamilyImpl(object): base_cls.__new__ = types.MethodType(new, base_cls) return cls - -class OsFamilyFuncImpl(object): - """ - Base class for os dependent function. Usage:: - - @OSFamilyFuncImpl("windows") - def os_foo(...):pass - - """ - _func_impls = {} - - def _createFunctionInstance(self, func): - self._func_impls[func.__module__ + "." + func.__name__ + "." + self.os_const] = func - - def thunk(*args, **kwargs): - fn_id_base = func.__module__ + "." + func.__name__ - fn_id = fn_id_base + "." + OSCheck.get_os_family() - if fn_id not in self._func_impls: - fn_id = fn_id_base + "." + OsFamilyImpl.DEFAULT - - fn = self._func_impls[fn_id] - return fn(*args, **kwargs) - return thunk - - def __init__(self, os_family): - self.os_const = os_family - - def __call__(self, func): - return self._createFunctionInstance(func) http://git-wip-us.apache.org/repos/asf/ambari/blob/fad56746/ambari-common/src/main/python/ambari_commons/os_linux.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/os_linux.py b/ambari-common/src/main/python/ambari_commons/os_linux.py index eca3c5d..38f3fb9 100644 --- a/ambari-common/src/main/python/ambari_commons/os_linux.py +++ b/ambari-common/src/main/python/ambari_commons/os_linux.py @@ -24,7 +24,7 @@ import pwd import shlex import subprocess -from ambari_commons.logging_utils import print_info_msg, print_warning_msg +from logging_utils import * NR_CHMOD_CMD = 'chmod {0} {1} {2}' @@ -33,15 +33,14 @@ NR_CHOWN_CMD = 'chown {0} {1} {2}' ULIMIT_CMD = "ulimit -n" -def os_run_os_command(cmd, env=None): +def run_os_command(cmd): print_info_msg('about to run command: ' + str(cmd)) if type(cmd) == str: cmd = shlex.split(cmd) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env + stderr=subprocess.PIPE ) (stdoutdata, stderrdata) = process.communicate() return process.returncode, stdoutdata, stderrdata @@ -65,17 +64,17 @@ def os_set_file_permissions(file, mod, recursive, user): else: params = "" command = NR_CHMOD_CMD.format(params, mod, file) - retcode, out, err = os_run_os_command(command) + retcode, out, err = run_os_command(command) if retcode != 0: print_warning_msg(WARN_MSG.format(command, file, err)) command = NR_CHOWN_CMD.format(params, user, file) - retcode, out, err = os_run_os_command(command) + retcode, out, err = run_os_command(command) if retcode != 0: print_warning_msg(WARN_MSG.format(command, file, err)) def os_set_open_files_limit(maxOpenFiles): command = "%s %s" % (ULIMIT_CMD, str(maxOpenFiles)) - os_run_os_command(command) + run_os_command(command) def os_getpass(prompt): http://git-wip-us.apache.org/repos/asf/ambari/blob/fad56746/ambari-common/src/main/python/ambari_commons/os_utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/os_utils.py b/ambari-common/src/main/python/ambari_commons/os_utils.py index 6e2c1d2..3f4819d 100644 --- a/ambari-common/src/main/python/ambari_commons/os_utils.py +++ b/ambari-common/src/main/python/ambari_commons/os_utils.py @@ -18,23 +18,20 @@ See the License for the specific language governing permissions and limitations under the License. ''' -import os import shutil import string +import os -from ambari_commons import OSCheck +from os_check import * if OSCheck.is_windows_family(): - from ambari_commons.os_windows import os_change_owner, os_getpass, os_is_root, os_run_os_command, \ - os_set_open_files_limit, os_set_file_permissions + from os_windows import * else: # MacOS not supported - from ambari_commons.os_linux import os_change_owner, os_getpass, os_is_root, os_run_os_command, \ - os_set_open_files_limit, os_set_file_permissions - pass + from os_linux import * -from ambari_commons.exceptions import FatalException -from ambari_commons.logging_utils import print_info_msg, print_warning_msg +from logging_utils import * +from exceptions import FatalException def is_valid_filepath(filepath): @@ -91,9 +88,6 @@ def set_file_permissions(file, mod, user, recursive): else: print_info_msg("File %s does not exist" % file) -def run_os_command(cmd, env=None): - return os_run_os_command(cmd, env) - def is_root(): return os_is_root() @@ -114,4 +108,4 @@ def find_in_path(file): full_path = os.path.join(dir, file) if os.path.exists(full_path): return full_path - raise Exception("File {} not found in PATH".format(file)) + raise Exception("File {} not found in PATH".format(file)) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/fad56746/ambari-common/src/main/python/ambari_commons/os_windows.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/os_windows.py b/ambari-common/src/main/python/ambari_commons/os_windows.py index 0cc4ea2..2fb98e4 100644 --- a/ambari-common/src/main/python/ambari_commons/os_windows.py +++ b/ambari-common/src/main/python/ambari_commons/os_windows.py @@ -17,35 +17,41 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ''' -import getpass import os -import random +import getpass import shlex import subprocess import sys -import tempfile import time +import win32api +import win32event +import win32service +import win32con +import win32serviceutil +import wmi +import random import string import ctypes +from win32security import * +from win32api import * +from winerror import ERROR_INVALID_HANDLE +from win32process import GetExitCodeProcess, STARTF_USESTDHANDLES, STARTUPINFO, CreateProcessAsUser +from win32event import WaitForSingleObject, INFINITE import msvcrt +import tempfile +from win32event import * +from win32api import CloseHandle -import pywintypes -import winerror -import win32api -import win32con -import win32event -import win32net -import win32netcon -import win32process -import win32security -import win32service -import win32serviceutil -import wmi +from ambari_commons.exceptions import * +from logging_utils import * -from ambari_commons.exceptions import FatalException -from ambari_commons.logging_utils import print_info_msg, print_warning_msg +from win32security import LsaOpenPolicy, POLICY_CREATE_ACCOUNT, POLICY_LOOKUP_NAMES, LookupAccountName, \ + LsaAddAccountRights, LsaRemoveAccountRights, SE_SERVICE_LOGON_NAME +from win32net import NetUserAdd +from win32netcon import USER_PRIV_USER, UF_NORMAL_ACCOUNT, UF_SCRIPT +import pywintypes SERVICE_STATUS_UNKNOWN = "unknown" SERVICE_STATUS_STARTING = "starting" @@ -71,17 +77,17 @@ class OSVERSIONINFOEXW(ctypes.Structure): ('wReserved', ctypes.c_byte)] def get_windows_version(): - """ - Gets the OS major and minor versions. Returns a tuple of - (OS_MAJOR, OS_MINOR). - """ - os_version = OSVERSIONINFOEXW() - os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version) - retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version)) - if retcode != 0: - raise Exception("Failed to get OS version") - - return os_version.dwMajorVersion, os_version.dwMinorVersion, os_version.dwBuildNumber + """ + Get's the OS major and minor versions. Returns a tuple of + (OS_MAJOR, OS_MINOR). + """ + os_version = OSVERSIONINFOEXW() + os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version) + retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version)) + if retcode != 0: + raise Exception("Failed to get OS version") + + return os_version.dwMajorVersion, os_version.dwMinorVersion, os_version.dwBuildNumber CHECK_FIREWALL_SCRIPT = """[string]$CName = $env:computername $reg = [Microsoft.Win32.RegistryKey]::OpenRemoteBaseKey("LocalMachine",$computer) @@ -110,25 +116,25 @@ def _get_files_output(out, err): def _safe_duplicate_handle(h): try: - h = win32api.DuplicateHandle(win32process.GetCurrentProcess(), - h, - win32process.GetCurrentProcess(), - 0, - True, - win32con.DUPLICATE_SAME_ACCESS) + h = DuplicateHandle(GetCurrentProcess(), + h, + GetCurrentProcess(), + 0, + True, + win32con.DUPLICATE_SAME_ACCESS) return True, h except Exception as exc: - if exc.winerror == winerror.ERROR_INVALID_HANDLE: + if exc.winerror == ERROR_INVALID_HANDLE: return True, None return False, None def run_os_command_impersonated(cmd, user, password, domain='.'): - si = win32process.STARTUPINFO() + si = STARTUPINFO() out_handle, err_handle, out_file, err_file = _create_tmp_files() - ok, si.hStdInput = _safe_duplicate_handle(win32api.GetStdHandle(win32api.STD_INPUT_HANDLE)) + ok, si.hStdInput = _safe_duplicate_handle(GetStdHandle(STD_INPUT_HANDLE)) if not ok: raise Exception("Unable to create StdInput for child process") @@ -139,29 +145,27 @@ def run_os_command_impersonated(cmd, user, password, domain='.'): if not ok: raise Exception("Unable to create StdErr for child process") - si.dwFlags = win32process.STARTF_USESTDHANDLES + si.dwFlags = STARTF_USESTDHANDLES si.lpDesktop = "" - user_token = win32security.LogonUser(user, domain, password, - win32con.LOGON32_LOGON_SERVICE, win32con.LOGON32_PROVIDER_DEFAULT) - primary_token = win32security.DuplicateTokenEx(user_token, - win32security.SecurityImpersonation, 0, win32security.TokenPrimary) - info = win32process.CreateProcessAsUser(primary_token, None, cmd, None, None, 1, 0, None, None, si) + user_token = LogonUser(user, domain, password, win32con.LOGON32_LOGON_SERVICE, win32con.LOGON32_PROVIDER_DEFAULT) + primary_token = DuplicateTokenEx(user_token, SecurityImpersonation, 0, TokenPrimary) + info = CreateProcessAsUser(primary_token, None, cmd, None, None, 1, 0, None, None, si) hProcess, hThread, dwProcessId, dwThreadId = info hThread.Close() try: - win32event.WaitForSingleObject(hProcess, win32event.INFINITE) + WaitForSingleObject(hProcess, INFINITE) except KeyboardInterrupt: pass out, err = _get_files_output(out_file, err_file) - exitcode = win32process.GetExitCodeProcess(hProcess) + exitcode = GetExitCodeProcess(hProcess) return exitcode, out, err -def os_run_os_command(cmd, env=None): +def run_os_command(cmd, env=None): if isinstance(cmd,basestring): cmd = cmd.replace("\\", "\\\\") cmd = shlex.split(cmd) @@ -182,13 +186,13 @@ def run_powershell_script(script_content): script_file = open(os.path.join(tmp_dir,random_filename+".ps1"),"w") script_file.write(script_content) script_file.close() - result = os_run_os_command("powershell -ExecutionPolicy unrestricted -File {0}".format(script_file.name)) + result = run_os_command("powershell -ExecutionPolicy unrestricted -File {0}".format(script_file.name)) os.remove(script_file.name) return result def os_change_owner(filePath, user): cmd = ['icacls', filePath, '/setowner', user] - retcode, outdata, errdata = os_run_os_command(cmd) + retcode, outdata, errdata = run_os_command(cmd) return retcode def os_is_root(): @@ -196,10 +200,9 @@ def os_is_root(): Checks whether the current user is a member of the Administrators group Returns True if yes, otherwise False ''' - retcode, out, err = os_run_os_command(WHOAMI_GROUPS) + retcode, out, err = run_os_command(WHOAMI_GROUPS) if retcode != 0: - err_msg = "Unable to check the current user's group memberships. " \ - "Command {0} returned exit code {1} with message: {2}".format(WHOAMI_GROUPS, retcode, err) + err_msg = "Unable to check the current user's group memberships. Command {0} returned exit code {1} with message: {2}".format(WHOAMI_GROUPS, retcode, err) print_warning_msg(err_msg) raise FatalException(retcode, err_msg) @@ -218,20 +221,20 @@ def os_set_file_permissions(file, mod, recursive, user): #else: # params = "" #command = NR_CHMOD_CMD.format(params, mod, file) - #retcode, out, err = os_run_os_command(command) + #retcode, out, err = run_os_command(command) #if retcode != 0: # print_warning_msg(WARN_MSG.format(command, file, err)) #command = NR_CHOWN_CMD.format(params, user, file) - #retcode, out, err = os_run_os_command(command) + #retcode, out, err = run_os_command(command) #if retcode != 0: # print_warning_msg(WARN_MSG.format(command, file, err)) # rights = mod # acls_remove_cmd = "icacls {0} /remove {1}".format(file, user) - # retcode, out, err = os_run_os_command(acls_remove_cmd) + # retcode, out, err = run_os_command(acls_remove_cmd) # if retcode == 0: # acls_modify_cmd = "icacls {0} /grant {1}:{2}".format(file, user, rights) - # retcode, out, err = os_run_os_command(acls_modify_cmd) + # retcode, out, err = run_os_command(acls_modify_cmd) return retcode @@ -245,6 +248,8 @@ def os_getpass(prompt, stream=None): if sys.stdin is not sys.__stdin__: return getpass.fallback_getpass(prompt, stream) + import msvcrt + for c in prompt: msvcrt.putch(c) @@ -482,8 +487,7 @@ class WinService(win32serviceutil.ServiceFramework): def _StopOrWaitForChildProcessToFinish(self, childProcess): #Wait for the child process to finish or for the stop event to be signaled - if(win32event.WAIT_OBJECT_0 == win32event.WaitForMultipleObjects([self._heventSvcStop, childProcess._handle], - False, win32event.INFINITE)): + if(win32event.WAIT_OBJECT_0 == win32event.WaitForMultipleObjects([self._heventSvcStop, childProcess._handle], False, win32event.INFINITE)): # The OS only detaches the child process when the master process exits. # We must kill it manually. try: @@ -499,24 +503,24 @@ class WinService(win32serviceutil.ServiceFramework): class SystemWideLock(object): def __init__(self, name): - self._mutex = win32event.CreateMutex(None, 0, name) + self._mutex = CreateMutex(None, 0, name) def lock(self, timeout=0): - result = win32event.WaitForSingleObject(self._mutex, timeout) - if result in [win32event.WAIT_TIMEOUT, win32event.WAIT_ABANDONED, win32event.WAIT_FAILED]: + result = WaitForSingleObject(self._mutex, timeout) + if result in [WAIT_TIMEOUT, WAIT_ABANDONED, WAIT_FAILED]: return False - elif result == win32event.WAIT_OBJECT_0: + elif result == WAIT_OBJECT_0: return True def unlock(self): try: - win32event.ReleaseMutex(self._mutex) + ReleaseMutex(self._mutex) return True except: return False def __del__(self): - win32api.CloseHandle(self._mutex) + CloseHandle(self._mutex) class UserHelper(object): ACTION_OK = 0 @@ -524,18 +528,17 @@ class UserHelper(object): ACTION_FAILED = -1 def __init__(self): - self._policy = win32security.LsaOpenPolicy(None, - win32security.POLICY_CREATE_ACCOUNT | win32security.POLICY_LOOKUP_NAMES) + self._policy = LsaOpenPolicy(None, POLICY_CREATE_ACCOUNT | POLICY_LOOKUP_NAMES) def create_user(self, name, password, comment="Ambari user"): user_info = {} user_info['name'] = name user_info['password'] = password - user_info['priv'] = win32netcon.USER_PRIV_USER + user_info['priv'] = USER_PRIV_USER user_info['comment'] = comment - user_info['flags'] = win32netcon.UF_NORMAL_ACCOUNT | win32netcon.UF_SCRIPT + user_info['flags'] = UF_NORMAL_ACCOUNT | UF_SCRIPT try: - win32net.NetUserAdd(None, 1, user_info) + NetUserAdd(None, 1, user_info) except pywintypes.error as e: if e.winerror == 2224: return UserHelper.USER_EXISTS, e.strerror @@ -545,16 +548,16 @@ class UserHelper(object): def add_user_privilege(self, name, privilege): try: - acc_sid = win32security.LookupAccountName(None, name)[0] - win32security.LsaAddAccountRights(self._policy, acc_sid, (privilege,)) + acc_sid = LookupAccountName(None, name)[0] + LsaAddAccountRights(self._policy, acc_sid, (privilege,)) except pywintypes.error as e: return UserHelper.ACTION_FAILED, e.strerror return UserHelper.ACTION_OK, "Privilege added." def remove_user_privilege(self, name, privilege): try: - acc_sid = win32security.LookupAccountName(None, name)[0] - win32security.LsaRemoveAccountRights(self._policy, acc_sid, 0, (privilege,)) + acc_sid = LookupAccountName(None, name)[0] + LsaRemoveAccountRights(self._policy, acc_sid, 0, (privilege,)) except pywintypes.error as e: return UserHelper.ACTION_FAILED, e.strerror return UserHelper.ACTION_OK, "Privilege removed." http://git-wip-us.apache.org/repos/asf/ambari/blob/fad56746/ambari-common/src/main/python/resource_management/core/providers/windows/system.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/windows/system.py b/ambari-common/src/main/python/resource_management/core/providers/windows/system.py index dac44ec..e7a98fc 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/windows/system.py +++ b/ambari-common/src/main/python/resource_management/core/providers/windows/system.py @@ -140,7 +140,7 @@ def QueryPrivilegeState(hToken, priv): for (id, attr) in privList: if id == privId: privState = attr - Logger.debug('Privilege state: {0}={1} ({2}) Enabled={3}'.format(privId, priv, LookupPrivilegeDisplayName(None, priv), privState)) + Logger.debug('Privilege state: {}={} ({}) Enabled={}'.format(privId, priv, LookupPrivilegeDisplayName(None, priv), privState)) return privState # Execute command. As windows hdp stack heavily relies on proper environment it is better to reload fresh environment @@ -182,7 +182,7 @@ def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish= if not ok: raise Exception("Unable to create StdErr for child process") - Logger.debug("Redirecting stdout to '{0}', stderr to '{1}'".format(out_file.name, err_file.name)) + Logger.debug("Redirecting stdout to '{}', stderr to '{}'".format(out_file.name, err_file.name)) si.dwFlags = win32con.STARTF_USESTDHANDLES si.lpDesktop = "" http://git-wip-us.apache.org/repos/asf/ambari/blob/fad56746/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py index 3f8e983..afc82bb 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py @@ -23,30 +23,17 @@ Ambari Agent __all__ = ["get_unique_id_and_date"] import datetime from resource_management.core import shell -from ambari_commons import os_check, OSConst -from ambari_commons.os_family_impl import OsFamilyImpl, OsFamilyFuncImpl - - -@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) -def get_volume_serial_number(): - from ambari_commons.os_windows import os_run_os_command - - id = "" - code, out, err = os_run_os_command("cmd /c vol C:") - for line in out.splitlines(): - if line.startswith(" Volume Serial Number is"): - id = line[25:] - - return id - -@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) -def get_volume_serial_number(): - out = shell.checked_call("hostid")[1].split('\n')[-1] # bugfix: take the lastline (stdin is not tty part cut) - id = out.strip() - return id - +from ambari_commons import os_check def get_unique_id_and_date(): - id = get_volume_serial_number() + if os_check.OSCheck.is_windows_family(): + from ambari_commons.os_windows import run_os_command + code, out, err = run_os_command("cmd /c vol C:") + for line in out.splitlines(): + if line.startswith(" Volume Serial Number is"): + id = line[25:] + else: + out = shell.checked_call("hostid")[1].split('\n')[-1] # bugfix: take the lastline (stdin is not tty part cut) + id = out.strip() now = datetime.datetime.now() date = now.strftime("%M%d%y") http://git-wip-us.apache.org/repos/asf/ambari/blob/fad56746/ambari-server/conf/unix/ambari.properties ---------------------------------------------------------------------- diff --git a/ambari-server/conf/unix/ambari.properties b/ambari-server/conf/unix/ambari.properties index 84c96a3..e29a6e2 100644 --- a/ambari-server/conf/unix/ambari.properties +++ b/ambari-server/conf/unix/ambari.properties @@ -22,23 +22,10 @@ security.server.keys_dir = /var/lib/ambari-server/keys resources.dir = /var/lib/ambari-server/resources shared.resources.dir = /usr/lib/ambari-server/lib/ambari_commons/resources custom.action.definitions = /var/lib/ambari-server/resources/custom_action_definitions - -java.releases=jdk1.6,jdk1.7 -jdk1.6.desc=Oracle JDK 1.6 jdk1.6.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-6u31-linux-x64.bin -jdk1.6.dest-file=jdk-6u31-linux-x64.bin -jdk1.6.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/jce_policy-6.zip -jdk1.6.jcpol-file=jce_policy-6.zip -jdk1.6.home=/usr/jdk64/ -jdk1.6.re=Creating (jdk.*)/jre -jdk1.7.desc=Oracle JDK 1.7 +jce_policy1.6.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jce_policy-6.zip jdk1.7.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-linux-x64.tar.gz -jdk1.7.dest-file=jdk-7u67-linux-x64.tar.gz -jdk1.7.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip -jdk1.7.jcpol-file=UnlimitedJCEPolicyJDK7.zip -jdk1.7.home=/usr/jdk64/ -jdk1.7.re=(jdk.*)/jre - +jce_policy1.7.url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip metadata.path=/var/lib/ambari-server/resources/stacks common.services.path=/var/lib/ambari-server/resources/common-services server.version.file=/var/lib/ambari-server/resources/version http://git-wip-us.apache.org/repos/asf/ambari/blob/fad56746/ambari-server/conf/windows/ambari.properties ---------------------------------------------------------------------- diff --git a/ambari-server/conf/windows/ambari.properties b/ambari-server/conf/windows/ambari.properties index 434e42a..3982bb9 100644 --- a/ambari-server/conf/windows/ambari.properties +++ b/ambari-server/conf/windows/ambari.properties @@ -23,14 +23,13 @@ custom.action.definitions=resources\\custom_action_definitions #Comma-separated list of JDK versions #java.releases=jdk1.8.20,jdk1.6.31 -java.releases=jdk1.7 -jdk1.7.desc=Oracle JDK 1.7 -jdk1.7.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-windows-x64.exe -jdk1.7.dest-file=jdk-7u67-windows-x64.exe -jdk1.7.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip -jdk1.7.jcpol-file=UnlimitedJCEPolicyJDK7.zip -jdk1.7.home=C:\\jdk1.7.0_67 -jdk1.7.re=(jdk.*)/jre +java.releases=jdk1.7.67 +jdk1.7.67.desc=Oracle JDK 1.7.67 +jdk1.7.67.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-windows-x64.exe +jdk1.7.67.dest-file=jdk-7u67-windows-x64.exe +jdk1.7.67.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip +jdk1.7.67.jcpol-file=UnlimitedJCEPolicyJDK7.zip +jdk1.7.67.home=C:\\jdk1.7.0_67 metadata.path=resources\\stacks common.services.path=resources\\common-services http://git-wip-us.apache.org/repos/asf/ambari/blob/fad56746/ambari-server/src/main/python/ambari-server-windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server-windows.py b/ambari-server/src/main/python/ambari-server-windows.py index 8940972..6c4f894 100644 --- a/ambari-server/src/main/python/ambari-server-windows.py +++ b/ambari-server/src/main/python/ambari-server-windows.py @@ -19,27 +19,20 @@ limitations under the License. ''' import optparse -import os -import subprocess -import sys - -from ambari_commons.ambari_service import AmbariService -from ambari_commons.exceptions import NonFatalException, FatalException -from ambari_commons.logging_utils import print_error_msg, print_info_msg, print_warning_msg -from ambari_commons.os_utils import remove_file, set_open_files_limit + +from ambari_commons.ambari_service import AmbariService, ENV_PYTHON_PATH +from ambari_commons.logging_utils import * +from ambari_commons.os_utils import remove_file from ambari_commons.os_windows import SvcStatusCallback -from ambari_server.dbConfiguration import DBMSConfigFactory +from ambari_server import utils +from ambari_server.dbConfiguration import DBMSConfig from ambari_server.resourceFilesKeeper import ResourceFilesKeeper, KeeperException -from ambari_server.serverConfiguration import find_jdk, get_ambari_properties, get_value_from_properties, \ - get_full_ambari_classpath, configDefaults, VERBOSE_OUTPUT_KEY, DEBUG_MODE_KEY, SUSPEND_START_MODE_KEY, \ - SERVER_OUT_FILE_KEY, RESOURCES_DIR_PROPERTY, RESOURCES_DIR_DEFAULT, STACK_LOCATION_KEY, STACK_LOCATION_DEFAULT -from ambari_server.serverSetup import setup, reset, is_server_running, upgrade, SERVICE_PASSWORD_KEY, SERVICE_USERNAME_KEY -from ambari_server.setupActions import SETUP_ACTION, START_ACTION, PSTART_ACTION, STOP_ACTION, RESET_ACTION, \ - STATUS_ACTION, UPGRADE_ACTION, UPGRADE_STACK_ACTION, LDAP_SETUP_ACTION, SETUP_SECURITY_ACTION, ACTION_REQUIRE_RESTART -from ambari_server.setupSecurity import setup_ambari_krb5_jaas, setup_https, setup_ldap, setup_master_key -from ambari_server.userInput import get_validated_string_input -from ambari_server.utils import check_reverse_lookup, save_pid +from ambari_server.serverConfiguration import * +from ambari_server.serverSetup import setup, reset, is_server_running, upgrade +from ambari_server.setupActions import * +from ambari_server.setupSecurity import * +from ambari_server.serverSetup_windows import SERVICE_PASSWORD_KEY, SERVICE_USERNAME_KEY # debug settings SERVER_START_DEBUG = False @@ -49,6 +42,11 @@ SUSPEND_START_MODE = False ambari_provider_module_option = "" ambari_provider_module = os.environ.get('AMBARI_PROVIDER_MODULE') +#Common setup or upgrade message +SETUP_OR_UPGRADE_MSG = "- If this is a new setup, then run the \"ambari-server setup\" command to create the user\n" \ +"- If this is an upgrade of an existing setup, run the \"ambari-server upgrade\" command.\n" \ +"Refer to the Ambari documentation for more information on setup and upgrade." + AMBARI_SERVER_DIE_MSG = "Ambari Server java process died with exitcode {0}. Check {1} for more information." if ambari_provider_module is not None: @@ -112,7 +110,7 @@ class AmbariServerService(AmbariService): if not self._StopOrWaitForChildProcessToFinish(childProc): return - pid_file_path = os.path.join(configDefaults.PID_DIR, PID_NAME) + pid_file_path = PID_DIR + os.sep + PID_NAME remove_file(pid_file_path) pass @@ -124,7 +122,7 @@ class AmbariServerService(AmbariService): outFilePath = properties[SERVER_OUT_FILE_KEY] if (outFilePath is None or outFilePath == ""): - outFilePath = configDefaults.SERVER_OUT_FILE + outFilePath = SERVER_OUT_FILE self._RedirectOutputStreamsToFile(outFilePath) pass @@ -159,7 +157,7 @@ def start(options): childProc.wait() - pid_file_path = os.path.join(configDefaults.PID_DIR, PID_NAME) + pid_file_path = PID_DIR + os.sep + PID_NAME remove_file(pid_file_path) # @@ -201,7 +199,7 @@ def server_process_main(options, scmStatus=None): except AttributeError: pass - if not check_reverse_lookup(): + if not utils.check_reverse_lookup(): print_warning_msg("The hostname was not found in the reverse DNS lookup. " "This may result in incorrect behavior. " "Please check the DNS setup and fix the issue.") @@ -210,11 +208,12 @@ def server_process_main(options, scmStatus=None): print_info_msg("Ambari Server is not running...") + conf_dir = get_conf_dir() jdk_path = find_jdk() if jdk_path is None: err = "No JDK found, please run the \"ambari-server setup\" " \ "command to install a JDK automatically or install any " \ - "JDK manually to " + configDefaults.JDK_INSTALL_DIR + "JDK manually to " + JDK_INSTALL_DIR raise FatalException(1, err) # Preparations @@ -237,15 +236,17 @@ def server_process_main(options, scmStatus=None): if scmStatus is not None: scmStatus.reportStartPending() - conf_dir = get_full_ambari_classpath() + conf_dir = os.path.abspath(conf_dir) + os.pathsep + get_ambari_classpath() + if conf_dir.find(' ') != -1: + conf_dir = '"' + conf_dir + '"' - java_exe = os.path.join(jdk_path, configDefaults.JAVA_EXE_SUBPATH) - pidfile = os.path.join(configDefaults.PID_DIR, PID_NAME) + java_exe = jdk_path + os.sep + JAVA_EXE_SUBPATH + pidfile = PID_DIR + os.sep + PID_NAME command_base = SERVER_START_CMD_DEBUG if (DEBUG_MODE or SERVER_START_DEBUG) else SERVER_START_CMD suspend_mode = 'y' if SUSPEND_START_MODE else 'n' command = command_base.format(conf_dir, suspend_mode) - if not os.path.exists(configDefaults.PID_DIR): - os.makedirs(configDefaults.PID_DIR, 0755) + if not os.path.exists(PID_DIR): + os.makedirs(PID_DIR, 0755) set_open_files_limit(get_ulimit_open_files()); @@ -263,18 +264,18 @@ def server_process_main(options, scmStatus=None): if pidJava <= 0: procJava.terminate() exitcode = procJava.returncode - exitfile = os.path.join(configDefaults.PID_DIR, EXITCODE_NAME) - save_pid(exitcode, exitfile) + exitfile = os.path.join(PID_DIR, EXITCODE_NAME) + utils.save_pid(exitcode, exitfile) if scmStatus is not None: scmStatus.reportStopPending() - raise FatalException(-1, AMBARI_SERVER_DIE_MSG.format(exitcode, configDefaults.SERVER_OUT_FILE)) + raise FatalException(-1, AMBARI_SERVER_DIE_MSG.format(exitcode, SERVER_OUT_FILE)) else: - save_pid(pidJava, pidfile) - print "Server PID at: "+ pidfile - print "Server out at: "+ configDefaults.SERVER_OUT_FILE - print "Server log at: "+ configDefaults.SERVER_LOG_FILE + utils.save_pid(pidJava, pidfile) + print "Server PID at: "+pidfile + print "Server out at: "+SERVER_OUT_FILE + print "Server log at: "+SERVER_LOG_FILE if scmStatus is not None: scmStatus.reportStarted() @@ -288,15 +289,13 @@ def server_process_main(options, scmStatus=None): #Wait until the status is 'started' or a configured timeout elapses #If the timeout has been reached, bail out with exception def ensure_dbms_is_running(options, properties, scmStatus): - factory = DBMSConfigFactory() - - dbms = factory.create(options, properties, "Ambari") + dbms = DBMSConfig.create(options, properties, "Ambari") if not dbms._is_jdbc_driver_installed(properties): raise FatalException(-1, "JDBC driver is not installed. Run ambari-server setup and try again.") dbms.ensure_dbms_is_running(options, properties, scmStatus) - dbms2 = factory.create(options, properties, "Metrics") + dbms2 = DBMSConfig.create(options, properties, "Metrics") if dbms2.database_host.lower() != dbms.database_host.lower(): dbms2.ensure_dbms_is_running(options, properties, scmStatus) pass @@ -500,7 +499,6 @@ def main(): #perform checks options.warnings = [] - options.exit_message = None options.must_set_database_options = False if are_cmd_line_db_args_blank(options):
