AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components +Fixed ambari_server import in TestAmbariServer.py, to allow future imports from modules in the ambari_server subdir +Moved SILENT, VERBOSE and DEBUG_MODE to logging_utils.py, to allow future use from modular code +Moved run_os_command to os_utils +Extended the set of test platform primitives +Optimized imports
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/16bee385 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/16bee385 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/16bee385 Branch: refs/heads/trunk Commit: 16bee3858711178dd4025b56211f03b11fa75954 Parents: fb947b0 Author: Florian Barca <[email protected]> Authored: Tue Jan 13 13:46:04 2015 -0800 Committer: Florian Barca <[email protected]> Committed: Tue Jan 13 13:46:04 2015 -0800 ---------------------------------------------------------------------- .../test/python/ambari_agent/TestHostInfo.py | 40 +- .../python/ambari_agent/TestRegistration.py | 6 +- .../src/main/python/ambari_commons/firewall.py | 21 +- .../main/python/ambari_commons/logging_utils.py | 32 +- .../src/main/python/ambari_commons/os_linux.py | 14 +- .../src/main/python/ambari_commons/os_utils.py | 21 +- .../main/python/ambari_commons/os_windows.py | 168 +- .../core/providers/windows/system.py | 4 +- .../functions/get_unique_id_and_date.py | 2 +- .../src/test/python/only_for_platform.py | 12 + ambari-server/conf/windows/ambari.properties | 1 + .../src/main/python/ambari-server-windows.py | 22 +- ambari-server/src/main/python/ambari-server.py | 57 +- .../ambari_server/dbConfiguration_linux.py | 8 +- .../python/ambari_server/serverConfiguration.py | 8 + .../main/python/ambari_server/serverSetup.py | 5 +- .../python/ambari_server/serverSetup_linux.py | 5 +- .../python/ambari_server/serverSetup_windows.py | 2 +- .../main/python/ambari_server/setupSecurity.py | 6 +- .../src/main/python/ambari_server/userInput.py | 12 +- .../custom_actions/scripts/install_packages.py | 8 +- .../src/test/python/TestAmbariServer.py | 2614 +++++++++--------- .../test/python/custom_actions/TestCheckHost.py | 18 +- 23 files changed, 1574 insertions(+), 1512 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py index 338d080..adeb798 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py +++ b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py @@ -31,7 +31,7 @@ from mock.mock import create_autospec import ambari_commons from ambari_commons import OSCheck import os -from only_for_platform import only_for_platform, get_platform, PLATFORM_WINDOWS, PLATFORM_LINUX +from only_for_platform import not_for_platform, get_platform, PLATFORM_WINDOWS, PLATFORM_LINUX if get_platform() != PLATFORM_WINDOWS: os_distro_value = ('Suse','11','Final') @@ -51,7 +51,7 @@ with patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_ @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) class TestHostInfo(TestCase): - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) @patch.object(OSCheck, 'get_os_family') @patch('resource_management.libraries.functions.packages_analyzer.subprocessWithTimeout') def test_analyze_zypper_out(self, spwt_mock, get_os_family_mock): @@ -82,7 +82,7 @@ class TestHostInfo(TestCase): self.assertTrue(installedPackages[3][2], "HDP") self.assertTrue(installedPackages[6][1], "11-38.13.9") - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) def test_getReposToRemove(self): l1 = ["Hortonworks Data Platform Utils Version - HDP-UTILS-1.1.0.15", "Ambari 1.x", "HDP"] l2 = ["Ambari", "HDP-UTIL"] @@ -96,7 +96,7 @@ class TestHostInfo(TestCase): self.assertTrue(1, len(l3)) self.assertEqual(l3[0], "HDP-1.3.0") - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) def test_perform_package_analysis(self): installedPackages = [ ["hadoop-a", "2.3", "HDP"], ["zk", "3.1", "HDP"], ["webhcat", "3.1", "HDP"], @@ -143,7 +143,7 @@ class TestHostInfo(TestCase): for package in expected: self.assertTrue(package in allPackages) - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) @patch.object(OSCheck, 'get_os_family') @patch('resource_management.libraries.functions.packages_analyzer.subprocessWithTimeout') def test_analyze_yum_output(self, subprocessWithTimeout_mock, get_os_family_mock): @@ -199,7 +199,7 @@ class TestHostInfo(TestCase): "koji-override-0/$releasever"]) self.assertFalse(package['repoName'] in ["AMBARI.dev-1.x"]) - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) @patch.object(OSCheck, 'get_os_family') @patch('resource_management.libraries.functions.packages_analyzer.subprocessWithTimeout') def test_analyze_yum_output_err(self, subprocessWithTimeout_mock, get_os_family_mock): @@ -230,7 +230,7 @@ class TestHostInfo(TestCase): self.assertTrue(item in names) - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) @patch('os.path.exists') @patch('__builtin__.open') @@ -253,7 +253,7 @@ class TestHostInfo(TestCase): self.assertTrue(newlist[1]['status'], "Invalid home directory") print(path_mock.mock_calls) - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) @patch.object(OSCheck, "get_os_type") @patch('os.umask') @patch.object(HostCheckReportFileHandler, 'writeHostCheckFile') @@ -296,7 +296,7 @@ class TestHostInfo(TestCase): self.assertTrue(0 < len(dict['installedPackages'])) self.assertTrue('agentTimeStampAtReporting' in dict['hostHealth']) - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) @patch.object(OSCheck, "get_os_type") @patch('os.umask') @patch.object(HostCheckReportFileHandler, 'writeHostCheckFile') @@ -399,7 +399,7 @@ class TestHostInfo(TestCase): result = host.dirType("/home") self.assertEquals(result, 'unknown') - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) @patch("os.path.exists") @patch("glob.glob") def test_hadoopVarRunCount(self, glob_glob_mock, os_path_exists_mock): @@ -414,7 +414,7 @@ class TestHostInfo(TestCase): result = hostInfo.hadoopVarRunCount() self.assertEquals(result, 0) - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) @patch("os.path.exists") @patch("glob.glob") def test_hadoopVarLogCount(self, glob_glob_mock, os_path_exists_mock): @@ -429,7 +429,7 @@ class TestHostInfo(TestCase): result = hostInfo.hadoopVarLogCount() self.assertEquals(result, 0) - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = ('redhat','11','Final'))) @patch("os.listdir", create=True, autospec=True) @patch("__builtin__.open", create=True, autospec=True) @@ -451,7 +451,7 @@ class TestHostInfo(TestCase): self.assertTrue(list[0]['hadoop']) self.assertEquals(list[0]['user'], 'user') - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) @patch("subprocess.Popen") @patch.object(Hardware, 'extractMountInfo') def test_osdiskAvailableSpace(self, extract_mount_info_mock, subproc_popen_mock): @@ -469,7 +469,7 @@ class TestHostInfo(TestCase): self.assertEquals(result, {}) - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) @patch.object(OSCheck, "get_os_type") @patch("subprocess.Popen") def test_checkLiveServices(self, subproc_popen, get_os_type_method): @@ -513,7 +513,7 @@ class TestHostInfo(TestCase): self.assertEquals(result[0]['name'], 'service1') self.assertTrue(len(result[0]['desc']) > 0) - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = ('redhat','11','Final'))) @patch("os.path.exists") @patch("os.listdir", create=True, autospec=True) @@ -536,11 +536,11 @@ class TestHostInfo(TestCase): self.assertEquals(result[0]['name'], 'config1') self.assertEquals(result[0]['target'], 'real_path_to_conf') - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) @patch.object(OSCheck, "get_os_family") @patch.object(OSCheck, "get_os_type") @patch.object(OSCheck, "get_os_major_version") - @patch.object(FirewallChecks, "run_os_command") + @patch("ambari_commons.firewall.run_os_command") def test_IpTablesRunning(self, run_os_command_mock, get_os_major_version_mock, get_os_type_mock, get_os_family_mock): get_os_type_mock.return_value = "" get_os_family_mock.return_value = OSConst.REDHAT_FAMILY @@ -572,18 +572,18 @@ class TestHostInfo(TestCase): self.assertFalse(hostInfo.checkReverseLookup()) - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) @patch.object(OSCheck, "get_os_family") @patch.object(OSCheck, "get_os_type") @patch.object(OSCheck, "get_os_major_version") - @patch.object(FirewallChecks, "run_os_command") + @patch("ambari_commons.firewall.run_os_command") def test_IpTablesStopped(self, run_os_command_mock, get_os_major_version_mock, get_os_type_mock, get_os_family_mock): get_os_type_mock.return_value = "" get_os_family_mock.return_value = OSConst.REDHAT_FAMILY run_os_command_mock.return_value = 3, "", "" self.assertFalse(Firewall().getFirewallObject().check_iptables()) - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = ('redhat','11','Final'))) @patch("os.path.isfile") @patch('__builtin__.open') http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-agent/src/test/python/ambari_agent/TestRegistration.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestRegistration.py b/ambari-agent/src/test/python/ambari_agent/TestRegistration.py index 41f9880..92e4b36 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestRegistration.py +++ b/ambari-agent/src/test/python/ambari_agent/TestRegistration.py @@ -23,7 +23,7 @@ import os import tempfile from mock.mock import patch from mock.mock import MagicMock -from only_for_platform import only_for_platform, PLATFORM_LINUX +from only_for_platform import not_for_platform, PLATFORM_WINDOWS with patch("platform.linux_distribution", return_value = ('Suse','11','Final')): from ambari_agent.Register import Register @@ -33,9 +33,9 @@ with patch("platform.linux_distribution", return_value = ('Suse','11','Final')): class TestRegistration(TestCase): - @only_for_platform(PLATFORM_LINUX) + @not_for_platform(PLATFORM_WINDOWS) @patch.object(Hardware, "_chk_mount", new = MagicMock(return_value=True)) - @patch.object(FirewallChecks, "run_os_command") + @patch("ambari_commons.firewall.run_os_command") @patch.object(OSCheck, "get_os_type") @patch.object(OSCheck, "get_os_version") def test_registration_build(self, get_os_version_mock, get_os_type_mock, run_os_cmd_mock): http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-common/src/main/python/ambari_commons/firewall.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/firewall.py b/ambari-common/src/main/python/ambari_commons/firewall.py index b73cc0c..76d9890 100644 --- a/ambari-common/src/main/python/ambari_commons/firewall.py +++ b/ambari-common/src/main/python/ambari_commons/firewall.py @@ -18,10 +18,8 @@ See the License for the specific language governing permissions and limitations under the License. ''' -import subprocess -import shlex from ambari_commons import OSCheck, OSConst - +from ambari_commons.os_utils import run_os_command class Firewall(object): def __init__(self): @@ -64,24 +62,11 @@ class FirewallChecks(object): def check_iptables(self): try: - retcode, out, err = self.run_os_command(self.get_command()) - return self.check_result(retcode, out, err) + self.returncode, self.stdoutdata, self.stderrdata = run_os_command(self.get_command()) + return self.check_result(self.returncode, self.stdoutdata, self.stderrdata) except OSError: return False - def run_os_command(self, cmd): - if type(cmd) == str: - cmd = shlex.split(cmd) - - process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.PIPE) - (stdoutdata, stderrdata) = process.communicate() - self.returncode = process.returncode - self.stdoutdata = stdoutdata - self.stderrdata = stderrdata - return self.returncode, self.stdoutdata, self.stderrdata - - class UbuntuFirewallChecks(FirewallChecks): def __init__(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-common/src/main/python/ambari_commons/logging_utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/logging_utils.py b/ambari-common/src/main/python/ambari_commons/logging_utils.py index 9d45fdb..af1390c 100644 --- a/ambari-common/src/main/python/ambari_commons/logging_utils.py +++ b/ambari-common/src/main/python/ambari_commons/logging_utils.py @@ -18,19 +18,43 @@ See the License for the specific language governing permissions and limitations under the License. ''' -VERBOSE = False -SILENT = False -DEBUG_MODE = False +_VERBOSE = False +_SILENT = False +_DEBUG_MODE = False # terminal styles BOLD_ON = '\033[1m' BOLD_OFF = '\033[0m' +def get_verbose(): + global _VERBOSE + return _VERBOSE + +def set_verbose(newVal): + global _VERBOSE + _VERBOSE = newVal + +def get_silent(): + global _SILENT + return _SILENT + +def set_silent(newVal): + global _SILENT + _SILENT = newVal + +def get_debug_mode(): + global _DEBUG_MODE + return _DEBUG_MODE + +def set_debug_mode(newVal): + global _DEBUG_MODE + _DEBUG_MODE = newVal + # # Prints an "info" messsage. # def print_info_msg(msg): - if VERBOSE: + if _VERBOSE: print("INFO: " + msg) # http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-common/src/main/python/ambari_commons/os_linux.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/os_linux.py b/ambari-common/src/main/python/ambari_commons/os_linux.py index 38f3fb9..8599a0d 100644 --- a/ambari-common/src/main/python/ambari_commons/os_linux.py +++ b/ambari-common/src/main/python/ambari_commons/os_linux.py @@ -24,7 +24,7 @@ import pwd import shlex import subprocess -from logging_utils import * +from ambari_commons.logging_utils import print_info_msg, print_warning_msg NR_CHMOD_CMD = 'chmod {0} {1} {2}' @@ -33,14 +33,16 @@ NR_CHOWN_CMD = 'chown {0} {1} {2}' ULIMIT_CMD = "ulimit -n" -def run_os_command(cmd): +def os_run_os_command(cmd, env=None, shell=False): print_info_msg('about to run command: ' + str(cmd)) if type(cmd) == str: cmd = shlex.split(cmd) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.PIPE + stderr=subprocess.PIPE, + env=env, + shell=shell ) (stdoutdata, stderrdata) = process.communicate() return process.returncode, stdoutdata, stderrdata @@ -64,17 +66,17 @@ def os_set_file_permissions(file, mod, recursive, user): else: params = "" command = NR_CHMOD_CMD.format(params, mod, file) - retcode, out, err = run_os_command(command) + retcode, out, err = os_run_os_command(command) if retcode != 0: print_warning_msg(WARN_MSG.format(command, file, err)) command = NR_CHOWN_CMD.format(params, user, file) - retcode, out, err = run_os_command(command) + retcode, out, err = os_run_os_command(command) if retcode != 0: print_warning_msg(WARN_MSG.format(command, file, err)) def os_set_open_files_limit(maxOpenFiles): command = "%s %s" % (ULIMIT_CMD, str(maxOpenFiles)) - run_os_command(command) + os_run_os_command(command) def os_getpass(prompt): http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-common/src/main/python/ambari_commons/os_utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/os_utils.py b/ambari-common/src/main/python/ambari_commons/os_utils.py index 942a920..c6a3832 100644 --- a/ambari-common/src/main/python/ambari_commons/os_utils.py +++ b/ambari-common/src/main/python/ambari_commons/os_utils.py @@ -18,20 +18,23 @@ See the License for the specific language governing permissions and limitations under the License. ''' +import os import shutil import string -import os -from os_check import * +from ambari_commons import OSCheck if OSCheck.is_windows_family(): - from os_windows import * + from ambari_commons.os_windows import os_change_owner, os_getpass, os_is_root, os_run_os_command, \ + os_set_open_files_limit, os_set_file_permissions else: # MacOS not supported - from os_linux import * + from ambari_commons.os_linux import os_change_owner, os_getpass, os_is_root, os_run_os_command, \ + os_set_open_files_limit, os_set_file_permissions + pass -from logging_utils import * -from exceptions import FatalException +from ambari_commons.exceptions import FatalException +from ambari_commons.logging_utils import print_info_msg, print_warning_msg def is_valid_filepath(filepath): @@ -91,6 +94,12 @@ def set_file_permissions(file, mod, user, recursive): else: print_info_msg("File %s does not exist" % file) +def run_os_command(cmd, env=None): + return os_run_os_command(cmd, env, False) + +def run_in_shell(cmd, env=None): + return os_run_os_command(cmd, env, True) + def is_root(): return os_is_root() http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-common/src/main/python/ambari_commons/os_windows.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/os_windows.py b/ambari-common/src/main/python/ambari_commons/os_windows.py index 7918a2f..e413b8e 100644 --- a/ambari-common/src/main/python/ambari_commons/os_windows.py +++ b/ambari-common/src/main/python/ambari_commons/os_windows.py @@ -17,41 +17,35 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ''' -import os import getpass +import os +import random import shlex import subprocess import sys +import tempfile import time -import win32api -import win32event -import win32service -import win32con -import win32serviceutil -import wmi -import random import string import ctypes -from win32security import * -from win32api import * -from winerror import ERROR_INVALID_HANDLE -from win32process import GetExitCodeProcess, STARTF_USESTDHANDLES, STARTUPINFO, CreateProcessAsUser -from win32event import WaitForSingleObject, INFINITE import msvcrt -import tempfile -from win32event import * -from win32api import CloseHandle - -from ambari_commons.exceptions import * -from logging_utils import * -from win32security import LsaOpenPolicy, POLICY_CREATE_ACCOUNT, POLICY_LOOKUP_NAMES, LookupAccountName, \ - LsaAddAccountRights, LsaRemoveAccountRights, SE_SERVICE_LOGON_NAME -from win32net import NetUserAdd -from win32netcon import USER_PRIV_USER, UF_NORMAL_ACCOUNT, UF_SCRIPT import pywintypes +import winerror +import win32api +import win32con +import win32event +import win32net +import win32netcon +import win32process +import win32security +import win32service +import win32serviceutil +import wmi + +from ambari_commons.exceptions import FatalException +from ambari_commons.logging_utils import print_info_msg, print_warning_msg SERVICE_STATUS_UNKNOWN = "unknown" SERVICE_STATUS_STARTING = "starting" @@ -63,6 +57,27 @@ SERVICE_STATUS_NOT_INSTALLED = "not installed" WHOAMI_GROUPS = "whoami /groups" ADMIN_ACCOUNT = "BUILTIN\\Administrators" +# +# os.symlink is not implemented in Windows. Patch it. +# +__CSL = None +def symlink(source, link_name): + '''symlink(source, link_name) + Creates a symbolic link pointing to source named link_name''' + global __CSL + if __CSL is None: + csl = ctypes.windll.kernel32.CreateSymbolicLinkW + csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32) + csl.restype = ctypes.c_ubyte + __CSL = csl + flags = 0 + if source is not None and os.path.isdir(source): + flags = 1 + if __CSL(link_name, source, flags) == 0: + raise ctypes.WinError() + +os.symlink = symlink + class OSVERSIONINFOEXW(ctypes.Structure): _fields_ = [('dwOSVersionInfoSize', ctypes.c_ulong), ('dwMajorVersion', ctypes.c_ulong), @@ -77,17 +92,17 @@ class OSVERSIONINFOEXW(ctypes.Structure): ('wReserved', ctypes.c_byte)] def get_windows_version(): - """ - Get's the OS major and minor versions. Returns a tuple of - (OS_MAJOR, OS_MINOR). - """ - os_version = OSVERSIONINFOEXW() - os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version) - retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version)) - if retcode != 0: - raise Exception("Failed to get OS version") - - return os_version.dwMajorVersion, os_version.dwMinorVersion, os_version.dwBuildNumber + """ + Gets the OS major and minor versions. Returns a tuple of + (OS_MAJOR, OS_MINOR). + """ + os_version = OSVERSIONINFOEXW() + os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version) + retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version)) + if retcode != 0: + raise Exception("Failed to get OS version") + + return os_version.dwMajorVersion, os_version.dwMinorVersion, os_version.dwBuildNumber CHECK_FIREWALL_SCRIPT = """[string]$CName = $env:computername $reg = [Microsoft.Win32.RegistryKey]::OpenRemoteBaseKey("LocalMachine",$computer) @@ -116,25 +131,25 @@ def _get_files_output(out, err): def _safe_duplicate_handle(h): try: - h = DuplicateHandle(GetCurrentProcess(), - h, - GetCurrentProcess(), - 0, - True, - win32con.DUPLICATE_SAME_ACCESS) + h = win32api.DuplicateHandle(win32process.GetCurrentProcess(), + h, + win32process.GetCurrentProcess(), + 0, + True, + win32con.DUPLICATE_SAME_ACCESS) return True, h except Exception as exc: - if exc.winerror == ERROR_INVALID_HANDLE: + if exc.winerror == winerror.ERROR_INVALID_HANDLE: return True, None return False, None def run_os_command_impersonated(cmd, user, password, domain='.'): - si = STARTUPINFO() + si = win32process.STARTUPINFO() out_handle, err_handle, out_file, err_file = _create_tmp_files() - ok, si.hStdInput = _safe_duplicate_handle(GetStdHandle(STD_INPUT_HANDLE)) + ok, si.hStdInput = _safe_duplicate_handle(win32api.GetStdHandle(win32api.STD_INPUT_HANDLE)) if not ok: raise Exception("Unable to create StdInput for child process") @@ -145,27 +160,29 @@ def run_os_command_impersonated(cmd, user, password, domain='.'): if not ok: raise Exception("Unable to create StdErr for child process") - si.dwFlags = STARTF_USESTDHANDLES + si.dwFlags = win32process.STARTF_USESTDHANDLES si.lpDesktop = "" - user_token = LogonUser(user, domain, password, win32con.LOGON32_LOGON_SERVICE, win32con.LOGON32_PROVIDER_DEFAULT) - primary_token = DuplicateTokenEx(user_token, SecurityImpersonation, 0, TokenPrimary) - info = CreateProcessAsUser(primary_token, None, cmd, None, None, 1, 0, None, None, si) + user_token = win32security.LogonUser(user, domain, password, + win32con.LOGON32_LOGON_SERVICE, win32con.LOGON32_PROVIDER_DEFAULT) + primary_token = win32security.DuplicateTokenEx(user_token, + win32security.SecurityImpersonation, 0, win32security.TokenPrimary) + info = win32process.CreateProcessAsUser(primary_token, None, cmd, None, None, 1, 0, None, None, si) hProcess, hThread, dwProcessId, dwThreadId = info hThread.Close() try: - WaitForSingleObject(hProcess, INFINITE) + win32event.WaitForSingleObject(hProcess, win32event.INFINITE) except KeyboardInterrupt: pass out, err = _get_files_output(out_file, err_file) - exitcode = GetExitCodeProcess(hProcess) + exitcode = win32process.GetExitCodeProcess(hProcess) return exitcode, out, err -def run_os_command(cmd, env=None): +def os_run_os_command(cmd, env=None, shell=False): if isinstance(cmd,basestring): cmd = cmd.replace("\\", "\\\\") cmd = shlex.split(cmd) @@ -173,7 +190,8 @@ def run_os_command(cmd, env=None): stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE, - env=env + env=env, + shell=shell ) (stdoutdata, stderrdata) = process.communicate() return process.returncode, stdoutdata, stderrdata @@ -186,13 +204,13 @@ def run_powershell_script(script_content): script_file = open(os.path.join(tmp_dir,random_filename+".ps1"),"w") script_file.write(script_content) script_file.close() - result = run_os_command("powershell -ExecutionPolicy unrestricted -File {0}".format(script_file.name)) + result = os_run_os_command("powershell -ExecutionPolicy unrestricted -File {0}".format(script_file.name)) os.remove(script_file.name) return result def os_change_owner(filePath, user): cmd = ['icacls', filePath, '/setowner', user] - retcode, outdata, errdata = run_os_command(cmd) + retcode, outdata, errdata = os_run_os_command(cmd) return retcode def os_is_root(): @@ -200,9 +218,10 @@ def os_is_root(): Checks whether the current user is a member of the Administrators group Returns True if yes, otherwise False ''' - retcode, out, err = run_os_command(WHOAMI_GROUPS) + retcode, out, err = os_run_os_command(WHOAMI_GROUPS) if retcode != 0: - err_msg = "Unable to check the current user's group memberships. Command {0} returned exit code {1} with message: {2}".format(WHOAMI_GROUPS, retcode, err) + err_msg = "Unable to check the current user's group memberships. " \ + "Command {0} returned exit code {1} with message: {2}".format(WHOAMI_GROUPS, retcode, err) print_warning_msg(err_msg) raise FatalException(retcode, err_msg) @@ -221,20 +240,20 @@ def os_set_file_permissions(file, mod, recursive, user): #else: # params = "" #command = NR_CHMOD_CMD.format(params, mod, file) - #retcode, out, err = run_os_command(command) + #retcode, out, err = os_run_os_command(command) #if retcode != 0: # print_warning_msg(WARN_MSG.format(command, file, err)) #command = NR_CHOWN_CMD.format(params, user, file) - #retcode, out, err = run_os_command(command) + #retcode, out, err = os_run_os_command(command) #if retcode != 0: # print_warning_msg(WARN_MSG.format(command, file, err)) # rights = mod # acls_remove_cmd = "icacls {0} /remove {1}".format(file, user) - # retcode, out, err = run_os_command(acls_remove_cmd) + # retcode, out, err = os_run_os_command(acls_remove_cmd) # if retcode == 0: # acls_modify_cmd = "icacls {0} /grant {1}:{2}".format(file, user, rights) - # retcode, out, err = run_os_command(acls_modify_cmd) + # retcode, out, err = os_run_os_command(acls_modify_cmd) return retcode @@ -248,8 +267,6 @@ def os_getpass(prompt, stream=None): if sys.stdin is not sys.__stdin__: return getpass.fallback_getpass(prompt, stream) - import msvcrt - for c in prompt: msvcrt.putch(c) @@ -506,24 +523,24 @@ class WinService(win32serviceutil.ServiceFramework): class SystemWideLock(object): def __init__(self, name): - self._mutex = CreateMutex(None, 0, name) + self._mutex = win32event.CreateMutex(None, 0, name) def lock(self, timeout=0): - result = WaitForSingleObject(self._mutex, timeout) - if result in [WAIT_TIMEOUT, WAIT_ABANDONED, WAIT_FAILED]: + result = win32event.WaitForSingleObject(self._mutex, timeout) + if result in [win32event.WAIT_TIMEOUT, win32event.WAIT_ABANDONED, win32event.WAIT_FAILED]: return False - elif result == WAIT_OBJECT_0: + elif result == win32event.WAIT_OBJECT_0: return True def unlock(self): try: - ReleaseMutex(self._mutex) + win32event.ReleaseMutex(self._mutex) return True except: return False def __del__(self): - CloseHandle(self._mutex) + win32api.CloseHandle(self._mutex) class UserHelper(object): ACTION_OK = 0 @@ -531,17 +548,18 @@ class UserHelper(object): ACTION_FAILED = -1 def __init__(self): - self._policy = LsaOpenPolicy(None, POLICY_CREATE_ACCOUNT | POLICY_LOOKUP_NAMES) + self._policy = win32security.LsaOpenPolicy(None, + win32security.POLICY_CREATE_ACCOUNT | win32security.POLICY_LOOKUP_NAMES) def create_user(self, name, password, comment="Ambari user"): user_info = {} user_info['name'] = name user_info['password'] = password - user_info['priv'] = USER_PRIV_USER + user_info['priv'] = win32netcon.USER_PRIV_USER user_info['comment'] = comment - user_info['flags'] = UF_NORMAL_ACCOUNT | UF_SCRIPT + user_info['flags'] = win32netcon.UF_NORMAL_ACCOUNT | win32netcon.UF_SCRIPT try: - NetUserAdd(None, 1, user_info) + win32net.NetUserAdd(None, 1, user_info) except pywintypes.error as e: if e.winerror == 2224: return UserHelper.USER_EXISTS, e.strerror @@ -551,16 +569,16 @@ class UserHelper(object): def add_user_privilege(self, name, privilege): try: - acc_sid = LookupAccountName(None, name)[0] - LsaAddAccountRights(self._policy, acc_sid, (privilege,)) + acc_sid = win32security.LookupAccountName(None, name)[0] + win32security.LsaAddAccountRights(self._policy, acc_sid, (privilege,)) except pywintypes.error as e: return UserHelper.ACTION_FAILED, e.strerror return UserHelper.ACTION_OK, "Privilege added." def remove_user_privilege(self, name, privilege): try: - acc_sid = LookupAccountName(None, name)[0] - LsaRemoveAccountRights(self._policy, acc_sid, 0, (privilege,)) + acc_sid = win32security.LookupAccountName(None, name)[0] + win32security.LsaRemoveAccountRights(self._policy, acc_sid, 0, (privilege,)) except pywintypes.error as e: return UserHelper.ACTION_FAILED, e.strerror return UserHelper.ACTION_OK, "Privilege removed." http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-common/src/main/python/resource_management/core/providers/windows/system.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/windows/system.py b/ambari-common/src/main/python/resource_management/core/providers/windows/system.py index e7a98fc..dac44ec 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/windows/system.py +++ b/ambari-common/src/main/python/resource_management/core/providers/windows/system.py @@ -140,7 +140,7 @@ def QueryPrivilegeState(hToken, priv): for (id, attr) in privList: if id == privId: privState = attr - Logger.debug('Privilege state: {}={} ({}) Enabled={}'.format(privId, priv, LookupPrivilegeDisplayName(None, priv), privState)) + Logger.debug('Privilege state: {0}={1} ({2}) Enabled={3}'.format(privId, priv, LookupPrivilegeDisplayName(None, priv), privState)) return privState # Execute command. As windows hdp stack heavily relies on proper environment it is better to reload fresh environment @@ -182,7 +182,7 @@ def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish= if not ok: raise Exception("Unable to create StdErr for child process") - Logger.debug("Redirecting stdout to '{}', stderr to '{}'".format(out_file.name, err_file.name)) + Logger.debug("Redirecting stdout to '{0}', stderr to '{1}'".format(out_file.name, err_file.name)) si.dwFlags = win32con.STARTF_USESTDHANDLES si.lpDesktop = "" http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py index afc82bb..c428a6d 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py @@ -26,7 +26,7 @@ from resource_management.core import shell from ambari_commons import os_check def get_unique_id_and_date(): if os_check.OSCheck.is_windows_family(): - from ambari_commons.os_windows import run_os_command + from ambari_commons.os_utils import run_os_command code, out, err = run_os_command("cmd /c vol C:") for line in out.splitlines(): if line.startswith(" Volume Serial Number is"): http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-common/src/test/python/only_for_platform.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/test/python/only_for_platform.py b/ambari-common/src/test/python/only_for_platform.py index a0b5442..1b08db5 100644 --- a/ambari-common/src/test/python/only_for_platform.py +++ b/ambari-common/src/test/python/only_for_platform.py @@ -33,3 +33,15 @@ def only_for_platform(system): if platform.system() == system: return obj return decorator + +def not_for_platform(system): + def decorator(obj): + if platform.system() != system: + return obj + return decorator + +def for_specific_platforms(systems): + def decorator(obj): + if platform.system() in systems: + return obj + return decorator http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-server/conf/windows/ambari.properties ---------------------------------------------------------------------- diff --git a/ambari-server/conf/windows/ambari.properties b/ambari-server/conf/windows/ambari.properties index 3982bb9..ec4da55 100644 --- a/ambari-server/conf/windows/ambari.properties +++ b/ambari-server/conf/windows/ambari.properties @@ -30,6 +30,7 @@ jdk1.7.67.dest-file=jdk-7u67-windows-x64.exe jdk1.7.67.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip jdk1.7.67.jcpol-file=UnlimitedJCEPolicyJDK7.zip jdk1.7.67.home=C:\\jdk1.7.0_67 +jdk1.7.67.re=(jdk.*)/jre metadata.path=resources\\stacks common.services.path=resources\\common-services http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-server/src/main/python/ambari-server-windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server-windows.py b/ambari-server/src/main/python/ambari-server-windows.py index 6c4f894..5f9d58f 100644 --- a/ambari-server/src/main/python/ambari-server-windows.py +++ b/ambari-server/src/main/python/ambari-server-windows.py @@ -19,10 +19,10 @@ limitations under the License. ''' import optparse +import subprocess from ambari_commons.ambari_service import AmbariService, ENV_PYTHON_PATH -from ambari_commons.logging_utils import * -from ambari_commons.os_utils import remove_file +from ambari_commons.logging_utils import get_verbose, set_verbose, get_silent, set_silent, get_debug_mode, set_debug_mode from ambari_commons.os_windows import SvcStatusCallback from ambari_server import utils @@ -42,11 +42,6 @@ SUSPEND_START_MODE = False ambari_provider_module_option = "" ambari_provider_module = os.environ.get('AMBARI_PROVIDER_MODULE') -#Common setup or upgrade message -SETUP_OR_UPGRADE_MSG = "- If this is a new setup, then run the \"ambari-server setup\" command to create the user\n" \ -"- If this is an upgrade of an existing setup, run the \"ambari-server upgrade\" command.\n" \ -"Refer to the Ambari documentation for more information on setup and upgrade." - AMBARI_SERVER_DIE_MSG = "Ambari Server java process died with exitcode {0}. Check {1} for more information." if ambari_provider_module is not None: @@ -173,22 +168,19 @@ def svcstart(): def server_process_main(options, scmStatus=None): # set verbose try: - global VERBOSE - VERBOSE = options.verbose + set_verbose(options.verbose) except AttributeError: pass # set silent try: - global SILENT - SILENT = options.silent + set_silent(options.silent) except AttributeError: pass # debug mode try: - global DEBUG_MODE - DEBUG_MODE = options.debug + set_debug_mode(options.debug) except AttributeError: pass @@ -242,7 +234,7 @@ def server_process_main(options, scmStatus=None): java_exe = jdk_path + os.sep + JAVA_EXE_SUBPATH pidfile = PID_DIR + os.sep + PID_NAME - command_base = SERVER_START_CMD_DEBUG if (DEBUG_MODE or SERVER_START_DEBUG) else SERVER_START_CMD + command_base = SERVER_START_CMD_DEBUG if (get_debug_mode() or SERVER_START_DEBUG) else SERVER_START_CMD suspend_mode = 'y' if SUSPEND_START_MODE else 'n' command = command_base.format(conf_dir, suspend_mode) if not os.path.exists(PID_DIR): @@ -305,7 +297,7 @@ def ensure_resources_are_organized(properties): resource_files_keeper = ResourceFilesKeeper(resources_location) try: print "Organizing resource files at {0}...".format(resources_location, - verbose=VERBOSE) + verbose=get_verbose()) resource_files_keeper.perform_housekeeping() except KeeperException, ex: msg = "Can not organize resource files at {0}: {1}".format( http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-server/src/main/python/ambari-server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py index a6366e7..6388dc6 100755 --- a/ambari-server/src/main/python/ambari-server.py +++ b/ambari-server/src/main/python/ambari-server.py @@ -39,16 +39,17 @@ import datetime import tempfile import random import pwd + +from ambari_commons.logging_utils import get_verbose, set_verbose, get_silent, set_silent, get_debug_mode, \ + set_debug_mode from ambari_server.resourceFilesKeeper import ResourceFilesKeeper, KeeperException import json import base64 -from threading import Thread + from ambari_commons import OSCheck, OSConst, Firewall from ambari_server import utils, BackupRestore # debug settings -VERBOSE = False -SILENT = False SERVER_START_DEBUG = False # ldap settings @@ -1390,7 +1391,7 @@ def execute_remote_script(args, scriptPath): tool = get_db_cli_tool(args) if not tool: # args.warnings.append('{0} not found. Please, run DDL script manually'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX])) - if VERBOSE: + if get_verbose(): print_warning_msg('{0} not found'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX])) return -1, "Client wasn't found", "Client wasn't found" CMD = get_remote_script_line(args, scriptPath, False) @@ -1405,7 +1406,7 @@ def get_remote_script_line(args, scriptPath, forPrint=True): tool = get_db_cli_tool(args) if not tool: # args.warnings.append('{0} not found. Please, run DDL script manually'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX])) - if VERBOSE or args.persistence_type == "remote": + if get_verbose() or args.persistence_type == "remote": print_warning_msg('{0} not found'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX])) return None if args.dbms == "postgres": @@ -2205,7 +2206,7 @@ def check_jdbc_drivers(args): JAVA_SHARE_PATH) if result == -1: - if SILENT: + if get_silent(): print_error_msg(msg) raise FatalException(-1, msg) else: @@ -2251,7 +2252,7 @@ def verify_setup_allowed(): isSecure = get_is_secure(properties) (isPersisted, masterKeyFile) = get_is_persisted(properties) - if isSecure and not isPersisted and SILENT: + if isSecure and not isPersisted and get_silent(): print "ERROR: Cannot run silent 'setup' with password encryption enabled " \ "and Master Key not persisted." print "Ambari Server 'setup' exiting." @@ -2432,8 +2433,7 @@ def proceedJDBCProperties(args): # def reset(args): #force reset if silent option provided - global SILENT - if SILENT: + if get_silent(): default = "yes" else: default = "no" @@ -2452,7 +2452,7 @@ def reset(args): "You will be required to re-configure the Ambari server " "and re-run the cluster wizard. \n" "Are you SURE you want to perform the reset " - "[yes/no] ({0})? ".format(default), SILENT) + "[yes/no] ({0})? ".format(default), get_silent()) okToRun = choice if not okToRun: @@ -2480,7 +2480,7 @@ def reset(args): raise NonFatalException("Please replace '*' symbols with password before running DDL`s!") else: # Run automatic reset only for embedded DB - okToRun = get_YN_input("Confirm server reset [yes/no]({0})? ".format(default), SILENT) + okToRun = get_YN_input("Confirm server reset [yes/no]({0})? ".format(default), get_silent()) if not okToRun: err = "Ambari Server 'reset' cancelled" @@ -2499,14 +2499,14 @@ def reset(args): raise FatalException(1, drop_errdata) if drop_errdata and PG_ERROR_BLOCKED in drop_errdata: raise FatalException(1, "Database is in use. Please, make sure all connections to the database are closed") - if drop_errdata and VERBOSE: + if drop_errdata and get_verbose(): print_warning_msg(drop_errdata) print_info_msg("About to run database setup") retcode, outdata, errdata = setup_db(args) - if errdata and VERBOSE: + if errdata and get_verbose(): print_warning_msg(errdata) if (errdata and 'ERROR' in errdata.upper()) or (drop_errdata and 'ERROR' in drop_errdata.upper()): - if not VERBOSE: + if not get_verbose(): raise NonFatalException("Non critical error in DDL, use --verbose for more information") else: raise NonFatalException("Non critical error in DDL") @@ -2625,7 +2625,7 @@ def start(args): environ[SECURITY_MASTER_KEY_LOCATION] = tempFilePath pidfile = PID_DIR + os.sep + PID_NAME - command_base = SERVER_START_CMD_DEBUG if (SERVER_DEBUG_MODE or SERVER_START_DEBUG) else SERVER_START_CMD + command_base = SERVER_START_CMD_DEBUG if (get_debug_mode() or SERVER_START_DEBUG) else SERVER_START_CMD command = "%s %s; %s" % (ULIMIT_CMD, str(get_ulimit_open_files()), command_base.format(jdk_path, conf_dir, @@ -2797,7 +2797,7 @@ def change_objects_owner(args): command[-1] = command[-1].format(database_name, 'ambari', new_owner) retcode, stdout, stderr = run_os_command(command) if not retcode == 0: - if VERBOSE: + if get_verbose(): if stdout: print_error_msg(stdout.strip()) if stderr: @@ -2968,7 +2968,7 @@ def status(args): # Prints an "info" messsage. # def print_info_msg(msg): - if VERBOSE: + if get_verbose(): print("INFO: " + msg) @@ -3001,7 +3001,7 @@ def get_YN_input(prompt, default): def get_choice_string_input(prompt, default, firstChoice, secondChoice): - if SILENT: + if get_silent(): print(prompt) return default choice = raw_input(prompt).lower() @@ -3021,7 +3021,7 @@ def get_validated_string_input(prompt, default, pattern, description, input = "" while not input: - if SILENT: + if get_silent(): print (prompt) input = default elif is_pass: @@ -3126,7 +3126,7 @@ def sync_ldap(): new_specs = [{"principal_type":"groups","sync_type":"specific","names":""}] get_ldap_event_spec_names(LDAP_SYNC_GROUPS, specs, new_specs) - if VERBOSE: + if get_verbose(): sys.stdout.write('\nCalling API ' + SERVER_API_LDAP_URL + ' : ' + str(bodies) + '\n') request.add_data(json.dumps(bodies)) @@ -3834,7 +3834,7 @@ def setup_https(args): 'root-level privileges' raise FatalException(4, err) args.exit_message = None - if not SILENT: + if not get_silent(): properties = get_ambari_properties() try: security_server_keys_dir = properties.get_property(SSL_KEY_DIR) @@ -3912,7 +3912,7 @@ def is_server_runing(): def setup_component_https(component, command, property, alias): - if not SILENT: + if not get_silent(): jdk_path = find_jdk() if jdk_path is None: @@ -4149,7 +4149,7 @@ def generate_random_string(length=SSL_KEY_PASSWORD_LENGTH): def get_validated_filepath_input(prompt, description, default=None): input = False while not input: - if SILENT: + if get_silent(): print (prompt) return default else: @@ -4380,7 +4380,7 @@ def refresh_stack_hash(): try: print "Organizing resource files at {0}...".format(resources_location, - verbose=VERBOSE) + verbose=get_verbose()) resource_files_keeper.perform_housekeeping() except KeeperException, ex: msg = "Can not organize resource files at {0}: {1}".format( @@ -4464,16 +4464,13 @@ def main(): (options, args) = parser.parse_args() # set verbose - global VERBOSE - VERBOSE = options.verbose + set_verbose(options.verbose) # set silent - global SILENT - SILENT = options.silent + set_silent(options.silent) # debug mode - global SERVER_DEBUG_MODE - SERVER_DEBUG_MODE = options.debug + set_debug_mode(options.debug) # set ldap_sync_all global LDAP_SYNC_ALL http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-server/src/main/python/ambari_server/dbConfiguration_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/dbConfiguration_linux.py b/ambari-server/src/main/python/ambari_server/dbConfiguration_linux.py index ce47fae..62853b7 100644 --- a/ambari-server/src/main/python/ambari_server/dbConfiguration_linux.py +++ b/ambari-server/src/main/python/ambari_server/dbConfiguration_linux.py @@ -513,14 +513,14 @@ class PGConfig(DBMSConfig): raise FatalException(1, drop_errdata) if drop_errdata and PG_ERROR_BLOCKED in drop_errdata: raise FatalException(1, "Database is in use. Please, make sure all connections to the database are closed") - if drop_errdata and VERBOSE: + if drop_errdata and get_verbose(): print_warning_msg(drop_errdata) print_info_msg("About to run database setup") retcode, outdata, errdata = setup_db(args) - if errdata and VERBOSE: + if errdata and get_verbose(): print_warning_msg(errdata) if (errdata and 'ERROR' in errdata.upper()) or (drop_errdata and 'ERROR' in drop_errdata.upper()): - if not VERBOSE: + if not get_verbose(): raise NonFatalException("Non critical error in DDL, use --verbose for more information") else: raise NonFatalException("Non critical error in DDL") @@ -631,7 +631,7 @@ class PGDatabase: tool = get_db_cli_tool(args) if not tool: # args.warnings.append('{0} not found. Please, run DDL script manually'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX])) - if VERBOSE: + if get_verbose(): print_warning_msg('{0} not found'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX])) return -1, "Client wasn't found", "Client wasn't found" http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-server/src/main/python/ambari_server/serverConfiguration.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py index 5e5c440..6c43669 100644 --- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py +++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py @@ -21,6 +21,7 @@ limitations under the License. import datetime import glob import re +import tempfile from ambari_commons.os_utils import * from ambari_commons.logging_utils import print_warning_msg, print_info_msg, print_error_msg @@ -120,6 +121,13 @@ STACK_LOCATION_DEFAULT = "resources" + os.sep + "stacks" # JDK JDK_RELEASES="java.releases" +#Common setup or upgrade message +SETUP_OR_UPGRADE_MSG = "- If this is a new setup, then run the \"ambari-server setup\" command to create the user\n" \ + "- If this is an upgrade of an existing setup, run the \"ambari-server upgrade\" command.\n" \ + "Refer to the Ambari documentation for more information on setup and upgrade." + +DEFAULT_DB_NAME = "ambari" + # configuration backup back_up_file_path = None http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-server/src/main/python/ambari_server/serverSetup.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverSetup.py b/ambari-server/src/main/python/ambari_server/serverSetup.py index 2b9c950..b541ffc 100644 --- a/ambari-server/src/main/python/ambari_server/serverSetup.py +++ b/ambari-server/src/main/python/ambari_server/serverSetup.py @@ -50,7 +50,7 @@ def verify_setup_allowed(): isSecure = get_is_secure(properties) (isPersisted, masterKeyFile) = get_is_persisted(properties) - if isSecure and not isPersisted and SILENT: + if isSecure and not isPersisted and get_silent(): print "ERROR: Cannot run silent 'setup' with password encryption enabled " \ "and Master Key not persisted." print "Ambari Server 'setup' exiting." @@ -526,6 +526,9 @@ def reset(options, serviceClass): def is_server_running(serviceClass): statusStr = serviceClass.QueryStatus() + from ambari_commons.os_windows import SERVICE_STATUS_STARTING, SERVICE_STATUS_RUNNING, SERVICE_STATUS_STOPPING, \ + SERVICE_STATUS_STOPPED, SERVICE_STATUS_NOT_INSTALLED + if statusStr in(SERVICE_STATUS_STARTING, SERVICE_STATUS_RUNNING, SERVICE_STATUS_STOPPING): return True, "" elif statusStr == SERVICE_STATUS_STOPPED: http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-server/src/main/python/ambari_server/serverSetup_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverSetup_linux.py b/ambari-server/src/main/python/ambari_server/serverSetup_linux.py index b5436e0..ea92a7d 100644 --- a/ambari-server/src/main/python/ambari_server/serverSetup_linux.py +++ b/ambari-server/src/main/python/ambari_server/serverSetup_linux.py @@ -131,9 +131,6 @@ JDBC_DB_DEFAULT_DRIVER = {"postgresql" : "postgresql-jdbc.jar", "mysql" : "mysql ORACLE_DB_ID_TYPES = ["Service Name", "SID"] -DEFAULT_DB_NAME = "ambari" - - MESSAGE_ERROR_NOT_ROOT = 'Ambari-server setup should be run with root-level privileges' MESSAGE_CHECK_FIREWALL = 'Checking iptables...' @@ -360,7 +357,7 @@ def os_setup_jdbc_drivers(args): JAVA_SHARE_PATH) if result == -1: - if SILENT: + if get_silent(): print_error_msg(msg) raise FatalException(-1, msg) else: http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-server/src/main/python/ambari_server/serverSetup_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverSetup_windows.py b/ambari-server/src/main/python/ambari_server/serverSetup_windows.py index a906ef5..00b7d0b 100644 --- a/ambari-server/src/main/python/ambari_server/serverSetup_windows.py +++ b/ambari-server/src/main/python/ambari_server/serverSetup_windows.py @@ -30,7 +30,7 @@ from ambari_commons import os_utils from ambari_commons.exceptions import * from ambari_commons.logging_utils import * -from ambari_commons.os_windows import run_os_command, UserHelper +from ambari_commons.os_windows import run_powershell_script, UserHelper, CHECK_FIREWALL_SCRIPT from ambari_server.dbConfiguration import DBMSConfig from ambari_server.serverConfiguration import * from ambari_server.serverConfiguration_windows import OUT_DIR http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-server/src/main/python/ambari_server/setupSecurity.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/setupSecurity.py b/ambari-server/src/main/python/ambari_server/setupSecurity.py index 57a3231..e6879b9 100644 --- a/ambari-server/src/main/python/ambari_server/setupSecurity.py +++ b/ambari-server/src/main/python/ambari_server/setupSecurity.py @@ -23,9 +23,11 @@ import random import socket import stat import sys +import tempfile import urllib2 from ambari_commons.exceptions import * +from ambari_commons.os_utils import run_os_command from serverConfiguration import * from setupActions import * from userInput import * @@ -774,7 +776,7 @@ def setup_https(args): 'root-level privileges' raise FatalException(4, err) args.exit_message = None - if not SILENT: + if not get_silent(): properties = get_ambari_properties() try: security_server_keys_dir = properties.get_property(SSL_KEY_DIR) @@ -824,7 +826,7 @@ def setup_https(args): def setup_component_https(component, command, property, alias): - if not SILENT: + if not get_silent(): jdk_path = find_jdk() if jdk_path is None: err = "No JDK found, please run the \"ambari-server setup\" " \ http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-server/src/main/python/ambari_server/userInput.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/userInput.py b/ambari-server/src/main/python/ambari_server/userInput.py index 7a35831..54ad6c8 100644 --- a/ambari-server/src/main/python/ambari_server/userInput.py +++ b/ambari-server/src/main/python/ambari_server/userInput.py @@ -22,7 +22,8 @@ import os import getpass import re -from ambari_commons.logging_utils import * +from ambari_commons.logging_utils import get_silent +from ambari_commons.os_utils import get_password # @@ -30,9 +31,6 @@ from ambari_commons.logging_utils import * # # return True if 'y' or False if 'n' # -from ambari_commons.os_utils import get_password - - def get_YN_input(prompt, default): yes = set(['yes', 'ye', 'y']) no = set(['no', 'n']) @@ -40,7 +38,7 @@ def get_YN_input(prompt, default): def get_choice_string_input(prompt, default, firstChoice, secondChoice): - if SILENT: + if get_silent(): print(prompt) return default choice = raw_input(prompt).lower() @@ -60,7 +58,7 @@ def get_validated_string_input(prompt, default, pattern, description, input = "" while not input: - if SILENT: + if get_silent(): print (prompt) input = default elif is_pass: @@ -96,7 +94,7 @@ def get_validated_string_input(prompt, default, pattern, description, def get_validated_filepath_input(prompt, description, default=None): input = False while not input: - if SILENT: + if get_silent(): print (prompt) return default else: http://git-wip-us.apache.org/repos/asf/ambari/blob/16bee385/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py index 3221cdb..83c8037 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py @@ -24,9 +24,9 @@ import json import sys import traceback from resource_management import * -from resource_management.libraries.functions.list_ambari_managed_repos import * +from resource_management.libraries.functions.list_ambari_managed_repos import list_ambari_managed_repos from ambari_commons.os_check import OSCheck, OSConst -from resource_management.libraries.functions import packages_analyzer +from resource_management.libraries.functions.packages_analyzer import allInstalledPackages class InstallPackages(Script): @@ -81,7 +81,7 @@ class InstallPackages(Script): packages_were_checked = False try: packages_installed_before = [] - packages_analyzer.allInstalledPackages(packages_installed_before) + allInstalledPackages(packages_installed_before) packages_installed_before = [package[0] for package in packages_installed_before] packages_were_checked = True for package in package_list: @@ -95,7 +95,7 @@ class InstallPackages(Script): # Remove already installed packages in case of fail if packages_were_checked and packages_installed_before: packages_installed_after = [] - packages_analyzer.allInstalledPackages(packages_installed_after) + allInstalledPackages(packages_installed_after) packages_installed_after = [package[0] for package in packages_installed_after] packages_installed_before = set(packages_installed_before) new_packages_installed = [package for package in packages_installed_after if package not in packages_installed_before]
