URL: https://github.com/freeipa/freeipa/pull/5069 Author: stanislavlevin Title: #5069: [Backport][ipa-4-8] [Azure] Add rawhide definitions Action: opened
PR body: """ This PR was opened manually because PR #5047 was pushed to master and backport to `ipa-4-8` is required. """ To pull the PR as Git branch: git remote add ghfreeipa https://github.com/freeipa/freeipa git fetch ghfreeipa pull/5069/head:pr5069 git checkout pr5069
From 3b319c1ae96173a437960f2986ebc0716a614a41 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Tue, 18 Aug 2020 10:28:59 +0300 Subject: [PATCH 1/8] Azure: Add Rawhide definitions - allow override variables template file with an externally provided one. This allows to add new Azure Pipeline which will point to a custom platform definition. Note: Azure's WebUI variables are runtime variables and not available at parsing time, that's why it's impossible to override template from WebUI in this case. - add Rawhide templates - add Dockerfile for build Rawhie Docker image for tests phase Note: 'fedora:rawhide' is too old, use for now 'registry.fedoraproject.org/fedora:rawhide'. See, https://bugzilla.redhat.com/show_bug.cgi?id=1869612 Signed-off-by: Stanislav Levin <s...@altlinux.org> --- .../Dockerfiles/Dockerfile.build.rawhide | 31 +++++++++++++++++++ ipatests/azure/azure-pipelines-rawhide.yml | 4 +++ ipatests/azure/azure-pipelines.yml | 6 +++- .../azure/templates/prepare-tox-fedora.yml | 2 +- .../azure/templates/variables-rawhide.yml | 23 ++++++++++++++ 5 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 ipatests/azure/Dockerfiles/Dockerfile.build.rawhide create mode 100644 ipatests/azure/azure-pipelines-rawhide.yml create mode 100644 ipatests/azure/templates/variables-rawhide.yml diff --git a/ipatests/azure/Dockerfiles/Dockerfile.build.rawhide b/ipatests/azure/Dockerfiles/Dockerfile.build.rawhide new file mode 100644 index 0000000000..3318c33fd1 --- /dev/null +++ b/ipatests/azure/Dockerfiles/Dockerfile.build.rawhide @@ -0,0 +1,31 @@ +# replace with 'fedora:rawhide' on fix: +# https://bugzilla.redhat.com/show_bug.cgi?id=1869612 +FROM registry.fedoraproject.org/fedora:rawhide +MAINTAINER [FreeIPA Developers freeipa-devel@lists.fedorahosted.org] +ENV container=docker LANG=en_US.utf8 LANGUAGE=en_US.utf8 LC_ALL=en_US.utf8 + +ADD dist /root +RUN echo 'deltarpm = false' >> /etc/dnf/dnf.conf \ + && dnf update -y dnf \ + && sed -i 's/%_install_langs \(.*\)/\0:fr/g' /etc/rpm/macros.image-language-conf \ + && dnf install -y systemd \ + && dnf install -y \ + firewalld \ + glibc-langpack-fr \ + glibc-langpack-en \ + iptables \ + nss-tools \ + openssh-server \ + sudo \ + wget \ + /root/rpms/*.rpm \ + && dnf clean all && rm -rf /root/rpms /root/srpms \ + && sed -i 's/.*PermitRootLogin .*/#&/g' /etc/ssh/sshd_config \ + && echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config \ + && systemctl enable sshd \ + && for i in /usr/lib/systemd/system/*-domainname.service; \ + do sed -i 's#^ExecStart=/#ExecStart=-/#' $i ; done + +STOPSIGNAL RTMIN+3 +VOLUME ["/freeipa", "/run", "/tmp"] +ENTRYPOINT [ "/usr/sbin/init" ] diff --git a/ipatests/azure/azure-pipelines-rawhide.yml b/ipatests/azure/azure-pipelines-rawhide.yml new file mode 100644 index 0000000000..e6fd16bb99 --- /dev/null +++ b/ipatests/azure/azure-pipelines-rawhide.yml @@ -0,0 +1,4 @@ +extends: + template: azure-pipelines.yml + parameters: + VARIABLES_FILE: 'templates/variables-rawhide.yml' diff --git a/ipatests/azure/azure-pipelines.yml b/ipatests/azure/azure-pipelines.yml index bc7e6bf913..8e7207d1e9 100644 --- a/ipatests/azure/azure-pipelines.yml +++ b/ipatests/azure/azure-pipelines.yml @@ -1,10 +1,14 @@ +parameters: +- name: VARIABLES_FILE + default: 'templates/variables.yml' + trigger: - master variables: - template: templates/variables-common.yml # platform specific variables, links to -- template: templates/variables.yml +- template: ${{ parameters.VARIABLES_FILE }} jobs: - job: Build diff --git a/ipatests/azure/templates/prepare-tox-fedora.yml b/ipatests/azure/templates/prepare-tox-fedora.yml index ee034b0989..4e27ac7557 100644 --- a/ipatests/azure/templates/prepare-tox-fedora.yml +++ b/ipatests/azure/templates/prepare-tox-fedora.yml @@ -1,6 +1,6 @@ steps: - script: | set -e - sudo dnf -y install nss-tools + sudo dnf -y install nss-tools python3-pip python3 -m pip install --user --upgrade pip setuptools pycodestyle displayName: Install Tox prerequisites diff --git a/ipatests/azure/templates/variables-rawhide.yml b/ipatests/azure/templates/variables-rawhide.yml new file mode 100644 index 0000000000..43201f1a34 --- /dev/null +++ b/ipatests/azure/templates/variables-rawhide.yml @@ -0,0 +1,23 @@ +variables: + IPA_PLATFORM: fedora + # the Docker public image to build IPA packages (rpms) + # + # replace with 'fedora:rawhide' on fix: + # https://bugzilla.redhat.com/show_bug.cgi?id=1869612 + DOCKER_BUILD_IMAGE: 'registry.fedoraproject.org/fedora:rawhide' + + # the Dockerfile to build Docker image for running IPA tests + DOCKER_DOCKERFILE: 'Dockerfile.build.rawhide' + + # the template to install IPA's buildtime dependencies + PREPARE_BUILD_TEMPLATE: ${{ format('prepare-build-{0}.yml', variables.IPA_PLATFORM) }} + + # the template to build IPA packages (rpms) + BUILD_TEMPLATE: ${{ format('build-{0}.yml', variables.IPA_PLATFORM) }} + PREPARE_TOX_TEMPLATE: ${{ format('prepare-tox-{0}.yml', variables.IPA_PLATFORM) }} + PREPARE_WEBUI_TEMPLATE: ${{ format('prepare-webui-{0}.yml', variables.IPA_PLATFORM) }} + + TOX_COMMAND: tox + + # Python version for UsePythonVersion@0 task + AZURE_PYTHON_VERSION: '3.9' From 936bf6065f627545e1fe0a9fc4d69eca81956a2d Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Tue, 18 Aug 2020 14:31:02 +0300 Subject: [PATCH 2/8] Azure: Drop dependency on UsePythonVersion task Python is provided by the Docker container image and it's no longer needed to bind mount host's Python into container. Signed-off-by: Stanislav Levin <s...@altlinux.org> --- ipatests/azure/azure-pipelines.yml | 8 -------- ipatests/azure/templates/variables-fedora.yml | 3 --- ipatests/azure/templates/variables-rawhide.yml | 3 --- 3 files changed, 14 deletions(-) diff --git a/ipatests/azure/azure-pipelines.yml b/ipatests/azure/azure-pipelines.yml index 8e7207d1e9..38d26815a9 100644 --- a/ipatests/azure/azure-pipelines.yml +++ b/ipatests/azure/azure-pipelines.yml @@ -89,10 +89,6 @@ jobs: options: --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --privileged --env container=docker steps: - template: templates/${{ variables.PREPARE_BUILD_TEMPLATE }} - - task: UsePythonVersion@0 - inputs: - versionSpec: ${{ variables.AZURE_PYTHON_VERSION }} - architecture: x64 - template: templates/${{ variables.PREPARE_TOX_TEMPLATE }} - script: | set -e @@ -116,10 +112,6 @@ jobs: options: --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --privileged --env container=docker steps: - template: templates/${{ variables.PREPARE_BUILD_TEMPLATE }} - - task: UsePythonVersion@0 - inputs: - versionSpec: ${{ variables.AZURE_PYTHON_VERSION }} - architecture: x64 - template: templates/${{ variables.PREPARE_WEBUI_TEMPLATE }} - script: | set -e diff --git a/ipatests/azure/templates/variables-fedora.yml b/ipatests/azure/templates/variables-fedora.yml index 9440942f73..3e8f878c22 100644 --- a/ipatests/azure/templates/variables-fedora.yml +++ b/ipatests/azure/templates/variables-fedora.yml @@ -15,6 +15,3 @@ variables: PREPARE_WEBUI_TEMPLATE: ${{ format('prepare-webui-{0}.yml', variables.IPA_PLATFORM) }} TOX_COMMAND: tox - - # Python version for UsePythonVersion@0 task - AZURE_PYTHON_VERSION: '3.8' diff --git a/ipatests/azure/templates/variables-rawhide.yml b/ipatests/azure/templates/variables-rawhide.yml index 43201f1a34..6cbc78d18d 100644 --- a/ipatests/azure/templates/variables-rawhide.yml +++ b/ipatests/azure/templates/variables-rawhide.yml @@ -18,6 +18,3 @@ variables: PREPARE_WEBUI_TEMPLATE: ${{ format('prepare-webui-{0}.yml', variables.IPA_PLATFORM) }} TOX_COMMAND: tox - - # Python version for UsePythonVersion@0 task - AZURE_PYTHON_VERSION: '3.9' From 27708e3c7dc0d7537650e52f16f774681bd61bcc Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Wed, 19 Aug 2020 17:51:07 +0300 Subject: [PATCH 3/8] Azure: base: Collect both install and uninstall logs Some applications remove their logs on uninstallation. As a result of this, Azure lost `install` logs. Signed-off-by: Stanislav Levin <s...@altlinux.org> --- .../azure/scripts/azure-run-base-tests.sh | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/ipatests/azure/scripts/azure-run-base-tests.sh b/ipatests/azure/scripts/azure-run-base-tests.sh index b3811dae69..a2129d2a45 100755 --- a/ipatests/azure/scripts/azure-run-base-tests.sh +++ b/ipatests/azure/scripts/azure-run-base-tests.sh @@ -5,6 +5,25 @@ # distro-specifics source "${IPA_TESTS_SCRIPTS}/variables.sh" +function collect_logs() { + if [ "$#" -ne 1 ]; then + printf "collect_logs: The path to output archive is required\n" + exit 1 + fi + local out_file="$1" + printf "Collecting logs\n" + journalctl -b --no-pager > systemd_journal.log + tar --ignore-failed-read -czf "$out_file" \ + /var/log/dirsrv \ + "$HTTPD_LOGDIR" \ + /var/log/ipa* \ + /var/log/krb5kdc.log \ + /var/log/pki \ + /var/log/samba \ + "$BIND_DATADIR" \ + systemd_journal.log +} + server_password=Secret123 echo "Installing FreeIPA master for the domain ${IPA_TESTS_DOMAIN} and realm ${IPA_TESTS_REALM}" @@ -53,6 +72,7 @@ if [ "$install_result" -eq 0 ] ; then else echo "ipa-server-install failed with code ${install_result}, skip IPA tests" fi +collect_logs ipaserver_install_logs.tar.gz echo "Potential Python 3 incompatibilities in the IPA framework:" grep -n -C5 BytesWarning "$HTTPD_ERRORLOG" || echo "Good, none detected" @@ -75,23 +95,12 @@ ipa-server-install --uninstall -U # second uninstall to verify that --uninstall without installation works ipa-server-install --uninstall -U +collect_logs ipaserver_uninstall_logs.tar.gz if [ "$install_result" -eq 0 ] ; then firewalld_cmd --remove-service={freeipa-ldap,freeipa-ldaps,dns} fi -echo "Collect the logs" -journalctl -b --no-pager > systemd_journal.log -tar --ignore-failed-read --remove-files -czf var_log.tar.gz \ - /var/log/dirsrv \ - "$HTTPD_LOGDIR" \ - /var/log/ipa* \ - /var/log/krb5kdc.log \ - /var/log/pki \ - /var/log/samba \ - "$BIND_DATADIR" \ - systemd_journal.log - echo "Report memory statistics" cat /sys/fs/cgroup/memory/memory.memsw.failcnt cat /sys/fs/cgroup/memory/memory.memsw.limit_in_bytes From 9d479fa37e2a58e31cbbea8ae429fa03f7f00384 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Thu, 27 Aug 2020 12:07:40 +0300 Subject: [PATCH 4/8] nss: Raise exception earlier on unsupported DB type For now FreeIPA handles explicit migration of NSS DB (dbm->sql). But Mozilla's NSS can be built without the support of legacy database (DBM). This implies that neither implicit nor explicit DB migration to SQL will work. So, eventually, this support will be removed from FreeIPA. With this patch, the instantiation of NSS with legacy db(if not supported by NSS) is forbidden. Fixes: https://pagure.io/freeipa/issue/8474 Signed-off-by: Stanislav Levin <s...@altlinux.org> --- ipapython/certdb.py | 23 ++++++++++++++----- ipatests/test_ipapython/test_certdb.py | 31 +++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/ipapython/certdb.py b/ipapython/certdb.py index 43245cd292..8e0faffdd0 100644 --- a/ipapython/certdb.py +++ b/ipapython/certdb.py @@ -26,10 +26,11 @@ import pwd import grp import re +import shutil import stat import tempfile +from ctypes.util import find_library from tempfile import NamedTemporaryFile -import shutil import cryptography.x509 @@ -73,6 +74,10 @@ ) +def nss_supports_dbm(): + return bool(find_library("nssdbm3")) + + def get_ca_nickname(realm, format=CA_NICKNAME_FMT): return format % realm @@ -252,15 +257,21 @@ class NSSDatabase: # Generic NSS DB code should be moved here. def __init__(self, nssdir=None, dbtype='auto'): - if nssdir is None: - self.secdir = tempfile.mkdtemp() - self._is_temporary = True - else: + if nssdir is not None: self.secdir = nssdir self._is_temporary = False - if dbtype == 'auto': + if dbtype == "auto": dbtype = self._detect_dbtype() + if dbtype == "dbm" and not nss_supports_dbm(): + raise ValueError( + "NSS is built without support of the legacy database(DBM)" + ) + + if nssdir is None: + self.secdir = tempfile.mkdtemp() + self._is_temporary = True + self.pwd_file = os.path.join(self.secdir, 'pwdfile.txt') self.dbtype = None self.certdb = self.keydb = self.secmod = None diff --git a/ipatests/test_ipapython/test_certdb.py b/ipatests/test_ipapython/test_certdb.py index e4366de6da..7c0558da81 100644 --- a/ipatests/test_ipapython/test_certdb.py +++ b/ipatests/test_ipapython/test_certdb.py @@ -4,7 +4,11 @@ import pytest -from ipapython.certdb import NSSDatabase, TRUSTED_PEER_TRUST_FLAGS +from ipapython.certdb import ( + NSSDatabase, + TRUSTED_PEER_TRUST_FLAGS, + nss_supports_dbm, +) from ipapython import ipautil from ipaplatform.osinfo import osinfo @@ -40,6 +44,10 @@ def create_selfsigned(nssdb): os.unlink(noisefile) +@pytest.mark.skipif( + not nss_supports_dbm(), + reason="NSS is built without support of the legacy database(DBM)", +) def test_dbm_tmp(): with NSSDatabase(dbtype='dbm') as nssdb: assert nssdb.dbtype == 'dbm' @@ -60,6 +68,19 @@ def test_dbm_tmp(): assert os.path.basename(nssdb.secmod) == 'secmod.db' +@pytest.mark.skipif( + nss_supports_dbm(), + reason="NSS is built with support of the legacy database(DBM)", +) +def test_dbm_raise(): + with pytest.raises(ValueError) as e: + NSSDatabase(dbtype="dbm") + assert ( + str(e.value) == "NSS is built without support of the legacy " + "database(DBM)" + ) + + def test_sql_tmp(): with NSSDatabase(dbtype='sql') as nssdb: assert nssdb.dbtype == 'sql' @@ -80,6 +101,10 @@ def test_sql_tmp(): assert os.path.basename(nssdb.secmod) == 'pkcs11.txt' +@pytest.mark.skipif( + not nss_supports_dbm(), + reason="NSS is built without support of the legacy database(DBM)", +) def test_convert_db(): with NSSDatabase(dbtype='dbm') as nssdb: assert nssdb.dbtype == 'dbm' @@ -115,6 +140,10 @@ def test_convert_db(): assert os.path.basename(nssdb.secmod) == 'pkcs11.txt' +@pytest.mark.skipif( + not nss_supports_dbm(), + reason="NSS is built without support of the legacy database(DBM)", +) def test_convert_db_nokey(): with NSSDatabase(dbtype='dbm') as nssdb: assert nssdb.dbtype == 'dbm' From f1253da215be1f4617fb924c13f54a6442d88558 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Wed, 26 Aug 2020 17:02:25 +0300 Subject: [PATCH 5/8] deps: Require `nss-tools` for make's fasttest target Otherwise, tests fail with: ``` E FileNotFoundError: [Errno 2] No such file or directory: '/usr/bin/certutil' ... =================================== short test summary info =================================== FAILED test_ipapython/test_certdb.py::test_dbm_tmp - FileNotFoundError: [Errno 2] No such fi... FAILED test_ipapython/test_certdb.py::test_sql_tmp - FileNotFoundError: [Errno 2] No such fi... FAILED test_ipapython/test_certdb.py::test_convert_db - FileNotFoundError: [Errno 2] No such... FAILED test_ipapython/test_certdb.py::test_convert_db_nokey - FileNotFoundError: [Errno 2] N... FAILED test_ipapython/test_certdb.py::test_auto_db - FileNotFoundError: [Errno 2] No such fi... FAILED test_ipapython/test_certdb.py::test_delete_cert_and_key - FileNotFoundError: [Errno 2... FAILED test_ipapython/test_certdb.py::test_check_validity - FileNotFoundError: [Errno 2] No ... ... ``` Signed-off-by: Stanislav Levin <s...@altlinux.org> --- freeipa.spec.in | 1 + 1 file changed, 1 insertion(+) diff --git a/freeipa.spec.in b/freeipa.spec.in index 3b09d6aa56..70f5bf7ae9 100755 --- a/freeipa.spec.in +++ b/freeipa.spec.in @@ -243,6 +243,7 @@ BuildRequires: python3-wheel # %if 0%{?with_lint} BuildRequires: jsl +BuildRequires: nss-tools BuildRequires: rpmlint BuildRequires: softhsm BuildRequires: keyutils From 71a8fe270c12ccd9ac65bfa4842dc6d8c0c82a34 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Thu, 27 Aug 2020 15:59:32 +0300 Subject: [PATCH 6/8] Azure: Increase verbosity for Tox task This allows to debug issues happened during packages installation: > -v, --verbose increase verbosity of reporting output. -vv mode turns off output redirection for package installation, above level two verbosity flags are passed through to pip (with two less level) (default: 0) Signed-off-by: Stanislav Levin <s...@altlinux.org> --- ipatests/azure/azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipatests/azure/azure-pipelines.yml b/ipatests/azure/azure-pipelines.yml index 38d26815a9..1ff4def3f3 100644 --- a/ipatests/azure/azure-pipelines.yml +++ b/ipatests/azure/azure-pipelines.yml @@ -96,7 +96,7 @@ jobs: export LANG=en_US.utf8 export LC_CTYPE=en_US.utf8 locale - $(TOX_COMMAND) -e py3,pypi,pylint3 + $(TOX_COMMAND) -e py3,pypi,pylint3 -vv displayName: Tox - task: PublishTestResults@2 inputs: From e73a4e4a992dfe4b6f987087f3288313b5d24c8e Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Thu, 27 Aug 2020 17:20:49 +0300 Subject: [PATCH 7/8] tox: Don't expand symlinks `virtualenv` < 20.0.0 copies system python binary into virt environment and then links `python` to it. While `virtualenv` >= 20.0.0 directly links `python` to system python binary (without copying). `realpath` by default expands symlinks. Thereby, pip attempts to install packages into the system's site-packages and fails with 'Permission denied' (non-privileged user). Fixes: https://pagure.io/freeipa/issue/8475 Signed-off-by: Stanislav Levin <s...@altlinux.org> --- .tox-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.tox-install.sh b/.tox-install.sh index 95f5c1e70a..ff22b4d79f 100755 --- a/.tox-install.sh +++ b/.tox-install.sh @@ -2,8 +2,8 @@ set -ex FLAVOR="$1" -ENVPYTHON="$(realpath "$2")" -ENVSITEPACKAGESDIR="$(realpath "$3")" +ENVPYTHON="$(realpath -s "$2")" +ENVSITEPACKAGESDIR="$(realpath -s "$3")" ENVDIR="$4" # 4...end are package requirements shift 4 From b7e6b511e3b71e872cf9bf37eb79dfb677927685 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Fri, 28 Aug 2020 16:31:10 +0300 Subject: [PATCH 8/8] dnspython: Add compatibility shim `dnspython` 2.0.0 has many changes and several deprecations like: ``` > dns.resolver.resolve() has been added, allowing control of whether search lists are used. dns.resolver.query() is retained for backwards compatibility, but deprecated. The default for search list behavior can be set at in the resolver object with the use_search_by_default parameter. The default is False. > dns.resolver.resolve_address() has been added, allowing easy address-to-name lookups. ``` The new class `DNSResolver`: - provides the compatibility layer - defaults the previous behavior (the search list configured in the system's resolver configuration is used for relative names) - defaults lifetime to 15sec (determines the number of seconds to spend trying to get an answer to the question) Fixes: https://pagure.io/freeipa/issue/8383 Signed-off-by: Stanislav Levin <s...@altlinux.org> --- ipaclient/discovery.py | 6 +- ipaclient/install/client.py | 9 +- ipalib/util.py | 22 +++-- ipapython/dnsutil.py | 92 ++++++++++++++++++- ipapython/ipautil.py | 19 ---- ipaserver/install/bindinstance.py | 11 ++- ipaserver/install/dns.py | 8 +- ipaserver/install/installutils.py | 13 ++- ipaserver/install/server/replicainstall.py | 10 +- ipaserver/plugins/cert.py | 4 +- ipaserver/plugins/dns.py | 24 +++-- ipaserver/plugins/host.py | 6 +- ipatests/pytest_ipa/integration/tasks.py | 5 +- .../test_integration/test_dns_locations.py | 6 +- ipatests/test_integration/test_dnssec.py | 6 +- 15 files changed, 157 insertions(+), 84 deletions(-) diff --git a/ipaclient/discovery.py b/ipaclient/discovery.py index c322e97124..fb344a34a8 100644 --- a/ipaclient/discovery.py +++ b/ipaclient/discovery.py @@ -24,11 +24,11 @@ import six -from dns import resolver, rdatatype +from dns import rdatatype from dns.exception import DNSException from ipalib import errors from ipalib.util import validate_domain_name -from ipapython.dnsutil import query_srv +from ipapython.dnsutil import query_srv, resolve from ipaplatform.paths import paths from ipapython.ipautil import valid_ip, realm_to_suffix @@ -562,7 +562,7 @@ def ipadnssearchkrbrealm(self, domain=None): logger.debug("Search DNS for TXT record of %s", qname) try: - answers = resolver.query(qname, rdatatype.TXT) + answers = resolve(qname, rdatatype.TXT) except DNSException as e: logger.debug("DNS record not found: %s", e.__class__.__name__) answers = [] diff --git a/ipaclient/install/client.py b/ipaclient/install/client.py index 5b4caa09a6..960e25e061 100644 --- a/ipaclient/install/client.py +++ b/ipaclient/install/client.py @@ -53,7 +53,7 @@ from ipaplatform.constants import constants from ipaplatform.paths import paths from ipaplatform.tasks import tasks -from ipapython import certdb, kernel_keyring, ipaldap, ipautil +from ipapython import certdb, kernel_keyring, ipaldap, ipautil, dnsutil from ipapython.admintool import ScriptError from ipapython.dn import DN from ipapython.install import typing @@ -1440,7 +1440,7 @@ def verify_dns_update(fqdn, ips): logger.debug('DNS resolver: Query: %s IN %s', fqdn, dns.rdatatype.to_text(record_type)) try: - answers = dns.resolver.query(fqdn, record_type) + answers = dnsutil.resolve(fqdn, record_type) except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN): logger.debug('DNS resolver: No record.') except dns.resolver.NoNameservers: @@ -1460,10 +1460,9 @@ def verify_dns_update(fqdn, ips): missing_reverse = [str(ip) for ip in ips] for ip in ips: ip_str = str(ip) - addr = dns.reversename.from_address(ip_str) - logger.debug('DNS resolver: Query: %s IN PTR', addr) + logger.debug('DNS resolver: Query: %s IN PTR', ip_str) try: - answers = dns.resolver.query(addr, dns.rdatatype.PTR) + answers = dnsutil.resolve_address(ip_str) except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN): logger.debug('DNS resolver: No record.') except dns.resolver.NoNameservers: diff --git a/ipalib/util.py b/ipalib/util.py index 786cfa71eb..dc22c02789 100644 --- a/ipalib/util.py +++ b/ipalib/util.py @@ -42,7 +42,7 @@ import subprocess import netaddr -from dns import resolver, rdatatype +from dns import rdatatype from dns.exception import DNSException from dns.resolver import NXDOMAIN from netaddr.core import AddrFormatError @@ -67,8 +67,12 @@ from ipaplatform.paths import paths from ipapython.ssh import SSHPublicKey from ipapython.dn import DN, RDN -from ipapython.dnsutil import DNSName -from ipapython.dnsutil import resolve_ip_addresses +from ipapython.dnsutil import ( + DNSName, + DNSResolver, + resolve, + resolve_ip_addresses, +) from ipapython.admintool import ScriptError if sys.version_info >= (3, 2): @@ -117,13 +121,13 @@ def has_soa_or_ns_record(domain): Returns True or False. """ try: - resolver.query(domain, rdatatype.SOA) + resolve(domain, rdatatype.SOA) soa_record_found = True except DNSException: soa_record_found = False try: - resolver.query(domain, rdatatype.NS) + resolve(domain, rdatatype.NS) ns_record_found = True except DNSException: ns_record_found = False @@ -797,7 +801,7 @@ def _resolve_record(owner, rtype, nameserver_ip=None, edns0=False, assert isinstance(nameserver_ip, str) or nameserver_ip is None assert isinstance(rtype, str) - res = dns.resolver.Resolver() + res = DNSResolver() if nameserver_ip: res.nameservers = [nameserver_ip] res.lifetime = timeout @@ -815,7 +819,7 @@ def _resolve_record(owner, rtype, nameserver_ip=None, edns0=False, elif edns0: res.use_edns(0, 0, 4096) - return res.query(owner, rtype) + return res.resolve(owner, rtype) def _validate_edns0_forwarder(owner, rtype, ip_addr, timeout=10): @@ -985,7 +989,7 @@ def detect_dns_zone_realm_type(api, domain): kerberos_record_name = kerberos_prefix + domain_suffix try: - result = resolver.query(kerberos_record_name, rdatatype.TXT) + result = resolve(kerberos_record_name, rdatatype.TXT) answer = result.response.answer # IPA domain will have only one _kerberos TXT record @@ -1012,7 +1016,7 @@ def detect_dns_zone_realm_type(api, domain): try: # The presence of this record is enough, return foreign in such case - resolver.query(ad_specific_record_name, rdatatype.SRV) + resolve(ad_specific_record_name, rdatatype.SRV) except DNSException: # If we could not detect type with certainty, return unknown return 'unknown' diff --git a/ipapython/dnsutil.py b/ipapython/dnsutil.py index aec1ec231a..7c4ef7acea 100644 --- a/ipapython/dnsutil.py +++ b/ipapython/dnsutil.py @@ -27,6 +27,7 @@ import dns.resolver import dns.rdataclass import dns.rdatatype +import dns.reversename import six @@ -39,6 +40,85 @@ logger = logging.getLogger(__name__) +ipa_resolver = None + + +def get_ipa_resolver(): + global ipa_resolver + if ipa_resolver is None: + ipa_resolver = DNSResolver() + return ipa_resolver + + +def resolve(*args, **kwargs): + return get_ipa_resolver().resolve(*args, **kwargs) + + +def resolve_address(*args, **kwargs): + return get_ipa_resolver().resolve_address(*args, **kwargs) + + +def zone_for_name(*args, **kwargs): + if "resolver" not in kwargs: + kwargs["resolver"] = get_ipa_resolver() + + return dns.resolver.zone_for_name(*args, **kwargs) + + +def reset_default_resolver(): + """Re-initialize ipa resolver. + """ + global ipa_resolver + ipa_resolver = DNSResolver() + + +class DNSResolver(dns.resolver.Resolver): + """DNS stub resolver compatible with both dnspython < 2.0.0 + and dnspython >= 2.0.0. + + Set `use_search_by_default` attribute to `True`, which + determines the default for whether the search list configured + in the system's resolver configuration is used for relative + names, and whether the resolver's domain may be added to relative + names. + + Increase the default lifetime which determines the number of seconds + to spend trying to get an answer to the question. dnspython 2.0.0 + changes this to 5sec, while the previous one was 30sec. + """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.reset_ipa_defaults() + self.resolve = getattr(super(), "resolve", self.query) + self.resolve_address = getattr( + super(), + "resolve_address", + self._resolve_address + ) + + def reset_ipa_defaults(self): + self.use_search_by_default = True + # the default is 5sec + self.lifetime = 15 + + def reset(self): + super().reset() + self.reset_ipa_defaults() + + def _resolve_address(self, ip_address, *args, **kwargs): + """Query nameservers for PTR records. + + :param ip_address: IPv4 or IPv6 address + :type ip_address: str + """ + return resolve( + dns.reversename.from_address(ip_address), + rdtype=dns.rdatatype.PTR, + *args, + **kwargs, + ) + + class DNSZoneAlreadyExists(dns.exception.DNSException): supp_kwargs = {'zone', 'ns'} fmt = (u"DNS zone {zone} already exists in DNS " @@ -321,7 +401,7 @@ def resolve_rrsets(fqdn, rdtypes): rrsets = [] for rdtype in rdtypes: try: - answer = dns.resolver.query(fqdn, rdtype) + answer = resolve(fqdn, rdtype) logger.debug('found %d %s records for %s: %s', len(answer), rdtype, @@ -363,7 +443,7 @@ def check_zone_overlap(zone, raise_on_error=True): return try: - containing_zone = dns.resolver.zone_for_name(zone) + containing_zone = zone_for_name(zone) except dns.exception.DNSException as e: msg = ("DNS check for domain %s failed: %s." % (zone, e)) if raise_on_error: @@ -374,7 +454,7 @@ def check_zone_overlap(zone, raise_on_error=True): if containing_zone == zone: try: - ns = [ans.to_text() for ans in dns.resolver.query(zone, 'NS')] + ns = [ans.to_text() for ans in resolve(zone, 'NS')] except dns.exception.DNSException as e: logger.debug("Failed to resolve nameserver(s) for domain %s: %s", zone, e) @@ -463,6 +543,8 @@ def query_srv(qname, resolver=None, **kwargs): :return: list of dns.rdtypes.IN.SRV.SRV instances """ if resolver is None: - resolver = dns.resolver - answer = resolver.query(qname, rdtype=dns.rdatatype.SRV, **kwargs) + resolve_f = resolve + else: + resolve_f = getattr(resolver, "resolve", resolver.query) + answer = resolve_f(qname, rdtype=dns.rdatatype.SRV, **kwargs) return sort_prio_weight(answer) diff --git a/ipapython/ipautil.py b/ipapython/ipautil.py index f29126c645..4d3bdead37 100644 --- a/ipapython/ipautil.py +++ b/ipapython/ipautil.py @@ -43,9 +43,6 @@ import collections import urllib -from dns import resolver, reversename -from dns.exception import DNSException - import six from six.moves import input @@ -1112,22 +1109,6 @@ def check_port_bindable(port, socket_type=socket.SOCK_STREAM): s.close() -def reverse_record_exists(ip_address): - """ - Checks if IP address have some reverse record somewhere. - Does not care where it points. - - Returns True/False - """ - reverse = reversename.from_address(str(ip_address)) - try: - resolver.query(reverse, "PTR") - except DNSException: - # really don't care what exception, PTR is simply unresolvable - return False - return True - - def config_replace_variables(filepath, replacevars=dict(), appendvars=dict(), removevars=None): """ diff --git a/ipaserver/install/bindinstance.py b/ipaserver/install/bindinstance.py index 90cc9b38bc..4c5e2d690b 100644 --- a/ipaserver/install/bindinstance.py +++ b/ipaserver/install/bindinstance.py @@ -30,9 +30,9 @@ import sys import time -import dns.resolver import ldap import six +from dns.exception import DNSException from ipaserver.dns_data_management import ( IPASystemRecords, @@ -320,10 +320,15 @@ def read_reverse_zone(default, ip_address, allow_zone_overlap=False): def get_auto_reverse_zones(ip_addresses, allow_zone_overlap=False): auto_zones = [] for ip in ip_addresses: - if ipautil.reverse_record_exists(ip): + try: + dnsutil.resolve_address(str(ip)) + except DNSException: + pass + else: # PTR exist there is no reason to create reverse zone logger.info("Reverse record for IP address %s already exists", ip) continue + default_reverse = get_reverse_zone_default(ip) if not allow_zone_overlap: try: @@ -1135,7 +1140,7 @@ def __setup_resolv_conf(self): else: # python DNS might have global resolver cached in this variable # we have to re-initialize it because resolv.conf has changed - dns.resolver.reset_default_resolver() + dnsutil.reset_default_resolver() def __generate_rndc_key(self): installutils.check_entropy() diff --git a/ipaserver/install/dns.py b/ipaserver/install/dns.py index 132d89f223..f7d793e1de 100644 --- a/ipaserver/install/dns.py +++ b/ipaserver/install/dns.py @@ -12,13 +12,9 @@ import enum import logging import os - -# absolute import is necessary because IPA module dns clashes with python-dns -from dns import resolver -import six - import sys +import six from subprocess import CalledProcessError from ipalib import api @@ -294,7 +290,7 @@ def install_check(standalone, api, replica, options, hostname): if not options.forwarders: options.forwarders = [] if options.auto_forwarders: - options.forwarders += resolver.get_default_resolver().nameservers + options.forwarders += dnsutil.get_ipa_resolver().nameservers elif standalone or not replica: options.forwarders = read_dns_forwarders() diff --git a/ipaserver/install/installutils.py b/ipaserver/install/installutils.py index a3274d5797..a46acf9f5f 100644 --- a/ipaserver/install/installutils.py +++ b/ipaserver/install/installutils.py @@ -38,7 +38,7 @@ from configparser import ConfigParser as SafeConfigParser from configparser import NoOptionError -from dns import resolver, rdatatype +from dns import rdatatype from dns.exception import DNSException import ldap import six @@ -53,6 +53,7 @@ from ipalib.util import validate_hostname from ipalib import api, errors, x509 from ipapython.dn import DN +from ipapython.dnsutil import resolve, get_ipa_resolver from ipaserver.install import certs, service, sysupgrade from ipaplatform import services from ipaplatform.paths import paths @@ -187,7 +188,7 @@ def verify_fqdn(host_name, no_host_dns=False, local_hostname=True): # Verify this is NOT a CNAME try: logger.debug('Check if %s is not a CNAME', host_name) - resolver.query(host_name, rdatatype.CNAME) + resolve(host_name, rdatatype.CNAME) raise HostReverseLookupError("The IPA Server Hostname cannot be a CNAME, only A and AAAA names are allowed.") except DNSException: pass @@ -284,11 +285,13 @@ def read_ip_addresses(): def read_dns_forwarders(): addrs = [] if ipautil.user_input("Do you want to configure DNS forwarders?", True): - print("Following DNS servers are configured in /etc/resolv.conf: %s" % - ", ".join(resolver.get_default_resolver().nameservers)) + print( + "Following DNS servers are configured in /etc/resolv.conf: %s" + % ", ".join(get_ipa_resolver().nameservers) + ) if ipautil.user_input("Do you want to configure these servers as DNS " "forwarders?", True): - addrs = resolver.default_resolver.nameservers[:] + addrs = get_ipa_resolver().nameservers[:] print("All DNS servers from /etc/resolv.conf were added. You can " "enter additional addresses now:") while True: diff --git a/ipaserver/install/server/replicainstall.py b/ipaserver/install/server/replicainstall.py index 2990f4382d..c6c438a04a 100644 --- a/ipaserver/install/server/replicainstall.py +++ b/ipaserver/install/server/replicainstall.py @@ -9,8 +9,6 @@ import dns.exception as dnsexception import dns.name as dnsname -import dns.resolver as dnsresolver -import dns.reversename as dnsreversename import os import shutil import socket @@ -28,6 +26,7 @@ from ipalib.install.kinit import kinit_keytab from ipapython import ipaldap, ipautil from ipapython.dn import DN +from ipapython.dnsutil import DNSResolver from ipapython.admintool import ScriptError from ipapython.ipachangeconf import IPAChangeConf from ipaplatform import services @@ -289,7 +288,7 @@ def check_dns_resolution(host_name, dns_servers): logger.error( 'Could not resolve any DNS server hostname: %s', dns_servers) return False - resolver = dnsresolver.Resolver() + resolver = DNSResolver() resolver.nameservers = server_ips logger.debug('Search DNS server %s (%s) for %s', @@ -299,7 +298,7 @@ def check_dns_resolution(host_name, dns_servers): addresses = set() for rtype in 'A', 'AAAA': try: - result = resolver.query(host_name, rtype) + result = resolver.resolve(host_name, rtype) except dnsexception.DNSException: rrset = [] else: @@ -327,8 +326,7 @@ def check_dns_resolution(host_name, dns_servers): checked.add(address) try: logger.debug('Check reverse address %s (%s)', address, host_name) - revname = dnsreversename.from_address(address) - rrset = resolver.query(revname, 'PTR').rrset + rrset = resolver.resolve_address(address).rrset except Exception as e: logger.debug('Check failed: %s %s', type(e).__name__, e) logger.error( diff --git a/ipaserver/plugins/cert.py b/ipaserver/plugins/cert.py index fe7ea34f5d..a5be24af90 100644 --- a/ipaserver/plugins/cert.py +++ b/ipaserver/plugins/cert.py @@ -1191,7 +1191,7 @@ def _san_ip_update_reachable(reachable, dnsname, cname_depth): """ fqdn = dnsutil.DNSName(dnsname).make_absolute() try: - zone = dnsutil.DNSName(resolver.zone_for_name(fqdn)) + zone = dnsutil.DNSName(dnsutil.zone_for_name(fqdn)) except resolver.NoNameservers: return # if there's no zone, there are no records name = fqdn.relativize(zone) @@ -1225,7 +1225,7 @@ def _ip_ptr_records(ip): """ rname = dnsutil.DNSName(reversename.from_address(ip)) try: - zone = dnsutil.DNSName(resolver.zone_for_name(rname)) + zone = dnsutil.DNSName(dnsutil.zone_for_name(rname)) name = rname.relativize(zone) result = api.Command['dnsrecord_show'](zone, name)['result'] except resolver.NoNameservers: diff --git a/ipaserver/plugins/dns.py b/ipaserver/plugins/dns.py index 275e3433e3..919402db1a 100644 --- a/ipaserver/plugins/dns.py +++ b/ipaserver/plugins/dns.py @@ -79,9 +79,15 @@ from ipaplatform import services from ipapython.dn import DN from ipapython.ipautil import CheckedIPAddress -from ipapython.dnsutil import check_zone_overlap, DNSZoneAlreadyExists -from ipapython.dnsutil import DNSName -from ipapython.dnsutil import related_to_auto_empty_zone +from ipapython.dnsutil import ( + check_zone_overlap, + DNSName, + DNSResolver, + DNSZoneAlreadyExists, + related_to_auto_empty_zone, + resolve, + zone_for_name, +) from ipaserver.dns_data_management import ( IPASystemRecords, IPADomainIsNotManagedByIPAError, @@ -542,7 +548,7 @@ def get_reverse_zone(ipaddr): ip = netaddr.IPAddress(str(ipaddr)) revdns = DNSName(unicode(ip.reverse_dns)) try: - revzone = DNSName(dns.resolver.zone_for_name(revdns)) + revzone = DNSName(zone_for_name(revdns)) except dns.resolver.NoNameservers: raise errors.NotFound( reason=_( @@ -3363,7 +3369,7 @@ def wait_for_modified_attr(self, ldap_rrset, rdtype, dns_name): :raises errors.DNSDataMismatch: if data in DNS and LDAP doesn't match :raises dns.exception.DNSException: if DNS resolution failed ''' - resolver = dns.resolver.Resolver() + resolver = DNSResolver() resolver.set_flags(0) # disable recursion (for NS RR checks) max_attempts = int(self.api.env['wait_for_dns']) warn_attempts = max_attempts // 2 @@ -3379,9 +3385,9 @@ def wait_for_modified_attr(self, ldap_rrset, rdtype, dns_name): log_fn = logger.warning attempt += 1 try: - dns_answer = resolver.query(dns_name, rdtype, - dns.rdataclass.IN, - raise_on_no_answer=False) + dns_answer = resolver.resolve(dns_name, rdtype, + dns.rdataclass.IN, + raise_on_no_answer=False) dns_rrset = None if rdtype == _NS: # NS records can be in Authority section (sometimes) @@ -4303,7 +4309,7 @@ def _warning_if_forwarders_do_not_work(self, result, new_zone, ipa_dns_ip = None for rdtype in (dns.rdatatype.A, dns.rdatatype.AAAA): try: - ans = dns.resolver.query(ipa_dns_masters[0], rdtype) + ans = resolve(ipa_dns_masters[0], rdtype) except dns.exception.DNSException: continue else: diff --git a/ipaserver/plugins/host.py b/ipaserver/plugins/host.py index c31eb59613..6756374db9 100644 --- a/ipaserver/plugins/host.py +++ b/ipaserver/plugins/host.py @@ -22,8 +22,6 @@ import logging -import dns.resolver - import six from ipalib import api, errors, util @@ -64,7 +62,7 @@ CheckedIPAddress, TMP_PWD_ENTROPY_BITS ) -from ipapython.dnsutil import DNSName +from ipapython.dnsutil import DNSName, zone_for_name from ipapython.ssh import SSHPublicKey from ipapython.dn import DN from ipapython import kerberos @@ -826,7 +824,7 @@ def pre_callback(self, ldap, dn, *keys, **options): if updatedns: # Remove A, AAAA, SSHFP and PTR records of the host fqdn_dnsname = DNSName(fqdn).make_absolute() - zone = DNSName(dns.resolver.zone_for_name(fqdn_dnsname)) + zone = DNSName(zone_for_name(fqdn_dnsname)) relative_hostname = fqdn_dnsname.relativize(zone) # Get all resources for this host diff --git a/ipatests/pytest_ipa/integration/tasks.py b/ipatests/pytest_ipa/integration/tasks.py index 21b196933f..fceac1b628 100755 --- a/ipatests/pytest_ipa/integration/tasks.py +++ b/ipatests/pytest_ipa/integration/tasks.py @@ -49,6 +49,7 @@ from ipapython import certdb from ipapython import ipautil +from ipapython.dnsutil import DNSResolver from ipaplatform.paths import paths from ipaplatform.services import knownservices from ipapython.dn import DN @@ -1436,7 +1437,7 @@ def resolve_record(nameserver, query, rtype="SOA", retry=True, timeout=100): :timeout: max period of time while method will try to resolve query (requires retry=True) """ - res = dns.resolver.Resolver() + res = DNSResolver() res.nameservers = [nameserver] res.lifetime = 10 # wait max 10 seconds for reply @@ -1444,7 +1445,7 @@ def resolve_record(nameserver, query, rtype="SOA", retry=True, timeout=100): while time.time() < wait_until: try: - ans = res.query(query, rtype) + ans = res.resolve(query, rtype) return ans except dns.exception.DNSException: if not retry: diff --git a/ipatests/test_integration/test_dns_locations.py b/ipatests/test_integration/test_dns_locations.py index 1337418b10..aa662abd3f 100644 --- a/ipatests/test_integration/test_dns_locations.py +++ b/ipatests/test_integration/test_dns_locations.py @@ -12,7 +12,7 @@ from ipatests.test_integration.base import IntegrationTest from ipatests.pytest_ipa.integration import tasks -from ipapython.dnsutil import DNSName +from ipapython.dnsutil import DNSName, DNSResolver from ipalib.constants import IPA_CA_RECORD logger = logging.getLogger(__name__) @@ -45,14 +45,14 @@ def resolve_records_from_server(rname, rtype, nameserver): error = None - res = dns.resolver.Resolver() + res = DNSResolver() res.nameservers = [nameserver] res.lifetime = 30 logger.info("Query: %s %s, nameserver %s", rname, rtype, nameserver) # lets try to query 3x for _i in range(3): try: - ans = res.query(rname, rtype) + ans = res.resolve(rname, rtype) logger.info("Answer: %s", ans.rrset) return ans.rrset except (dns.resolver.NXDOMAIN, dns.resolver.Timeout) as e: diff --git a/ipatests/test_integration/test_dnssec.py b/ipatests/test_integration/test_dnssec.py index ccaa7f9b54..62ed62efc8 100644 --- a/ipatests/test_integration/test_dnssec.py +++ b/ipatests/test_integration/test_dnssec.py @@ -10,13 +10,13 @@ import time import dns.dnssec -import dns.resolver import dns.name from ipatests.test_integration.base import IntegrationTest from ipatests.pytest_ipa.integration import tasks from ipatests.pytest_ipa.integration.firewall import Firewall from ipaplatform.paths import paths +from ipapython.dnsutil import DNSResolver logger = logging.getLogger(__name__) @@ -33,7 +33,7 @@ def resolve_with_dnssec(nameserver, query, rtype="SOA"): - res = dns.resolver.Resolver() + res = DNSResolver() res.nameservers = [nameserver] res.lifetime = 10 # wait max 10 seconds for reply # enable Authenticated Data + Checking Disabled flags @@ -42,7 +42,7 @@ def resolve_with_dnssec(nameserver, query, rtype="SOA"): # enable EDNS v0 + enable DNSSEC-Ok flag res.use_edns(0, dns.flags.DO, 0) - ans = res.query(query, rtype) + ans = res.resolve(query, rtype) return ans
_______________________________________________ FreeIPA-devel mailing list -- freeipa-devel@lists.fedorahosted.org To unsubscribe send an email to freeipa-devel-le...@lists.fedorahosted.org Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/ List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines List Archives: https://lists.fedorahosted.org/archives/list/freeipa-devel@lists.fedorahosted.org
