Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package benji for openSUSE:Factory checked in at 2021-06-02 22:12:23 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/benji (Old) and /work/SRC/openSUSE:Factory/.benji.new.1898 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "benji" Wed Jun 2 22:12:23 2021 rev:13 rq:896938 version:0.15.0 Changes: -------- --- /work/SRC/openSUSE:Factory/benji/benji.changes 2021-03-16 15:46:16.605216817 +0100 +++ /work/SRC/openSUSE:Factory/.benji.new.1898/benji.changes 2021-06-02 22:12:49.404066580 +0200 @@ -1,0 +2,12 @@ +Wed Jun 2 15:13:02 UTC 2021 - Michael Vetter <[email protected]> + +- Update to 0.15.0: + * Helm chart: Migrate Helm chart to version 2 of the chart + format (only compatible with Helm 3) + * Helm chart: Add option to set activeDeadlineSeconds for + cronjobs (benji.cronJob.activeDeadlineSeconds) (#108) + * Update container images to fix CVE-2021-20288 in the + Ceph client code + * Revalidate invalid blocks during deep-scrubbing (#105) + +------------------------------------------------------------------- Old: ---- v0.14.1.tar.gz New: ---- v0.15.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ benji.spec ++++++ --- /var/tmp/diff_new_pack.XyO2wr/_old 2021-06-02 22:12:50.048064987 +0200 +++ /var/tmp/diff_new_pack.XyO2wr/_new 2021-06-02 22:12:50.048064987 +0200 @@ -17,7 +17,7 @@ Name: benji -Version: 0.14.1 +Version: 0.15.0 Release: 0 Summary: Deduplicating block based backup software License: LGPL-3.0-only ++++++ v0.14.1.tar.gz -> v0.15.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/benji-0.14.1/CHANGES.md new/benji-0.15.0/CHANGES.md --- old/benji-0.14.1/CHANGES.md 2021-03-13 11:00:35.000000000 +0100 +++ new/benji-0.15.0/CHANGES.md 2021-06-01 18:42:59.000000000 +0200 @@ -1,3 +1,10 @@ +## 0.15.0, 01.06.2021 + +* Helm chart: Migrate Helm chart to version 2 of the chart format (only compatible with Helm 3) +* Helm chart: Add option to set `activeDeadlineSeconds` for cronjobs (`benji.cronJob.activeDeadlineSeconds`) (#108) +* Update container images to fix CVE-2021-20288 in the Ceph client code +* Revalidate invalid blocks during deep-scrubbing (#105) + ## 0.14.1, 13.03.2021 * Fixed wrong container image repository reference in Helm chart diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/benji-0.14.1/README.rst new/benji-0.15.0/README.rst --- old/benji-0.14.1/README.rst 2021-03-13 11:00:35.000000000 +0100 +++ new/benji-0.15.0/README.rst 2021-06-01 18:42:59.000000000 +0200 @@ -1,4 +1,4 @@ -.. image:: https://img.shields.io/github/workflow/status/elemental-lf/benji/test-build-publish?style=plastic +.. image:: https://img.shields.io/github/workflow/status/elemental-lf/benji/All-in-One/master?style=plastic :target: https://github.com/elemental-lf/benji/actions?query=branch%3Amaster .. image:: https://img.shields.io/pypi/l/benji.svg?style=plastic&label=License diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/benji-0.14.1/charts/benji-k8s/Chart.yaml new/benji-0.15.0/charts/benji-k8s/Chart.yaml --- old/benji-0.14.1/charts/benji-k8s/Chart.yaml 2021-03-13 11:00:35.000000000 +0100 +++ new/benji-0.15.0/charts/benji-k8s/Chart.yaml 2021-06-01 18:42:59.000000000 +0200 @@ -1,7 +1,16 @@ -apiVersion: v1 +apiVersion: v2 description: Benji Backup for Kubernetes name: benji-k8s -version: 0.1.0 +version: 0.2.0 maintainers: - name: Elemental email: [email protected] +dependencies: + - name: postgresql + version: 10.1.1 + repository: https://charts.bitnami.com/bitnami + condition: postgresql.enabled + - name: prometheus-pushgateway + alias: pushgateway + version: 1.5.0 + repository: https://prometheus-community.github.io/helm-charts diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/benji-0.14.1/charts/benji-k8s/requirements.yaml new/benji-0.15.0/charts/benji-k8s/requirements.yaml --- old/benji-0.14.1/charts/benji-k8s/requirements.yaml 2021-03-13 11:00:35.000000000 +0100 +++ new/benji-0.15.0/charts/benji-k8s/requirements.yaml 1970-01-01 01:00:00.000000000 +0100 @@ -1,9 +0,0 @@ -dependencies: - - name: postgresql - version: 10.1.1 - repository: https://charts.bitnami.com/bitnami - condition: postgresql.enabled - - name: prometheus-pushgateway - alias: pushgateway - version: 1.5.0 - repository: https://prometheus-community.github.io/helm-charts diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/benji-0.14.1/charts/benji-k8s/templates/cronjobs.yaml new/benji-0.15.0/charts/benji-k8s/templates/cronjobs.yaml --- old/benji-0.14.1/charts/benji-k8s/templates/cronjobs.yaml 2021-03-13 11:00:35.000000000 +0100 +++ new/benji-0.15.0/charts/benji-k8s/templates/cronjobs.yaml 2021-06-01 18:42:59.000000000 +0200 @@ -26,6 +26,9 @@ app.kubernetes.io/component: cronjob-{{ .name }} spec: backoffLimit: 0 + {{- if $.Values.benji.cronJob.activeDeadlineSeconds }} + activeDeadlineSeconds: {{ $.Values.benji.cronJob.activeDeadlineSeconds }} + {{- end }} template: metadata: labels: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/benji-0.14.1/charts/benji-k8s/values.yaml new/benji-0.15.0/charts/benji-k8s/values.yaml --- old/benji-0.14.1/charts/benji-k8s/values.yaml 2021-03-13 11:00:35.000000000 +0100 +++ new/benji-0.15.0/charts/benji-k8s/values.yaml 2021-06-01 18:42:59.000000000 +0200 @@ -29,6 +29,9 @@ - name: file module: file + cronJob: + activeDeadlineSeconds: null + crontab: - name: backup-all schedule: "*/10 * * * *" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/benji-0.14.1/docs/source/_static/quickstart.cast new/benji-0.15.0/docs/source/_static/quickstart.cast --- old/benji-0.14.1/docs/source/_static/quickstart.cast 2021-03-13 11:00:35.000000000 +0100 +++ new/benji-0.15.0/docs/source/_static/quickstart.cast 2021-06-01 18:42:59.000000000 +0200 @@ -159,7 +159,7 @@ [40.756207, "o", "u"] [40.980378, "o", "p"] [42.148175, "o", "\r\n"] -[42.822204, "o", " INFO: $ /benji/bin/benji backup file://TESTFILE myfirsttestbackup\r\n"] +[42.822204, "o", " INFO: $ /benji/bin/benji backup file:TESTFILE myfirsttestbackup\r\n"] [43.005395, "o", " INFO: Marked version V0000000001 as unprotected.\r\n"] [43.099467, "o", " INFO: Backed up 1/10 blocks (10.0%)\r\n"] [43.138277, "o", " INFO: Backed up 2/10 blocks (20.0%)\r\n"] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/benji-0.14.1/docs/source/quickstart.rst new/benji-0.15.0/docs/source/quickstart.rst --- old/benji-0.14.1/docs/source/quickstart.rst 2021-03-13 11:00:35.000000000 +0100 +++ new/benji-0.15.0/docs/source/quickstart.rst 2021-06-01 18:42:59.000000000 +0200 @@ -76,7 +76,7 @@ 3. Backup the image (works similar with a device):: - $ benji backup file://TESTFILE myfirsttestbackup + $ benji backup file:TESTFILE myfirsttestbackup INFO: $ benji backup file://TESTFILE myfirsttestbackup INFO: Backed up 1/10 blocks (10.0%) INFO: Backed up 2/10 blocks (20.0%) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/benji-0.14.1/images/benji/Dockerfile new/benji-0.15.0/images/benji/Dockerfile --- old/benji-0.14.1/images/benji/Dockerfile 2021-03-13 11:00:35.000000000 +0100 +++ new/benji-0.15.0/images/benji/Dockerfile 2021-06-01 18:42:59.000000000 +0200 @@ -1,6 +1,6 @@ FROM centos:7 AS build -ARG CEPH_CODENAME="nautilus" +ARG CEPH_CODENAME="octopus" ARG CEPH_DISTRO="el7" ENV VENV_DIR /benji @@ -9,6 +9,7 @@ RUN sed -i -e "s/{ceph-release}/$CEPH_CODENAME/" -e "s/{distro}/$CEPH_DISTRO/" /etc/yum.repos.d/ceph.repo RUN rpm --import 'https://download.ceph.com/keys/release.asc' && \ + ulimit -n 1024 && \ yum install -y tzdata epel-release && \ yum update -y && \ yum install -y git gcc make \ @@ -48,6 +49,7 @@ COPY --from=build /etc/yum.repos.d/ceph.repo /etc/yum.repos.d/ceph.repo RUN rpm --import 'https://download.ceph.com/keys/release.asc' && \ + ulimit -n 1024 && \ yum install -y tzdata epel-release && \ yum update -y && \ yum install -y python36 && \ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/benji-0.14.1/setup.py new/benji-0.15.0/setup.py --- old/benji-0.14.1/setup.py 2021-03-13 11:00:35.000000000 +0100 +++ new/benji-0.15.0/setup.py 2021-06-01 18:42:59.000000000 +0200 @@ -49,7 +49,7 @@ zip_safe=False, # ONLY because of alembic.ini. The rest is zip-safe. install_requires=[ 'PrettyTable>=0.7.2,<1', - 'sqlalchemy>=1.2.6,<2', + 'sqlalchemy>=1.4.14,<2', 'setproctitle>=1.1.8,<2', 'python-dateutil>=2.6.0,<3', 'alembic>=1.0.5,<2', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/benji-0.14.1/src/benji/_static_version.py new/benji-0.15.0/src/benji/_static_version.py --- old/benji-0.14.1/src/benji/_static_version.py 2021-03-13 11:00:35.000000000 +0100 +++ new/benji-0.15.0/src/benji/_static_version.py 2021-06-01 18:42:59.000000000 +0200 @@ -8,5 +8,5 @@ version = "__use_git__" # These values are only set if the distribution was created with 'git archive' -refnames = "HEAD -> master, tag: v0.14.1" -git_hash = "810c235" +refnames = "tag: v0.15.0" +git_hash = "f541c75" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/benji-0.14.1/src/benji/benji.py new/benji-0.15.0/src/benji/benji.py --- old/benji-0.14.1/src/benji/benji.py 2021-03-13 11:00:35.000000000 +0100 +++ new/benji-0.15.0/src/benji/benji.py 2021-06-01 18:42:59.000000000 +0200 @@ -278,7 +278,7 @@ logger.error('Block {} (UID {}) is invalid: {}{}'.format( entry.block.idx, entry.block.uid, entry, f' Caused by: {entry.__cause__}' if entry.__cause__ else '')) - affected_version_uids.extend(Version.set_block_invalid(entry.block.uid)) + affected_version_uids.extend(Version.set_block_valid(entry.block.uid, False)) valid = False continue else: @@ -291,7 +291,7 @@ except (KeyError, ValueError) as exception: logger.error('Metadata check failed, block {} (UID {}) is invalid: {}'.format( block.idx, block.uid, exception)) - affected_version_uids.extend(Version.set_block_invalid(block.uid)) + affected_version_uids.extend(Version.set_block_valid(block.uid, False)) valid = False continue @@ -366,7 +366,7 @@ logger.error('Block {} (UID {}) is invalid: {}{}'.format( entry.block.idx, entry.block.uid, entry, f' Caused by: {entry.__cause__}' if entry.__cause__ else '')) - affected_version_uids.extend(Version.set_block_invalid(entry.block.uid)) + affected_version_uids.extend(Version.set_block_valid(entry.block.uid, False)) valid = False continue else: @@ -379,7 +379,7 @@ except (KeyError, ValueError) as exception: logger.error('Metadata check failed, block {} of version {} (UID {}) is invalid: {}'.format( block.idx, version_uid, block.uid, exception)) - Version.set_block_invalid(block.uid) + Version.set_block_valid(block.uid, False) valid = False continue @@ -389,7 +389,7 @@ 'Checksum mismatch during deep-scrub of block {} of version {} (UID {}) (is: {}... should-be: {}...).'.format( block.idx, version_uid, block.uid, data_checksum[:16], cast(str, block.checksum)[:16])) # We know that block.checksum is set - affected_version_uids.extend(Version.set_block_invalid(block.uid)) + affected_version_uids.extend(Version.set_block_valid(block.uid, False)) valid = False continue @@ -405,6 +405,13 @@ # correct then the source is probably invalid. source_mismatch = True + if not block.valid: + logger.info('Block {} of version {} (UID {}) passed revalidation, marking it as valid.'.format( + block.idx, version_uid, block.uid)) + Version.set_block_valid(block.uid, True) + + # Only add the block to the history if it is valid. This ensure that this block will also get flagged + # again when deep-scrubbing other versions containing it. if history: history.add(version.storage_id, block.uid) @@ -611,7 +618,7 @@ logger.error('Storage backend read failed: {}'.format(entry)) # If it really is a data inconsistency mark blocks invalid if isinstance(entry, (KeyError, ValueError)): - Version.set_block_invalid(block.uid) + Version.set_block_valid(block.uid, False) continue else: raise entry @@ -626,7 +633,7 @@ storage.check_block_metadata(block=block, data_length=len(data), metadata=metadata) except (KeyError, ValueError) as exception: logger.error('Metadata check failed, block is invalid: {}'.format(exception)) - Version.set_block_invalid(block.uid) + Version.set_block_valid(block.uid, False) continue data_checksum = self._block_hash.data_hexdigest(data) @@ -635,7 +642,7 @@ 'block.valid: {}). Block restored is invalid.'.format( block.idx, block.uid, data_checksum[:16], cast(str, block.checksum)[:16], block.valid)) # We know that block.checksum is set - Version.set_block_invalid(block.uid) + Version.set_block_valid(block.uid, False) else: logger.debug('Restored block {} successfully ({} bytes).'.format(block.idx, block.size)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/benji-0.14.1/src/benji/database.py new/benji-0.15.0/src/benji/database.py --- old/benji-0.14.1/src/benji/database.py 2021-03-13 11:00:35.000000000 +0100 +++ new/benji-0.15.0/src/benji/database.py 2021-06-01 18:42:59.000000000 +0200 @@ -32,10 +32,6 @@ from alembic.config import Config as alembic_config_Config from alembic.runtime.environment import EnvironmentContext from alembic.script import ScriptDirectory -from sqlalchemy import func -from sqlalchemy.orm import object_session, sessionmaker, scoped_session, aliased -from sqlalchemy.orm.collections import attribute_mapped_collection - from benji.config import Config from benji.exception import InputDataError, InternalError, AlreadyLocked, UsageError, ConfigurationError from benji.logging import logger @@ -43,6 +39,9 @@ from benji.storage.key import StorageKeyMixIn from benji.utils import InputValidation from benji.versions import VERSIONS +from sqlalchemy import func +from sqlalchemy.orm import object_session, sessionmaker, scoped_session, aliased +from sqlalchemy.orm.collections import attribute_mapped_collection # SQLite 3 supports checking of foreign keys but it needs to be enabled explicitly! @@ -59,6 +58,9 @@ impl = sqlalchemy.DateTime + # This is non-cacheable as it may be a relative time specifications and so is dependent on the current time. + cache_ok = False + def process_bind_param(self, value: Optional[Union[datetime.datetime, str]], dialect) -> Optional[datetime.datetime]: if isinstance(value, datetime.datetime): if value.tzinfo is None: @@ -111,6 +113,8 @@ impl = sqlalchemy.Integer + cache_ok = True + def process_bind_param(self, value: Optional[Union[int, str, VersionStatus]], dialect) -> Optional[int]: if value is None: return None @@ -161,6 +165,8 @@ impl = sqlalchemy.String(255) + cache_ok = True + def process_bind_param(self, value: Optional[str], dialect) -> Optional[str]: if value is None or isinstance(value, str): return value @@ -178,6 +184,8 @@ impl = sqlalchemy.LargeBinary + cache_ok = True + def process_bind_param(self, value: Optional[str], dialect) -> Optional[bytes]: if value is not None: return unhexlify(value) @@ -471,22 +479,30 @@ raise @classmethod - def set_block_invalid(cls, block_uid: BlockUid) -> List[VersionUid]: + def set_block_valid(cls, block_uid: BlockUid, valid: bool) -> List[VersionUid]: try: - affected_version_uids_query = Session.query(sqlalchemy.distinct( - Version.uid).label('uid')).join(Block).filter(Block.uid == block_uid) - affected_version_uids = [row.uid for row in affected_version_uids_query] - assert len(affected_version_uids) > 0 - - Session.query(Block).filter(Block.uid == block_uid).update({'valid': False}, synchronize_session=False) - Session.commit() - - logger.error('Marked block with UID {} as invalid. Affected versions: {}.'.format( - block_uid, ', '.join(affected_version_uids))) + # Can't use DISTINCT here as PostgreSQL doesn't support DISTINCT together with FOR UPDATE. + affected_version_uids_query = Session.query(cls.uid.label('uid')).join(Block).filter(Block.uid == block_uid) + if not valid: + affected_version_uids_query = affected_version_uids_query.with_for_update() + # Use a set to replace the missing DISTINCT above. + affected_version_uids = set([row.uid for row in affected_version_uids_query]) + + Session.query(Block).filter(Block.uid == block_uid).update({'valid': valid}, synchronize_session=False) + + if len(affected_version_uids) > 0: + logger.error('Marked block with UID {} as {} in all affected versions: {}.'.format( + block_uid, 'valid' if valid else 'invalid', ', '.join(affected_version_uids))) + + # We won't mark any versions as valid because they might contain other invalid blocks. + if not valid: + Session.query(cls).filter(cls.uid.in_(affected_version_uids)).update( + {'status': VersionStatus.invalid}, synchronize_session=False) + else: + # Version could have been deleted in the meantime + logger.warning('No version was affected by marking block with UID {} as {}.'.format( + block_uid, 'valid' if valid else 'invalid')) - for version_uid in affected_version_uids: - version = Version.get_by_uid(version_uid) - version.set(status=VersionStatus.invalid) Session.commit() except: Session.rollback() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/benji-0.14.1/src/benji/tests/test_database.py new/benji-0.15.0/src/benji/tests/test_database.py --- old/benji-0.14.1/src/benji/tests/test_database.py 2021-03-13 11:00:35.000000000 +0100 +++ new/benji-0.15.0/src/benji/tests/test_database.py 2021-06-01 18:42:59.000000000 +0200 @@ -379,7 +379,7 @@ datetime.datetime.now(tz=tz.tzlocal()).strftime("%Y-%m-%dT%H:%M:%S"))) self.assertEqual(3, len(versions)) - def test_set_block_invalid(self): + def test_set_block_valid(self): Storage.sync('s-1', storage_id=1) versions = [] good_uid = BlockUid(1, 2) @@ -404,7 +404,11 @@ versions.append(version) - Version.set_block_invalid(bad_uid) + for i in range(6): + self.assertEqual(VersionStatus.valid, versions[i].status) + self.assertTrue(list(versions[i].blocks)[0].valid) + + Version.set_block_valid(bad_uid, False) for i in range(3): self.assertEqual(VersionStatus.invalid, versions[i].status) @@ -412,6 +416,16 @@ for i in range(3, 6): self.assertEqual(VersionStatus.valid, versions[i].status) + self.assertTrue(list(versions[i].blocks)[0].valid) + + Version.set_block_valid(bad_uid, True) + + for i in range(3): + self.assertEqual(VersionStatus.invalid, versions[i].status) + self.assertTrue(list(versions[i].blocks)[0].valid) + + for i in range(3, 6): + self.assertEqual(VersionStatus.valid, versions[i].status) self.assertTrue(list(versions[i].blocks)[0].valid) def test_version_blocks_count(self):
