Hello community,
here is the log from the commit of package benji for openSUSE:Factory checked
in at 2020-09-07 21:35:36
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/benji (Old)
and /work/SRC/openSUSE:Factory/.benji.new.3399 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "benji"
Mon Sep 7 21:35:36 2020 rev:9 rq:832695 version:0.10.0
Changes:
--------
--- /work/SRC/openSUSE:Factory/benji/benji.changes 2020-07-28
17:26:52.858009748 +0200
+++ /work/SRC/openSUSE:Factory/.benji.new.3399/benji.changes 2020-09-07
21:35:54.473396100 +0200
@@ -1,0 +2,14 @@
+Mon Sep 7 08:04:41 UTC 2020 - Michael Vetter <mvet...@suse.com>
+
+- Update to 0.10.0:
+ * Helm chart changes:
+ - Change chart's requirements to use URL based repository references.
+ This should help when deploying Benji via FluxCD's helm-operator (#89)
+ - Fix rendering error when specifying a nodeSelector, node
+ affinities or tolerations. (#90)
+ - Use API group rbac.authorization.k8s.io/v1 for RBAC related resources
+ * Add new transform module aes_256_gcm_ecc (#86)
+ * Add support for discovering RBD images provisioned by Ceph's CSI
+ provisioner to the benji-backup-pvc script (#91)
+
+-------------------------------------------------------------------
Old:
----
v0.9.0.tar.gz
New:
----
v0.10.0.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ benji.spec ++++++
--- /var/tmp/diff_new_pack.UYsXrg/_old 2020-09-07 21:35:56.089396848 +0200
+++ /var/tmp/diff_new_pack.UYsXrg/_new 2020-09-07 21:35:56.089396848 +0200
@@ -17,7 +17,7 @@
Name: benji
-Version: 0.9.0
+Version: 0.10.0
Release: 0
Summary: Deduplicating block based backup software
License: LGPL-3.0-only
++++++ v0.9.0.tar.gz -> v0.10.0.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/.travis.yml new/benji-0.10.0/.travis.yml
--- old/benji-0.9.0/.travis.yml 2020-07-27 11:20:08.000000000 +0200
+++ new/benji-0.10.0/.travis.yml 2020-09-05 13:36:45.000000000 +0200
@@ -1,4 +1,4 @@
-dist: xenial
+dist: bionic
language: python
cache:
@@ -40,7 +40,7 @@
install:
- travis_retry pip install --upgrade setuptools pip
- - travis_retry pip install '.[s3,b2,compression,readcache,dev,doc]'
+ - travis_retry pip install '.[s3,b2,compression,dev,doc]'
script:
- pip freeze
@@ -57,18 +57,28 @@
include:
- stage: test
install:
- - travis_retry pip install --upgrade setuptools pip
+ - travis_retry pip install --upgrade 'setuptools>=50.2.0' pip
script:
- SKIP_DOCKER_PUSH=1 DOCKER_BUILDKIT=0 VERSION_VARIANT=miniver
DOCKERFILE_PATH=images/benji/Dockerfile maint-scripts/docker-build
- SKIP_DOCKER_PUSH=1 DOCKER_BUILDKIT=0 VERSION_VARIANT=miniver
DOCKERFILE_PATH=images/benji-k8s/Dockerfile maint-scripts/docker-build
- stage: test
before_install: skip
install:
- - HELM_VERSION=2.12.3
+ - HELM_VERSION=2.16.7
- curl --retry 5 -sLO
https://storage.googleapis.com/kubernetes-helm/helm-v$HELM_VERSION-linux-amd64.tar.gz
- tar --strip-components=1 -xzvvf
helm-v$HELM_VERSION-linux-amd64.tar.gz linux-amd64/helm
- rm -f helm-v$HELM_VERSION-linux-amd64.tar.gz
- chmod a+x helm
+ script:
+ - ./helm lint charts/benji-k8s
+ - stage: test
+ before_install: skip
+ install:
+ - HELM_VERSION=3.3.1
+ - curl --retry 5 -sLO
https://get.helm.sh/helm-v$HELM_VERSION-linux-amd64.tar.gz
+ - tar --strip-components=1 -xzvvf
helm-v$HELM_VERSION-linux-amd64.tar.gz linux-amd64/helm
+ - rm -f helm-v$HELM_VERSION-linux-amd64.tar.gz
+ - chmod a+x helm
script:
- ./helm lint charts/benji-k8s
- stage: push
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/CHANGES.md new/benji-0.10.0/CHANGES.md
--- old/benji-0.9.0/CHANGES.md 2020-07-27 11:20:08.000000000 +0200
+++ new/benji-0.10.0/CHANGES.md 2020-09-05 13:36:45.000000000 +0200
@@ -1,5 +1,24 @@
+## 0.10.0, 05.09.2020
+
+Notable changes:
+
+* Helm chart changes:
+
+ * Change chart's requirements to use URL based repository references. This
should help when deploying Benji via
+ FluxCD's helm-operator for example. (#89)
+
+ * Fix rendering error when specifying a nodeSelector, node affinities or
tolerations. (#90)
+
+ * Use API group rbac.authorization.k8s.io/v1 for RBAC related resources
+
+* Add new transform module `aes_256_gcm_ecc` (#86)
+
+* Add support for discovering RBD images provisioned by Ceph's CSI provisioner
to the `benji-backup-pvc` script (#91)
+
## 0.9.0, 27.07.2020
+Notable changes:
+
* Sparse blocks are no longer explicitly represented in the database. This
greatly speeds up the initial step
of backup creation if the backup is based on an older version, and a lot of
the blocks are sparse. It also reduces
database size and speeds up database operations overall due to the reduced
number of rows in the blocks table. No
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/charts/benji-k8s/requirements.yaml
new/benji-0.10.0/charts/benji-k8s/requirements.yaml
--- old/benji-0.9.0/charts/benji-k8s/requirements.yaml 2020-07-27
11:20:08.000000000 +0200
+++ new/benji-0.10.0/charts/benji-k8s/requirements.yaml 2020-09-05
13:36:45.000000000 +0200
@@ -1,9 +1,9 @@
dependencies:
- name: postgresql
version: 7.7.2
- repository: "@stable"
+ repository: https://kubernetes-charts.storage.googleapis.com/
condition: postgresql.enabled
- name: prometheus-pushgateway
alias: pushgateway
version: 1.2.6
- repository: "@stable"
+ repository: https://kubernetes-charts.storage.googleapis.com/
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/benji-0.9.0/charts/benji-k8s/templates/cluster-role-binding.yaml
new/benji-0.10.0/charts/benji-k8s/templates/cluster-role-binding.yaml
--- old/benji-0.9.0/charts/benji-k8s/templates/cluster-role-binding.yaml
2020-07-27 11:20:08.000000000 +0200
+++ new/benji-0.10.0/charts/benji-k8s/templates/cluster-role-binding.yaml
2020-09-05 13:36:45.000000000 +0200
@@ -1,6 +1,6 @@
---
kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ tuple . "" | include "benji.fullname" }}
labels:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/benji-0.9.0/charts/benji-k8s/templates/cluster-role.yaml
new/benji-0.10.0/charts/benji-k8s/templates/cluster-role.yaml
--- old/benji-0.9.0/charts/benji-k8s/templates/cluster-role.yaml
2020-07-27 11:20:08.000000000 +0200
+++ new/benji-0.10.0/charts/benji-k8s/templates/cluster-role.yaml
2020-09-05 13:36:45.000000000 +0200
@@ -1,4 +1,4 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ tuple . "" | include "benji.fullname" }}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/charts/benji-k8s/templates/cronjobs.yaml
new/benji-0.10.0/charts/benji-k8s/templates/cronjobs.yaml
--- old/benji-0.9.0/charts/benji-k8s/templates/cronjobs.yaml 2020-07-27
11:20:08.000000000 +0200
+++ new/benji-0.10.0/charts/benji-k8s/templates/cronjobs.yaml 2020-09-05
13:36:45.000000000 +0200
@@ -72,16 +72,13 @@
hostPath:
path: /usr/share/zoneinfo/{{ $.Values.timeZone }}
{{ toYaml $.Values.benji.volumes | nindent 12 }}
- {{- with $.Values.benji.nodeSelector -}}
- nodeSelector:
- {{ toYaml . | nindent 12 }}
+ {{- with $.Values.benji.nodeSelector }}
+ nodeSelector: {{ toYaml . | nindent 12 }}
{{- end -}}
- {{- with $.Values.benji.affinity -}}
- affinity:
- {{ toYaml . | nindent 12 }}
+ {{- with $.Values.benji.affinity }}
+ affinity: {{ toYaml . | nindent 12 }}
{{- end -}}
- {{- with $.Values.benji.tolerations -}}
- tolerations:
- {{ toYaml . | nindent 12 }}
+ {{- with $.Values.benji.tolerations }}
+ tolerations: {{ toYaml . | nindent 12 }}
{{- end -}}
{{- end -}}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/docs/source/configuration.rst
new/benji-0.10.0/docs/source/configuration.rst
--- old/benji-0.9.0/docs/source/configuration.rst 2020-07-27
11:20:08.000000000 +0200
+++ new/benji-0.10.0/docs/source/configuration.rst 2020-09-05
13:36:45.000000000 +0200
@@ -448,6 +448,47 @@
the salt and iteration count after writing encrypted data objects, they
cannot be decrypted anymore.
+
+Transform Module aes_256_gcm_ecc
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This module encrypts each data block using the ``aes_256_gcm`` module
+(see above). Instead of specifying a symmetric master key, the encryption
+key is encrypted asymmetrically using elliptic curve cryptography. The
+implementation is based on the algorithm outlined in the ebook
+`Practical Cryptography for Developers
<https://cryptobook.nakov.com/asymmetric-key-ciphers/ecc-encryption-decryption>`_.
+
+The idea of this module is that the configuration used for backup does not
+need to include the private key of the ECC key pair and so cannot decrypt any
+of the data once it has been stored. For restore and deep-scrubbing operations
+the private key is still needed but this key can be confined to a separate
+specifically secured installation of Benji.
+
+The ``aes_256_gcm_ecc`` module supports the following configuration options:
+
+* name: **eccKey**
+* type: BASE64-encoded DER representation of a ECC key (public or private)
+* default: none
+
+Specify the encryption keyfile to encrypt a block's symmetric key.
+For backup (encryption-only) this should be the public key only.
+For restore (decryption) this must include the private key.
+
+* name: **eccCurve**
+* type: string
+* default: ``NIST P-384``
+
+The ECC curve to use. Valid values are 'NIST P-256', 'NIST P-384' and 'NIST
P-521'.
+
+Example python code to create a valid ECC key for **eccKey** (using
``PyCryptodome``)::
+
+ from base64 import b64encode
+ from Crypto.PublicKey import ECC
+ key = ECC.generate(curve='NIST P-384')
+ public_key = b64encode(key.public_key().export_key(format='DER',
compress=True)).decode('ascii'))
+ private_key = b64encode(key.export_key(format='DER',
compress=True).decode('ascii'))
+
+
Storage Modules
---------------
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/docs/source/installation.rst
new/benji-0.10.0/docs/source/installation.rst
--- old/benji-0.9.0/docs/source/installation.rst 2020-07-27
11:20:08.000000000 +0200
+++ new/benji-0.10.0/docs/source/installation.rst 2020-09-05
13:36:45.000000000 +0200
@@ -72,7 +72,6 @@
- ``s3``: AWS S3 object storage support
- ``b2``: Backblaze's B2 Cloud object storage support
- ``compression``: Compression support
-- ``readcache``: Disk caching support
Specify any extra extra features as a comma delimited list in square brackets
after the package URL::
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/etc/benji.yaml
new/benji-0.10.0/etc/benji.yaml
--- old/benji-0.9.0/etc/benji.yaml 2020-07-27 11:20:08.000000000 +0200
+++ new/benji-0.10.0/etc/benji.yaml 2020-09-05 13:36:45.000000000 +0200
@@ -74,7 +74,14 @@
# kdfSalt:
# kdfIterations:
# password:
+# or
# masterKey:
+#
+# - name:
+# module: aes_256_gcm_ecc
+# configuration:
+# eccCurve:
+# eccKey:
io:
#
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/images/benji-k8s/bin/benji-backup-pvc
new/benji-0.10.0/images/benji-k8s/bin/benji-backup-pvc
--- old/benji-0.9.0/images/benji-k8s/bin/benji-backup-pvc 2020-07-27
11:20:08.000000000 +0200
+++ new/benji-0.10.0/images/benji-k8s/bin/benji-backup-pvc 2020-09-05
13:36:45.000000000 +0200
@@ -321,6 +321,19 @@
driver = pv.spec.flex_volume.driver
if driver.startswith('ceph.rook.io/') and options.get('pool') and
options.get('image'):
pool, image = options['pool'], options['image']
+ elif (hasattr(pv.spec, 'csi')
+ and hasattr(pv.spec.csi, 'driver')
+ and pv.spec.csi.driver == 'rook-ceph.rbd.csi.ceph.com'
+ and hasattr(pv.spec.csi, 'volume_handle')
+ and pv.spec.csi.volume_handle
+ and hasattr(pv.spec.csi, 'volume_attributes')
+ and pv.spec.csi.volume_attributes.get('pool')):
+ attributes = pv.spec.csi.volume_attributes
+ volume_handle_parts = pv.spec.csi.volume_handle.split('-')
+ if len(volume_handle_parts) >= 9:
+ image_prefix = attributes.get('volumeNamePrefix', 'csi-vol-')
+ image_suffix =
'-'.join(volume_handle_parts[len(volume_handle_parts)-5:])
+ pool, image = attributes['pool'], image_prefix + image_suffix
if pool is None or image is None:
continue
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/setup.py new/benji-0.10.0/setup.py
--- old/benji-0.9.0/setup.py 2020-07-27 11:20:08.000000000 +0200
+++ new/benji-0.10.0/setup.py 2020-09-05 13:36:45.000000000 +0200
@@ -1,8 +1,5 @@
# -*- encoding: utf-8 -*-
-try:
- from setuptools import setup, find_packages
-except ImportError:
- from distutils.core import setup, find_packages
+from setuptools import setup, find_packages
with open('README.rst', 'r', encoding='utf-8') as fh:
long_description = fh.read()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/src/benji/_static_version.py
new/benji-0.10.0/src/benji/_static_version.py
--- old/benji-0.9.0/src/benji/_static_version.py 2020-07-27
11:20:08.000000000 +0200
+++ new/benji-0.10.0/src/benji/_static_version.py 2020-09-05
13:36:45.000000000 +0200
@@ -8,5 +8,5 @@
version = "__use_git__"
# These values are only set if the distribution was created with 'git archive'
-refnames = "tag: v0.9.0"
-git_hash = "e675435"
+refnames = "HEAD -> master, tag: v0.10.0"
+git_hash = "894738d"
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/src/benji/_version.py
new/benji-0.10.0/src/benji/_version.py
--- old/benji-0.9.0/src/benji/_version.py 2020-07-27 11:20:08.000000000
+0200
+++ new/benji-0.10.0/src/benji/_version.py 2020-09-05 13:36:45.000000000
+0200
@@ -4,10 +4,10 @@
import os
import subprocess
from collections import namedtuple
-from distutils.command.build_py import build_py as build_py_orig
from typing import List, Any, Dict, Optional, Sequence, Tuple
from setuptools.command.sdist import sdist as sdist_orig
+from setuptools.command.build_py import build_py as build_py_orig
Version = namedtuple('Version', ('release', 'dev', 'labels'))
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/src/benji/commands.py
new/benji-0.10.0/src/benji/commands.py
--- old/benji-0.9.0/src/benji/commands.py 2020-07-27 11:20:08.000000000
+0200
+++ new/benji-0.10.0/src/benji/commands.py 2020-09-05 13:36:45.000000000
+0200
@@ -37,9 +37,7 @@
if labels:
label_add, label_remove =
InputValidation.parse_and_validate_labels(labels)
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
+ with Benji(self.config) as benji_obj:
hints = None
if rbd_hints:
logger.debug(f'Loading RBD hints from file {rbd_hints}.')
@@ -69,10 +67,7 @@
if self.machine_output:
benji_obj.export_any({'versions': [backup_version]},
sys.stdout,
- ignore_relationships=[((Version,),
('blocks',))])
- finally:
- if benji_obj:
- benji_obj.close()
+ ignore_relationships=(((Version,),
('blocks',)),))
def restore(self, version_uid: str, destination: str, sparse: bool, force:
bool, database_less: bool,
storage: str) -> None:
@@ -80,119 +75,89 @@
raise benji.exception.UsageError('Specifying a storage location is
only supported for database-less restores.')
version_uid_obj = VersionUid(version_uid)
- benji_obj = None
- try:
- benji_obj = Benji(self.config, in_memory_database=database_less)
+ with Benji(self.config, in_memory_database=database_less) as benji_obj:
if database_less:
benji_obj.metadata_restore([version_uid_obj], storage)
benji_obj.restore(version_uid_obj, destination, sparse, force)
- finally:
- if benji_obj:
- benji_obj.close()
def protect(self, version_uids: List[str]) -> None:
version_uid_objs = [VersionUid(version_uid) for version_uid in
version_uids]
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
+ with Benji(self.config) as benji_obj:
for version_uid in version_uid_objs:
benji_obj.protect(version_uid, protected=True)
- finally:
- if benji_obj:
- benji_obj.close()
def unprotect(self, version_uids: List[str]) -> None:
version_uid_objs = [VersionUid(version_uid) for version_uid in
version_uids]
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
+ with Benji(self.config) as benji_obj:
for version_uid in version_uid_objs:
benji_obj.protect(version_uid, protected=False)
- finally:
- if benji_obj:
- benji_obj.close()
def rm(self, version_uids: List[str], force: bool, keep_metadata_backup:
bool, override_lock: bool) -> None:
version_uid_objs = [VersionUid(version_uid) for version_uid in
version_uids]
disallow_rm_when_younger_than_days =
self.config.get('disallowRemoveWhenYounger', types=int)
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
+ with Benji(self.config) as benji_obj:
for version_uid in version_uid_objs:
benji_obj.rm(version_uid,
force=force,
disallow_rm_when_younger_than_days=disallow_rm_when_younger_than_days,
keep_metadata_backup=keep_metadata_backup,
override_lock=override_lock)
- finally:
- if benji_obj:
- benji_obj.close()
def scrub(self, version_uid: str, block_percentage: int) -> None:
version_uid_obj = VersionUid(version_uid)
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
- benji_obj.scrub(version_uid_obj, block_percentage=block_percentage)
- except benji.exception.ScrubbingError:
- assert benji_obj is not None
- if self.machine_output:
- benji_obj.export_any(
- {
- 'versions':
[benji_obj.get_version_by_uid(version_uid=version_uid_obj)],
- 'errors':
[benji_obj.get_version_by_uid(version_uid=version_uid_obj)]
- },
- sys.stdout,
- ignore_relationships=[((Version,), ('blocks',))])
- raise
- else:
- if self.machine_output:
- benji_obj.export_any(
- {
- 'versions':
[benji_obj.get_version_by_uid(version_uid=version_uid_obj)],
- 'errors': []
- },
- sys.stdout,
- ignore_relationships=[((Version,), ('blocks',))])
- finally:
- if benji_obj:
- benji_obj.close()
+ with Benji(self.config) as benji_obj:
+ try:
+ benji_obj.scrub(version_uid_obj,
block_percentage=block_percentage)
+ except benji.exception.ScrubbingError:
+ assert benji_obj is not None
+ if self.machine_output:
+ benji_obj.export_any(
+ {
+ 'versions':
[benji_obj.get_version_by_uid(version_uid=version_uid_obj)],
+ 'errors':
[benji_obj.get_version_by_uid(version_uid=version_uid_obj)]
+ },
+ sys.stdout,
+ ignore_relationships=(((Version,), ('blocks',)),))
+ raise
+ else:
+ if self.machine_output:
+ benji_obj.export_any(
+ {
+ 'versions':
[benji_obj.get_version_by_uid(version_uid=version_uid_obj)],
+ 'errors': []
+ },
+ sys.stdout,
+ ignore_relationships=(((Version,), ('blocks',)),))
def deep_scrub(self, version_uid: str, source: str, block_percentage: int)
-> None:
version_uid_obj = VersionUid(version_uid)
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
- benji_obj.deep_scrub(version_uid_obj, source=source,
block_percentage=block_percentage)
- except benji.exception.ScrubbingError:
- assert benji_obj is not None
- if self.machine_output:
- benji_obj.export_any(
- {
- 'versions':
[benji_obj.get_version_by_uid(version_uid=version_uid_obj)],
- 'errors':
[benji_obj.get_version_by_uid(version_uid=version_uid_obj)]
- },
- sys.stdout,
- ignore_relationships=[((Version,), ('blocks',))])
- raise
- else:
- if self.machine_output:
- benji_obj.export_any(
- {
- 'versions':
[benji_obj.get_version_by_uid(version_uid=version_uid_obj)],
- 'errors': []
- },
- sys.stdout,
- ignore_relationships=[((Version,), ('blocks',))])
- finally:
- if benji_obj:
- benji_obj.close()
+ with Benji(self.config) as benji_obj:
+ try:
+ benji_obj.deep_scrub(version_uid_obj, source=source,
block_percentage=block_percentage)
+ except benji.exception.ScrubbingError:
+ assert benji_obj is not None
+ if self.machine_output:
+ benji_obj.export_any(
+ {
+ 'versions':
[benji_obj.get_version_by_uid(version_uid=version_uid_obj)],
+ 'errors':
[benji_obj.get_version_by_uid(version_uid=version_uid_obj)]
+ },
+ sys.stdout,
+ ignore_relationships=(((Version,), ('blocks',)),))
+ raise
+ else:
+ if self.machine_output:
+ benji_obj.export_any(
+ {
+ 'versions':
[benji_obj.get_version_by_uid(version_uid=version_uid_obj)],
+ 'errors': []
+ },
+ sys.stdout,
+ ignore_relationships=(((Version,), ('blocks',)),))
def _batch_scrub(self, method: str, filter_expression: Optional[str],
version_percentage: int,
block_percentage: int, group_label: Optional[str]) ->
None:
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
+ with Benji(self.config) as benji_obj:
versions, errors = getattr(benji_obj, method)(filter_expression,
version_percentage, block_percentage,
group_label)
if errors:
@@ -212,10 +177,7 @@
'errors': []
},
sys.stdout,
- ignore_relationships=[((Version,),
('blocks',))])
- finally:
- if benji_obj:
- benji_obj.close()
+ ignore_relationships=(((Version,),
('blocks',)),))
def batch_scrub(self, filter_expression: Optional[str],
version_percentage: int, block_percentage: int,
group_label: Optional[str]) -> None:
@@ -280,9 +242,7 @@
print(tbl)
def ls(self, filter_expression: Optional[str], include_labels: bool,
include_stats: bool) -> None:
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
+ with Benji(self.config) as benji_obj:
versions = benji_obj.find_versions_with_filter(filter_expression)
if self.machine_output:
@@ -293,23 +253,13 @@
)
else:
self._ls_versions_table_output(versions, include_labels,
include_stats)
- finally:
- if benji_obj:
- benji_obj.close()
def cleanup(self, override_lock: bool) -> None:
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
+ with Benji(self.config) as benji_obj:
benji_obj.cleanup(override_lock=override_lock)
- finally:
- if benji_obj:
- benji_obj.close()
def metadata_export(self, filter_expression: Optional[str], output_file:
Optional[str], force: bool) -> None:
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
+ with Benji(self.config) as benji_obj:
version_uid_objs = [version.uid for version in
benji_obj.find_versions_with_filter(filter_expression)]
if output_file is None:
benji_obj.metadata_export(version_uid_objs, sys.stdout)
@@ -319,42 +269,24 @@
with open(output_file, 'w') as f:
benji_obj.metadata_export(version_uid_objs, f)
- finally:
- if benji_obj:
- benji_obj.close()
def metadata_backup(self, filter_expression: str, force: bool = False) ->
None:
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
+ with Benji(self.config) as benji_obj:
version_uid_objs = [version.uid for version in
benji_obj.find_versions_with_filter(filter_expression)]
benji_obj.metadata_backup(version_uid_objs, overwrite=force)
- finally:
- if benji_obj:
- benji_obj.close()
def metadata_import(self, input_file: str = None) -> None:
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
+ with Benji(self.config) as benji_obj:
if input_file is None:
benji_obj.metadata_import(sys.stdin)
else:
with open(input_file, 'r') as f:
benji_obj.metadata_import(f)
- finally:
- if benji_obj:
- benji_obj.close()
def metadata_restore(self, version_uids: List[str], storage: str = None)
-> None:
version_uid_objs = [VersionUid(version_uid) for version_uid in
version_uids]
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
+ with Benji(self.config) as benji_obj:
benji_obj.metadata_restore(version_uid_objs, storage)
- finally:
- if benji_obj:
- benji_obj.close()
@staticmethod
def _metadata_ls_table_output(version_uids: List[VersionUid]):
@@ -366,9 +298,7 @@
print(tbl)
def metadata_ls(self, storage: str = None) -> None:
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
+ with Benji(self.config) as benji_obj:
version_uids = benji_obj.metadata_ls(storage)
if self.machine_output:
json.dump(
@@ -378,16 +308,11 @@
)
else:
self._metadata_ls_table_output(version_uids)
- finally:
- if benji_obj:
- benji_obj.close()
def label(self, version_uid: str, labels: List[str]) -> None:
version_uid_obj = VersionUid(version_uid)
label_add, label_remove =
InputValidation.parse_and_validate_labels(labels)
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
+ with Benji(self.config) as benji_obj:
for name, value in label_add:
benji_obj.add_label(version_uid_obj, name, value)
for name in label_remove:
@@ -397,23 +322,16 @@
version_uid_obj, ', '.join('{}={}'.format(name, value) for
name, value in label_add)))
if label_remove:
logger.info('Removed label(s) from version {}:
{}.'.format(version_uid_obj, ', '.join(label_remove)))
- finally:
- if benji_obj:
- benji_obj.close()
def database_init(self) -> None:
- benji_obj = Benji(self.config, init_database=True)
- benji_obj.close()
+ Benji(self.config, init_database=True).close()
def database_migrate(self) -> None:
- benji_obj = Benji(self.config, migrate_database=True)
- benji_obj.close()
+ Benji(self.config, migrate_database=True).close()
def enforce_retention_policy(self, rules_spec: str, filter_expression:
str, dry_run: bool,
keep_metadata_backup: bool, group_label:
Optional[str]) -> None:
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
+ with Benji(self.config) as benji_obj:
dismissed_versions =
benji_obj.enforce_retention_policy(filter_expression=filter_expression,
rules_spec=rules_spec,
dry_run=dry_run,
@@ -424,23 +342,15 @@
'versions': dismissed_versions,
},
sys.stdout,
- ignore_relationships=[((Version,),
('blocks',))])
- finally:
- if benji_obj:
- benji_obj.close()
+ ignore_relationships=(((Version,),
('blocks',)),))
def nbd(self, bind_address: str, bind_port: str, read_only: bool) -> None:
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
+ with Benji(self.config) as benji_obj:
store = BenjiStore(benji_obj)
addr = (bind_address, bind_port)
server = NbdServer(addr, store, read_only)
logger.info("Starting to serve NBD on %s:%s" % (addr[0], addr[1]))
server.serve_forever()
- finally:
- if benji_obj:
- benji_obj.close()
def version_info(self) -> None:
if not self.machine_output:
@@ -486,9 +396,7 @@
print(tbl)
def storage_stats(self, storage_name: str = None) -> None:
- benji_obj = None
- try:
- benji_obj = Benji(self.config)
+ with Benji(self.config) as benji_obj:
objects_count, objects_size = benji_obj.storage_stats(storage_name)
if self.machine_output:
@@ -499,9 +407,6 @@
print(json.dumps(result, indent=4))
else:
self._ls_storage_stats_table_output(objects_count,
objects_size)
- finally:
- if benji_obj:
- benji_obj.close()
@staticmethod
def _storage_usage_table_output(usage: Dict[str, Dict[str, int]]) -> None:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/src/benji/logging.py
new/benji-0.10.0/src/benji/logging.py
--- old/benji-0.9.0/src/benji/logging.py 2020-07-27 11:20:08.000000000
+0200
+++ new/benji-0.10.0/src/benji/logging.py 2020-09-05 13:36:45.000000000
+0200
@@ -6,7 +6,7 @@
import sys
import threading
import warnings
-from typing import Optional, Dict
+from typing import Dict
import structlog
from structlog._frames import _find_first_app_frame_and_name
@@ -52,18 +52,11 @@
structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
]
-structlog.configure(
- processors=_sl_processors,
- context_class=dict,
- logger_factory=structlog.stdlib.LoggerFactory(),
- wrapper_class=structlog.stdlib.BoundLogger,
- cache_logger_on_first_use=True,
-)
-
-def init_logging(logfile: Optional[str],
- console_level: str,
- console_formatter: str = "console-plain",
+def init_logging(*,
+ logfile: str = None,
+ console_level: str = 'INFO',
+ console_formatter: str = 'json',
logfile_formatter: str = 'legacy') -> None:
logging_config: Dict = {
@@ -139,27 +132,9 @@
logging.config.dictConfig(logging_config)
- # silence alembic
- logging.getLogger('alembic').setLevel(logging.WARN)
- # silence boto3
- # See https://github.com/boto/boto3/issues/521
- logging.getLogger('boto3').setLevel(logging.WARN)
- logging.getLogger('botocore').setLevel(logging.WARN)
- logging.getLogger('nose').setLevel(logging.WARN)
- # This disables ResourceWarnings from boto3 which are normal
- # See: https://github.com/boto/boto3/issues/454
- warnings.filterwarnings("ignore",
- category=ResourceWarning,
-
message=r'unclosed.*<(?:ssl.SSLSocket|socket\.socket).*>')
- # silence b2
- logging.getLogger('b2').setLevel(logging.WARN)
-
- if os.getenv('BENJI_DEBUG_SQL') == '1':
- logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
-
# Source:
https://stackoverflow.com/questions/6234405/logging-uncaught-exceptions-in-python/16993115#16993115
-def handle_exception(exc_type, exc_value, exc_traceback):
+def _handle_exception(exc_type, exc_value, exc_traceback):
if issubclass(exc_type, KeyboardInterrupt):
sys.__excepthook__(exc_type, exc_value, exc_traceback)
return
@@ -167,4 +142,30 @@
logger.error("Uncaught exception", exc_info=(exc_type, exc_value,
exc_traceback))
-sys.excepthook = handle_exception
+sys.excepthook = _handle_exception
+
+structlog.configure(
+ processors=_sl_processors,
+ context_class=dict,
+ logger_factory=structlog.stdlib.LoggerFactory(),
+ wrapper_class=structlog.stdlib.BoundLogger,
+ cache_logger_on_first_use=True,
+)
+
+init_logging()
+
+# silence alembic
+logging.getLogger('alembic').setLevel(logging.WARN)
+# silence boto3
+# See https://github.com/boto/boto3/issues/521
+logging.getLogger('boto3').setLevel(logging.WARN)
+logging.getLogger('botocore').setLevel(logging.WARN)
+logging.getLogger('nose').setLevel(logging.WARN)
+# This disables ResourceWarnings from boto3 which are normal
+# See: https://github.com/boto/boto3/issues/454
+warnings.filterwarnings("ignore", category=ResourceWarning,
message=r'unclosed.*<(?:ssl.SSLSocket|socket\.socket).*>')
+# silence b2
+logging.getLogger('b2').setLevel(logging.WARN)
+
+if os.getenv('BENJI_DEBUG_SQL') == '1':
+ logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/src/benji/scripts/benji.py
new/benji-0.10.0/src/benji/scripts/benji.py
--- old/benji-0.9.0/src/benji/scripts/benji.py 2020-07-27 11:20:08.000000000
+0200
+++ new/benji-0.10.0/src/benji/scripts/benji.py 2020-09-05 13:36:45.000000000
+0200
@@ -305,9 +305,15 @@
else:
config = Config()
- init_logging(config.get('logFile', types=(str, type(None))),
+ console_formatter = 'console-colored'
+ if args.machine_output:
+ console_formatter = 'json'
+ elif args.no_color:
+ console_formatter = 'console-plain'
+
+ init_logging(logfile=config.get('logFile', types=(str, type(None))),
console_level=args.log_level,
- console_formatter='console-plain' if args.no_color else
'console-colored')
+ console_formatter=console_formatter)
IOFactory.initialize(config)
StorageFactory.initialize(config)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/src/benji/tests/test_transform_ecc.py
new/benji-0.10.0/src/benji/tests/test_transform_ecc.py
--- old/benji-0.9.0/src/benji/tests/test_transform_ecc.py 1970-01-01
01:00:00.000000000 +0100
+++ new/benji-0.10.0/src/benji/tests/test_transform_ecc.py 2020-09-05
13:36:45.000000000 +0200
@@ -0,0 +1,68 @@
+import base64
+from unittest import TestCase
+
+from Crypto.PublicKey import ECC
+
+from benji.config import ConfigDict
+from benji.transform.aes_256_gcm_ecc import Transform
+
+
+class TestEccTransform(TestCase):
+
+ @staticmethod
+ def _get_transform_args(key):
+ conf = ConfigDict()
+ conf['eccKey'] =
base64.b64encode(Transform._pack_envelope_key(key)).decode('ascii')
+ conf['eccCurve'] = key.curve
+ return Transform(name='EccTest', config=None,
module_configuration=conf)
+
+ @classmethod
+ def _get_transform(cls):
+ curve = 'NIST P-384'
+ return cls._get_transform_args(ECC.generate(curve=curve))
+
+ def setUp(self):
+ self.ecc_transform = self._get_transform()
+
+ def test_decryption(self):
+ data = b'THIS IS A TEST'
+ enc_data, materials = self.ecc_transform.encapsulate(data=data)
+ self.assertTrue(enc_data)
+ self.assertNotEqual(enc_data, data)
+
+ dec_data = self.ecc_transform.decapsulate(data=enc_data,
materials=materials)
+ self.assertEqual(dec_data, data)
+
+ def test_encryption_random(self):
+ ecc_transform_ref = self._get_transform()
+
+ data = b'THIS IS A TEST'
+
+ enc_data, materials = self.ecc_transform.encapsulate(data=data)
+ enc_data_ref, materials_ref = ecc_transform_ref.encapsulate(data=data)
+
+ self.assertNotEqual(enc_data, enc_data_ref)
+ self.assertNotEqual(materials['envelope_key'],
materials_ref['envelope_key'])
+
+ def test_envelope_key(self):
+ data = b'THIS IS A TEST'
+
+ enc_data, materials = self.ecc_transform.encapsulate(data=data)
+
+ envelope_key =
self.ecc_transform._unpack_envelope_key(base64.b64decode(materials['envelope_key']))
+ self.assertFalse(envelope_key.has_private())
+
+ def test_pubkey_only_encryption(self):
+ curve = 'NIST P-384'
+ ecc_transform =
self._get_transform_args(ECC.generate(curve=curve).public_key())
+ data = b'THIS IS A TEST'
+ enc_data, materials = ecc_transform.encapsulate(data=data)
+ self.assertTrue(enc_data)
+ self.assertNotEqual(enc_data, data)
+
+ def test_pubkey_only_decryption(self):
+ curve = 'NIST P-384'
+ ecc_transform =
self._get_transform_args(ECC.generate(curve=curve).public_key())
+ data = b'THIS IS A TEST'
+ enc_data, materials = ecc_transform.encapsulate(data=data)
+ self.assertRaises(ValueError, ecc_transform.decapsulate,
data=enc_data, materials=materials)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/src/benji/tests/testcase.py
new/benji-0.10.0/src/benji/tests/testcase.py
--- old/benji-0.9.0/src/benji/tests/testcase.py 2020-07-27 11:20:08.000000000
+0200
+++ new/benji-0.10.0/src/benji/tests/testcase.py 2020-09-05
13:36:45.000000000 +0200
@@ -45,8 +45,7 @@
def setUp(self):
self.testpath = self.TestPath()
- init_logging(None,
- logging.WARN if os.environ.get('UNITTEST_QUIET', False)
else logging.DEBUG,
+ init_logging(console_level=logging.WARN if
os.environ.get('UNITTEST_QUIET', False) else logging.DEBUG,
console_formatter='console-plain')
self.config =
Config(ad_hoc_config=self.CONFIG.format(testpath=self.testpath.path))
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/src/benji/transform/aes_256_gcm.py
new/benji-0.10.0/src/benji/transform/aes_256_gcm.py
--- old/benji-0.9.0/src/benji/transform/aes_256_gcm.py 2020-07-27
11:20:08.000000000 +0200
+++ new/benji-0.10.0/src/benji/transform/aes_256_gcm.py 2020-09-05
13:36:45.000000000 +0200
@@ -11,6 +11,7 @@
class Transform(TransformBase):
+ AES_KEY_LEN = 32
def __init__(self, *, config: Config, name: str, module_configuration:
ConfigDict) -> None:
super().__init__(config=config, name=name,
module_configuration=module_configuration)
@@ -19,7 +20,7 @@
if master_key_encoded is not None:
master_key = base64.b64decode(master_key_encoded)
- if len(master_key) != 32:
+ if len(master_key) != self.AES_KEY_LEN:
raise ValueError('Key masterKey has the wrong length. It must
be 32 bytes long and encoded as BASE64.')
self._master_key = master_key
@@ -30,15 +31,21 @@
self._master_key = derive_key(salt=kdf_salt,
iterations=kdf_iterations, key_length=32, password=password)
+ def _create_envelope_key(self) -> Tuple[bytes, bytes]:
+ envelope_key = get_random_bytes(self.AES_KEY_LEN)
+ encrypted_key = aes_wrap_key(self._master_key, envelope_key)
+ return envelope_key, encrypted_key
+
+ def _derive_envelope_key(self, encrypted_key: bytes) -> bytes:
+ return aes_unwrap_key(self._master_key, encrypted_key)
+
def encapsulate(self, *, data: bytes) -> Tuple[Optional[bytes],
Optional[Dict]]:
- envelope_key = get_random_bytes(32)
+ envelope_key, encrypted_key = self._create_envelope_key()
envelope_iv = get_random_bytes(16)
encryptor = AES.new(envelope_key, AES.MODE_GCM, nonce=envelope_iv)
- envelope_key = aes_wrap_key(self._master_key, envelope_key)
-
materials = {
- 'envelope_key': base64.b64encode(envelope_key).decode('ascii'),
+ 'envelope_key': base64.b64encode(encrypted_key).decode('ascii'),
'iv': base64.b64encode(envelope_iv).decode('ascii'),
}
@@ -59,8 +66,8 @@
raise ValueError('Encryption materials IV iv has wrong length of
{}. It must be 16 bytes long.'.format(
len(iv)))
- envelope_key = aes_unwrap_key(self._master_key, envelope_key)
- if len(envelope_key) != 32:
+ envelope_key = self._derive_envelope_key(envelope_key)
+ if len(envelope_key) != self.AES_KEY_LEN:
raise ValueError(
'Encryption materials key envelope_key has wrong length of {}.
It must be 32 bytes long.'.format(
len(envelope_key)))
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/benji-0.9.0/src/benji/transform/aes_256_gcm_ecc.py
new/benji-0.10.0/src/benji/transform/aes_256_gcm_ecc.py
--- old/benji-0.9.0/src/benji/transform/aes_256_gcm_ecc.py 1970-01-01
01:00:00.000000000 +0100
+++ new/benji-0.10.0/src/benji/transform/aes_256_gcm_ecc.py 2020-09-05
13:36:45.000000000 +0200
@@ -0,0 +1,67 @@
+import base64
+from hashlib import sha256
+from typing import Dict, Tuple, Optional
+
+from Crypto.PublicKey import ECC
+
+from benji.config import Config, ConfigDict
+from benji.logging import logger
+from benji.transform.aes_256_gcm import Transform as TransformAES
+
+
+class Transform(TransformAES):
+
+ def __init__(self, *, config: Config, name: str, module_configuration:
ConfigDict) -> None:
+ ecc_key_der: str = Config.get_from_dict(module_configuration,
'eccKey', types=str)
+ ecc_curve: Optional[str] = Config.get_from_dict(module_configuration,
'eccCurve', 'NIST P-384', types=str)
+
+ ecc_key = self._unpack_envelope_key(base64.b64decode(ecc_key_der))
+
+ if ecc_key.curve != ecc_curve:
+ raise ValueError(f'Key eccKey does not match the eccCurve setting
(found: {ecc_key.curve}, expected: {ecc_curve}).')
+
+ self._ecc_key = ecc_key
+ self._ecc_curve = ecc_key.curve
+
+ point_q_len = self._ecc_key.pointQ.size_in_bytes()
+ if point_q_len < self.AES_KEY_LEN:
+ raise ValueError(f'Size of point Q is smaller than the AES key
length, which reduces security ({point_q_len} < {self.AES_KEY_LEN}).')
+
+ # Note: We don't actually have a "master" aes key, because the key is
derived from the ECC key
+ # and set before calling the parent's encapsulate/decapsulate method.
+ aes_config = module_configuration.copy()
+ aes_config['masterKey'] = base64.b64encode(b'\x00' *
self.AES_KEY_LEN).decode('ascii')
+ super().__init__(config=config, name=name,
module_configuration=aes_config)
+
+ @staticmethod
+ def _pack_envelope_key(key: ECC.EccKey) -> bytes:
+ return key.export_key(format='DER', compress=True)
+
+ @staticmethod
+ def _unpack_envelope_key(key: bytes) -> ECC.EccKey:
+ return ECC.import_key(key)
+
+ @staticmethod
+ def _ecc_point_to_key(point: ECC.EccPoint) -> bytes:
+ sha = sha256(int.to_bytes(int(point.x), point.size_in_bytes(), 'big'))
+ sha.update(int.to_bytes(int(point.y), point.size_in_bytes(), 'big'))
+ return sha.digest()
+
+ def _create_envelope_key(self) -> Tuple[bytes, bytes]:
+ cipher_privkey = ECC.generate(curve=self._ecc_curve)
+ shared_key = self._ecc_point_to_key(self._ecc_key.pointQ *
cipher_privkey.d)
+ return shared_key, self._pack_envelope_key(cipher_privkey.public_key())
+
+ def _derive_envelope_key(self, cipher_pubkey: bytes) -> bytes:
+ ecc_point = self._unpack_envelope_key(cipher_pubkey)
+ return self._ecc_point_to_key(ecc_point.pointQ * self._ecc_key.d)
+
+ def encapsulate(self, *, data: bytes) -> Tuple[Optional[bytes],
Optional[Dict]]:
+ if self._ecc_key.has_private():
+ logger.warning('ECC key loaded from config includes private key
data, which is not needed for encryption.')
+ return super().encapsulate(data=data)
+
+ def decapsulate(self, *, data: bytes, materials: Dict) -> bytes:
+ if not self._ecc_key.has_private():
+ raise ValueError('ECC key loaded from config does not include
private key data, cannot proceed.')
+ return super().decapsulate(data=data, materials=materials)
