Hello community,
here is the log from the commit of package openSUSE-release-tools for
openSUSE:Factory checked in at 2019-11-10 22:44:15
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/openSUSE-release-tools (Old)
and /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new.2990 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "openSUSE-release-tools"
Sun Nov 10 22:44:15 2019 rev:244 rq:747076 version:20191108.6b1fd618
Changes:
--------
---
/work/SRC/openSUSE:Factory/openSUSE-release-tools/openSUSE-release-tools.changes
2019-11-09 23:42:38.873501388 +0100
+++
/work/SRC/openSUSE:Factory/.openSUSE-release-tools.new.2990/openSUSE-release-tools.changes
2019-11-10 22:44:19.393626690 +0100
@@ -1,0 +2,52 @@
+Fri Nov 08 23:15:03 UTC 2019 - [email protected]
+
+- Update to version 20191108.6b1fd618:
+ * dist/obs/attribute-update: provide script for updating attribute
definitions.
+
+-------------------------------------------------------------------
+Fri Nov 08 22:15:04 UTC 2019 - [email protected]
+
+- Update to version 20191108.4d09eda6:
+ * docs/origin-manager: provide comprehensive documentation.
+
+-------------------------------------------------------------------
+Fri Nov 08 22:09:01 UTC 2019 - [email protected]
+
+- Update to version 20191108.2a32702b:
+ * osclib/origin: policy_input_calculate(): s/unkown/unknown/.
+ * dist/obs: remove namespace from attribute names.
+
+-------------------------------------------------------------------
+Fri Nov 08 21:45:33 UTC 2019 - [email protected]
+
+- Update to version 20191108.03a954ae:
+ * osclib/origin: origin_update(): support blacklist patterns for initial
updates.
+ * osc-origin: extract osrt_origin_update_packages().
+
+-------------------------------------------------------------------
+Fri Nov 08 21:21:31 UTC 2019 - [email protected]
+
+- Update to version 20191108.a83e40e0:
+ * osclib/origin: origin_history(): include non-annotated requests.
+
+-------------------------------------------------------------------
+Fri Nov 08 20:52:37 UTC 2019 - [email protected]
+
+- Update to version 20191108.a0488c3f:
+ * osclib/origin: origin_annotation_dump(): include pending information.
+
+-------------------------------------------------------------------
+Fri Nov 08 20:25:46 UTC 2019 - [email protected]
+
+- Update to version 20191108.d271224d:
+ * osclib/core: request_action_list_maintenance_incident(): support another
form.
+ * osclib/conf: configure maintenance staging review options for
origin-manager.
+ * osclib/origin: add maintenance staging equivalent reviews to
pending_submission_allowed_reviews.
+ * osclib/origin: drop repo-checker from pending_submission_allowed_reviews.
+ * osclib/origin: provide pending_submission_allowed_reviews_update option.
+ * osclib/origin: reviews_filter_allowed(): support negation rules.
+ * osclib/origin: provide reviews_filter_allowed().
+ * osclib/origin: project_source_pending(): include psuedo incident review.
+ * osclib/core: reviews_remaining(): provide incident_psuedo option.
+
+-------------------------------------------------------------------
Old:
----
openSUSE-release-tools-20191108.c7e1800d.obscpio
New:
----
openSUSE-release-tools-20191108.6b1fd618.obscpio
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ openSUSE-release-tools.spec ++++++
--- /var/tmp/diff_new_pack.IgRhNR/_old 2019-11-10 22:44:20.469628087 +0100
+++ /var/tmp/diff_new_pack.IgRhNR/_new 2019-11-10 22:44:20.469628087 +0100
@@ -20,7 +20,7 @@
%define source_dir openSUSE-release-tools
%define announcer_filename factory-package-news
Name: openSUSE-release-tools
-Version: 20191108.c7e1800d
+Version: 20191108.6b1fd618
Release: 0
Summary: Tools to aid in staging and release work for openSUSE/SUSE
License: GPL-2.0-or-later AND MIT
++++++ _servicedata ++++++
--- /var/tmp/diff_new_pack.IgRhNR/_old 2019-11-10 22:44:20.497628123 +0100
+++ /var/tmp/diff_new_pack.IgRhNR/_new 2019-11-10 22:44:20.501628128 +0100
@@ -1,6 +1,6 @@
<servicedata>
<service name="tar_scm">
<param
name="url">https://github.com/openSUSE/openSUSE-release-tools.git</param>
- <param
name="changesrevision">c7e1800d3009933937ae104dbe226c23c1543a69</param>
+ <param
name="changesrevision">45e0989b28e1ab5abb37e29d2f197cb58d638895</param>
</service>
</servicedata>
++++++ openSUSE-release-tools-20191108.c7e1800d.obscpio ->
openSUSE-release-tools-20191108.6b1fd618.obscpio ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/OSRT:OriginUpdateDelay.xml
new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/OSRT:OriginUpdateDelay.xml
---
old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/OSRT:OriginUpdateDelay.xml
2019-11-08 20:49:24.000000000 +0100
+++
new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/OSRT:OriginUpdateDelay.xml
2019-11-09 00:14:13.000000000 +0100
@@ -1,4 +1,4 @@
-<definition name="OSRT:OriginUpdateDelay" namespace="OSRT">
+<definition name="OriginUpdateDelay" namespace="OSRT">
<description>OriginManager update delay frequency in seconds (minimum time
since source change)</description>
<count>1</count>
<modifiable_by role="maintainer"/>
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/OSRT:OriginUpdateFrequency.xml
new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/OSRT:OriginUpdateFrequency.xml
---
old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/OSRT:OriginUpdateFrequency.xml
2019-11-08 20:49:24.000000000 +0100
+++
new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/OSRT:OriginUpdateFrequency.xml
2019-11-09 00:14:13.000000000 +0100
@@ -1,4 +1,4 @@
-<definition name="OSRT:OriginUpdateFrequency" namespace="OSRT">
+<definition name="OriginUpdateFrequency" namespace="OSRT">
<description>OriginManager update frequency in seconds (minimum time since
last request)</description>
<count>1</count>
<modifiable_by role="maintainer"/>
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/OSRT:OriginUpdateInitialBlacklist.xml
new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/OSRT:OriginUpdateInitialBlacklist.xml
---
old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/OSRT:OriginUpdateInitialBlacklist.xml
1970-01-01 01:00:00.000000000 +0100
+++
new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/OSRT:OriginUpdateInitialBlacklist.xml
2019-11-09 00:14:13.000000000 +0100
@@ -0,0 +1,5 @@
+<definition name="OriginUpdateInitialBlacklist" namespace="OSRT">
+ <description>OriginManager update initial blacklist patterns (filters
potential new package submissions)</description>
+ <count>1</count>
+ <modifiable_by role="maintainer"/>
+</definition>
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/OSRT:OriginUpdateSupersede.xml
new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/OSRT:OriginUpdateSupersede.xml
---
old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/OSRT:OriginUpdateSupersede.xml
2019-11-08 20:49:24.000000000 +0100
+++
new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/OSRT:OriginUpdateSupersede.xml
2019-11-09 00:14:13.000000000 +0100
@@ -1,4 +1,4 @@
-<definition name="OSRT:OSRT:OriginUpdateSupersede" namespace="OSRT">
+<definition name="OriginUpdateSupersede" namespace="OSRT">
<description>OriginManager allowed to supersede existing requests (true or
false)</description>
<count>1</count>
<modifiable_by role="maintainer"/>
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/attribute-update
new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/attribute-update
--- old/openSUSE-release-tools-20191108.c7e1800d/dist/obs/attribute-update
1970-01-01 01:00:00.000000000 +0100
+++ new/openSUSE-release-tools-20191108.6b1fd618/dist/obs/attribute-update
2019-11-09 00:14:13.000000000 +0100
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+if [ $# -ne 2 ] ; then
+ echo "usage: $(basename "$0") <APIURL> <group>"
+ echo
+ echo " ./attribute-update obs factory-staging"
+ echo " ./attribute-update ibs sle-staging-managers"
+ exit 1
+fi
+
+for file in *.xml ; do
+ read -p "update $(basename "$file")? [y/n]: " response
+ if [ "$response" != "y" ] ; then
+ continue
+ fi
+
+ temp="/tmp/$(basename "$file")"
+ sed "5i<modifiable_by group='$2'/>" "$file" > "$temp"
+ path="$(basename "$file" | sed 's|:|/|' | sed 's|.xml||')"
+ echo "PUT /attribute/$path/_meta"
+ osc -A "$1" api -X PUT -T "$temp" "/attribute/$path/_meta"
+ rm "$temp"
+done
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20191108.c7e1800d/docs/origin-manager.md
new/openSUSE-release-tools-20191108.6b1fd618/docs/origin-manager.md
--- old/openSUSE-release-tools-20191108.c7e1800d/docs/origin-manager.md
1970-01-01 01:00:00.000000000 +0100
+++ new/openSUSE-release-tools-20191108.6b1fd618/docs/origin-manager.md
2019-11-09 00:14:13.000000000 +0100
@@ -0,0 +1,484 @@
+# Origin Manager
+
+The primary function of origin manager is, as the name implies, to _manage_
the _origin_ of packages. In other words to keep track of from what project a
package originates, submit updates, review requests to detect origin changes,
and enforce origin specific policies like adding appropriate reviews.
+
+The primary configuration for all parts of the tool is the `OSRT:OriginConfig`
attribute to be placed on the target project. Once the config is created all
relevant caches, listeners, and cron jobs will automatically include the
project. The only exception is the `origin-manager` user should be added as a
project _reviewer_ in order to activate the review portion of the tool.
Conversely either removing the config or locking the project will disable all
services. Locking the project will preserve the caches used by the web
interface, but not update them which allows for archived viewing.
+
+Additionally, there are some other `OSRT:Origin*` attributes used to override
the `OSRT:OriginConfig` outside of the target project and standard
`OSRT:Config` relevant options.
+
+## Core concepts
+
+The following core concepts are important for understanding the operation of
the tool and its configuration.
+
+### Origin
+
+The primary concept within origin manager is an _origin_ which is simply a
project from which a package may originate. The origin config will contain a
prioritized list of possible origins. In order to determine the origin for a
package each origin is searched in priority order for the package and a
matching source revision. The first origin containing a match is considered the
origin of the package.
+
+An origin can change for a package without the source of the package changing
in the target project in one of two ways. Either a devel project can be set on
the target package in conjunction with an allowed `<devel>` origin or the
package may change in a potential origin project. If the matching sources are
accepted into a higher priority origin then the package will be consider as
coming from the new origin. This automatic switching properly encapsulates the
rather complicated interrelated product workflow.
+
+Alternatively, if the `OSRT:OriginConfig` is changed that may also change a
package's origin.
+
+### Workaround
+
+A _workaround_ is a _pseudo origin_ used to track a package that used to match
an origin, but had a source change accepted that matched no origin. The change
to a workaround requires human review via the `fallback-group`. If the source
later shows up in the origin project it will automatically switch back to not
being considered a workaround.
+
+## Configuration
+
+As noted above all configuration for the tool is read from OBS attributes.
+
+### OSRT:OriginConfig
+
+Any target project that expects origin manager to manage it must contain this
config. The config is expected to be valid _yaml_ of the following format.
+
+```yaml
+origins:
+- origin1:
+ policy_option1: policy_option1_value
+
+global_option1: global_option1_value
+```
+
+Note the origins are a list of single key-value pairs. The list allows the
order to be maintained by _yaml_ parsers over a simple key-value set, but makes
for a bit more cumbersome markup. The following demonstrates the default config.
+
+```yaml
+# Wait instead of declining an initial source submission that cannot be found
in
+# any origin.
+unknown_origin_wait: false
+
+# List of prioritized origins and their policy overrides.
+origins: []
+
+# User that will perform reviews (ie. user under which ./origin-manager.py will
+# run). If not specified nor available from default reference 'origin-manager'
+# will be used.
+review-user: <config:origin-manager-review-user>
+
+# Group from which a review will be requested as a fallback whenever automated
+# approval is not possible.
+fallback-group: <config:origin-manager-fallback-group>
+
+# Fallback workaround project primarily used for migration to origin-manager.
+fallback-workaround: {}
+```
+
+The following is the default policy for an origin. The policy of the new
origin is applied to requests and when determining the current origin
(especially relevant to pending* options).
+
+```yaml
+# Additional by_group reviews to be added to requests from this origin. Special
+# keys supported are: fallback and maintainer. The fallback key will be
replaced
+# by the fallback-group global configuration value and the maintainer key will
+# be replaced by the maintainer from the origin package (by_package review) or
+# the devel project for the origin package if configured. Maintainer reviews
+# are skipped if the request creator is a maintainer.
+additional_reviews: []
+
+# Submit updates for packages from this origin and allow forward (based on
+# revision order) submissions without fallback review. If disabled updates will
+# not be submitted automatically and manually submitted updates will require
+# fallback review.
+automatic_updates: true
+
+# Submit new packages from this origin (highest priority, enabled, origin
wins).
+# New packages will only be submitted once. Any prior requests will block
future
+# submission of the package until it is accepted.
+automatic_updates_initial: false
+
+# Submit updates even if an existing request is still pending.
+automatic_updates_supersede: true
+
+# Delay update submission by a minimum of N seconds after a source change.
+automatic_updates_delay: 0
+
+# Cap updates at a minimum of N seconds since the last request.
+automatic_updates_frequency: 0
+
+# Always add a review for the package maintainer.
+maintainer_review_always: false
+
+# Add a review for the package maintainer on the initial submission of package.
+maintainer_review_initial: true
+
+# Approve source submissions that match pending submissions in the origin
+# project of either the new state or with only allowed reviews
+# (pending_submission_allowed_reviews). Otherwise, consider as originating from
+# the project containing the pending request and wait for the conditions to be
+# met. Disable to require submissions to be accepted before accepting review.
+pending_submission_allow: false
+
+# Consider pending submissions for the purposes of determining origin, but wait
+# for requests to be accepted before accepting review.
+pending_submission_consider: false
+
+# List of reviews to allow for the purposes of accepting review when
+# pending_submission_allow is enabled.
+pending_submission_allowed_reviews:
+# Non-maintenance projects:
+- <config_source:staging>*
+# Maintenance projects:
+- <config_source:review-install-check>
+- <config_source:review-openqa>
+
+# Control how early pending requests in an origin project are considered for
+# submission to the target project. Submit pending requests with a set of
+# allowed reviews, but still wait for the above reviews before being accepted.
+# For example !<config:review-team> could be added to require the review team
+# approval before mirroring a pending request.
+'pending_submission_allowed_reviews_update':
+- '!maintenance_incident'
+```
+
+The following _special_ origins are supported where `*` denotes all and `~`
indicates a workaround (explained later).
+
+- `*`: applied to all origins above it
+- `*~`: applied to all workaround origins above it and generates them if they
do not exist
+- `prefix*suffix`: describes a product family which is expanded into specific
projects
+- `origin1~`: describes a specific origin workaround
+- `<devel>`: replaced with the devel project for a given package if set
+
+To add an origin with a completely default policy use `{}` as shown below.
+
+```yaml
+origins:
+- projectFoo: {}
+```
+
+#### Config references
+
+As one may have noticed, both the global options and policy defaults contain
config references denoted by `<config:*>` or `<config_source:*>`. References
are replaced with the corresponding value from the `OSRT:Config` for the
relevant project.
+
+- `config`: read from the target project
+- `config_source`: read from the source project (relative to a source change
request)
+
+If no value is found in the relevant config the value is replaced by a blank
which in the case of lists removes the entry.
+
+A reference may be used in conjunction with additional characters. This is
useful for the `pending_submission_allowed_reviews` policy option which
defaults to including `<config_source:staging>*`. This will allow any review
that starts with `<config_source:staging>`. For example, when evaluated in the
context of `openSUSE:Leap:15.2` for a request from `openSUSE:Factory` this will
be evaluated as `openSUSE:Factory:Staging*` which will allow staging reviews.
+
+The reference abstraction is especially useful for cross product family
origins. For example _SLE_ can use `openSUSE.org:openSUSE:Factory` as an origin
and automatically have the appropriate staging reviews ignored based on the
cross-OBS-instance config.
+
+#### Workarounds
+
+When a package is considered a workaround for an origin a `~` will be appended
to the origin. Workarounds, just like any origin are only allowed if they are
found in the list of origins from the config. Workarounds may be configured
with different settings, such as `additional_reviews`, than the non-workaround
origin.
+
+Workarounds of any origin may be allowed by adding `*~` origin at the end of
the list.
+
+#### Family expansion
+
+Instead of defining specific origin projects for an entire family the config
supports family expansion. Note that this has become less useful since layered
projects such as maintenance projects have their history stacked on parents for
origin finding purposes. As such it is typically sufficient to include just the
previous product's `:Update` project instead of the whole family.
+
+Looking at the config from `openSUSE:Leap:15.1` helps explain this feature
best. The config contained the following family expansions.
+
+- `SUSE:SLE-15*`
+- `openSUSE:Leap:15*`
+
+These were expanded into the following.
+
+- SUSE:SLE-15-SP1:Update
+- SUSE:SLE-15-SP1:GA
+- SUSE:SLE-15:Update
+- SUSE:SLE-15:GA
+- openSUSE:Leap:15.0:Update
+- openSUSE:Leap:15.0
+
+Note that the _Leap_ expansion automatically did not include the target
project nor its `:Update` project as they are considered above it. The `*~`
origin in the config meant that each of these was also duplicated for a
workaround origin.
+
+Family expansion can also be used with a suffix like what was done in
`openSUSE:Leap:15.1:NonFree`.
+
+- openSUSE:Leap:15*:NonFree
+
+The above expanded into the following.
+
+- openSUSE:Leap:15.0:NonFree:Update
+- openSUSE:Leap:15.0:NonFree
+
+#### Fallback workaround
+
+The `fallback-workaround` global option is used as a fallback project which
will be considered a workaround for another origin. This was designed for the
migration to origin manager before which a `:SLE-workarounds` subproject was
used to house workaround sources.
+
+The option requires two keys to be set: `project` and `origin`. The `project`
key is the project to search for matching sources and the `origin` is what
origin of that will be used if matched. For example, `openSUSE:Leap:15.1` used
the following.
+
+```yaml
+fallback-workaround:
+ origin: 'SUSE:SLE-15-SP1:GA~'
+ project: 'openSUSE:Leap:15.1:SLE-workarounds'
+```
+
+The above config meant that if sources matched no origin, but matched those in
`openSUSE:Leap:15.1:SLE-workarounds` the origin was considered
`SUSE:SLE-15-SP1:GA~`.
+
+### Override attributes
+
+The following attributes are provided to override their corresponding policy
option. These attributes are searched for in the following order of precedence.
+
+- origin package
+- origin project
+- target package
+- target project
+
+If no attributes are found the `OSRT:OriginConfig` policy values for the
matched origin are used.
+
+- `OSRT:OriginUpdateSkip`: `automatic_updates = false`
+- `OSRT:OriginUpdateSupersede`: `automatic_updates_supersede`
+- `OSRT:OriginUpdateDelay`: `automatic_updates_delay`
+- `OSRT:OriginUpdateFrequency`: `automatic_updates_frequency`
+
+The `OSRT:OriginUpdateSkip` attribute can also be added to a target project to
disable updates entirely, otherwise if the `OSRT:OriginConfig` includes at
least one origin with `automatic_updates` then the project will be included in
the update jobs.
+
+### Initial submission blacklist
+
+For target projects containing origins with `automatic_updates_initial`
enabled the `OSRT:OriginUpdateInitialBlacklist` attribute can be utilized to
blacklist new packages from being submitted. The blacklist does not stop those
packages from being submitted manually or updated if they are included in the
target project. The blacklist only limits the new packages considered for
initial submission.
+
+The blacklist supports one entry per line where entries can be a python
compatible regular expression and is placed on the target project to be applied
to all origins. For example, the following config would be relevant for _Leap_
which has `automatic_updates_initial` enabled for _SLE_, but does not want
_SLE_ specific packages.
+
+```
+.*SLE.*
+.*SLED.*
+.*SLES.*
+.*suse-manager.*
+.*spacewalk.*
+kernel-livepatch-.*
+system-role-.*
+sca-.*
+patterns-server-enterprise
+unified-installer-release
+```
+
+### OSRT:Config
+
+A few options relevant to origin manager are standard options and thus
available via the `OSRT:Config` attribute.
+
+```ini
+# Group whose members are allowed to issue override commands. The staging-group
+# is also included regardless of what value is set.
+originmanager-override-group = ''
+
+# Minimum time in seconds to wait before reviewing source submissions. The
+# default is 30 minutes with the intention of allowing parrallel submissions
+# to origin projects and downstream projects that allow pending requests.
+# Without the wait downstream projects can be reviewed before the upstream
+# requests have been created. 30 minutes is also less than the 50 minutes that
+# staging-bot waits which insure the quick strategy can still be utilized.
+originmanager-request-age-min = 1800
+```
+
+## Command line interface
+
+A CLI is provided for origin manager via the `osc-plugin-origin` package. See
`osc origin --help` for complete reference. A few especially useful examples
are included below.
+
+### config --origins-only
+
+The `config` command shows the expanded configuration for a given project. The
`--origins-only` flag is useful for ensuring origin expansions are working as
expected, but the full output is useful for ensuring policy overrides are
working.
+
+```
+$ osc origin -p openSUSE:Leap:15.2 config --origins-only
+```
+
+```
+<devel>
+<devel>~
+SUSE:SLE-15-SP2:GA
+SUSE:SLE-15-SP2:GA~
+openSUSE:Leap:15.1:Update
+openSUSE:Leap:15.1:Update~
+openSUSE:Leap:15.1
+openSUSE:Leap:15.1~
+openSUSE:Factory
+openSUSE:Factory~
+```
+
+### package --debug
+
+It can be helpful to understand the origin search path which can be done via
the `--debug` flag on the `package` command.
+
+```
+$ osc origin -p openSUSE:Leap:15.2 package --debug adminer
+```
+
+```
+[D] origin_find: openSUSE:Leap:15.2/adminer with source 3bd0d78 (True, True,
True)
+[D] source_contain: openSUSE:Leap:15.1:Update 16c519a == 3bd0d78
+[D] source_contain: openSUSE:Leap:15.1:Update 440c7f2 == 3bd0d78
+[D] source_contain: openSUSE:Leap:15.1:Update 440c7f2 == 3bd0d78
+[D] source_contain: openSUSE:Leap:15.1:Update 4c26333 == 3bd0d78
+[D] source_contain: openSUSE:Leap:15.1:Update eb15f78 == 3bd0d78
+[D] source_contain: openSUSE:Leap:15.1 16c519a == 3bd0d78
+[D] source_contain: openSUSE:Leap:15.1 440c7f2 == 3bd0d78
+[D] source_contain: openSUSE:Leap:15.1 440c7f2 == 3bd0d78
+[D] source_contain: openSUSE:Leap:15.1 4c26333 == 3bd0d78
+[D] source_contain: openSUSE:Leap:15.1 eb15f78 == 3bd0d78
+[D] source_contain: openSUSE:Factory 3bd0d78 ==
3bd0d78 (match)
+openSUSE:Factory
+```
+
+Which shows that the latest revision from `openSUSE:Factory` is the first
match and thus the origin.
+
+### potentials
+
+Understanding the _potential_ origins of a package is also useful.
+
+```
+$ osc origin -p openSUSE:Leap:15.2 potentials kernel-source
+```
+
+```
+origin version
+Kernel:openSUSE-15.2 5.3.8
+openSUSE:Leap:15.1:Update unknown
+openSUSE:Leap:15.1 4.12.14
+openSUSE:Factory 5.3.8
+```
+
+### history
+
+Knowning the origin history of a package can also be enlightening.
+
+```
+$ osc origin -p openSUSE:Leap:15.2 history kernel-source
+```
+
+```
+origin state request
+Kernel:openSUSE-15.2 declined 745644
+Kernel:openSUSE-15.2 review 745323
+Kernel:openSUSE-15.2 accepted 744385
+Kernel:openSUSE-15.2 declined 743969
+Kernel:openSUSE-15.2 declined 743419
+Kernel:openSUSE-15.2 declined 743414
+Kernel:openSUSE-15.2 accepted 742724
+Kernel:openSUSE-15.2 accepted 737241
+Kernel:openSUSE-15.2 accepted 733609
+Kernel:openSUSE-15.2 superseded 733094
+Kernel:openSUSE-15.2 superseded 733005
+Kernel:openSUSE-15.2 accepted 732312
+Kernel:openSUSE-15.2 accepted 731459
+openSUSE:Leap:15.1:Update~ accepted 728396
+openSUSE:Leap:15.1:Update~ accepted 725631
+openSUSE:Leap:15.1:Update accepted 724147
+openSUSE:Leap:15.1:Update accepted 721159
+```
+
+## Review bot
+
+As with all `ReviewBot` based bots they can be debugged locally by using the
`--debug` and `--dry` options. Origin manager provides detailed debug output
for understanding both the origin search and policy evaluation. The following
example demonstrates a request that is changing the origin from
`openSUSE:Leap:15.1:Update` to `openSUSE:Factory`.
+
+```
+$ ./origin-manager.py --dry --debug id 746065
+```
+
+```
+[I] checking 746065
+[D] origin_find: openSUSE:Leap:15.2/virtualbox with source d79bf01 (False,
True, True)
+[D] source_contain: openSUSE:Leap:15.1:Update 1655653 == d79bf01
+[D] source_contain: openSUSE:Leap:15.1:Update f642fe0 == d79bf01
+[D] source_contain: openSUSE:Leap:15.1:Update 22bd811 == d79bf01
+[D] source_contain: openSUSE:Leap:15.1:Update 48ab129 == d79bf01
+[D] source_contain: openSUSE:Leap:15.1:Update dd5c97b == d79bf01
+[D] source_contain: openSUSE:Leap:15.1 48ab129 == d79bf01
+[D] source_contain: openSUSE:Leap:15.1 dd5c97b == d79bf01
+[D] source_contain: openSUSE:Leap:15.1 da823e6 == d79bf01
+[D] source_contain: openSUSE:Leap:15.1 28ae8a7 == d79bf01
+[D] source_contain: openSUSE:Leap:15.1 0bc96d1 == d79bf01
+[D] source_contain: openSUSE:Factory d79bf01 ==
d79bf01 (match)
+[D] origin_find: openSUSE:Leap:15.2/virtualbox with source 1655653 (True,
True, True)
+[D] source_contain: openSUSE:Leap:15.1:Update 1655653 ==
1655653 (match)
+[D] policy_evaluate:
+
+# policy
+additional_reviews: []
+automatic_updates: true
+automatic_updates_delay: 0
+automatic_updates_frequency: 0
+automatic_updates_initial: false
+automatic_updates_supersede: true
+maintainer_review_always: false
+maintainer_review_initial: true
+pending_submission_allow: false
+pending_submission_allowed_reviews:
+- openSUSE:Factory:Staging*
+pending_submission_allowed_reviews_update:
+- '!maintenance_incident'
+pending_submission_consider: true
+
+# inputs
+direction: unknown
+higher_priority: false
+new_package: false
+origin_change: true
+pending_submission: 'False'
+same_family: false
+
+PolicyResult(wait=False, accept=True, reviews={'fallback': 'Changing to a
lower priority origin.'}, comments=[])
+[D] skipped adding duplicate review for origin-reviewers
+[D] broadening search to include any state on 746065
+[D] no previous comment to replace on 746065
+[I] 746065 accepted: origin: openSUSE:Factory
+origin_old: openSUSE:Leap:15.1:Update
+
+[D] 746065 review not changed
+```
+
+From the `PolicyResult` it is clear that the review will be accepted with the
addition of a review for the fallback group since `Changing to a lower priority
origin.`.
+
+### Commands
+
+There are a two commands that can be used to control the review bot from an
OBS request. The commands follow the standard reply syntax `@<user> <message>`
and can be placed in the request description or a comment on the request.
Commands issued by users not part of either the `staging-group` or
`originmanager-override-group` will be ignored. In order to be accepted
commands must match the following regular expression.
+
+```regex
+^@(?P<user>[^ ,:]+)[,:]? (?P<args>.*)$
+```
+
+The standard `override` command is available to force acceptance when the
submission does not match any allowed origin, but this should be avoided if
possible. The `unknown_origin_wait` global option can be enabled to make it
easier to utilize the override feature.
+
+The origin manager specific command `change_devel` is also available with an
expanded user pool to include the request creator. The command tells origin
manager to consider the origin as a devel project from the request source
project. Alternatively, a different origin project can be optionally specified.
+
+For example, a request originating from `projectDevel` would consider the
origin for review purposes as if the devel on the package was set to
`projectDevel` using the following command.
+
+```
+@origin-manager change_devel
+```
+
+The same request would consider the origin for review purposes as if the devel
on the package was set to `projectFoo` using the following command.
+
+```
+@origin-manager change_devel projectFoo
+```
+
+## Automatic updates
+
+For projects with at least one origin having `automatic_updates` enabled
source changes from enabled origins will be automatically submitted to the
target project. There is both a cron job that runs once daily and an event
listener that will make submissions immediately. The listener considers the
same control options as the cron job so if something like
`automatic_updates_delay` is configured the listener will not make updates for
that project and instead they will fallback to the cron job. The cron job also
handles configuration changes and backfilling newly managed projects.
+
+### Automatic change_devel
+
+If a source submission is accepted, but during update an origin cannot be
matched the annotation on the most recent request will be checked to see if it
was considered as devel project. If such an annotation exists a `change_devel`
request will automatically be created to match the annotation. After the
request is accept future updates will be submitted as appropriate.
+
+## Web interface
+
+A _web interface_ is provided using the _OBS operator_ server which wraps the
_origin CLI plugin_. The web interface is backed by a cache which is updated
trice-weekly and allows for quickly retrieving the full list of origins for all
packages within a project.
+
+In conjunction to the primary web interface a
[userscript](../userscript/README.md) is provided to automatically display
origin information on the OBS web interface. Both the package view and request
views are supplemented and provide links to the web interface. **Note that one
must have an active OBS session for either to work.**
+
+### Layout
+
+The interface condenses a rather large amount of information into one screen
and can be overwhelming at first glance. The following is a description of the
columns from left the right.
+
+#### Package list
+
+A list of all _source_ packages within a target project are shown with their
respective origin. The _revisions_ column denotes the type of the last 10
commits: green matches origin in target, red only in origin, and gray for
target revisions that do not match origin. The _request_ column shows the
highest open request for that package in the target project.
+
+#### Potentials and history
+
+The middle column shows the potential origins for the selected package along
with the version within that origin. The two icons are external diff (via OBS)
and submit to target from origin.
+
+The origin history shows source change requests for the selected package
against the target project. If an annotation is present the origin from the
annotation is extracted otherwise the source project is shown.
+
+#### Diff
+
+The last column shows a diff between the selected potential origin and the
target project. When a request from the history is also selected it shows the
diff between the potential origin and the request source.
+
+### Filtering and sorting
+
+One of the most useful features of the web interface is _filtering and
sorting_ which allows for a number different package states to be identified.
+
+The _origin_ column can be sorted to see packages grouped by origin or
filtered to show workarounds by entering `~`. Additionally, specific origins
can be filtered or even the `None` origin.
+
+The _revisions_ column can be sorted to show the most _behind_ (most red
revisions) packages first with the most _worked around_ (most gray revisions)
packages next. This is useful for seeing if updates are behind and determining
the reason (usually due to bot or human decline).
+
+The _request_ column can be sorted to place packages with requests at the top
of the list which is useful when doing lots of reviews against open requests
and thus flipping between requests and the origin manager web interface.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20191108.c7e1800d/osc-origin.py
new/openSUSE-release-tools-20191108.6b1fd618/osc-origin.py
--- old/openSUSE-release-tools-20191108.c7e1800d/osc-origin.py 2019-11-08
20:49:24.000000000 +0100
+++ new/openSUSE-release-tools-20191108.6b1fd618/osc-origin.py 2019-11-09
00:14:13.000000000 +0100
@@ -376,24 +376,7 @@
return
if len(packages) == 0:
- packages = set(package_list_kind_filtered(apiurl, opts.project))
-
- # Include packages from origins with initial update enabled to allow
for
- # potential new package submissions.
- for origin in origin_updatable_initial(apiurl, opts.project):
- for package in package_list(apiurl, origin):
- # Only add missing package if it does not exist in target
- # project. If it exists in target then it is not a source
- # package (since origin list is filtered to source) and should
- # not be updated. This also properly avoids submitting a
package
- # that is a subpackage in target, but is a source package in an
- # origin project.
- if package in packages or entity_exists(apiurl, opts.project,
package):
- continue
-
- # No sense submitting a non-source package (most expensive).
- if package_kind(apiurl, origin, package) == 'source':
- packages.add(package)
+ packages = osrt_origin_update_packages(apiurl, opts.project)
for package in packages:
print('checking for updates to {}/{}...'.format(opts.project, package))
@@ -401,3 +384,25 @@
request_future = origin_update(apiurl, opts.project, package)
if request_future:
request_future.print_and_create(opts.dry)
+
+def osrt_origin_update_packages(apiurl, project):
+ packages = set(package_list_kind_filtered(apiurl, project))
+
+ # Include packages from origins with initial update enabled to allow for
+ # potential new package submissions.
+ for origin in origin_updatable_initial(apiurl, project):
+ for package in package_list(apiurl, origin):
+ # Only add missing package if it does not exist in target
+ # project. If it exists in target then it is not a source
+ # package (since origin list is filtered to source) and should
+ # not be updated. This also properly avoids submitting a package
+ # that is a subpackage in target, but is a source package in an
+ # origin project.
+ if package in packages or entity_exists(apiurl, project, package):
+ continue
+
+ # No sense submitting a non-source package (most expensive).
+ if package_kind(apiurl, origin, package) == 'source':
+ packages.add(package)
+
+ return packages
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20191108.c7e1800d/osclib/conf.py
new/openSUSE-release-tools-20191108.6b1fd618/osclib/conf.py
--- old/openSUSE-release-tools-20191108.c7e1800d/osclib/conf.py 2019-11-08
20:49:24.000000000 +0100
+++ new/openSUSE-release-tools-20191108.6b1fd618/osclib/conf.py 2019-11-09
00:14:13.000000000 +0100
@@ -120,6 +120,8 @@
'repo_checker-arch-whitelist': 'x86_64',
'repo_checker-no-filter': 'True',
'repo_checker-package-comment-devel': 'True',
+ 'review-install-check': 'maintenance-installcheck',
+ 'review-openqa': 'qam-openqa',
},
r'openSUSE:(?P<project>Backports:(?P<version>[^:]+))$': {
'staging': 'openSUSE:%(project)s:Staging',
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20191108.c7e1800d/osclib/core.py
new/openSUSE-release-tools-20191108.6b1fd618/osclib/core.py
--- old/openSUSE-release-tools-20191108.c7e1800d/osclib/core.py 2019-11-08
20:49:24.000000000 +0100
+++ new/openSUSE-release-tools-20191108.6b1fd618/osclib/core.py 2019-11-09
00:14:13.000000000 +0100
@@ -759,12 +759,20 @@
return None
-def reviews_remaining(request):
+def reviews_remaining(request, incident_psuedo=False):
reviews = []
for review in request.reviews:
if review.state != 'accepted':
reviews.append(review_short(review))
+ if incident_psuedo:
+ # Add review in the same style as the staging review used for non
+ # maintenance projects to allow for the same wait on review.
+ for action in request.actions:
+ if action.type == 'maintenance_incident':
+ reviews.append('maintenance_incident')
+ break
+
return reviews
def review_short(review):
@@ -903,7 +911,15 @@
# Before being assigned to an incident.
xpath_project = xpath_join(xpath_project,
'action/target/@project="{}"'.format(
maintenance_project))
- xpath_project = xpath_join(xpath_project,
'action/source/@package="{}"'.format(package), op='and', inner=True)
+
+ xpath_project_package = ''
+ xpath_project_package = xpath_join(
+ xpath_project_package,
'action/source/@package="{}"'.format(package))
+ xpath_project_package = xpath_join(
+ xpath_project_package, 'action/source/@package="{}"'.format(
+ package_repository), op='or', inner=True)
+
+ xpath_project = xpath_join(xpath_project,
f'({xpath_project_package})', op='and', inner=True)
xpath = xpath_join(xpath, xpath_project, op='or',
nexpr_parentheses=True)
xpath_project = ''
@@ -933,7 +949,8 @@
for action in request.actions:
if action.type == 'maintenance_incident' and
action.tgt_releaseproject == project and (
- (action.tgt_package is None and action.src_package == package)
or
+ (action.tgt_package is None and
+ (action.src_package == package or action.src_package ==
package_repository)) or
(action.tgt_package == package_repository)):
yield request, action
break
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20191108.c7e1800d/osclib/origin.py
new/openSUSE-release-tools-20191108.6b1fd618/osclib/origin.py
--- old/openSUSE-release-tools-20191108.c7e1800d/osclib/origin.py
2019-11-08 20:49:24.000000000 +0100
+++ new/openSUSE-release-tools-20191108.6b1fd618/osclib/origin.py
2019-11-09 00:14:13.000000000 +0100
@@ -51,8 +51,16 @@
'pending_submission_allow': False,
'pending_submission_consider': False,
'pending_submission_allowed_reviews': [
+ # Non-maintenance projects:
'<config_source:staging>*',
- '<config_source:repo-checker>',
+ # Maintenance projects:
+ '<config_source:review-install-check>',
+ '<config_source:review-openqa>',
+ ],
+ # Submit pending requests with a set of allowed reviews, but still wait for
+ # the above reviews before being accepted.
+ 'pending_submission_allowed_reviews_update': [
+ '!maintenance_incident',
],
}
@@ -277,7 +285,7 @@
if source_hash_consider == source_hash:
return PendingRequestInfo(
request_remote_identifier(apiurl, apiurl_remote,
request.reqid),
- reviews_remaining(request))
+ reviews_remaining(request, True))
return False
@@ -319,7 +327,14 @@
return None
def origin_annotation_dump(origin_info_new, origin_info_old, override=False,
raw=False):
- data = {'origin': str(origin_info_new.project) if origin_info_new else
'None'}
+ data = {}
+ if origin_info_new is None:
+ data['origin'] = str(origin_info_new)
+ else:
+ data['origin'] = str(origin_info_new.project)
+ if origin_info_new.pending:
+ data['pending'] = origin_info_new.pending.identifier
+
if origin_info_old and origin_info_new.project != origin_info_old.project:
data['origin_old'] = str(origin_info_old.project)
@@ -403,9 +418,11 @@
def policy_get_preprocess(apiurl, origin, policy):
project = origin.rstrip('~')
config_project = Config.get(apiurl, project)
- policy['pending_submission_allowed_reviews'] = list(filter(None, [
- config_resolve_variable(v, config_project, 'config_source')
- for v in policy['pending_submission_allowed_reviews']]))
+ for suffix in ('', '_update'):
+ key = 'pending_submission_allowed_reviews{}'.format(suffix)
+ policy[key] = list(filter(None, [
+ config_resolve_variable(v, config_project, 'config_source')
+ for v in policy[key]]))
return policy
@@ -462,7 +479,7 @@
else:
inputs['direction'] = 'forward' if index_new < index_old
else 'backward'
except ValueError:
- inputs['direction'] = 'unkown'
+ inputs['direction'] = 'unknown'
return inputs
@@ -519,18 +536,32 @@
return result
def policy_input_evaluate_reviews_not_allowed(policy, inputs):
+ return
reviews_filter_allowed(inputs['pending_submission'].reviews_remaining,
+ policy['pending_submission_allowed_reviews'])
+
+def reviews_filter_allowed(reviews_remaining, allowed_reviews):
reviews_not_allowed = []
- for review_remaining in inputs['pending_submission'].reviews_remaining:
- allowed = False
- for review_allowed in policy['pending_submission_allowed_reviews']:
+ for review_remaining in reviews_remaining:
+ allowed = None
+ not_rule = False
+ for review_allowed in allowed_reviews:
+ result = True
+ if review_allowed.startswith('!'):
+ not_rule = True
+ review_allowed = review_allowed[1:]
+ result = False
+
if review_allowed.endswith('*') and
review_remaining.startswith(review_allowed[:-1]):
- allowed = True
+ allowed = result
break
if review_remaining == review_allowed:
- allowed = True
+ allowed = result
break
- if not allowed:
+ # If not the allowed case then add the review to not allowed list.
+ # Allowed if either matches an allow rule or does not match anything
+ # when at least one not rule is present.
+ if not (allowed or (not_rule and allowed is None)):
reviews_not_allowed.append(review_remaining)
return reviews_not_allowed
@@ -599,11 +630,10 @@
request_actions = request_action_list_source(apiurl, target_project,
package, states=['all'])
for request, action in sorted(request_actions, key=lambda i: i[0].reqid,
reverse=True):
annotation = origin_annotation_load(request, action, user)
- if not annotation:
- continue
-
history.append({
- 'origin': annotation.get('origin', 'None'),
+ # Not completely accurate, but more useful to have non-annotated
+ # entries than to exclude them.
+ 'origin': annotation['origin'] if annotation else
action.src_project,
'request': request.reqid,
'state': request.state.name,
'source_project': action.src_project,
@@ -655,6 +685,9 @@
return request_create_delete(apiurl, target_project, package,
message)
if not exists:
+ if origin_update_initial_blacklisted(apiurl, target_project,
package):
+ return False
+
message = 'Submitting new package from highest potential origin.'
return request_create_submit(apiurl, origin, package,
target_project, message=message,
ignore_if_any_request=True)
@@ -691,7 +724,7 @@
if policy['pending_submission_allow']:
request_id = origin_update_pending(
- apiurl, origin_info.project, package, target_project, supersede,
frequency)
+ apiurl, origin_info.project, package, target_project, policy,
supersede, frequency)
if request_id:
return request_id
@@ -699,11 +732,16 @@
return request_create_submit(apiurl, origin_info.project, package,
target_project, message=message,
supersede=supersede, frequency=frequency)
-def origin_update_pending(apiurl, origin_project, package, target_project,
supersede, frequency):
+def origin_update_pending(apiurl, origin_project, package, target_project,
policy, supersede, frequency):
apiurl_remote, project_remote = project_remote_apiurl(apiurl,
origin_project)
request_actions = request_action_list_source(
apiurl_remote, project_remote, package, include_release=True)
for request, action in sorted(request_actions, key=lambda i: i[0].reqid,
reverse=True):
+ reviews_not_allowed = reviews_filter_allowed(
+ reviews_remaining(request, True),
policy['pending_submission_allowed_reviews_update'])
+ if len(reviews_not_allowed):
+ continue
+
identifier = request_remote_identifier(apiurl, apiurl_remote,
request.reqid)
message = 'Newer pending source available from package origin. See
{}.'.format(identifier)
src_project = project_remote_prefixed(apiurl, apiurl_remote,
action.src_project)
@@ -737,6 +775,28 @@
return values
+def origin_update_initial_blacklisted(apiurl, target_project, package):
+ patterns = origin_update_initial_blacklisted_patterns(apiurl,
target_project)
+ for pattern in patterns:
+ if pattern.match(package):
+ return True
+
+ return False
+
+@memoize(session=True)
+def origin_update_initial_blacklisted_patterns(apiurl, target_project):
+ patterns = []
+
+ blacklist = attribute_value_load(apiurl, target_project,
'OriginUpdateInitialBlacklist')
+ if blacklist:
+ for entry in blacklist.strip().splitlines():
+ if not entry:
+ continue
+
+ patterns.append(re.compile(entry, re.IGNORECASE))
+
+ return patterns
+
@memoize(session=True)
def origin_updatable(apiurl):
""" List of origin managed projects that can be updated. """
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20191108.c7e1800d/tests/origin_tests.py
new/openSUSE-release-tools-20191108.6b1fd618/tests/origin_tests.py
--- old/openSUSE-release-tools-20191108.c7e1800d/tests/origin_tests.py
2019-11-08 20:49:24.000000000 +0100
+++ new/openSUSE-release-tools-20191108.6b1fd618/tests/origin_tests.py
2019-11-09 00:14:13.000000000 +0100
@@ -363,6 +363,7 @@
upstream3_project = self.randomString('upstream3')
package1 = self.randomString('package1')
package2 = self.randomString('package2')
+ package3 = self.randomString('package3')
target_package1 = self.wf.create_package(self.target_project, package1)
upstream1_package1 = self.wf.create_package(upstream1_project,
package1)
@@ -375,6 +376,9 @@
upstream3_package2 = self.wf.create_package(upstream3_project,
package2)
upstream3_package2.create_commit()
+ upstream1_package3 = self.wf.create_package(upstream1_project,
package3)
+ upstream1_package3.create_commit()
+
attribute_value_save(self.wf.apiurl, upstream1_project,
'ApprovedRequestSource', '', 'OBS')
attribute_value_save(self.wf.apiurl, upstream2_project,
'ApprovedRequestSource', '', 'OBS')
attribute_value_save(self.wf.apiurl, upstream3_project,
'ApprovedRequestSource', '', 'OBS')
@@ -425,6 +429,14 @@
self.assertEqual(request_future, False)
self.osc_user_pop()
+ # Ensure blacklist prevents initial package submission.
+ self.wf.create_attribute_type('OSRT', 'OriginUpdateInitialBlacklist',
1)
+ attribute_value_save(self.wf.apiurl, self.target_project,
'OriginUpdateInitialBlacklist', package3)
+ self.assertNoUpdate(package3)
+
+ attribute_value_delete(self.wf.apiurl, self.target_project,
'OriginUpdateInitialBlacklist')
+ self.assertUpdate(package3)
+
def test_automatic_update_modes(self):
self.remote_config_set_age_minimum()
++++++ openSUSE-release-tools.obsinfo ++++++
--- /var/tmp/diff_new_pack.IgRhNR/_old 2019-11-10 22:44:21.205629042 +0100
+++ /var/tmp/diff_new_pack.IgRhNR/_new 2019-11-10 22:44:21.205629042 +0100
@@ -1,5 +1,5 @@
name: openSUSE-release-tools
-version: 20191108.c7e1800d
-mtime: 1573242564
-commit: c7e1800d3009933937ae104dbe226c23c1543a69
+version: 20191108.6b1fd618
+mtime: 1573254853
+commit: 6b1fd6180011733457cc47bab1ff66e15e25b3fa
