Hello community, here is the log from the commit of package python-yq for openSUSE:Factory checked in at 2020-03-05 23:22:27 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-yq (Old) and /work/SRC/openSUSE:Factory/.python-yq.new.26092 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-yq" Thu Mar 5 23:22:27 2020 rev:6 rq:781772 version:2.10.0 Changes: -------- --- /work/SRC/openSUSE:Factory/python-yq/python-yq.changes 2019-10-31 18:18:21.662203418 +0100 +++ /work/SRC/openSUSE:Factory/.python-yq.new.26092/python-yq.changes 2020-03-05 23:22:34.541320600 +0100 @@ -1,0 +2,11 @@ +Thu Mar 5 08:32:23 UTC 2020 - Tomáš Chvátal <[email protected]> + +- Update to 2.10.0: + * Add support for in-place editing (yq -yi) + * Add argcomplete integration + * Docs: Migrate from RTD to gh-pages + * Add -Y/--yaml-roundtrip for preserving YAML styles and tags +- Add patch to work with new PyYAML: + * pyaml53.patch + +------------------------------------------------------------------- Old: ---- yq-2.8.1.tar.gz New: ---- pyaml53.patch yq-2.10.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-yq.spec ++++++ --- /var/tmp/diff_new_pack.TBMOKj/_old 2020-03-05 23:22:38.313322673 +0100 +++ /var/tmp/diff_new_pack.TBMOKj/_new 2020-03-05 23:22:38.317322676 +0100 @@ -1,7 +1,7 @@ # # spec file for package python-yq # -# Copyright (c) 2019 SUSE LINUX GmbH, Nuernberg, Germany. +# Copyright (c) 2020 SUSE LLC # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -18,17 +18,19 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-yq -Version: 2.8.1 +Version: 2.10.0 Release: 0 Summary: Command-line YAML processor - jq wrapper for YAML documents License: Apache-2.0 URL: https://github.com/kislyuk/yq Source: https://files.pythonhosted.org/packages/source/y/yq/yq-%{version}.tar.gz +Patch0: pyaml53.patch BuildRequires: %{python_module setuptools} BuildRequires: fdupes BuildRequires: python-rpm-macros Requires: jq Requires: python-PyYAML >= 3.11 +Requires: python-argcomplete >= 1.8.1 Requires: python-setuptools Requires: python-toml >= 0.9.4 Requires: python-xmltodict >= 0.11.0 @@ -37,6 +39,7 @@ BuildArch: noarch # SECTION test requirements BuildRequires: %{python_module PyYAML >= 3.11} +BuildRequires: %{python_module argcomplete >= 1.8.1} BuildRequires: %{python_module toml >= 0.9.4} BuildRequires: %{python_module xmltodict >= 0.11.0} BuildRequires: jq @@ -48,6 +51,7 @@ %prep %setup -q -n yq-%{version} +%patch0 -p1 %build %python_build ++++++ pyaml53.patch ++++++ >From 5754ef46981d688dd249f96fc55ceaf4cd51f130 Mon Sep 17 00:00:00 2001 From: "Daniel M. Capella" <[email protected]> Date: Sat, 29 Feb 2020 16:10:50 -0500 Subject: [PATCH] Fix test_datetimes for PyYAML >=5.3 https://github.com/yaml/pyyaml/pull/163 --- test/test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/test.py b/test/test.py index 12fa806..8897196 100755 --- a/test/test.py +++ b/test/test.py @@ -3,7 +3,7 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import os, sys, unittest, tempfile, json, io, platform, subprocess +import os, sys, unittest, tempfile, json, io, platform, subprocess, yaml sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from yq import yq, cli # noqa @@ -116,7 +116,10 @@ def test_multidocs(self): def test_datetimes(self): self.assertEqual(self.run_yq("- 2016-12-20T22:07:36Z\n", ["."]), "") - self.assertEqual(self.run_yq("- 2016-12-20T22:07:36Z\n", ["-y", "."]), "- '2016-12-20T22:07:36'\n") + if yaml.__version__ < '5.3': + self.assertEqual(self.run_yq("- 2016-12-20T22:07:36Z\n", ["-y", "."]), "- '2016-12-20T22:07:36'\n") + else: + self.assertEqual(self.run_yq("- 2016-12-20T22:07:36Z\n", ["-y", "."]), "- '2016-12-20T22:07:36+00:00'\n") self.assertEqual(self.run_yq("2016-12-20", ["."]), "") self.assertEqual(self.run_yq("2016-12-20", ["-y", "."]), "'2016-12-20'\n") ++++++ yq-2.8.1.tar.gz -> yq-2.10.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yq-2.8.1/PKG-INFO new/yq-2.10.0/PKG-INFO --- old/yq-2.8.1/PKG-INFO 2019-10-28 17:16:45.000000000 +0100 +++ new/yq-2.10.0/PKG-INFO 2019-12-23 17:52:29.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: yq -Version: 2.8.1 +Version: 2.10.0 Summary: Command-line YAML/XML processor - jq wrapper for YAML/XML documents Home-page: https://github.com/kislyuk/yq Author: Andrey Kislyuk @@ -31,20 +31,81 @@ yq .foo.bar input.yml - By default, no conversion of ``jq`` output is done. Use the ``--yaml-output``/``-y`` argument to convert it back into YAML:: + By default, no conversion of ``jq`` output is done. Use the ``--yaml-output``/``-y`` option to convert it back into YAML:: cat input.yml | yq -y .foo.bar - Use the ``--width``/``-w`` argument to pass the line wrap width for string literals. All other command line arguments are - forwarded to ``jq``. ``yq`` forwards the exit code ``jq`` produced, unless there was an error in YAML parsing, in which case - the exit code is 1. See the `jq manual <https://stedolan.github.io/jq/manual/>`_ for more details on ``jq`` features and - options. - - YAML `tags <http://www.yaml.org/spec/1.2/spec.html#id2764295>`_ in the input are ignored (any nested data is treated as - untagged). Key order is preserved. + Mapping key order is preserved. By default, custom `YAML tags <http://www.yaml.org/spec/1.2/spec.html#id2764295>`_ and + `styles <https://yaml.org/spec/current.html#id2509255>`_ in the input are ignored. Use the ``--yaml-roundtrip``/``-Y`` + option to preserve YAML tags and styles by representing them as extra items in their enclosing mappings and sequences + while in JSON:: + + yq -Y .foo.bar input.yml + + Use the ``--width``/``-w`` option to pass the line wrap width for string literals. With ``-y/-Y``, files can be edited + in place like with ``sed -i``: ``yq -yi .foo=1 *.yml``. All other command line arguments are forwarded to ``jq``. ``yq`` + forwards the exit code ``jq`` produced, unless there was an error in YAML parsing, in which case the exit code is 1. + See the `jq manual <https://stedolan.github.io/jq/manual/>`_ for more details on ``jq`` features and options. Because YAML treats JSON as a dialect of YAML, you can use yq to convert JSON to YAML: ``yq -y . < in.json > out.yml``. + Preserving tags and styles using the ``-Y`` (``--yaml-roundtrip``) option + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + The ``-Y`` option helps preserve custom `string styles <https://yaml-multiline.info/>`_ and + `tags <https://camel.readthedocs.io/en/latest/yamlref.html#tags>`_ in your document. For exmaple, consider the following + document (an `AWS CloudFormation <https://aws.amazon.com/cloudformation/>`_ template fragment):: + + Resources: + ElasticLoadBalancer: + Type: 'AWS::ElasticLoadBalancing::LoadBalancer' + Properties: + AvailabilityZones: !GetAZs '' + Instances: + - !Ref Ec2Instance1 + - !Ref Ec2Instance2 + Description: >- + Load balancer for Big Important Service. + + Good thing it's managed by this template. + + Passing this document through ``yq -y .Resources.ElasticLoadBalancer`` will drop custom tags, such as ``!Ref``, + and styles, such as the `folded <https://yaml-multiline.info/>`_ style of the ``Description`` field:: + + Type: AWS::ElasticLoadBalancing::LoadBalancer + Properties: + AvailabilityZones: '' + Instances: + - Ec2Instance1 + - Ec2Instance2 + Description: 'Load balancer for Big Important Service. + + Good thing it''s managed by this template.' + + By contrast, passing it through ``yq -Y .Resources.ElasticLoadBalancer`` will preserve tags and styles:: + + Type: 'AWS::ElasticLoadBalancing::LoadBalancer' + Properties: + AvailabilityZones: !GetAZs '' + Instances: + - !Ref 'Ec2Instance1' + - !Ref 'Ec2Instance2' + Description: >- + Load balancer for Big Important Service. + + Good thing it's managed by this template. + + To accomplish this in ``-Y`` mode, yq carries extra metadata (mapping pairs and sequence values) in the JSON + representation of your document for any custom tags or styles that it finds. When converting the JSON back into YAML, it + parses this metadata, re-applies the tags and styles, and discards the extra pairs and values. + + .. warning :: + + The ``-Y`` option is incompatible with jq filters that do not expect the extra information injected into the document + to preserve the YAML formatting. For example, a jq filter that counts entries in the Instances array will come up with + 4 entries instead of 2. A filter that expects all array entries to be mappings may break due to the presence of string + metadata keys. Check your jq filter for compatibility/semantic validity when using the ``-Y`` option. + XML support ----------- ``yq`` also supports XML. The ``yq`` package installs an executable, ``xq``, which @@ -67,7 +128,7 @@ Links ----- * `Project home page (GitHub) <https://github.com/kislyuk/yq>`_ - * `Documentation (Read the Docs) <https://yq.readthedocs.io/en/latest/>`_ + * `Documentation <https://kislyuk.github.io/yq/>`_ * `Package distribution (PyPI) <https://pypi.python.org/pypi/yq>`_ * `Change log <https://github.com/kislyuk/yq/blob/master/Changes.rst>`_ * `jq <https://stedolan.github.io/jq/>`_ - the command-line JSON processor utility powering ``yq`` @@ -80,16 +141,14 @@ ------- Licensed under the terms of the `Apache License, Version 2.0 <http://www.apache.org/licenses/LICENSE-2.0>`_. - .. image:: https://img.shields.io/travis/kislyuk/yq.svg - :target: https://travis-ci.org/kislyuk/yq + .. image:: https://github.com/kislyuk/yq/workflows/Python%20package/badge.svg + :target: https://github.com/kislyuk/yq/actions .. image:: https://codecov.io/github/kislyuk/yq/coverage.svg?branch=master :target: https://codecov.io/github/kislyuk/yq?branch=master .. image:: https://img.shields.io/pypi/v/yq.svg :target: https://pypi.python.org/pypi/yq .. image:: https://img.shields.io/pypi/l/yq.svg :target: https://pypi.python.org/pypi/yq - .. image:: https://readthedocs.org/projects/yq/badge/?version=latest - :target: https://yq.readthedocs.io/ Platform: UNKNOWN Classifier: Intended Audience :: Developers @@ -98,10 +157,10 @@ Classifier: Operating System :: POSIX Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3.3 -Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 Classifier: Topic :: Software Development :: Libraries :: Python Modules Provides-Extra: test Provides-Extra: toml diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yq-2.8.1/README.rst new/yq-2.10.0/README.rst --- old/yq-2.8.1/README.rst 2019-10-16 18:29:39.000000000 +0200 +++ new/yq-2.10.0/README.rst 2019-12-23 17:43:58.000000000 +0100 @@ -23,20 +23,81 @@ yq .foo.bar input.yml -By default, no conversion of ``jq`` output is done. Use the ``--yaml-output``/``-y`` argument to convert it back into YAML:: +By default, no conversion of ``jq`` output is done. Use the ``--yaml-output``/``-y`` option to convert it back into YAML:: cat input.yml | yq -y .foo.bar -Use the ``--width``/``-w`` argument to pass the line wrap width for string literals. All other command line arguments are -forwarded to ``jq``. ``yq`` forwards the exit code ``jq`` produced, unless there was an error in YAML parsing, in which case -the exit code is 1. See the `jq manual <https://stedolan.github.io/jq/manual/>`_ for more details on ``jq`` features and -options. - -YAML `tags <http://www.yaml.org/spec/1.2/spec.html#id2764295>`_ in the input are ignored (any nested data is treated as -untagged). Key order is preserved. +Mapping key order is preserved. By default, custom `YAML tags <http://www.yaml.org/spec/1.2/spec.html#id2764295>`_ and +`styles <https://yaml.org/spec/current.html#id2509255>`_ in the input are ignored. Use the ``--yaml-roundtrip``/``-Y`` +option to preserve YAML tags and styles by representing them as extra items in their enclosing mappings and sequences +while in JSON:: + + yq -Y .foo.bar input.yml + +Use the ``--width``/``-w`` option to pass the line wrap width for string literals. With ``-y/-Y``, files can be edited +in place like with ``sed -i``: ``yq -yi .foo=1 *.yml``. All other command line arguments are forwarded to ``jq``. ``yq`` +forwards the exit code ``jq`` produced, unless there was an error in YAML parsing, in which case the exit code is 1. +See the `jq manual <https://stedolan.github.io/jq/manual/>`_ for more details on ``jq`` features and options. Because YAML treats JSON as a dialect of YAML, you can use yq to convert JSON to YAML: ``yq -y . < in.json > out.yml``. +Preserving tags and styles using the ``-Y`` (``--yaml-roundtrip``) option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``-Y`` option helps preserve custom `string styles <https://yaml-multiline.info/>`_ and +`tags <https://camel.readthedocs.io/en/latest/yamlref.html#tags>`_ in your document. For exmaple, consider the following +document (an `AWS CloudFormation <https://aws.amazon.com/cloudformation/>`_ template fragment):: + + Resources: + ElasticLoadBalancer: + Type: 'AWS::ElasticLoadBalancing::LoadBalancer' + Properties: + AvailabilityZones: !GetAZs '' + Instances: + - !Ref Ec2Instance1 + - !Ref Ec2Instance2 + Description: >- + Load balancer for Big Important Service. + + Good thing it's managed by this template. + +Passing this document through ``yq -y .Resources.ElasticLoadBalancer`` will drop custom tags, such as ``!Ref``, +and styles, such as the `folded <https://yaml-multiline.info/>`_ style of the ``Description`` field:: + + Type: AWS::ElasticLoadBalancing::LoadBalancer + Properties: + AvailabilityZones: '' + Instances: + - Ec2Instance1 + - Ec2Instance2 + Description: 'Load balancer for Big Important Service. + + Good thing it''s managed by this template.' + +By contrast, passing it through ``yq -Y .Resources.ElasticLoadBalancer`` will preserve tags and styles:: + + Type: 'AWS::ElasticLoadBalancing::LoadBalancer' + Properties: + AvailabilityZones: !GetAZs '' + Instances: + - !Ref 'Ec2Instance1' + - !Ref 'Ec2Instance2' + Description: >- + Load balancer for Big Important Service. + + Good thing it's managed by this template. + +To accomplish this in ``-Y`` mode, yq carries extra metadata (mapping pairs and sequence values) in the JSON +representation of your document for any custom tags or styles that it finds. When converting the JSON back into YAML, it +parses this metadata, re-applies the tags and styles, and discards the extra pairs and values. + +.. warning :: + + The ``-Y`` option is incompatible with jq filters that do not expect the extra information injected into the document + to preserve the YAML formatting. For example, a jq filter that counts entries in the Instances array will come up with + 4 entries instead of 2. A filter that expects all array entries to be mappings may break due to the presence of string + metadata keys. Check your jq filter for compatibility/semantic validity when using the ``-Y`` option. + XML support ----------- ``yq`` also supports XML. The ``yq`` package installs an executable, ``xq``, which @@ -59,7 +120,7 @@ Links ----- * `Project home page (GitHub) <https://github.com/kislyuk/yq>`_ -* `Documentation (Read the Docs) <https://yq.readthedocs.io/en/latest/>`_ +* `Documentation <https://kislyuk.github.io/yq/>`_ * `Package distribution (PyPI) <https://pypi.python.org/pypi/yq>`_ * `Change log <https://github.com/kislyuk/yq/blob/master/Changes.rst>`_ * `jq <https://stedolan.github.io/jq/>`_ - the command-line JSON processor utility powering ``yq`` @@ -72,13 +133,11 @@ ------- Licensed under the terms of the `Apache License, Version 2.0 <http://www.apache.org/licenses/LICENSE-2.0>`_. -.. image:: https://img.shields.io/travis/kislyuk/yq.svg - :target: https://travis-ci.org/kislyuk/yq +.. image:: https://github.com/kislyuk/yq/workflows/Python%20package/badge.svg + :target: https://github.com/kislyuk/yq/actions .. image:: https://codecov.io/github/kislyuk/yq/coverage.svg?branch=master :target: https://codecov.io/github/kislyuk/yq?branch=master .. image:: https://img.shields.io/pypi/v/yq.svg :target: https://pypi.python.org/pypi/yq .. image:: https://img.shields.io/pypi/l/yq.svg :target: https://pypi.python.org/pypi/yq -.. image:: https://readthedocs.org/projects/yq/badge/?version=latest - :target: https://yq.readthedocs.io/ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yq-2.8.1/setup.py new/yq-2.10.0/setup.py --- old/yq-2.8.1/setup.py 2019-10-28 17:16:14.000000000 +0100 +++ new/yq-2.10.0/setup.py 2019-12-23 17:47:08.000000000 +0100 @@ -6,7 +6,7 @@ setup( name="yq", - version="2.8.1", + version="2.10.0", url="https://github.com/kislyuk/yq";, license="Apache Software License", author="Andrey Kislyuk", @@ -16,7 +16,8 @@ install_requires=[ "setuptools", "PyYAML >= 3.11", - "xmltodict >= 0.11.0" + "xmltodict >= 0.11.0", + "argcomplete >= 1.8.1" ], tests_require=tests_require, extras_require={ @@ -39,10 +40,10 @@ "Operating System :: POSIX", "Programming Language :: Python", "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3.3", - "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Topic :: Software Development :: Libraries :: Python Modules" ] ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yq-2.8.1/test/bomb.yml new/yq-2.10.0/test/bomb.yml --- old/yq-2.8.1/test/bomb.yml 1970-01-01 01:00:00.000000000 +0100 +++ new/yq-2.10.0/test/bomb.yml 2019-11-30 16:47:11.000000000 +0100 @@ -0,0 +1,9 @@ +a: &a ["lol","lol","lol","lol","lol","lol","lol","lol","lol"] +b: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a] +c: &c [*b,*b,*b,*b,*b,*b,*b,*b,*b] +d: &d [*c,*c,*c,*c,*c,*c,*c,*c,*c] +e: &e [*d,*d,*d,*d,*d,*d,*d,*d,*d] +f: &f [*e,*e,*e,*e,*e,*e,*e,*e,*e] +g: &g [*f,*f,*f,*f,*f,*f,*f,*f,*f] +h: &h [*g,*g,*g,*g,*g,*g,*g,*g,*g] +i: &i [*h,*h,*h,*h,*h,*h,*h,*h,*h] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yq-2.8.1/test/cfn.yml new/yq-2.10.0/test/cfn.yml --- old/yq-2.8.1/test/cfn.yml 1970-01-01 01:00:00.000000000 +0100 +++ new/yq-2.10.0/test/cfn.yml 2019-11-06 23:54:40.000000000 +0100 @@ -0,0 +1,92 @@ +AWSTemplateFormatVersion: '2010-09-09' +Metadata: + License: Apache-2.0 +Description: 'AWS CloudFormation Sample Template S3_Website_With_CloudFront_Distribution: + Sample template showing how to create a website with a custom DNS name, hosted on + Amazon S3 and served via Amazone CloudFront. It assumes you already have a Hosted + Zone registered with Amazon Route 53. **WARNING** This template creates an Amazon + Route 53 DNS record, an S3 bucket and a CloudFront distribution. You will be billed + for the AWS resources used if you create a stack from this template.' +Parameters: + HostedZone: + Type: String + Description: The DNS name of an existing Amazon Route 53 hosted zone + AllowedPattern: (?!-)[a-zA-Z0-9-.]{1,63}(?<!-) + ConstraintDescription: must be a valid DNS zone name. +Mappings: + Region2S3WebsiteSuffix: + us-east-1: + Suffix: .s3-website-us-east-1.amazonaws.com + us-west-1: + Suffix: .s3-website-us-west-1.amazonaws.com + us-west-2: + Suffix: .s3-website-us-west-2.amazonaws.com + eu-west-1: + Suffix: .s3-website-eu-west-1.amazonaws.com + ap-northeast-1: + Suffix: .s3-website-ap-northeast-1.amazonaws.com + ap-northeast-2: + Suffix: .s3-website-ap-northeast-2.amazonaws.com + ap-southeast-1: + Suffix: .s3-website-ap-southeast-1.amazonaws.com + ap-southeast-2: + Suffix: .s3-website-ap-southeast-2.amazonaws.com + ap-south-1: + Suffix: .s3-website-ap-south-1.amazonaws.com + us-east-2: + Suffix: .s3-website-us-east-2.amazonaws.com + sa-east-1: + Suffix: .s3-website-sa-east-1.amazonaws.com + cn-north-1: + Suffix: .s3-website.cn-north-1.amazonaws.com.cn + eu-central-1: + Suffix: .s3-website.eu-central-1.amazonaws.com +Resources: + S3BucketForWebsiteContent: + Type: AWS::S3::Bucket + Properties: + AccessControl: PublicRead + WebsiteConfiguration: + IndexDocument: index.html + ErrorDocument: error.html + WebsiteCDN: + Type: AWS::CloudFront::Distribution + Properties: + DistributionConfig: + Comment: CDN for S3-backed website + Aliases: + - !Join ['', [!Ref 'AWS::StackName', !Ref 'AWS::AccountId', ., !Ref 'AWS::Region', + ., !Ref 'HostedZone']] + Enabled: 'true' + DefaultCacheBehavior: + ForwardedValues: + QueryString: 'true' + TargetOriginId: only-origin + ViewerProtocolPolicy: allow-all + DefaultRootObject: index.html + Origins: + - CustomOriginConfig: + HTTPPort: '80' + HTTPSPort: '443' + OriginProtocolPolicy: http-only + DomainName: !Join ['', [!Ref 'S3BucketForWebsiteContent', !FindInMap [ + Region2S3WebsiteSuffix, !Ref 'AWS::Region', Suffix]]] + Id: only-origin + WebsiteDNSName: + Type: AWS::Route53::RecordSet + Properties: + HostedZoneName: !Join ['', [!Ref 'HostedZone', .]] + Comment: CNAME redirect custom name to CloudFront distribution + Name: !Join ['', [!Ref 'AWS::StackName', !Ref 'AWS::AccountId', ., !Ref 'AWS::Region', + ., !Ref 'HostedZone']] + Type: CNAME + TTL: '900' + ResourceRecords: + - !GetAtt [WebsiteCDN, DomainName] +Outputs: + WebsiteURL: + Value: !Join ['', ['http://', !Ref 'WebsiteDNSName']] + Description: The URL of the newly created website + BucketName: + Value: !Ref 'S3BucketForWebsiteContent' + Description: Name of S3 bucket to hold website content diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yq-2.8.1/test/test.py new/yq-2.10.0/test/test.py --- old/yq-2.8.1/test/test.py 2019-10-28 16:41:04.000000000 +0100 +++ new/yq-2.10.0/test/test.py 2019-12-23 17:38:41.000000000 +0100 @@ -131,6 +131,29 @@ tf.seek(0) self.assertEqual(self.run_yq("", ["-y", ".xyz.foo", self.fd_path(tf)]), 'bar\n...\n') + def test_roundtrip_yaml(self): + cfn_filename = os.path.join(os.path.dirname(__file__), "cfn.yml") + with io.open(cfn_filename) as fh: + self.assertEqual(self.run_yq("", ["-Y", ".", cfn_filename]), fh.read()) + + @unittest.skipIf(sys.version_info < (3, 5), "Skipping feature incompatible with Python 2") + def test_in_place(self): + with tempfile.NamedTemporaryFile() as tf, tempfile.NamedTemporaryFile() as tf2: + tf.write(b"- foo\n- bar\n") + tf.seek(0) + tf2.write(b"- foo\n- bar\n") + tf2.seek(0) + self.run_yq("", ["-i", "-y", ".[0]", tf.name, tf2.name]) + self.assertEqual(tf.read(), b'foo\n...\n') + self.assertEqual(tf2.read(), b'foo\n...\n') + + # Files do not get overwritten on error (DeferredOutputStream logic) + self.run_yq("", ["-i", "-y", tf.name, tf2.name], expect_exit_codes=[3]) + tf.seek(0) + tf2.seek(0) + self.assertEqual(tf.read(), b'foo\n...\n') + self.assertEqual(tf2.read(), b'foo\n...\n') + @unittest.expectedFailure def test_times(self): """ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yq-2.8.1/yq/__init__.py new/yq-2.10.0/yq/__init__.py --- old/yq-2.8.1/yq/__init__.py 2019-10-28 16:32:15.000000000 +0100 +++ new/yq-2.10.0/yq/__init__.py 2019-12-23 17:39:16.000000000 +0100 @@ -5,34 +5,21 @@ See https://github.com/kislyuk/yq for more information. """ +# PYTHON_ARGCOMPLETE_OK + from __future__ import absolute_import, division, print_function, unicode_literals import sys, argparse, subprocess, json from collections import OrderedDict from datetime import datetime, date, time -import yaml - -from .version import __version__ +import yaml, argcomplete -class Parser(argparse.ArgumentParser): - def print_help(self): - yq_help = argparse.ArgumentParser.format_help(self).splitlines() - print("\n".join(["usage: yq [options] <jq filter> [YAML file...]"] + yq_help[1:] + [""])) - try: - subprocess.check_call(["jq", "--help"]) - except Exception: - pass - -class OrderedLoader(yaml.SafeLoader): - pass - -class OrderedIndentlessDumper(yaml.SafeDumper): - pass - -class OrderedDumper(yaml.SafeDumper): - def increase_indent(self, flow=False, indentless=False): - return super(OrderedDumper, self).increase_indent(flow, False) +from .compat import USING_PYTHON2, open +from .parser import get_parser, jq_arg_spec +from .loader import get_loader +from .dumper import get_dumper +from .version import __version__ # noqa class JSONDateTimeEncoder(json.JSONEncoder): def default(self, o): @@ -40,104 +27,57 @@ return o.isoformat() return json.JSONEncoder.default(self, o) -def construct_mapping(loader, node): - loader.flatten_mapping(node) - return OrderedDict(loader.construct_pairs(node)) - -def represent_dict_order(dumper, data): - return dumper.represent_mapping("tag:yaml.org,2002:map", data.items()) - def decode_docs(jq_output, json_decoder): while jq_output: doc, pos = json_decoder.raw_decode(jq_output) jq_output = jq_output[pos + 1:] yield doc -def parse_unknown_tags(loader, tag_suffix, node): - if isinstance(node, yaml.nodes.ScalarNode): - return loader.construct_scalar(node) - elif isinstance(node, yaml.nodes.SequenceNode): - return loader.construct_sequence(node) - elif isinstance(node, yaml.nodes.MappingNode): - return construct_mapping(loader, node) - -OrderedLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping) -OrderedLoader.add_multi_constructor('', parse_unknown_tags) - -for dumper in OrderedIndentlessDumper, OrderedDumper: - dumper.add_representer(OrderedDict, represent_dict_order) - -# jq arguments that consume positionals must be listed here to avoid our parser mistaking them for our positionals -jq_arg_spec = {"--indent": 1, "-f": 1, "--from-file": 1, "-L": 1, "--arg": 2, "--argjson": 2, "--slurpfile": 2, - "--argfile": 2, "--rawfile": 2, "--args": argparse.REMAINDER, "--jsonargs": argparse.REMAINDER} - -# Detection for Python 2 -USING_PYTHON2 = True if sys.version_info < (3, 0) else False - -def get_parser(program_name): - # By default suppress these help strings and only enable them in the specific programs. - yaml_output_help, width_help, indentless_help = argparse.SUPPRESS, argparse.SUPPRESS, argparse.SUPPRESS - xml_output_help, xml_dtd_help, xml_root_help = argparse.SUPPRESS, argparse.SUPPRESS, argparse.SUPPRESS - toml_output_help = argparse.SUPPRESS - - if program_name == "yq": - current_language = "YAML" - yaml_output_help = "Transcode jq JSON output back into YAML and emit it" - width_help = "When using --yaml-output, specify string wrap width" - indentless_help = 'When using --yaml-output, indent block style lists (sequences) with 0 spaces instead of 2' - elif program_name == "xq": - current_language = "XML" - xml_output_help = "Transcode jq JSON output back into XML and emit it" - xml_dtd_help = "Preserve XML Document Type Definition (disables streaming of multiple docs)" - xml_root_help = "When transcoding back to XML, envelope the output in an element with this name" - elif program_name == "tq": - current_language = "TOML" - toml_output_help = "Transcode jq JSON output back into TOML and emit it" - else: - raise Exception("Unknown program name") - - description = __doc__.replace("yq", program_name).replace("YAML", current_language) - parser_args = dict(prog=program_name, description=description, formatter_class=argparse.RawTextHelpFormatter) - if sys.version_info >= (3, 5): - parser_args.update(allow_abbrev=False) # required to disambiguate options listed in jq_arg_spec - parser = Parser(**parser_args) - parser.add_argument("--output-format", default="json", help=argparse.SUPPRESS) - parser.add_argument("--yaml-output", "--yml-output", "-y", dest="output_format", action="store_const", const="yaml", - help=yaml_output_help) - parser.add_argument("--width", "-w", type=int, help=width_help) - parser.add_argument("--indentless-lists", "--indentless", action="store_true", help=indentless_help) - parser.add_argument("--xml-output", "-x", dest="output_format", action="store_const", const="xml", - help=xml_output_help) - parser.add_argument("--xml-dtd", action="store_true", help=xml_dtd_help) - parser.add_argument("--xml-root", help=xml_root_help) - parser.add_argument("--toml-output", "-t", dest="output_format", action="store_const", const="toml", - help=toml_output_help) - parser.add_argument("--version", action="version", version="%(prog)s {version}".format(version=__version__)) - - for arg in jq_arg_spec: - parser.add_argument(arg, nargs=jq_arg_spec[arg], dest=arg, action="append", help=argparse.SUPPRESS) - - parser.add_argument("jq_filter") - parser.add_argument("input_streams", nargs="*", type=argparse.FileType(), metavar="files", default=[sys.stdin]) - return parser - def xq_cli(): cli(input_format="xml", program_name="xq") def tq_cli(): cli(input_format="toml", program_name="tq") +class DeferredOutputStream: + def __init__(self, name, mode="w"): + self.name = name + self.mode = mode + self._fh = None + + @property + def fh(self): + if self._fh is None: + self._fh = open(self.name, self.mode) + return self._fh + + def flush(self): + if self._fh is not None: + return self.fh.flush() + + def close(self): + if self._fh is not None: + return self.fh.close() + + def __getattr__(self, a): + return getattr(self.fh, a) + def cli(args=None, input_format="yaml", program_name="yq"): - parser = get_parser(program_name) + parser = get_parser(program_name, __doc__) + argcomplete.autocomplete(parser) args, jq_args = parser.parse_known_args(args=args) for i, arg in enumerate(jq_args): if arg.startswith("-") and not arg.startswith("--"): + if "i" in arg: + args.in_place = True if "y" in arg: args.output_format = "yaml" + elif "Y" in arg: + args.output_format = "annotated_yaml" elif "x" in arg: args.output_format = "xml" - jq_args[i] = arg.replace("x", "").replace("y", "") + jq_args[i] = arg.replace("i", "").replace("x", "").replace("y", "").replace("Y", "") if args.output_format != "json": jq_args[i] = jq_args[i].replace("C", "") if jq_args[i] == "-": @@ -163,15 +103,35 @@ jq_filter_arg_loc = jq_args.index('--jsonargs') + 1 jq_args.insert(jq_filter_arg_loc, args.jq_filter) delattr(args, "jq_filter") + in_place = args.in_place + delattr(args, "in_place") if sys.stdin.isatty() and not args.input_streams: return parser.print_help() - yq(input_format=input_format, program_name=program_name, jq_args=jq_args, **vars(args)) + yq_args = dict(input_format=input_format, program_name=program_name, jq_args=jq_args, **vars(args)) + if in_place: + if USING_PYTHON2: + sys.exit("{}: -i/--in-place is not compatible with Python 2".format(program_name)) + if args.output_format not in {"yaml", "annotated_yaml"}: + sys.exit("{}: -i/--in-place can only be used with -y/-Y".format(program_name)) + input_streams = yq_args.pop("input_streams") + if len(input_streams) == 1 and input_streams[0].name == "<stdin>": + msg = "{}: -i/--in-place can only be used with filename arguments, not on standard input" + sys.exit(msg.format(program_name)) + for i, input_stream in enumerate(input_streams): + def exit_handler(arg=None): + if arg: + sys.exit(arg) + if i < len(input_streams): + yq_args["exit_func"] = exit_handler + yq(input_streams=[input_stream], output_stream=DeferredOutputStream(input_stream.name), **yq_args) + else: + yq(**yq_args) def yq(input_streams=None, output_stream=None, input_format="yaml", output_format="json", - program_name="yq", width=None, indentless_lists=False, xml_root=None, xml_dtd=False, jq_args=frozenset(), - exit_func=None): + program_name="yq", width=None, indentless_lists=False, xml_root=None, xml_dtd=False, + jq_args=frozenset(), exit_func=None): if not input_streams: input_streams = [sys.stdin] if not output_stream: @@ -195,10 +155,12 @@ # TODO: enable true streaming in this branch (with asyncio, asyncproc, a multi-shot variant of # subprocess.Popen._communicate, etc.) # See https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python + use_annotations = True if output_format == "annotated_yaml" else False input_docs = [] for input_stream in input_streams: if input_format == "yaml": - input_docs.extend(yaml.load_all(input_stream, Loader=OrderedLoader)) + loader = get_loader(use_annotations=use_annotations) + input_docs.extend(yaml.load_all(input_stream, Loader=loader)) elif input_format == "xml": import xmltodict input_docs.append(xmltodict.parse(input_stream.read(), disable_entities=True)) @@ -210,9 +172,9 @@ input_payload = "\n".join(json.dumps(doc, cls=JSONDateTimeEncoder) for doc in input_docs) jq_out, jq_err = jq.communicate(input_payload) json_decoder = json.JSONDecoder(object_pairs_hook=OrderedDict) - if output_format == "yaml": - dumper_class = OrderedIndentlessDumper if indentless_lists else OrderedDumper - yaml.dump_all(decode_docs(jq_out, json_decoder), stream=output_stream, Dumper=dumper_class, + if output_format == "yaml" or output_format == "annotated_yaml": + yaml.dump_all(decode_docs(jq_out, json_decoder), stream=output_stream, + Dumper=get_dumper(use_annotations=use_annotations, indentless=indentless_lists), width=width, allow_unicode=True, default_flow_style=False) elif output_format == "xml": import xmltodict @@ -249,8 +211,9 @@ toml.dump(doc, output_stream) else: if input_format == "yaml": + loader = get_loader(use_annotations=False) for input_stream in input_streams: - for doc in yaml.load_all(input_stream, Loader=OrderedLoader): + for doc in yaml.load_all(input_stream, Loader=loader): json.dump(doc, jq.stdin, cls=JSONDateTimeEncoder) jq.stdin.write("\n") elif input_format == "xml": diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yq-2.8.1/yq/compat.py new/yq-2.10.0/yq/compat.py --- old/yq-2.8.1/yq/compat.py 1970-01-01 01:00:00.000000000 +0100 +++ new/yq-2.10.0/yq/compat.py 2019-12-23 17:34:12.000000000 +0100 @@ -0,0 +1,12 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + +import sys, io + +USING_PYTHON2 = True if sys.version_info < (3, 0) else False + +if USING_PYTHON2: + str = unicode # noqa + open = io.open +else: + str = str + open = open diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yq-2.8.1/yq/dumper.py new/yq-2.10.0/yq/dumper.py --- old/yq-2.8.1/yq/dumper.py 1970-01-01 01:00:00.000000000 +0100 +++ new/yq-2.10.0/yq/dumper.py 2019-11-04 19:36:00.000000000 +0100 @@ -0,0 +1,74 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + +import re +from collections import OrderedDict + +import yaml + +from .compat import str +from .loader import hash_key + +class OrderedIndentlessDumper(yaml.SafeDumper): + pass + +class OrderedDumper(yaml.SafeDumper): + def increase_indent(self, flow=False, indentless=False): + return super(OrderedDumper, self).increase_indent(flow, False) + +yaml_value_annotation_re = re.compile(r"^__yq_(?P<type>tag|style)_(?P<key>.+)__$") +yaml_item_annotation_re = re.compile(r"^__yq_(?P<type>tag|style)_(?P<key>\d+)_(?P<value>.+)__$") + +def get_dumper(use_annotations=False, indentless=False): + def represent_dict(dumper, data): + pairs, custom_styles, custom_tags = [], {}, {} + for k, v in data.items(): + if use_annotations and isinstance(k, str): + value_annotation = yaml_value_annotation_re.match(k) + if value_annotation and value_annotation.group("type") == "style": + custom_styles[value_annotation.group("key")] = v + continue + elif value_annotation and value_annotation.group("type") == "tag": + custom_tags[value_annotation.group("key")] = v + continue + pairs.append((k, v)) + mapping = dumper.represent_mapping("tag:yaml.org,2002:map", pairs) + if use_annotations: + for k, v in mapping.value: + hashed_key = hash_key(k.value) + if hashed_key in custom_styles: + if isinstance(v, yaml.nodes.ScalarNode): + v.style = custom_styles[hashed_key] + elif custom_styles[hashed_key] == "flow": + v.flow_style = True + if hashed_key in custom_tags: + v.tag = custom_tags[hashed_key] + return mapping + + def represent_list(dumper, data): + raw_list, custom_styles, custom_tags = [], {}, {} + for v in data: + if use_annotations and isinstance(v, str): + annotation = yaml_item_annotation_re.match(v) + if annotation and annotation.group("type") == "style": + custom_styles[annotation.group("key")] = annotation.group("value") + continue + elif annotation and annotation.group("type") == "tag": + custom_tags[annotation.group("key")] = annotation.group("value") + continue + raw_list.append(v) + sequence = dumper.represent_list(raw_list) + if use_annotations: + for i, v in enumerate(sequence.value): + if str(i) in custom_styles: + if isinstance(v, yaml.nodes.ScalarNode): + v.style = custom_styles[str(i)] + elif custom_styles[str(i)] == "flow": + v.flow_style = True + if str(i) in custom_tags: + v.tag = custom_tags[str(i)] + return sequence + + dumper = OrderedIndentlessDumper if indentless else OrderedDumper + dumper.add_representer(OrderedDict, represent_dict) + dumper.add_representer(list, represent_list) + return dumper diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yq-2.8.1/yq/loader.py new/yq-2.10.0/yq/loader.py --- old/yq-2.8.1/yq/loader.py 1970-01-01 01:00:00.000000000 +0100 +++ new/yq-2.10.0/yq/loader.py 2019-12-01 16:52:31.000000000 +0100 @@ -0,0 +1,58 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + +import yaml +from base64 import b64encode +from collections import OrderedDict +from hashlib import sha224 + +from .compat import str + +def hash_key(key): + return b64encode(sha224(key.encode() if isinstance(key, str) else key).digest()).decode() + +class OrderedLoader(yaml.SafeLoader): + pass + +def get_loader(use_annotations=False): + def construct_sequence(loader, node): + annotations = [] + for i, v_node in enumerate(node.value): + if not use_annotations: + break + if v_node.tag and v_node.tag.startswith("!") and not v_node.tag.startswith("!!") and len(v_node.tag) > 1: + annotations.append("__yq_tag_{}_{}__".format(i, v_node.tag)) + if isinstance(v_node, yaml.nodes.ScalarNode) and v_node.style: + annotations.append("__yq_style_{}_{}__".format(i, v_node.style)) + elif isinstance(v_node, (yaml.nodes.SequenceNode, yaml.nodes.MappingNode)) and v_node.flow_style is True: + annotations.append("__yq_style_{}_{}__".format(i, "flow")) + return [loader.construct_object(i) for i in node.value] + annotations + + def construct_mapping(loader, node): + loader.flatten_mapping(node) # TODO: is this needed? + pairs = [] + for k_node, v_node in node.value: + key = loader.construct_object(k_node) + value = loader.construct_object(v_node) + pairs.append((key, value)) + if not (use_annotations and isinstance(key, (str, bytes))): + continue + if v_node.tag and v_node.tag.startswith("!") and not v_node.tag.startswith("!!") and len(v_node.tag) > 1: + pairs.append(("__yq_tag_{}__".format(hash_key(key)), v_node.tag)) + if isinstance(v_node, yaml.nodes.ScalarNode) and v_node.style: + pairs.append(("__yq_style_{}__".format(hash_key(key)), v_node.style)) + elif isinstance(v_node, (yaml.nodes.SequenceNode, yaml.nodes.MappingNode)) and v_node.flow_style is True: + pairs.append(("__yq_style_{}__".format(hash_key(key)), "flow")) + return OrderedDict(pairs) + + def parse_unknown_tags(loader, tag_suffix, node): + if isinstance(node, yaml.nodes.ScalarNode): + return loader.construct_scalar(node) + elif isinstance(node, yaml.nodes.SequenceNode): + return construct_sequence(loader, node) + elif isinstance(node, yaml.nodes.MappingNode): + return construct_mapping(loader, node) + + OrderedLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping) + OrderedLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG, construct_sequence) + OrderedLoader.add_multi_constructor('', parse_unknown_tags) + return OrderedLoader diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yq-2.8.1/yq/parser.py new/yq-2.10.0/yq/parser.py --- old/yq-2.8.1/yq/parser.py 1970-01-01 01:00:00.000000000 +0100 +++ new/yq-2.10.0/yq/parser.py 2019-12-23 05:06:54.000000000 +0100 @@ -0,0 +1,74 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + +import sys, argparse, subprocess + +from .version import __version__ + +# jq arguments that consume positionals must be listed here to avoid our parser mistaking them for our positionals +jq_arg_spec = {"--indent": 1, "-f": 1, "--from-file": 1, "-L": 1, "--arg": 2, "--argjson": 2, "--slurpfile": 2, + "--argfile": 2, "--rawfile": 2, "--args": argparse.REMAINDER, "--jsonargs": argparse.REMAINDER} + +class Parser(argparse.ArgumentParser): + def print_help(self): + yq_help = argparse.ArgumentParser.format_help(self).splitlines() + print("\n".join(["usage: yq [options] <jq filter> [YAML file...]"] + yq_help[1:] + [""])) + try: + subprocess.check_call(["jq", "--help"]) + except Exception: + pass + +def get_parser(program_name, description): + # By default suppress these help strings and only enable them in the specific programs. + yaml_output_help, yaml_roundtrip_help, width_help, indentless_help = (argparse.SUPPRESS, argparse.SUPPRESS, + argparse.SUPPRESS, argparse.SUPPRESS) + xml_output_help, xml_dtd_help, xml_root_help = argparse.SUPPRESS, argparse.SUPPRESS, argparse.SUPPRESS + toml_output_help = argparse.SUPPRESS + + if program_name == "yq": + current_language = "YAML" + yaml_output_help = "Transcode jq JSON output back into YAML and emit it" + yaml_roundtrip_help = """Transcode jq JSON output back into YAML and emit it. +Preserve YAML tags and styles by representing them as extra items +in their enclosing mappings and sequences while in JSON. This option +is incompatible with jq filters that do not expect these extra items.""" + width_help = "When using --yaml-output, specify string wrap width" + indentless_help = """When using --yaml-output, indent block style lists (sequences) +with 0 spaces instead of 2""" + elif program_name == "xq": + current_language = "XML" + xml_output_help = "Transcode jq JSON output back into XML and emit it" + xml_dtd_help = "Preserve XML Document Type Definition (disables streaming of multiple docs)" + xml_root_help = "When transcoding back to XML, envelope the output in an element with this name" + elif program_name == "tq": + current_language = "TOML" + toml_output_help = "Transcode jq JSON output back into TOML and emit it" + else: + raise Exception("Unknown program name") + + description = description.replace("yq", program_name).replace("YAML", current_language) + parser_args = dict(prog=program_name, description=description, formatter_class=argparse.RawTextHelpFormatter) + if sys.version_info >= (3, 5): + parser_args.update(allow_abbrev=False) # required to disambiguate options listed in jq_arg_spec + parser = Parser(**parser_args) + parser.add_argument("--output-format", default="json", help=argparse.SUPPRESS) + parser.add_argument("--yaml-output", "--yml-output", "-y", dest="output_format", action="store_const", const="yaml", + help=yaml_output_help) + parser.add_argument("--yaml-roundtrip", "--yml-roundtrip", "-Y", dest="output_format", action="store_const", + const="annotated_yaml", help=yaml_roundtrip_help) + parser.add_argument("--width", "-w", type=int, help=width_help) + parser.add_argument("--indentless-lists", "--indentless", action="store_true", help=indentless_help) + parser.add_argument("--xml-output", "-x", dest="output_format", action="store_const", const="xml", + help=xml_output_help) + parser.add_argument("--xml-dtd", action="store_true", help=xml_dtd_help) + parser.add_argument("--xml-root", help=xml_root_help) + parser.add_argument("--toml-output", "-t", dest="output_format", action="store_const", const="toml", + help=toml_output_help) + parser.add_argument("--in-place", "-i", action="store_true", help="Edit files in place (no backup - use caution)") + parser.add_argument("--version", action="version", version="%(prog)s {version}".format(version=__version__)) + + for arg in jq_arg_spec: + parser.add_argument(arg, nargs=jq_arg_spec[arg], dest=arg, action="append", help=argparse.SUPPRESS) + + parser.add_argument("jq_filter") + parser.add_argument("input_streams", nargs="*", type=argparse.FileType(), metavar="files", default=[sys.stdin]) + return parser diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yq-2.8.1/yq/version.py new/yq-2.10.0/yq/version.py --- old/yq-2.8.1/yq/version.py 2019-10-28 17:16:41.000000000 +0100 +++ new/yq-2.10.0/yq/version.py 2019-12-23 17:52:24.000000000 +0100 @@ -1 +1 @@ -__version__ = '2.8.1' +__version__ = '2.10.0' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yq-2.8.1/yq.egg-info/PKG-INFO new/yq-2.10.0/yq.egg-info/PKG-INFO --- old/yq-2.8.1/yq.egg-info/PKG-INFO 2019-10-28 17:16:45.000000000 +0100 +++ new/yq-2.10.0/yq.egg-info/PKG-INFO 2019-12-23 17:52:28.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: yq -Version: 2.8.1 +Version: 2.10.0 Summary: Command-line YAML/XML processor - jq wrapper for YAML/XML documents Home-page: https://github.com/kislyuk/yq Author: Andrey Kislyuk @@ -31,20 +31,81 @@ yq .foo.bar input.yml - By default, no conversion of ``jq`` output is done. Use the ``--yaml-output``/``-y`` argument to convert it back into YAML:: + By default, no conversion of ``jq`` output is done. Use the ``--yaml-output``/``-y`` option to convert it back into YAML:: cat input.yml | yq -y .foo.bar - Use the ``--width``/``-w`` argument to pass the line wrap width for string literals. All other command line arguments are - forwarded to ``jq``. ``yq`` forwards the exit code ``jq`` produced, unless there was an error in YAML parsing, in which case - the exit code is 1. See the `jq manual <https://stedolan.github.io/jq/manual/>`_ for more details on ``jq`` features and - options. - - YAML `tags <http://www.yaml.org/spec/1.2/spec.html#id2764295>`_ in the input are ignored (any nested data is treated as - untagged). Key order is preserved. + Mapping key order is preserved. By default, custom `YAML tags <http://www.yaml.org/spec/1.2/spec.html#id2764295>`_ and + `styles <https://yaml.org/spec/current.html#id2509255>`_ in the input are ignored. Use the ``--yaml-roundtrip``/``-Y`` + option to preserve YAML tags and styles by representing them as extra items in their enclosing mappings and sequences + while in JSON:: + + yq -Y .foo.bar input.yml + + Use the ``--width``/``-w`` option to pass the line wrap width for string literals. With ``-y/-Y``, files can be edited + in place like with ``sed -i``: ``yq -yi .foo=1 *.yml``. All other command line arguments are forwarded to ``jq``. ``yq`` + forwards the exit code ``jq`` produced, unless there was an error in YAML parsing, in which case the exit code is 1. + See the `jq manual <https://stedolan.github.io/jq/manual/>`_ for more details on ``jq`` features and options. Because YAML treats JSON as a dialect of YAML, you can use yq to convert JSON to YAML: ``yq -y . < in.json > out.yml``. + Preserving tags and styles using the ``-Y`` (``--yaml-roundtrip``) option + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + The ``-Y`` option helps preserve custom `string styles <https://yaml-multiline.info/>`_ and + `tags <https://camel.readthedocs.io/en/latest/yamlref.html#tags>`_ in your document. For exmaple, consider the following + document (an `AWS CloudFormation <https://aws.amazon.com/cloudformation/>`_ template fragment):: + + Resources: + ElasticLoadBalancer: + Type: 'AWS::ElasticLoadBalancing::LoadBalancer' + Properties: + AvailabilityZones: !GetAZs '' + Instances: + - !Ref Ec2Instance1 + - !Ref Ec2Instance2 + Description: >- + Load balancer for Big Important Service. + + Good thing it's managed by this template. + + Passing this document through ``yq -y .Resources.ElasticLoadBalancer`` will drop custom tags, such as ``!Ref``, + and styles, such as the `folded <https://yaml-multiline.info/>`_ style of the ``Description`` field:: + + Type: AWS::ElasticLoadBalancing::LoadBalancer + Properties: + AvailabilityZones: '' + Instances: + - Ec2Instance1 + - Ec2Instance2 + Description: 'Load balancer for Big Important Service. + + Good thing it''s managed by this template.' + + By contrast, passing it through ``yq -Y .Resources.ElasticLoadBalancer`` will preserve tags and styles:: + + Type: 'AWS::ElasticLoadBalancing::LoadBalancer' + Properties: + AvailabilityZones: !GetAZs '' + Instances: + - !Ref 'Ec2Instance1' + - !Ref 'Ec2Instance2' + Description: >- + Load balancer for Big Important Service. + + Good thing it's managed by this template. + + To accomplish this in ``-Y`` mode, yq carries extra metadata (mapping pairs and sequence values) in the JSON + representation of your document for any custom tags or styles that it finds. When converting the JSON back into YAML, it + parses this metadata, re-applies the tags and styles, and discards the extra pairs and values. + + .. warning :: + + The ``-Y`` option is incompatible with jq filters that do not expect the extra information injected into the document + to preserve the YAML formatting. For example, a jq filter that counts entries in the Instances array will come up with + 4 entries instead of 2. A filter that expects all array entries to be mappings may break due to the presence of string + metadata keys. Check your jq filter for compatibility/semantic validity when using the ``-Y`` option. + XML support ----------- ``yq`` also supports XML. The ``yq`` package installs an executable, ``xq``, which @@ -67,7 +128,7 @@ Links ----- * `Project home page (GitHub) <https://github.com/kislyuk/yq>`_ - * `Documentation (Read the Docs) <https://yq.readthedocs.io/en/latest/>`_ + * `Documentation <https://kislyuk.github.io/yq/>`_ * `Package distribution (PyPI) <https://pypi.python.org/pypi/yq>`_ * `Change log <https://github.com/kislyuk/yq/blob/master/Changes.rst>`_ * `jq <https://stedolan.github.io/jq/>`_ - the command-line JSON processor utility powering ``yq`` @@ -80,16 +141,14 @@ ------- Licensed under the terms of the `Apache License, Version 2.0 <http://www.apache.org/licenses/LICENSE-2.0>`_. - .. image:: https://img.shields.io/travis/kislyuk/yq.svg - :target: https://travis-ci.org/kislyuk/yq + .. image:: https://github.com/kislyuk/yq/workflows/Python%20package/badge.svg + :target: https://github.com/kislyuk/yq/actions .. image:: https://codecov.io/github/kislyuk/yq/coverage.svg?branch=master :target: https://codecov.io/github/kislyuk/yq?branch=master .. image:: https://img.shields.io/pypi/v/yq.svg :target: https://pypi.python.org/pypi/yq .. image:: https://img.shields.io/pypi/l/yq.svg :target: https://pypi.python.org/pypi/yq - .. image:: https://readthedocs.org/projects/yq/badge/?version=latest - :target: https://yq.readthedocs.io/ Platform: UNKNOWN Classifier: Intended Audience :: Developers @@ -98,10 +157,10 @@ Classifier: Operating System :: POSIX Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3.3 -Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 Classifier: Topic :: Software Development :: Libraries :: Python Modules Provides-Extra: test Provides-Extra: toml diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yq-2.8.1/yq.egg-info/SOURCES.txt new/yq-2.10.0/yq.egg-info/SOURCES.txt --- old/yq-2.8.1/yq.egg-info/SOURCES.txt 2019-10-28 17:16:45.000000000 +0100 +++ new/yq-2.10.0/yq.egg-info/SOURCES.txt 2019-12-23 17:52:28.000000000 +0100 @@ -3,8 +3,14 @@ README.rst setup.cfg setup.py +test/bomb.yml +test/cfn.yml test/test.py yq/__init__.py +yq/compat.py +yq/dumper.py +yq/loader.py +yq/parser.py yq/version.py yq.egg-info/PKG-INFO yq.egg-info/SOURCES.txt diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/yq-2.8.1/yq.egg-info/requires.txt new/yq-2.10.0/yq.egg-info/requires.txt --- old/yq-2.8.1/yq.egg-info/requires.txt 2019-10-28 17:16:45.000000000 +0100 +++ new/yq-2.10.0/yq.egg-info/requires.txt 2019-12-23 17:52:28.000000000 +0100 @@ -1,6 +1,7 @@ setuptools PyYAML>=3.11 xmltodict>=0.11.0 +argcomplete>=1.8.1 [test] coverage
