ARIA-23 Add initial CSAR support

Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/d7addbc7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/d7addbc7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/d7addbc7

Branch: refs/heads/ARIA-30-SQL-based-storage-implementation
Commit: d7addbc7f8637170e10dc9dffd09fb1dc38189f0
Parents: b54478b
Author: Dan Kilman <d...@gigaspaces.com>
Authored: Thu Nov 17 12:43:49 2016 +0200
Committer: Dan Kilman <d...@gigaspaces.com>
Committed: Wed Nov 30 09:55:05 2016 +0200

----------------------------------------------------------------------
 aria/cli/args_parser.py |  44 +++++++++++
 aria/cli/cli.py         |   6 ++
 aria/cli/commands.py    |  71 +++++++++++++++++-
 aria/cli/csar.py        | 171 +++++++++++++++++++++++++++++++++++++++++++
 tox.ini                 |   1 +
 5 files changed, 292 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d7addbc7/aria/cli/args_parser.py
----------------------------------------------------------------------
diff --git a/aria/cli/args_parser.py b/aria/cli/args_parser.py
index 56fd074..8eacf05 100644
--- a/aria/cli/args_parser.py
+++ b/aria/cli/args_parser.py
@@ -69,6 +69,9 @@ def config_parser(parser=None):
     add_execute_parser(sub_parser)
     add_parse_parser(sub_parser)
     add_spec_parser(sub_parser)
+    add_csar_create_parser(sub_parser)
+    add_csar_open_parser(sub_parser)
+    add_csar_validate_parser(sub_parser)
     return parser
 
 
@@ -199,3 +202,44 @@ def add_spec_parser(spec):
         '--csv',
         action='store_true',
         help='output as CSV')
+
+
+@sub_parser_decorator(
+    name='csar-create',
+    help='Create a CSAR file from a TOSCA service template directory',
+    formatter_class=SmartFormatter)
+def add_csar_create_parser(parse):
+    parse.add_argument(
+        'source',
+        help='Service template directory')
+    parse.add_argument(
+        'entry',
+        help='Entry definition file relative to service template directory')
+    parse.add_argument(
+        '-d', '--destination',
+        help='Output CSAR zip destination',
+        required=True)
+
+
+@sub_parser_decorator(
+    name='csar-open',
+    help='Extracts a CSAR file to a TOSCA service template directory',
+    formatter_class=SmartFormatter)
+def add_csar_open_parser(parse):
+    parse.add_argument(
+        'source',
+        help='CSAR file location')
+    parse.add_argument(
+        '-d', '--destination',
+        help='Output directory to extract the CSAR into',
+        required=True)
+
+
+@sub_parser_decorator(
+    name='csar-validate',
+    help='Validates a CSAR file',
+    formatter_class=SmartFormatter)
+def add_csar_validate_parser(parse):
+    parse.add_argument(
+        'source',
+        help='CSAR file location')

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d7addbc7/aria/cli/cli.py
----------------------------------------------------------------------
diff --git a/aria/cli/cli.py b/aria/cli/cli.py
index ad9784c..c5830d5 100644
--- a/aria/cli/cli.py
+++ b/aria/cli/cli.py
@@ -33,6 +33,9 @@ from .commands import (
     ExecuteCommand,
     ParseCommand,
     SpecCommand,
+    CSARCreateCommand,
+    CSAROpenCommand,
+    CSARValidateCommand,
 )
 
 __version__ = '0.1.0'
@@ -50,6 +53,9 @@ class AriaCli(LoggerMixin):
             'execute': ExecuteCommand.with_logger(base_logger=self.logger),
             'parse': ParseCommand.with_logger(base_logger=self.logger),
             'spec': SpecCommand.with_logger(base_logger=self.logger),
+            'csar-create': 
CSARCreateCommand.with_logger(base_logger=self.logger),
+            'csar-open': CSAROpenCommand.with_logger(base_logger=self.logger),
+            'csar-validate': 
CSARValidateCommand.with_logger(base_logger=self.logger),
         }
 
     def __enter__(self):

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d7addbc7/aria/cli/commands.py
----------------------------------------------------------------------
diff --git a/aria/cli/commands.py b/aria/cli/commands.py
index 17a2564..3426bb0 100644
--- a/aria/cli/commands.py
+++ b/aria/cli/commands.py
@@ -21,6 +21,8 @@ import json
 import os
 import sys
 import csv
+import shutil
+import tempfile
 from glob import glob
 from importlib import import_module
 
@@ -43,11 +45,12 @@ from ..parser.consumption import (
     Inputs,
     Instance
 )
-from ..parser.loading import (UriLocation, URI_LOADER_PREFIXES)
+from ..parser.loading import (LiteralLocation, UriLocation, 
URI_LOADER_PREFIXES)
 from ..utils.application import StorageManager
 from ..utils.caching import cachedmethod
 from ..utils.console import (puts, Colored, indent)
 from ..utils.imports import (import_fullname, import_modules)
+from . import csar
 from .exceptions import (
     AriaCliFormatInputsError,
     AriaCliYAMLInputsError,
@@ -394,3 +397,69 @@ class SpecCommand(BaseCommand):
                         with indent(2):
                             for k, v in details.iteritems():
                                 puts('%s: %s' % (Colored.magenta(k), v))
+
+
+class BaseCSARCommand(BaseCommand):
+
+    @staticmethod
+    def _parse_and_dump(reader):
+        context = ConsumptionContext()
+        context.loading.prefixes += [os.path.join(reader.destination, 
'definitions')]
+        context.presentation.location = 
LiteralLocation(reader.entry_definitions_yaml)
+        chain = ConsumerChain(context, (Read, Validate, Model, Instance))
+        chain.consume()
+        if context.validation.dump_issues():
+            raise RuntimeError('Validation failed')
+        dumper = chain.consumers[-1]
+        dumper.dump()
+
+    def _read(self, source, destination):
+        reader = csar.read(
+            source=source,
+            destination=destination,
+            logger=self.logger)
+        self.logger.info(
+            'Path: {r.destination}\n'
+            'TOSCA meta file version: {r.meta_file_version}\n'
+            'CSAR Version: {r.csar_version}\n'
+            'Created By: {r.created_by}\n'
+            'Entry definitions: {r.entry_definitions}'
+            .format(r=reader))
+        self._parse_and_dump(reader)
+
+    def _validate(self, source):
+        workdir = tempfile.mkdtemp()
+        try:
+            self._read(
+                source=source,
+                destination=workdir)
+        finally:
+            shutil.rmtree(workdir, ignore_errors=True)
+
+
+class CSARCreateCommand(BaseCSARCommand):
+
+    def __call__(self, args_namespace, unknown_args):
+        super(CSARCreateCommand, self).__call__(args_namespace, unknown_args)
+        csar.write(
+            source=args_namespace.source,
+            entry=args_namespace.entry,
+            destination=args_namespace.destination,
+            logger=self.logger)
+        self._validate(args_namespace.destination)
+
+
+class CSAROpenCommand(BaseCSARCommand):
+
+    def __call__(self, args_namespace, unknown_args):
+        super(CSAROpenCommand, self).__call__(args_namespace, unknown_args)
+        self._read(
+            source=args_namespace.source,
+            destination=args_namespace.destination)
+
+
+class CSARValidateCommand(BaseCSARCommand):
+
+    def __call__(self, args_namespace, unknown_args):
+        super(CSARValidateCommand, self).__call__(args_namespace, unknown_args)
+        self._validate(args_namespace.source)

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d7addbc7/aria/cli/csar.py
----------------------------------------------------------------------
diff --git a/aria/cli/csar.py b/aria/cli/csar.py
new file mode 100644
index 0000000..933df17
--- /dev/null
+++ b/aria/cli/csar.py
@@ -0,0 +1,171 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import pprint
+import tempfile
+import zipfile
+
+import requests
+from ruamel import yaml
+
+
+META_FILE = 'TOSCA-Metadata/TOSCA.meta'
+META_FILE_VERSION_KEY = 'TOSCA-Meta-File-Version'
+META_FILE_VERSION_VALUE = '1.0'
+META_CSAR_VERSION_KEY = 'CSAR-Version'
+META_CSAR_VERSION_VALUE = '1.1'
+META_CREATED_BY_KEY = 'Created-By'
+META_CREATED_BY_VALUE = 'ARIA'
+META_ENTRY_DEFINITIONS_KEY = 'Entry-Definitions'
+BASE_METADATA = {
+    META_FILE_VERSION_KEY: META_FILE_VERSION_VALUE,
+    META_CSAR_VERSION_KEY: META_CSAR_VERSION_VALUE,
+    META_CREATED_BY_KEY: META_CREATED_BY_VALUE,
+}
+
+
+def write(source, entry, destination, logger):
+    source = os.path.expanduser(source)
+    destination = os.path.expanduser(destination)
+    entry_definitions = os.path.join(source, entry)
+    meta_file = os.path.join(source, META_FILE)
+    if not os.path.isdir(source):
+        raise ValueError('{0} is not a directory. Please specify the service 
template '
+                         'directory.'.format(source))
+    if not os.path.isfile(entry_definitions):
+        raise ValueError('{0} does not exists. Please specify a valid entry 
point.'
+                         .format(entry_definitions))
+    if os.path.exists(destination):
+        raise ValueError('{0} already exists. Please provide a path to where 
the CSAR should be '
+                         'created.'.format(destination))
+    if os.path.exists(meta_file):
+        raise ValueError('{0} already exists. This commands generates a meta 
file for you. Please '
+                         'remove the existing metafile.'.format(meta_file))
+    metadata = BASE_METADATA.copy()
+    metadata[META_ENTRY_DEFINITIONS_KEY] = entry
+    logger.debug('Compressing root directory to ZIP')
+    with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as f:
+        for root, _, files in os.walk(source):
+            for file in files:
+                file_full_path = os.path.join(root, file)
+                file_relative_path = os.path.relpath(file_full_path, source)
+                logger.debug('Writing to archive: 
{0}'.format(file_relative_path))
+                f.write(file_full_path, file_relative_path)
+        logger.debug('Writing new metadata file to {0}'.format(META_FILE))
+        f.writestr(META_FILE, yaml.dump(metadata, default_flow_style=False))
+
+
+class _CSARReader(object):
+
+    def __init__(self, source, destination, logger):
+        self.logger = logger
+        if os.path.isdir(destination) and os.listdir(destination):
+            raise ValueError('{0} already exists and is not empty. '
+                             'Please specify the location where the CSAR '
+                             'should be extracted.'.format(destination))
+        downloaded_csar = '://' in source
+        if downloaded_csar:
+            file_descriptor, download_target = tempfile.mkstemp()
+            os.close(file_descriptor)
+            self._download(source, download_target)
+            source = download_target
+        self.source = os.path.expanduser(source)
+        self.destination = os.path.expanduser(destination)
+        self.metadata = {}
+        try:
+            if not os.path.exists(self.source):
+                raise ValueError('{0} does not exists. Please specify a valid 
CSAR path.'
+                                 .format(self.source))
+            if not zipfile.is_zipfile(self.source):
+                raise ValueError('{0} is not a valid 
CSAR.'.format(self.source))
+            self._extract()
+            self._read_metadata()
+            self._validate()
+        finally:
+            if downloaded_csar:
+                os.remove(self.source)
+
+    @property
+    def created_by(self):
+        return self.metadata.get(META_CREATED_BY_KEY)
+
+    @property
+    def csar_version(self):
+        return self.metadata.get(META_CSAR_VERSION_KEY)
+
+    @property
+    def meta_file_version(self):
+        return self.metadata.get(META_FILE_VERSION_KEY)
+
+    @property
+    def entry_definitions(self):
+        return self.metadata.get(META_ENTRY_DEFINITIONS_KEY)
+
+    @property
+    def entry_definitions_yaml(self):
+        with open(os.path.join(self.destination, self.entry_definitions)) as f:
+            return yaml.load(f)
+
+    def _extract(self):
+        self.logger.debug('Extracting CSAR contents')
+        if not os.path.exists(self.destination):
+            os.mkdir(self.destination)
+        with zipfile.ZipFile(self.source) as f:
+            f.extractall(self.destination)
+        self.logger.debug('CSAR contents successfully extracted')
+
+    def _read_metadata(self):
+        csar_metafile = os.path.join(self.destination, META_FILE)
+        if not os.path.exists(csar_metafile):
+            raise ValueError('Metadata file {0} is missing from the 
CSAR'.format(csar_metafile))
+        self.logger.debug('CSAR metadata file: {0}'.format(csar_metafile))
+        self.logger.debug('Attempting to parse CSAR metadata YAML')
+        with open(csar_metafile) as f:
+            self.metadata.update(yaml.load(f))
+        self.logger.debug('CSAR 
metadata:\n{0}'.format(pprint.pformat(self.metadata)))
+
+    def _validate(self):
+        def validate_key(key, expected=None):
+            if not self.metadata.get(key):
+                raise ValueError('{0} is missing from the metadata 
file.'.format(key))
+            actual = str(self.metadata[key])
+            if expected and actual != expected:
+                raise ValueError('{0} is expected to be {1} in the metadata 
file while it is in '
+                                 'fact {2}.'.format(key, expected, actual))
+        validate_key(META_FILE_VERSION_KEY, expected=META_FILE_VERSION_VALUE)
+        validate_key(META_CSAR_VERSION_KEY, expected=META_CSAR_VERSION_VALUE)
+        validate_key(META_CREATED_BY_KEY)
+        validate_key(META_ENTRY_DEFINITIONS_KEY)
+        self.logger.debug('CSAR entry definitions: 
{0}'.format(self.entry_definitions))
+        entry_definitions_path = os.path.join(self.destination, 
self.entry_definitions)
+        if not os.path.isfile(entry_definitions_path):
+            raise ValueError('The entry definitions {0} referenced by the 
metadata file does not '
+                             'exist.'.format(entry_definitions_path))
+
+    def _download(self, url, target):
+        response = requests.get(url, stream=True)
+        if response.status_code != 200:
+            raise ValueError('Server at {0} returned a {1} status code'
+                             .format(url, response.status_code))
+        self.logger.info('Downloading {0} to {1}'.format(url, target))
+        with open(target, 'wb') as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:
+                    f.write(chunk)
+
+
+def read(source, destination, logger):
+    return _CSARReader(source=source, destination=destination, logger=logger)

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d7addbc7/tox.ini
----------------------------------------------------------------------
diff --git a/tox.ini b/tox.ini
index 2efc329..8355b19 100644
--- a/tox.ini
+++ b/tox.ini
@@ -34,3 +34,4 @@ commands=pylint --rcfile=aria/.pylintrc 
--disable=fixme,missing-docstring --igno
 
 [testenv:pylint_tests]
 commands=pylint --rcfile=tests/.pylintrc --disable=fixme,missing-docstring 
tests
+

Reply via email to