Repository: incubator-ariatosca Updated Branches: refs/heads/ARIA-23-integrate-csar-packager c5be808d9 -> 37bc10265 (forced update)
ARIA-23 Add initial CSAR support Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/37bc1026 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/37bc1026 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/37bc1026 Branch: refs/heads/ARIA-23-integrate-csar-packager Commit: 37bc102650f8935478bebfc4867213f2d1c3cfc0 Parents: b33c70e Author: Dan Kilman <[email protected]> Authored: Thu Nov 17 12:43:49 2016 +0200 Committer: Dan Kilman <[email protected]> Committed: Tue Nov 29 11:43:44 2016 +0200 ---------------------------------------------------------------------- aria/cli/args_parser.py | 44 +++++++++++ aria/cli/cli.py | 6 ++ aria/cli/commands.py | 71 +++++++++++++++++- aria/cli/csar.py | 171 +++++++++++++++++++++++++++++++++++++++++++ tox.ini | 1 + 5 files changed, 292 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/37bc1026/aria/cli/args_parser.py ---------------------------------------------------------------------- diff --git a/aria/cli/args_parser.py b/aria/cli/args_parser.py index 56fd074..8eacf05 100644 --- a/aria/cli/args_parser.py +++ b/aria/cli/args_parser.py @@ -69,6 +69,9 @@ def config_parser(parser=None): add_execute_parser(sub_parser) add_parse_parser(sub_parser) add_spec_parser(sub_parser) + add_csar_create_parser(sub_parser) + add_csar_open_parser(sub_parser) + add_csar_validate_parser(sub_parser) return parser @@ -199,3 +202,44 @@ def add_spec_parser(spec): '--csv', action='store_true', help='output as CSV') + + +@sub_parser_decorator( + name='csar-create', + help='Create a CSAR file from a TOSCA service template directory', + formatter_class=SmartFormatter) +def add_csar_create_parser(parse): + parse.add_argument( + 'source', + help='Service template directory') + parse.add_argument( + 'entry', + help='Entry definition file relative to service template directory') + parse.add_argument( + '-d', '--destination', + help='Output CSAR zip destination', + required=True) + + +@sub_parser_decorator( + name='csar-open', + help='Extracts a CSAR file to a TOSCA service template directory', + formatter_class=SmartFormatter) +def add_csar_open_parser(parse): + parse.add_argument( + 'source', + help='CSAR file location') + parse.add_argument( + '-d', '--destination', + help='Output directory to extract the CSAR into', + required=True) + + +@sub_parser_decorator( + name='csar-validate', + help='Validates a CSAR file', + formatter_class=SmartFormatter) +def add_csar_validate_parser(parse): + parse.add_argument( + 'source', + help='CSAR file location') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/37bc1026/aria/cli/cli.py ---------------------------------------------------------------------- diff --git a/aria/cli/cli.py b/aria/cli/cli.py index ad9784c..c5830d5 100644 --- a/aria/cli/cli.py +++ b/aria/cli/cli.py @@ -33,6 +33,9 @@ from .commands import ( ExecuteCommand, ParseCommand, SpecCommand, + CSARCreateCommand, + CSAROpenCommand, + CSARValidateCommand, ) __version__ = '0.1.0' @@ -50,6 +53,9 @@ class AriaCli(LoggerMixin): 'execute': ExecuteCommand.with_logger(base_logger=self.logger), 'parse': ParseCommand.with_logger(base_logger=self.logger), 'spec': SpecCommand.with_logger(base_logger=self.logger), + 'csar-create': CSARCreateCommand.with_logger(base_logger=self.logger), + 'csar-open': CSAROpenCommand.with_logger(base_logger=self.logger), + 'csar-validate': CSARValidateCommand.with_logger(base_logger=self.logger), } def __enter__(self): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/37bc1026/aria/cli/commands.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands.py b/aria/cli/commands.py index 57118a7..e969e06 100644 --- a/aria/cli/commands.py +++ b/aria/cli/commands.py @@ -21,6 +21,8 @@ import json import os import sys import csv +import shutil +import tempfile from glob import glob from importlib import import_module @@ -43,11 +45,12 @@ from ..parser.consumption import ( Inputs, Instance ) -from ..parser.loading import (UriLocation, URI_LOADER_PREFIXES) +from ..parser.loading import (LiteralLocation, UriLocation, URI_LOADER_PREFIXES) from ..utils.application import StorageManager from ..utils.caching import cachedmethod from ..utils.console import (puts, Colored, indent) from ..utils.imports import (import_fullname, import_modules) +from . import csar from .exceptions import ( AriaCliFormatInputsError, AriaCliYAMLInputsError, @@ -388,3 +391,69 @@ class SpecCommand(BaseCommand): with indent(2): for k, v in details.iteritems(): puts('%s: %s' % (Colored.magenta(k), v)) + + +class BaseCSARCommand(BaseCommand): + + @staticmethod + def _parse_and_dump(reader): + context = ConsumptionContext() + context.loading.prefixes += [os.path.join(reader.destination, 'definitions')] + context.presentation.location = LiteralLocation(reader.entry_definitions_yaml) + chain = ConsumerChain(context, (Read, Validate, Model, Instance)) + chain.consume() + if context.validation.dump_issues(): + raise RuntimeError('Validation failed') + dumper = chain.consumers[-1] + dumper.dump() + + def _read(self, source, destination): + reader = csar.read( + source=source, + destination=destination, + logger=self.logger) + self.logger.info( + 'Path: {r.destination}\n' + 'TOSCA meta file version: {r.meta_file_version}\n' + 'CSAR Version: {r.csar_version}\n' + 'Created By: {r.created_by}\n' + 'Entry definitions: {r.entry_definitions}' + .format(r=reader)) + self._parse_and_dump(reader) + + def _validate(self, source): + workdir = tempfile.mkdtemp() + try: + self._read( + source=source, + destination=workdir) + finally: + shutil.rmtree(workdir, ignore_errors=True) + + +class CSARCreateCommand(BaseCSARCommand): + + def __call__(self, args_namespace, unknown_args): + super(CSARCreateCommand, self).__call__(args_namespace, unknown_args) + csar.write( + source=args_namespace.source, + entry=args_namespace.entry, + destination=args_namespace.destination, + logger=self.logger) + self._validate(args_namespace.destination) + + +class CSAROpenCommand(BaseCSARCommand): + + def __call__(self, args_namespace, unknown_args): + super(CSAROpenCommand, self).__call__(args_namespace, unknown_args) + self._read( + source=args_namespace.source, + destination=args_namespace.destination) + + +class CSARValidateCommand(BaseCSARCommand): + + def __call__(self, args_namespace, unknown_args): + super(CSARValidateCommand, self).__call__(args_namespace, unknown_args) + self._validate(args_namespace.source) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/37bc1026/aria/cli/csar.py ---------------------------------------------------------------------- diff --git a/aria/cli/csar.py b/aria/cli/csar.py new file mode 100644 index 0000000..4300184 --- /dev/null +++ b/aria/cli/csar.py @@ -0,0 +1,171 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pprint +import tempfile +import zipfile + +import requests +from ruamel import yaml + + +META_FILE = 'TOSCA-Metadata/TOSCA.meta' +META_FILE_VERSION_KEY = 'TOSCA-Meta-File-Version' +META_FILE_VERSION_VALUE = '1.0' +META_CSAR_VERSION_KEY = 'CSAR-Version' +META_CSAR_VERSION_VALUE = '1.1' +META_CREATED_BY_KEY = 'Created-By' +META_CREATED_BY_VALUE = 'ARIA' +META_ENTRY_DEFINITIONS_KEY = 'Entry-Definitions' +BASE_METADATA = { + META_FILE_VERSION_KEY: META_FILE_VERSION_VALUE, + META_CSAR_VERSION_KEY: META_CSAR_VERSION_VALUE, + META_CREATED_BY_KEY: META_CREATED_BY_VALUE, +} + + +def write(source, entry, destination, logger): + source = os.path.expanduser(source) + destination = os.path.expanduser(destination) + entry_definitions = os.path.join(source, entry) + meta_file = os.path.join(source, META_FILE) + if not os.path.isdir(source): + raise ValueError('{0} is not a directory. Please specify the service template ' + 'directory.'.format(source)) + if not os.path.isfile(entry_definitions): + raise ValueError('{0} does not exists. Please specify a valid entry point.' + .format(entry_definitions)) + if os.path.exists(destination): + raise ValueError('{0} already exists. Please provide a path to where the CSAR should be ' + 'created.'.format(destination)) + if os.path.exists(meta_file): + raise ValueError('{0} already exists. This commands generates a meta file for you. Please ' + 'remove the existing metafile.'.format(meta_file)) + metadata = BASE_METADATA.copy() + metadata[META_ENTRY_DEFINITIONS_KEY] = entry + logger.debug('Compressing root directory to ZIP') + with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as f: + for root, _, files in os.walk(source): + for file in files: + file_full_path = os.path.join(root, file) + file_relative_path = os.path.relpath(file_full_path, source) + logger.debug('Writing to archive: {0}'.format(file_relative_path)) + f.write(file_full_path, file_relative_path) + logger.debug('Writing new metadata file to {0}'.format(META_FILE)) + f.writestr(META_FILE, yaml.dump(metadata, default_flow_style=False)) + + +class _CSARReader(object): + + def __init__(self, source, destination, logger): + self.logger = logger + if os.path.isdir(destination) and os.listdir(destination): + raise ValueError('{0} already exists and is not empty. ' + 'Please specify the location where the CSAR ' + 'should be extracted.'.format(destination)) + downloaded_csar = '://' in source + if downloaded_csar: + fd, download_target = tempfile.mkstemp() + os.close(fd) + self._download(source, download_target) + source = download_target + self.source = os.path.expanduser(source) + self.destination = os.path.expanduser(destination) + self.metadata = {} + try: + if not os.path.exists(self.source): + raise ValueError('{0} does not exists. Please specify a valid CSAR path.' + .format(self.source)) + if not zipfile.is_zipfile(self.source): + raise ValueError('{0} is not a valid CSAR.'.format(self.source)) + self._extract() + self._read_metadata() + self._validate() + finally: + if downloaded_csar: + os.remove(self.source) + + @property + def created_by(self): + return self.metadata.get(META_CREATED_BY_KEY) + + @property + def csar_version(self): + return self.metadata.get(META_CSAR_VERSION_KEY) + + @property + def meta_file_version(self): + return self.metadata.get(META_FILE_VERSION_KEY) + + @property + def entry_definitions(self): + return self.metadata.get(META_ENTRY_DEFINITIONS_KEY) + + @property + def entry_definitions_yaml(self): + with open(os.path.join(self.destination, self.entry_definitions)) as f: + return yaml.load(f) + + def _extract(self): + self.logger.debug('Extracting CSAR contents') + if not os.path.exists(self.destination): + os.mkdir(self.destination) + with zipfile.ZipFile(self.source) as f: + f.extractall(self.destination) + self.logger.debug('CSAR contents successfully extracted') + + def _read_metadata(self): + csar_metafile = os.path.join(self.destination, META_FILE) + if not os.path.exists(csar_metafile): + raise ValueError('Metadata file {0} is missing from the CSAR'.format(csar_metafile)) + self.logger.debug('CSAR metadata file: {0}'.format(csar_metafile)) + self.logger.debug('Attempting to parse CSAR metadata YAML') + with open(csar_metafile) as f: + self.metadata.update(yaml.load(f)) + self.logger.debug('CSAR metadata:\n{0}'.format(pprint.pformat(self.metadata))) + + def _validate(self): + def validate_key(key, expected=None): + if not self.metadata.get(key): + raise ValueError('{0} is missing from the metadata file.'.format(key)) + actual = str(self.metadata[key]) + if expected and actual != expected: + raise ValueError('{0} is expected to be {1} in the metadata file while it is in ' + 'fact {2}.'.format(key, expected, actual)) + validate_key(META_FILE_VERSION_KEY, expected=META_FILE_VERSION_VALUE) + validate_key(META_CSAR_VERSION_KEY, expected=META_CSAR_VERSION_VALUE) + validate_key(META_CREATED_BY_KEY) + validate_key(META_ENTRY_DEFINITIONS_KEY) + self.logger.debug('CSAR entry definitions: {0}'.format(self.entry_definitions)) + entry_definitions_path = os.path.join(self.destination, self.entry_definitions) + if not os.path.isfile(entry_definitions_path): + raise ValueError('The entry definitions {0} referenced by the metadata file does not ' + 'exist.'.format(entry_definitions_path)) + + def _download(self, url, target): + response = requests.get(url, stream=True) + if response.status_code != requests.codes.ok: + raise ValueError('Server at {0} returned a {1} status code' + .format(url, response.status_code)) + self.logger.info('Downloading {0} to {1}'.format(url, target)) + with open(target, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + + +def read(source, destination, logger): + return _CSARReader(source=source, destination=destination, logger=logger) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/37bc1026/tox.ini ---------------------------------------------------------------------- diff --git a/tox.ini b/tox.ini index 2efc329..8355b19 100644 --- a/tox.ini +++ b/tox.ini @@ -34,3 +34,4 @@ commands=pylint --rcfile=aria/.pylintrc --disable=fixme,missing-docstring --igno [testenv:pylint_tests] commands=pylint --rcfile=tests/.pylintrc --disable=fixme,missing-docstring tests +
