This is an automated email from the git hooks/post-receive script. misterc-guest pushed a commit to annotated tag upstream/1.0.20170810192106 in repository cwltool.
commit 29cb30f1e1f21955dd181927f4fb2af476797ec1 Author: Michael R. Crusoe <michael.cru...@gmail.com> Date: Fri Aug 11 04:47:40 2017 -0700 New upstream version 1.0.20170810192106 --- PKG-INFO | 2 +- cwltool.egg-info/PKG-INFO | 2 +- cwltool.egg-info/SOURCES.txt | 28 ++ cwltool.egg-info/pbr.json | 1 + cwltool.egg-info/requires.txt | 16 +- cwltool/builder.py | 34 ++- cwltool/draft2tool.py | 64 +++-- cwltool/expression.py | 19 +- cwltool/job.py | 15 +- cwltool/main.py | 4 +- cwltool/pack.py | 5 + cwltool/pathmapper.py | 44 ++- cwltool/process.py | 1 + cwltool/sandboxjs.py | 12 +- cwltool/schemas/draft-3/index.md | 0 .../salad/schema_salad/metaschema/metaschema2.yml | 317 +++++++++++++++++++++ cwltool/stdfsaccess.py | 2 + setup.cfg | 3 +- setup.py | 2 +- tests/cat.cwl | 8 + tests/cat2.cwl | 9 + tests/env.cwl | 5 + tests/item1.yml | 3 + tests/output.txt | 16 ++ tests/test_http_input.py | 26 ++ tests/test_pack.py | 27 ++ tests/value | 1 + tests/wf/4c637b77-8130-4158-807c-ccc78ea7b563 | 1 + tests/wf/c5b8320e-6e31-4bbb-8453-03b054b3254e | 1 + tests/wf/conditional.cwl | 22 ++ tests/wf/emptyscatter.cwl | 18 ++ tests/wf/emptyscatter.yml | 1 + tests/wf/foo1.txt | 1 + tests/wf/foo3.txt | 1 + tests/wf/fooa.txt | 1 + tests/wf/foob.txt | 1 + tests/wf/foofoo5.txt.txt | 1 + tests/wf/hello-workflow.cwl | 38 +++ tests/wf/hello_single_tool.cwl | 9 + tests/wf/loop.cwl | 36 +++ tests/wf/output.txt | 16 ++ tests/wf/revsort-ovr2-job.json | 15 + tests/wf/value | 1 + tests/wf/whale.txt | 16 ++ tests/writeable.cwl | 11 + 45 files changed, 790 insertions(+), 66 deletions(-) diff --git a/PKG-INFO b/PKG-INFO index e5d73f5..059a944 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cwltool -Version: 1.0.20170803160545 +Version: 1.0.20170810192106 Summary: Common workflow language reference implementation Home-page: https://github.com/common-workflow-language/cwltool Author: Common workflow language working group diff --git a/cwltool.egg-info/PKG-INFO b/cwltool.egg-info/PKG-INFO index e5d73f5..059a944 100644 --- a/cwltool.egg-info/PKG-INFO +++ b/cwltool.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cwltool -Version: 1.0.20170803160545 +Version: 1.0.20170810192106 Summary: Common workflow language reference implementation Home-page: https://github.com/common-workflow-language/cwltool Author: Common workflow language working group diff --git a/cwltool.egg-info/SOURCES.txt b/cwltool.egg-info/SOURCES.txt index 0308f25..51d419e 100644 --- a/cwltool.egg-info/SOURCES.txt +++ b/cwltool.egg-info/SOURCES.txt @@ -36,6 +36,7 @@ cwltool.egg-info/PKG-INFO cwltool.egg-info/SOURCES.txt cwltool.egg-info/dependency_links.txt cwltool.egg-info/entry_points.txt +cwltool.egg-info/pbr.json cwltool.egg-info/requires.txt cwltool.egg-info/top_level.txt cwltool.egg-info/zip-safe @@ -50,6 +51,7 @@ cwltool/schemas/draft-3/UserGuide.yml cwltool/schemas/draft-3/Workflow.yml cwltool/schemas/draft-3/concepts.md cwltool/schemas/draft-3/contrib.md +cwltool/schemas/draft-3/index.md cwltool/schemas/draft-3/index.yml cwltool/schemas/draft-3/intro.md cwltool/schemas/draft-3/invocation.md @@ -103,6 +105,7 @@ cwltool/schemas/v1.0/salad/schema_salad/metaschema/map_res_proc.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/map_res_schema.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/map_res_src.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/metaschema.yml +cwltool/schemas/v1.0/salad/schema_salad/metaschema/metaschema2.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/metaschema_base.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/salad.md cwltool/schemas/v1.0/salad/schema_salad/metaschema/typedsl_res.yml @@ -148,10 +151,15 @@ cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/vocab_res_src.yml tests/2.fasta tests/2.fastq tests/__init__.py +tests/cat.cwl +tests/cat2.cwl tests/echo-cwlrun-job.yaml tests/echo-job.yaml tests/echo.cwl +tests/env.cwl +tests/item1.yml tests/listing-job.yml +tests/output.txt tests/random_lines.cwl tests/random_lines_job.json tests/random_lines_mapping.cwl @@ -170,31 +178,49 @@ tests/test_docker_warning.py tests/test_examples.py tests/test_ext.py tests/test_fetch.py +tests/test_http_input.py tests/test_js_sandbox.py tests/test_pack.py tests/test_pathmapper.py tests/test_relax_path_checks.py tests/test_toolargparse.py tests/util.py +tests/value +tests/writeable.cwl tests/tmp1/tmp2/tmp3/.gitkeep +tests/wf/4c637b77-8130-4158-807c-ccc78ea7b563 tests/wf/badout1.cwl tests/wf/badout2.cwl tests/wf/badout3.cwl +tests/wf/c5b8320e-6e31-4bbb-8453-03b054b3254e tests/wf/cat.cwl +tests/wf/conditional.cwl tests/wf/default_path.cwl tests/wf/echo.cwl tests/wf/empty.ttl +tests/wf/emptyscatter.cwl +tests/wf/emptyscatter.yml tests/wf/expect_packed.cwl +tests/wf/foo1.txt +tests/wf/foo3.txt +tests/wf/fooa.txt +tests/wf/foob.txt +tests/wf/foofoo5.txt.txt +tests/wf/hello-workflow.cwl tests/wf/hello.txt +tests/wf/hello_single_tool.cwl tests/wf/listing_deep.cwl tests/wf/listing_none.cwl tests/wf/listing_shallow.cwl tests/wf/listing_v1_0.cwl +tests/wf/loop.cwl tests/wf/missing_cwlVersion.cwl tests/wf/mut.cwl tests/wf/mut2.cwl tests/wf/mut3.cwl +tests/wf/output.txt tests/wf/revsort-job.json +tests/wf/revsort-ovr2-job.json tests/wf/revsort.cwl tests/wf/revtool.cwl tests/wf/scatterfail.cwl @@ -204,5 +230,7 @@ tests/wf/updatedir_inplace.cwl tests/wf/updateval.cwl tests/wf/updateval.py tests/wf/updateval_inplace.cwl +tests/wf/value tests/wf/wffail.cwl +tests/wf/whale.txt tests/wf/wrong_cwlVersion.cwl \ No newline at end of file diff --git a/cwltool.egg-info/pbr.json b/cwltool.egg-info/pbr.json new file mode 100644 index 0000000..75a451d --- /dev/null +++ b/cwltool.egg-info/pbr.json @@ -0,0 +1 @@ +{"is_release": false, "git_version": "0bb2f210"} \ No newline at end of file diff --git a/cwltool.egg-info/requires.txt b/cwltool.egg-info/requires.txt index 5e81b5a..df3e6f8 100644 --- a/cwltool.egg-info/requires.txt +++ b/cwltool.egg-info/requires.txt @@ -1,11 +1,11 @@ setuptools -requests>=1.0 -ruamel.yaml<0.15,>=0.12.4 -rdflib<4.3.0,>=4.2.2 -shellescape<3.5,>=3.4.1 -schema-salad<3,>=2.6 -typing>=3.5.3 -six>=1.8.0 +requests >= 2.12.4 +ruamel.yaml >= 0.12.4, < 0.15 +rdflib >= 4.2.2, < 4.3.0 +shellescape >= 3.4.1, < 3.5 +schema-salad >= 2.6, < 3 +typing >= 3.5.3 +six >= 1.8.0 [deps] -galaxy-lib>=17.09.3 +galaxy-lib >= 17.09.3 diff --git a/cwltool/builder.py b/cwltool/builder.py index c74e2e6..20f67b2 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import copy import os +import logging from typing import Any, Callable, Dict, List, Text, Type, Union import six @@ -18,6 +19,8 @@ from .pathmapper import (PathMapper, get_listing, normalizeFilesDirs, from .stdfsaccess import StdFsAccess from .utils import aslist, get_feature, docker_windows_path_adjust, onWindows +_logger = logging.getLogger("cwltool") + AvroSchemaFromJSONData = avro.schema.make_avsc_object CONTENT_LIMIT = 64 * 1024 @@ -49,6 +52,7 @@ class Builder(object): self.make_fs_access = None # type: Type[StdFsAccess] self.debug = False # type: bool self.mutation_manager = None # type: MutationManager + self.force_docker_pull = False # type: bool # One of "no_listing", "shallow_listing", "deep_listing" # Will be default "no_listing" for CWL v1.1 @@ -145,18 +149,25 @@ class Builder(object): datum["secondaryFiles"] = [] for sf in aslist(schema["secondaryFiles"]): if isinstance(sf, dict) or "$(" in sf or "${" in sf: - secondary_eval = self.do_eval(sf, context=datum) - if isinstance(secondary_eval, string_types): - sfpath = {"location": secondary_eval, - "class": "File"} - else: - sfpath = secondary_eval - else: - sfpath = {"location": substitute(datum["location"], sf), "class": "File"} - if isinstance(sfpath, list): - datum["secondaryFiles"].extend(sfpath) + sfpath = self.do_eval(sf, context=datum) else: - datum["secondaryFiles"].append(sfpath) + sfpath = substitute(datum["basename"], sf) + for sfname in aslist(sfpath): + found = False + for d in datum["secondaryFiles"]: + if not d.get("basename"): + d["basename"] = d["location"][d["location"].rindex("/")+1:] + if d["basename"] == sfname: + found = True + if not found: + if isinstance(sfname, dict): + datum["secondaryFiles"].append(sfname) + else: + datum["secondaryFiles"].append({ + "location": datum["location"][0:datum["location"].rindex("/")+1]+sfname, + "basename": sfname, + "class": "File"}) + normalizeFilesDirs(datum["secondaryFiles"]) def _capture_files(f): @@ -245,4 +256,5 @@ class Builder(object): self.resources, context=context, pull_image=pull_image, timeout=self.timeout, + force_docker_pull=self.force_docker_pull, debug=self.debug) diff --git a/cwltool/draft2tool.py b/cwltool/draft2tool.py index d2288f4..682012c 100644 --- a/cwltool/draft2tool.py +++ b/cwltool/draft2tool.py @@ -35,7 +35,7 @@ ACCEPTLIST_EN_STRICT_RE = re.compile(r"^[a-zA-Z0-9._+-]+$") ACCEPTLIST_EN_RELAXED_RE = re.compile(r".*") # Accept anything ACCEPTLIST_RE = ACCEPTLIST_EN_STRICT_RE DEFAULT_CONTAINER_MSG="""We are on Microsoft Windows and not all components of this CWL description have a -container specified. This means that these steps will be executed in the default container, +container specified. This means that these steps will be executed in the default container, which is %s. Note, this could affect portability if this CWL description relies on non-POSIX features @@ -116,17 +116,26 @@ def revmap_file(builder, outdir, f): if not split.scheme: outdir = file_uri(str(outdir)) + # builder.outdir is the inner (container/compute node) output directory + # outdir is the outer (host/storage system) output directory + if "location" in f: if f["location"].startswith("file://"): path = convert_pathsep_to_unix(uri_file_path(f["location"])) revmap_f = builder.pathmapper.reversemap(path) + if revmap_f and not builder.pathmapper.mapper(revmap_f[0]).type.startswith("Writable"): f["basename"] = os.path.basename(path) - f["location"] = revmap_f[0] + f["location"] = revmap_f[1] elif path == builder.outdir: f["location"] = outdir elif path.startswith(builder.outdir): f["location"] = builder.fs_access.join(outdir, path[len(builder.outdir) + 1:]) + elif f["location"].startswith(outdir): + revmap_f = builder.pathmapper.reversemap(builder.fs_access.join(builder.outdir, f["location"][len(outdir) + 1:])) + if revmap_f and not builder.pathmapper.mapper(revmap_f[0]).type.startswith("Writable"): + f["basename"] = os.path.basename(path) + f["location"] = revmap_f[1] return f if "path" in f: @@ -190,7 +199,7 @@ class CommandLineTool(Process): super(CommandLineTool, self).__init__(toolpath_object, **kwargs) self.find_default_container = kwargs.get("find_default_container", None) - def makeJobRunner(self, use_container=True): # type: (Optional[bool]) -> JobBase + def makeJobRunner(self, use_container=True, **kwargs): # type: (Optional[bool], **Any) -> JobBase dockerReq, _ = self.get_requirement("DockerRequirement") if not dockerReq and use_container: if self.find_default_container: @@ -216,7 +225,8 @@ class CommandLineTool(Process): def makePathMapper(self, reffiles, stagedir, **kwargs): # type: (List[Any], Text, **Any) -> PathMapper - return PathMapper(reffiles, kwargs["basedir"], stagedir) + return PathMapper(reffiles, kwargs["basedir"], stagedir, + separateDirs=kwargs.get("separateDirs", True)) def updatePathmap(self, outdir, pathmap, fn): # type: (Text, PathMapper, Dict) -> None @@ -325,9 +335,10 @@ class CommandLineTool(Process): reffiles = copy.deepcopy(builder.files) - j = self.makeJobRunner(kwargs.get("use_container")) + j = self.makeJobRunner(**kwargs) j.builder = builder j.joborder = builder.job + j.make_pathmapper = self.makePathMapper j.stdin = None j.stderr = None j.stdout = None @@ -350,6 +361,7 @@ class CommandLineTool(Process): if "stagedir" in make_path_mapper_kwargs: make_path_mapper_kwargs = make_path_mapper_kwargs.copy() del make_path_mapper_kwargs["stagedir"] + builder.pathmapper = self.makePathMapper(reffiles, builder.stagedir, **make_path_mapper_kwargs) builder.requirements = j.requirements @@ -566,7 +578,12 @@ class CommandLineTool(Process): elif gb.startswith("/"): raise WorkflowException("glob patterns must not start with '/'") try: + prefix = fs_access.glob(outdir) r.extend([{"location": g, + "path": fs_access.join(builder.outdir, g[len(prefix[0])+1:]), + "basename": os.path.basename(g), + "nameroot": os.path.splitext(os.path.basename(g))[0], + "nameext": os.path.splitext(os.path.basename(g))[1], "class": "File" if fs_access.isfile(g) else "Directory"} for g in fs_access.glob(fs_access.join(outdir, gb))]) except (OSError, IOError) as e: @@ -576,12 +593,14 @@ class CommandLineTool(Process): raise for files in r: + rfile = files.copy() + revmap(rfile) if files["class"] == "Directory": ll = builder.loadListing or (binding and binding.get("loadListing")) if ll and ll != "no_listing": get_listing(fs_access, files, (ll == "deep_listing")) else: - with fs_access.open(files["location"], "rb") as f: + with fs_access.open(rfile["location"], "rb") as f: contents = b"" if binding.get("loadContents") or compute_checksum: contents = f.read(CONTENT_LIMIT) @@ -625,28 +644,39 @@ class CommandLineTool(Process): else: r = r[0] - # Ensure files point to local references outside of the run environment - adjustFileObjs(r, cast( # known bug in mypy - # https://github.com/python/mypy/issues/797 - Callable[[Any], Any], revmap)) - if "secondaryFiles" in schema: with SourceLine(schema, "secondaryFiles", WorkflowException): for primary in aslist(r): if isinstance(primary, dict): - primary["secondaryFiles"] = [] + primary.setdefault("secondaryFiles", []) + pathprefix = primary["path"][0:primary["path"].rindex("/")+1] for sf in aslist(schema["secondaryFiles"]): if isinstance(sf, dict) or "$(" in sf or "${" in sf: sfpath = builder.do_eval(sf, context=primary) - if isinstance(sfpath, string_types): - sfpath = revmap({"location": sfpath, "class": "File"}) + subst = False else: - sfpath = {"location": substitute(primary["location"], sf), "class": "File"} - + sfpath = sf + subst = True for sfitem in aslist(sfpath): - if fs_access.exists(sfitem["location"]): + if isinstance(sfitem, string_types): + if subst: + sfitem = {"path": substitute(primary["path"], sfitem)} + else: + sfitem = {"path": pathprefix+sfitem} + if "path" in sfitem and "location" not in sfitem: + revmap(sfitem) + if fs_access.isfile(sfitem["location"]): + sfitem["class"] = "File" + primary["secondaryFiles"].append(sfitem) + elif fs_access.isdir(sfitem["location"]): + sfitem["class"] = "Directory" primary["secondaryFiles"].append(sfitem) + # Ensure files point to local references outside of the run environment + adjustFileObjs(r, cast( # known bug in mypy + # https://github.com/python/mypy/issues/797 + Callable[[Any], Any], revmap)) + if not r and optional: r = None diff --git a/cwltool/expression.py b/cwltool/expression.py index 777e9f9..4db5aa1 100644 --- a/cwltool/expression.py +++ b/cwltool/expression.py @@ -153,8 +153,8 @@ def next_seg(remain, obj): # type: (Text, Any) -> Any return obj -def evaluator(ex, jslib, obj, fullJS=False, timeout=None, debug=False): - # type: (Text, Text, Dict[Text, Any], bool, int, bool) -> JSON +def evaluator(ex, jslib, obj, fullJS=False, timeout=None, force_docker_pull=False, debug=False): + # type: (Text, Text, Dict[Text, Any], bool, int, bool, bool) -> JSON m = param_re.match(ex) if m: if m.end(1)+1 == len(ex) and m.group(1) == "null": @@ -164,7 +164,7 @@ def evaluator(ex, jslib, obj, fullJS=False, timeout=None, debug=False): except Exception as w: raise WorkflowException("%s%s" % (m.group(1), w)) elif fullJS: - return sandboxjs.execjs(ex, jslib, timeout=timeout, debug=debug) + return sandboxjs.execjs(ex, jslib, timeout=timeout, force_docker_pull=force_docker_pull, debug=debug) else: raise sandboxjs.JavascriptException( "Syntax error in parameter reference '%s' or used Javascript code without specifying InlineJavascriptRequirement.", @@ -172,8 +172,9 @@ def evaluator(ex, jslib, obj, fullJS=False, timeout=None, debug=False): def interpolate(scan, rootvars, - timeout=None, fullJS=None, jslib="", debug=False): - # type: (Text, Dict[Text, Any], int, bool, Union[str, Text], bool) -> JSON + timeout=None, fullJS=None, jslib="",force_docker_pull=False, + debug=False): + # type: (Text, Dict[Text, Any], int, bool, Union[str, Text], bool, bool) -> JSON scan = scan.strip() parts = [] w = scanner(scan) @@ -182,7 +183,8 @@ def interpolate(scan, rootvars, if scan[w[0]] == '$': e = evaluator(scan[w[0] + 1:w[1]], jslib, rootvars, fullJS=fullJS, - timeout=timeout, debug=debug) + timeout=timeout, force_docker_pull=force_docker_pull, + debug=debug) if w[0] == 0 and w[1] == len(scan): return e leaf = json.dumps(e, sort_keys=True) @@ -200,8 +202,8 @@ def interpolate(scan, rootvars, def do_eval(ex, jobinput, requirements, outdir, tmpdir, resources, - context=None, pull_image=True, timeout=None, debug=False): - # type: (Union[dict, AnyStr], Dict[Text, Union[Dict, List, Text]], List[Dict[Text, Any]], Text, Text, Dict[Text, Union[int, Text]], Any, bool, int, bool) -> Any + context=None, pull_image=True, timeout=None, force_docker_pull=False, debug=False): + # type: (Union[dict, AnyStr], Dict[Text, Union[Dict, List, Text]], List[Dict[Text, Any]], Text, Text, Dict[Text, Union[int, Text]], Any, bool, int, bool, bool) -> Any runtime = copy.copy(resources) runtime["tmpdir"] = docker_windows_path_adjust(tmpdir) @@ -227,6 +229,7 @@ def do_eval(ex, jobinput, requirements, outdir, tmpdir, resources, timeout=timeout, fullJS=fullJS, jslib=jslib, + force_docker_pull=force_docker_pull, debug=debug) except Exception as e: raise WorkflowException("Expression evaluation error:\n%s" % e) diff --git a/cwltool/job.py b/cwltool/job.py index 7227e99..4e82b7e 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -132,6 +132,7 @@ class JobBase(object): self.name = None # type: Text self.command_line = None # type: List[Text] self.pathmapper = None # type: PathMapper + self.make_pathmapper = None # type: Callable[..., PathMapper] self.generatemapper = None # type: PathMapper self.collect_outputs = None # type: Union[Callable[[Any], Any], functools.partial[Any]] self.output_callback = None # type: Callable[[Any, Any], Any] @@ -142,7 +143,7 @@ class JobBase(object): self.stagedir = None # type: Text self.inplace_update = None # type: bool - def _setup(self): # type: () -> None + def _setup(self, kwargs): # type: (Dict) -> None if not os.path.exists(self.outdir): os.makedirs(self.outdir) @@ -154,8 +155,12 @@ class JobBase(object): "file." % (knownfile, self.pathmapper.mapper(knownfile)[0])) if self.generatefiles["listing"]: - self.generatemapper = PathMapper(cast(List[Any], self.generatefiles["listing"]), - self.outdir, self.outdir, separateDirs=False) + make_path_mapper_kwargs = kwargs + if "basedir" in make_path_mapper_kwargs: + make_path_mapper_kwargs = make_path_mapper_kwargs.copy() + del make_path_mapper_kwargs["basedir"] + self.generatemapper = self.make_pathmapper(cast(List[Any], self.generatefiles["listing"]), + self.outdir, basedir=self.outdir, separateDirs=False, **make_path_mapper_kwargs) _logger.debug(u"[job %s] initial work dir %s", self.name, json.dumps({p: self.generatemapper.mapper(p) for p in self.generatemapper.files()}, indent=4)) @@ -275,7 +280,7 @@ class CommandLineJob(JobBase): rm_tmpdir=True, move_outputs="move", **kwargs): # type: (bool, bool, bool, Text, **Any) -> None - self._setup() + self._setup(kwargs) env = self.environment if not os.path.exists(self.tmpdir): @@ -369,7 +374,7 @@ class DockerCommandLineJob(JobBase): "Docker is not available for this tool, try --no-container" " to disable Docker: %s" % e) - self._setup() + self._setup(kwargs) runtime = [u"docker", u"run", u"-i"] diff --git a/cwltool/main.py b/cwltool/main.py index 9eaeab1..526c6c2 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -226,7 +226,9 @@ def arg_parser(): # type: () -> argparse.ArgumentParser exgroup.add_argument("--make-template", action="store_true", help="Generate a template input object") - + parser.add_argument("--force-docker-pull", action="store_true", + default=False, help="Pull latest docker image even if" + " it is locally present", dest="force_docker_pull") parser.add_argument("workflow", type=Text, nargs="?", default=None) parser.add_argument("job_order", nargs=argparse.REMAINDER) diff --git a/cwltool/pack.py b/cwltool/pack.py index b51d7ad..898f548 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -157,4 +157,9 @@ def pack(document_loader, processobj, uri, metadata): import_embed(packed, set()) + if len(packed["$graph"]) == 1: + # duplicate 'cwlVersion' inside $graph when there is a single item + # because we're printing contents inside '$graph' rather than whole dict + packed["$graph"][0]["cwlVersion"] = packed["cwlVersion"] + return packed diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 6802a91..dd7c09a 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -5,6 +5,11 @@ import os import stat import uuid from functools import partial +from tempfile import NamedTemporaryFile + +import requests +from cachecontrol import CacheControl +from cachecontrol.caches import FileCache from typing import Any, Callable, Dict, Iterable, List, Set, Text, Tuple, Union import schema_salad.validate as validate @@ -139,6 +144,29 @@ def trim_listing(obj): if obj.get("location", "").startswith("file://") and "listing" in obj: del obj["listing"] +# Download http Files +def downloadHttpFile(httpurl): + # type: (Text) -> Text + cache_session = None + if "XDG_CACHE_HOME" in os.environ: + directory = os.environ["XDG_CACHE_HOME"] + elif "HOME" in os.environ: + directory = os.environ["HOME"] + else: + directory = os.path.expanduser('~') + + cache_session = CacheControl( + requests.Session(), + cache=FileCache( + os.path.join(directory, ".cache", "cwltool"))) + + r = cache_session.get(httpurl, stream=True) + with NamedTemporaryFile(mode='wb', delete=False) as f: + for chunk in r.iter_content(chunk_size=16384): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + r.close() + return f.name class PathMapper(object): """Mapping of files from relative path provided in the file to a tuple of @@ -208,14 +236,18 @@ class PathMapper(object): self._pathmap[obj["location"]] = MapperEnt(obj["contents"], tgt, "CreateFile", staged) else: with SourceLine(obj, "location", validate.ValidationException): - # Dereference symbolic links deref = ab - st = os.lstat(deref) - while stat.S_ISLNK(st.st_mode): - rl = os.readlink(deref) - deref = rl if os.path.isabs(rl) else os.path.join( - os.path.dirname(deref), rl) + if urllib.parse.urlsplit(deref).scheme in ['http','https']: + deref = downloadHttpFile(path) + else: + # Dereference symbolic links st = os.lstat(deref) + while stat.S_ISLNK(st.st_mode): + rl = os.readlink(deref) + deref = rl if os.path.isabs(rl) else os.path.join( + os.path.dirname(deref), rl) + st = os.lstat(deref) + self._pathmap[path] = MapperEnt(deref, tgt, "WritableFile" if copy else "File", staged) self.visitlisting(obj.get("secondaryFiles", []), stagedir, basedir, copy=copy, staged=staged) diff --git a/cwltool/process.py b/cwltool/process.py index 30fdefb..0e76607 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -560,6 +560,7 @@ class Process(six.with_metaclass(abc.ABCMeta, object)): builder.make_fs_access = kwargs.get("make_fs_access") or StdFsAccess builder.fs_access = builder.make_fs_access(kwargs["basedir"]) + builder.force_docker_pull = kwargs.get("force_docker_pull") loadListingReq, _ = self.get_requirement("http://commonwl.org/cwltool#LoadListingRequirement") if loadListingReq: diff --git a/cwltool/sandboxjs.py b/cwltool/sandboxjs.py index a4c92be..fa08519 100644 --- a/cwltool/sandboxjs.py +++ b/cwltool/sandboxjs.py @@ -53,12 +53,11 @@ def check_js_threshold_version(working_alias): return False -def new_js_proc(): - # type: () -> subprocess.Popen +def new_js_proc(force_docker_pull=False): + # type: (bool) -> subprocess.Popen res = resource_stream(__name__, 'cwlNodeEngine.js') nodecode = res.read() - required_node_version, docker = (False,)*2 nodejs = None trynodes = ("nodejs", "node") @@ -86,10 +85,11 @@ def new_js_proc(): try: nodeimg = "node:slim" global have_node_slim + if not have_node_slim: dockerimgs = subprocess.check_output(["docker", "images", "-q", nodeimg]).decode('utf-8') # if output is an empty string - if len(dockerimgs.split("\n")) <= 1: + if (len(dockerimgs.split("\n")) <= 1) or force_docker_pull: # pull node:slim docker container nodejsimg = subprocess.check_output(["docker", "pull", nodeimg]).decode('utf-8') _logger.info("Pulled Docker image %s %s", nodeimg, nodejsimg) @@ -124,10 +124,10 @@ def new_js_proc(): return nodejs -def execjs(js, jslib, timeout=None, debug=False): # type: (Union[Mapping, Text], Any, int, bool) -> JSON +def execjs(js, jslib, timeout=None, force_docker_pull=False, debug=False): # type: (Union[Mapping, Text], Any, int, bool, bool) -> JSON if not hasattr(localdata, "proc") or localdata.proc.poll() is not None or onWindows(): - localdata.proc = new_js_proc() + localdata.proc = new_js_proc(force_docker_pull=force_docker_pull) nodejs = localdata.proc diff --git a/cwltool/schemas/draft-3/index.md b/cwltool/schemas/draft-3/index.md new file mode 100644 index 0000000..e69de29 diff --git a/cwltool/schemas/v1.0/salad/schema_salad/metaschema/metaschema2.yml b/cwltool/schemas/v1.0/salad/schema_salad/metaschema/metaschema2.yml new file mode 100644 index 0000000..c928928 --- /dev/null +++ b/cwltool/schemas/v1.0/salad/schema_salad/metaschema/metaschema2.yml @@ -0,0 +1,317 @@ +$base: "https://w3id.org/cwl/salad#" + +$namespaces: + sld: "https://w3id.org/cwl/salad#" + dct: "http://purl.org/dc/terms/" + rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#" + rdfs: "http://www.w3.org/2000/01/rdf-schema#" + xsd: "http://www.w3.org/2001/XMLSchema#" + +$graph: + +- name: "Semantic_Annotations_for_Linked_Avro_Data" + type: documentation + doc: + - $include: salad.md + - $import: field_name.yml + - $import: ident_res.yml + - $import: link_res.yml + - $import: vocab_res.yml + - $include: import_include.md + +- name: "Link_Validation" + type: documentation + doc: | + # Link validation + + Once a document has been preprocessed, an implementation may validate + links. The link validation traversal may visit fields which the schema + designates as link fields and check that each URI references an existing + object in the current document, an imported document, file system, or + network resource. Failure to validate links may be a fatal error. Link + validation behavior for individual fields may be modified by `identity` and + `noLinkCheck` in the `jsonldPredicate` section of the field schema. + + +- name: "Schema_validation" + type: documentation + doc: "" + + +# - name: "JSON_LD_Context" +# type: documentation +# doc: | +# # Generating JSON-LD Context + +# How to generate the json-ld context... + + +- $import: metaschema_base.yml + +- name: JsonldPredicate + type: record + doc: | + Attached to a record field to define how the parent record field is handled for + URI resolution and JSON-LD context generation. + fields: + - name: _id + type: string? + jsonldPredicate: + _id: sld:_id + _type: "@id" + identity: true + doc: | + The predicate URI that this field corresponds to. + Corresponds to JSON-LD `@id` directive. + - name: _type + type: string? + doc: | + The context type hint, corresponds to JSON-LD `@type` directive. + + * If the value of this field is `@id` and `identity` is false or + unspecified, the parent field must be resolved using the link + resolution rules. If `identity` is true, the parent field must be + resolved using the identifier expansion rules. + + * If the value of this field is `@vocab`, the parent field must be + resolved using the vocabulary resolution rules. + + - name: _container + type: string? + doc: | + Structure hint, corresponds to JSON-LD `@container` directive. + - name: identity + type: boolean? + doc: | + If true and `_type` is `@id` this indicates that the parent field must + be resolved according to identity resolution rules instead of link + resolution rules. In addition, the field value is considered an + assertion that the linked value exists; absence of an object in the loaded document + with the URI is not an error. + - name: noLinkCheck + type: boolean? + doc: | + If true, this indicates that link validation traversal must stop at + this field. This field (it is is a URI) or any fields under it (if it + is an object or array) are not subject to link checking. + - name: mapSubject + type: string? + doc: | + If the value of the field is a JSON object, it must be transformed + into an array of JSON objects, where each key-value pair from the + source JSON object is a list item, the list items must be JSON objects, + and the key is assigned to the field specified by `mapSubject`. + - name: mapPredicate + type: string? + doc: | + Only applies if `mapSubject` is also provided. If the value of the + field is a JSON object, it is transformed as described in `mapSubject`, + with the addition that when the value of a map item is not an object, + the item is transformed to a JSON object with the key assigned to the + field specified by `mapSubject` and the value assigned to the field + specified by `mapPredicate`. + - name: refScope + type: int? + doc: | + If the field contains a relative reference, it must be resolved by + searching for valid document references in each successive parent scope + in the document fragment. For example, a reference of `foo` in the + context `#foo/bar/baz` will first check for the existence of + `#foo/bar/baz/foo`, followed by `#foo/bar/foo`, then `#foo/foo` and + then finally `#foo`. The first valid URI in the search order shall be + used as the fully resolved value of the identifier. The value of the + refScope field is the specified number of levels from the containing + identifer scope before starting the search, so if `refScope: 2` then + "baz" and "bar" must be stripped to get the base `#foo` and search + `#foo/foo` and the `#foo`. The last scope searched must be the top + level scope before determining if the identifier cannot be resolved. + - name: typeDSL + type: boolean? + doc: | + Field must be expanded based on the the Schema Salad type DSL. + + +- name: SpecializeDef + type: record + fields: + - name: specializeFrom + type: string + doc: "The data type to be replaced" + jsonldPredicate: + _id: "sld:specializeFrom" + _type: "@id" + refScope: 1 + + - name: specializeTo + type: string + doc: "The new data type to replace with" + jsonldPredicate: + _id: "sld:specializeTo" + _type: "@id" + refScope: 1 + + +- name: NamedType + type: record + abstract: true + fields: + - name: name + type: string + jsonldPredicate: "@id" + doc: "The identifier for this type" + + +- name: DocType + type: record + abstract: true + fields: + - name: doc + type: + - string? + - string[]? + doc: "A documentation string for this type, or an array of strings which should be concatenated." + jsonldPredicate: "rdfs:comment" + + - name: docParent + type: string? + doc: | + Hint to indicate that during documentation generation, documentation + for this type should appear in a subsection under `docParent`. + jsonldPredicate: + _id: "sld:docParent" + _type: "@id" + + - name: docChild + type: + - string? + - string[]? + doc: | + Hint to indicate that during documentation generation, documentation + for `docChild` should appear in a subsection under this type. + jsonldPredicate: + _id: "sld:docChild" + _type: "@id" + + - name: docAfter + type: string? + doc: | + Hint to indicate that during documentation generation, documentation + for this type should appear after the `docAfter` section at the same + level. + jsonldPredicate: + _id: "sld:docAfter" + _type: "@id" + + +- name: SchemaDefinedType + type: record + extends: DocType + doc: | + Abstract base for schema-defined types. + abstract: true + fields: + - name: jsonldPredicate + type: + - string? + - JsonldPredicate? + doc: | + Annotate this type with linked data context. + jsonldPredicate: sld:jsonldPredicate + + - name: documentRoot + type: boolean? + doc: | + If true, indicates that the type is a valid at the document root. At + least one type in a schema must be tagged with `documentRoot: true`. + + +- name: SaladRecordField + type: record + extends: RecordField + doc: "A field of a record." + fields: + - name: jsonldPredicate + type: + - string? + - JsonldPredicate? + doc: | + Annotate this type with linked data context. + jsonldPredicate: "sld:jsonldPredicate" + + +- name: SaladRecordSchema + type: record + extends: [NamedType, RecordSchema, SchemaDefinedType] + documentRoot: true + specialize: + RecordField: SaladRecordField + fields: + - name: abstract + type: boolean? + doc: | + If true, this record is abstract and may be used as a base for other + records, but is not valid on its own. + + - name: extends + type: + - string? + - string[]? + jsonldPredicate: + _id: "sld:extends" + _type: "@id" + refScope: 1 + doc: | + Indicates that this record inherits fields from one or more base records. + + - name: specialize + type: + - SpecializeDef[]? + doc: | + Only applies if `extends` is declared. Apply type specialization using the + base record as a template. For each field inherited from the base + record, replace any instance of the type `specializeFrom` with + `specializeTo`. + jsonldPredicate: + _id: "sld:specialize" + mapSubject: specializeFrom + mapPredicate: specializeTo + +- name: SaladEnumSchema + type: record + extends: [EnumSchema, SchemaDefinedType] + documentRoot: true + doc: | + Define an enumerated type. + fields: + - name: extends + type: + - string? + - string[]? + jsonldPredicate: + _id: "sld:extends" + _type: "@id" + refScope: 1 + doc: | + Indicates that this enum inherits symbols from a base enum. + + +- name: Documentation + type: record + extends: [NamedType, DocType] + documentRoot: true + doc: | + A documentation section. This type exists to facilitate self-documenting + schemas but has no role in formal validation. + fields: + type: + doc: {foo: "Must be `documentation`"} + type: + name: Documentation_symbol + type: enum + symbols: + - "sld:documentation" + jsonldPredicate: + _id: "sld:type" + _type: "@vocab" + typeDSL: true + refScope: 2 diff --git a/cwltool/stdfsaccess.py b/cwltool/stdfsaccess.py index df5056b..72016a8 100644 --- a/cwltool/stdfsaccess.py +++ b/cwltool/stdfsaccess.py @@ -13,6 +13,8 @@ from schema_salad.ref_resolver import file_uri, uri_file_path def abspath(src, basedir): # type: (Text, Text) -> Text if src.startswith(u"file://"): ab = six.text_type(uri_file_path(str(src))) + elif urllib.parse.urlsplit(src).scheme in ['http','https']: + return src else: if basedir.startswith(u"file://"): ab = src if os.path.isabs(src) else basedir+ '/'+ src diff --git a/setup.cfg b/setup.cfg index e30e824..b2580d2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,6 +13,7 @@ addopts = --ignore cwltool/schemas testpaths = tests [egg_info] -tag_build = .20170803160545 +tag_build = .20170810192106 tag_date = 0 +tag_svn_revision = 0 diff --git a/setup.py b/setup.py index 1045b3d..989ccb2 100755 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ setup(name='cwltool', include_package_data=True, install_requires=[ 'setuptools', - 'requests >= 1.0', + 'requests >= 2.12.4', 'ruamel.yaml >= 0.12.4, < 0.15', 'rdflib >= 4.2.2, < 4.3.0', 'shellescape >= 3.4.1, < 3.5', diff --git a/tests/cat.cwl b/tests/cat.cwl new file mode 100644 index 0000000..93af517 --- /dev/null +++ b/tests/cat.cwl @@ -0,0 +1,8 @@ +cwlVersion: v1.0 +class: CommandLineTool +inputs: + - id: inp + type: File + inputBinding: {} +outputs: [] +baseCommand: cat diff --git a/tests/cat2.cwl b/tests/cat2.cwl new file mode 100644 index 0000000..29d42a8 --- /dev/null +++ b/tests/cat2.cwl @@ -0,0 +1,9 @@ +cwlVersion: v1.0 +class: CommandLineTool +inputs: + - id: inp + type: Directory + inputBinding: + valueFrom: $(self.listing[0].path) +outputs: [] +baseCommand: cat diff --git a/tests/env.cwl b/tests/env.cwl new file mode 100644 index 0000000..97d2b31 --- /dev/null +++ b/tests/env.cwl @@ -0,0 +1,5 @@ +class: CommandLineTool +cwlVersion: v1.0 +inputs: [] +outputs: [] +baseCommand: env \ No newline at end of file diff --git a/tests/item1.yml b/tests/item1.yml new file mode 100644 index 0000000..6f30df9 --- /dev/null +++ b/tests/item1.yml @@ -0,0 +1,3 @@ +inp: + class: File + location: "keep:6bf7c0d0f95d3ca89ce7a1d64ea4ff0a+56" \ No newline at end of file diff --git a/tests/output.txt b/tests/output.txt new file mode 100644 index 0000000..97b096c --- /dev/null +++ b/tests/output.txt @@ -0,0 +1,16 @@ +yrev hsirehc ,rehto ro emit emos ,eerged rieht ni nem lla tsomla ,ti +ylteiuq I ;drows sih nopu flesmih sworht otaC hsiruolf lacihposolihp +yllacidohtem dna ,teerts eht otni gnippets yletarebiled morf em tneverp +wenk tub yeht fI .siht ni gnisirprus gnihton si erehT .pihs eht ot ekat +teg sopyh ym revenehw yllaicepse dna ;teem I larenuf yreve fo raer eht +pu gnignirb dna ,sesuoheraw niffoc erofeb gnisuap yliratnulovni flesym +ot elpicnirp larom gnorts a seriuqer ti taht ,em fo dnah reppu na hcus +no em tseretni ot ralucitrap gnihton dna ,esrup ym ni yenom on ro elttil +gnivah--ylesicerp gnol woh dnim reven--oga sraey emoS .leamhsI em llaC +gnitaluger dna neelps eht ffo gnivird fo evah I yaw a si tI .dlrow eht +fo trap yretaw eht ees dna elttil a tuoba lias dluow I thguoht I ,erohs +dnif I revenehw ;luos ym ni rebmevoN ylzzird ,pmad a si ti revenehw +aes ot teg ot emit hgih ti tnuocca I ,neht--ffo stah s'elpoep gnikconk +a htiW .llab dna lotsip rof etutitsbus ym si sihT .nac I sa noos sa +;htuom eht tuoba mirg gniworg flesym dnif I revenehW .noitalucric eht +.em htiw naeco eht sdrawot sgnileef emas eht ylraen diff --git a/tests/test_http_input.py b/tests/test_http_input.py new file mode 100644 index 0000000..e3a2981 --- /dev/null +++ b/tests/test_http_input.py @@ -0,0 +1,26 @@ +from __future__ import absolute_import +import unittest +import os +import tempfile +from cwltool.pathmapper import PathMapper + + +class TestHttpInput(unittest.TestCase): + def test_http_path_mapping(self): + class SubPathMapper(PathMapper): + def __init__(self, referenced_files, basedir, stagedir): + super(SubPathMapper, self).__init__(referenced_files, basedir, stagedir) + input_file_path = "https://raw.githubusercontent.com/common-workflow-language/cwltool/master/tests/2.fasta" + tempdir = tempfile.mkdtemp() + base_file = [{ + "class": "File", + "location": "https://raw.githubusercontent.com/common-workflow-language/cwltool/master/tests/2.fasta", + "basename": "chr20.fa" + }] + path_map_obj = SubPathMapper(base_file, os.getcwd(), tempdir) + + self.assertIn(input_file_path,path_map_obj._pathmap) + assert os.path.exists(path_map_obj._pathmap[input_file_path].resolved) == 1 + with open(path_map_obj._pathmap[input_file_path].resolved) as f: + self.assertIn(">Sequence 561 BP; 135 A; 106 C; 98 G; 222 T; 0 other;",f.read()) + f.close() \ No newline at end of file diff --git a/tests/test_pack.py b/tests/test_pack.py index e49cd21..34a14e6 100644 --- a/tests/test_pack.py +++ b/tests/test_pack.py @@ -5,6 +5,7 @@ import unittest from functools import partial import cwltool.pack +from cwltool.main import print_pack as print_pack import cwltool.workflow from cwltool.load_tool import fetch_document, validate_document from cwltool.main import makeRelative @@ -33,3 +34,29 @@ class TestPack(unittest.TestCase): del expect_packed["$schemas"] self.assertEqual(expect_packed, packed) + + def test_pack_missing_cwlVersion(self): + """Test to ensure the generated pack output is not missing + the `cwlVersion` in case of single tool workflow and single step workflow""" + # Since diff is longer than 3174 characters + self.maxDiff = None + + # Testing single tool workflow + document_loader, workflowobj, uri = fetch_document( + get_data("tests/wf/hello_single_tool.cwl")) + document_loader, avsc_names, processobj, metadata, uri = validate_document( + document_loader, workflowobj, uri) + # generate pack output dict + packed = json.loads(print_pack(document_loader, processobj, uri, metadata)) + + self.assertEqual('v1.0', packed["cwlVersion"]) + + # Testing single step workflow + document_loader, workflowobj, uri = fetch_document( + get_data("tests/wf/hello-workflow.cwl")) + document_loader, avsc_names, processobj, metadata, uri = validate_document( + document_loader, workflowobj, uri) + # generate pack output dict + packed = json.loads(print_pack(document_loader, processobj, uri, metadata)) + + self.assertEqual('v1.0', packed["cwlVersion"]) diff --git a/tests/value b/tests/value new file mode 100644 index 0000000..dc7b54a --- /dev/null +++ b/tests/value @@ -0,0 +1 @@ +33 \ No newline at end of file diff --git a/tests/wf/4c637b77-8130-4158-807c-ccc78ea7b563 b/tests/wf/4c637b77-8130-4158-807c-ccc78ea7b563 new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/tests/wf/4c637b77-8130-4158-807c-ccc78ea7b563 @@ -0,0 +1 @@ +2 diff --git a/tests/wf/c5b8320e-6e31-4bbb-8453-03b054b3254e b/tests/wf/c5b8320e-6e31-4bbb-8453-03b054b3254e new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/tests/wf/c5b8320e-6e31-4bbb-8453-03b054b3254e @@ -0,0 +1 @@ +1 diff --git a/tests/wf/conditional.cwl b/tests/wf/conditional.cwl new file mode 100644 index 0000000..57f3654 --- /dev/null +++ b/tests/wf/conditional.cwl @@ -0,0 +1,22 @@ +cwlVersion: v1.0 +class: Workflow +inputs: + a: int +outputs: [] +steps: + step1: + in: + a: a + cond: + - case: $(a > 1) + run: greater.cwl + - case: default + run: default.cwl + out: [out] + step2: + in: + a: a + cond: + "$(inputs.a > 1)": greater.cwl + default: default.cwl + out: [out] diff --git a/tests/wf/emptyscatter.cwl b/tests/wf/emptyscatter.cwl new file mode 100644 index 0000000..10a17c1 --- /dev/null +++ b/tests/wf/emptyscatter.cwl @@ -0,0 +1,18 @@ +class: Workflow +cwlVersion: v1.0 +requirements: + ScatterFeatureRequirement: {} +inputs: + inp: string[] +outputs: + f: + type: File[] + outputSource: boo/out + +steps: + boo: + in: + r: inp + out: [out] + scatter: r + run: echo.cwl diff --git a/tests/wf/emptyscatter.yml b/tests/wf/emptyscatter.yml new file mode 100644 index 0000000..dd3e18c --- /dev/null +++ b/tests/wf/emptyscatter.yml @@ -0,0 +1 @@ +inp: [] \ No newline at end of file diff --git a/tests/wf/foo1.txt b/tests/wf/foo1.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/tests/wf/foo1.txt @@ -0,0 +1 @@ +1 diff --git a/tests/wf/foo3.txt b/tests/wf/foo3.txt new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/tests/wf/foo3.txt @@ -0,0 +1 @@ +3 diff --git a/tests/wf/fooa.txt b/tests/wf/fooa.txt new file mode 100644 index 0000000..7898192 --- /dev/null +++ b/tests/wf/fooa.txt @@ -0,0 +1 @@ +a diff --git a/tests/wf/foob.txt b/tests/wf/foob.txt new file mode 100644 index 0000000..6178079 --- /dev/null +++ b/tests/wf/foob.txt @@ -0,0 +1 @@ +b diff --git a/tests/wf/foofoo5.txt.txt b/tests/wf/foofoo5.txt.txt new file mode 100644 index 0000000..2182af9 --- /dev/null +++ b/tests/wf/foofoo5.txt.txt @@ -0,0 +1 @@ +foo5.txt diff --git a/tests/wf/hello-workflow.cwl b/tests/wf/hello-workflow.cwl new file mode 100644 index 0000000..08c333b --- /dev/null +++ b/tests/wf/hello-workflow.cwl @@ -0,0 +1,38 @@ +#!/usr/bin/env cwl-runner + +cwlVersion: v1.0 +class: Workflow + +label: "Hello World" +doc: "Outputs a message using echo" + +inputs: + usermessage: string + +outputs: + response: + outputSource: step0/response + type: File + +steps: + step0: + run: + class: CommandLineTool + inputs: + message: + type: string + doc: "The message to print" + default: "Hello World" + inputBinding: + position: 1 + baseCommand: echo + arguments: + - "-n" + - "-e" + stdout: response.txt + outputs: + response: + type: stdout + in: + message: usermessage + out: [response] \ No newline at end of file diff --git a/tests/wf/hello_single_tool.cwl b/tests/wf/hello_single_tool.cwl new file mode 100644 index 0000000..af0c4de --- /dev/null +++ b/tests/wf/hello_single_tool.cwl @@ -0,0 +1,9 @@ +cwlVersion: v1.0 +class: CommandLineTool +baseCommand: echo +inputs: + message: + type: string + inputBinding: + position: 1 +outputs: [] diff --git a/tests/wf/loop.cwl b/tests/wf/loop.cwl new file mode 100644 index 0000000..b56ac8b --- /dev/null +++ b/tests/wf/loop.cwl @@ -0,0 +1,36 @@ +cwlVersion: v1.0 +class: Workflow +inputs: + a: int +outputs: [] +steps: + step1: + in: + a: a + out: [out] + while: $(inputs.a > 1) + run: blah.cwl + update: + - source: out + dest: a + step2: + in: + a: a + out: [out] + while: $(inputs.a > 1) + run: blah.cwl + update: + a: out + step3: + in: + a: a + out: + out: + default: "foo" + scatter: a + while: $(inputs.a < 3) + cond: + "$(inputs.a < 1)": inc.cwl + else: default.cwl + update: + a: out diff --git a/tests/wf/output.txt b/tests/wf/output.txt new file mode 100644 index 0000000..97b096c --- /dev/null +++ b/tests/wf/output.txt @@ -0,0 +1,16 @@ +yrev hsirehc ,rehto ro emit emos ,eerged rieht ni nem lla tsomla ,ti +ylteiuq I ;drows sih nopu flesmih sworht otaC hsiruolf lacihposolihp +yllacidohtem dna ,teerts eht otni gnippets yletarebiled morf em tneverp +wenk tub yeht fI .siht ni gnisirprus gnihton si erehT .pihs eht ot ekat +teg sopyh ym revenehw yllaicepse dna ;teem I larenuf yreve fo raer eht +pu gnignirb dna ,sesuoheraw niffoc erofeb gnisuap yliratnulovni flesym +ot elpicnirp larom gnorts a seriuqer ti taht ,em fo dnah reppu na hcus +no em tseretni ot ralucitrap gnihton dna ,esrup ym ni yenom on ro elttil +gnivah--ylesicerp gnol woh dnim reven--oga sraey emoS .leamhsI em llaC +gnitaluger dna neelps eht ffo gnivird fo evah I yaw a si tI .dlrow eht +fo trap yretaw eht ees dna elttil a tuoba lias dluow I thguoht I ,erohs +dnif I revenehw ;luos ym ni rebmevoN ylzzird ,pmad a si ti revenehw +aes ot teg ot emit hgih ti tnuocca I ,neht--ffo stah s'elpoep gnikconk +a htiW .llab dna lotsip rof etutitsbus ym si sihT .nac I sa noos sa +;htuom eht tuoba mirg gniworg flesym dnif I revenehW .noitalucric eht +.em htiw naeco eht sdrawot sgnileef emas eht ylraen diff --git a/tests/wf/revsort-ovr2-job.json b/tests/wf/revsort-ovr2-job.json new file mode 100644 index 0000000..4928b9d --- /dev/null +++ b/tests/wf/revsort-ovr2-job.json @@ -0,0 +1,15 @@ +{ + "input": { + "class": "File", + "location": "whale.txt" + }, + "cwltool:overrides": [ + { + "overrideTarget": "sorttool.cwl", + "override": [{ + "class": "DockerRequirement", + "dockerPull": "ubuntu:14.04" + }] + } + ] +} diff --git a/tests/wf/value b/tests/wf/value new file mode 100644 index 0000000..43c451e --- /dev/null +++ b/tests/wf/value @@ -0,0 +1 @@ +54 \ No newline at end of file diff --git a/tests/wf/whale.txt b/tests/wf/whale.txt new file mode 100644 index 0000000..425d1ed --- /dev/null +++ b/tests/wf/whale.txt @@ -0,0 +1,16 @@ +Call me Ishmael. Some years ago--never mind how long precisely--having +little or no money in my purse, and nothing particular to interest me on +shore, I thought I would sail about a little and see the watery part of +the world. It is a way I have of driving off the spleen and regulating +the circulation. Whenever I find myself growing grim about the mouth; +whenever it is a damp, drizzly November in my soul; whenever I find +myself involuntarily pausing before coffin warehouses, and bringing up +the rear of every funeral I meet; and especially whenever my hypos get +such an upper hand of me, that it requires a strong moral principle to +prevent me from deliberately stepping into the street, and methodically +knocking people's hats off--then, I account it high time to get to sea +as soon as I can. This is my substitute for pistol and ball. With a +philosophical flourish Cato throws himself upon his sword; I quietly +take to the ship. There is nothing surprising in this. If they but knew +it, almost all men in their degree, some time or other, cherish very +nearly the same feelings towards the ocean with me. diff --git a/tests/writeable.cwl b/tests/writeable.cwl new file mode 100644 index 0000000..a635e79 --- /dev/null +++ b/tests/writeable.cwl @@ -0,0 +1,11 @@ +class: CommandLineTool +cwlVersion: v1.0 +baseCommand: [ls, -l] +requirements: + InitialWorkDirRequirement: + listing: + - entry: $(inputs.blah) + writable: true +inputs: + blah: Directory +outputs: [] \ No newline at end of file -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/cwltool.git _______________________________________________ debian-med-commit mailing list debian-med-commit@lists.alioth.debian.org http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/debian-med-commit