Merge branch 'shell-commands'

Author: Peter Amstutz

cwltool/draft2tool.py

@@ -20,6 +20,7 @@
 import urlparse
 import tempfile
 from builder import CONTENT_LIMIT, substitute
+import shellescape
 
 _logger = logging.getLogger("cwltool")
 
@@ -155,7 +156,18 @@ def job(self, joborder, input_basedir, output_callback, **kwargs):
             for t in evr["envDef"]:
                 j.environment[t["envName"]] = builder.do_eval(t["envValue"])
 
-        j.command_line = flatten(map(builder.generate_arg, builder.bindings))
+        shellcmd, _ = self.get_requirement("ShellCommandRequirement")
+        if shellcmd:
+            cmd = []
+            for b in builder.bindings:
+                arg = builder.generate_arg(b)
+                if b.get("shellQuote", True):
+                    arg = [shellescape.quote(a) for a in aslist(arg)]
+                cmd.extend(aslist(arg))
+            j.command_line = ["/bin/sh", "-c", " ".join(cmd)]
+            print j.command_line
+        else:
+            j.command_line = flatten(map(builder.generate_arg, builder.bindings))
 
         j.pathmapper = builder.pathmapper
         j.collect_outputs = functools.partial(self.collect_output_ports, self.tool["outputs"], builder)

cwltool/process.py

@@ -28,7 +28,8 @@
                                 "CreateFileRequirement",
                                 "ScatterFeatureRequirement",
                                 "SubworkflowFeatureRequirement",
-                                "MultipleInputFeatureRequirement"]
+                                "MultipleInputFeatureRequirement",
+                                "ShellCommandRequirement"]
 
 def get_schema():
     f = resource_stream(__name__, 'schemas/draft-3/cwl-avro.yml')

cwltool/schemas/CODE_OF_CONDUCT.md

@@ -0,0 +1,108 @@
+CWL Code of Conduct
+===================
+
+The CWL Project is dedicated to providing a harassment-free experience for
+everyone. We do not tolerate harassment of participants in any form.
+
+This code of conduct applies to all CWL Project spaces both online and off: the
+Google Group, the Gitter chat room, the Google Hangouts chats, and any other
+CWL spaces. Anyone who violates this code of conduct may be sanctioned or
+expelled from these spaces at the discretion of the CWL Leadership Team.
+
+Some CWL Project spaces may have additional rules in place, which will be
+made clearly available to participants. Participants are responsible for
+knowing and abiding by these rules.
+
+Harassment includes, but is not limited to:
+
+ - Offensive comments related to gender, gender identity and expression, sexual
+orientation, disability, mental illness, neuro(a)typicality, physical
+appearance, body size, age, race, or religion.
+ - Unwelcome comments regarding a person’s lifestyle choices and practices,
+including those related to food, health, parenting, drugs, and employment.
+ - Deliberate misgendering or use of [dead](https://www.quora.com/What-is-deadnaming/answer/Nancy-C-Walker)
+or rejected names.
+ - Gratuitous or off-topic sexual images or behaviour  in spaces where they’re not
+appropriate.
+ - Physical contact and simulated physical contact (eg, textual descriptions like
+“\*hug\*” or “\*backrub\*”) without consent or after a request to stop.
+ - Threats of violence.
+ - Incitement of violence towards any individual, including encouraging a person
+to commit suicide or to engage in self-harm.
+ - Deliberate intimidation.
+ - Stalking or following.
+ - Harassing photography or recording, including logging online activity for
+harassment purposes.
+ - Sustained disruption of discussion.
+ - Unwelcome sexual attention.
+ - Pattern of inappropriate social contact, such as requesting/assuming
+inappropriate levels of intimacy with others
+ - Continued one-on-one communication after requests to cease.
+ - Deliberate “outing” of any aspect of a person’s identity without their consent
+except as necessary to protect vulnerable people from intentional abuse.
+ - Publication of non-harassing private communication.
+
+The CWL Project prioritizes marginalized people’s safety over privileged
+people’s comfort. The CWL Leadeship Team will not act on complaints regarding:
+
+ - ‘Reverse’ -isms, including ‘reverse racism,’ ‘reverse sexism,’ and ‘cisphobia’
+ - Reasonable communication of boundaries, such as “leave me alone,” “go away,” or
+“I’m not discussing this with you.”
+ - Communicating in a [tone](http://geekfeminism.wikia.com/wiki/Tone_argument)
+you don’t find congenial
+
+Reporting
+---------
+
+If you are being harassed by a member of the CWL Project, notice that someone
+else is being harassed, or have any other concerns, please contact the CWL
+Leadership Team at [email protected]. If person who is harassing
+you is on the team, they will recuse themselves from handling your incident. We
+will respond as promptly as we can.
+
+This code of conduct applies to CWL Project spaces, but if you are being
+harassed by a member of CWL Project outside our spaces, we still want to
+know about it. We will take all good-faith reports of harassment by CWL Project
+members, especially the CWL Leadership Team, seriously. This includes harassment
+outside our spaces and harassment that took place at any point in time. The
+abuse team reserves the right to exclude people from the CWL Project based on
+their past behavior, including behavior outside CWL Project spaces and
+behavior towards people who are not in the CWL Project.
+
+In order to protect volunteers from abuse and burnout, we reserve the right to
+reject any report we believe to have been made in bad faith. Reports intended
+to silence legitimate criticism may be deleted without response.
+
+We will respect confidentiality requests for the purpose of protecting victims
+of abuse. At our discretion, we may publicly name a person about whom we’ve
+received harassment complaints, or privately warn third parties about them, if
+we believe that doing so will increase the safety of CWL Project members or
+the general public. We will not name harassment victims without their
+affirmative consent.
+
+Consequences
+------------
+
+Participants asked to stop any harassing behavior are expected to comply
+immediately.
+
+If a participant engages in harassing behavior, the CWL Leadership Team may
+take any action they deem appropriate, up to and including expulsion from all
+CWL Project spaces and identification of the participant as a harasser to other
+CWL Project members or the general public.
+
+This anti-harassment policy is based on the [example policy from the Geek
+Feminism wiki](http://geekfeminism.wikia.com/wiki/Community_anti-harassment/Policy),
+created by the Geek Feminism community.
+
+CWL Leadership Team
+-------------------
+
+As a stop gap measure until a more formal governance structure is adopted, the
+following individuals make up the leadership of the CWL Project: Peter Amstutz,
+John Chilton, Michael R. Crusoe, and Nebojša Tijanić.
+
+To report an issue with anyone on the team you can escalate to Ward Vandewege
+ (Curoverse) [email protected], Anton Nekrutenko (Galaxy)
+anton AT bx DOT psu DOT edu, C. Titus Brown (UC Davis) [email protected], or
+Brandi Davis-Dusenbery (Seven Bridges Genomics) [email protected].

cwltool/schemas/CONFORMANCE_TESTS.md

@@ -14,7 +14,7 @@ Test [1/1]
 All tests passed
 ```
 
-The cwltool relies on node.js. To install on ubuntu, 
+The cwltool relies on node.js. To install on ubuntu,
 
 ```
 $ sudo apt-get install nodejs npm
@@ -51,3 +51,9 @@ All tests passed
 
 All tool tests succeeded
 ```
+
+_NOTE_: For running on OSX systems, you'll need to install coreutils via brew. This will add to your
+system some needed GNU-like tools like `greadlink`.
+
+1. If you haven't already, install [brew](http://brew.sh/) package manager in your mac
+2. Run `brew install coreutils`

cwltool/schemas/README.md

@@ -26,22 +26,38 @@ http://common-workflow-language.github.io/draft-3/
 
 ## Implementations
 
-Implementations working on or currently capable of running the Common Workflow Language:
-
-* [Reference implementation (Python)](https://github.com/common-workflow-language/cwltool)
-* [Rabix](https://github.com/rabix/rabix)
-* [Arvados](https://arvados.org)
-* [Parallel Recipes](https://github.com/yvdriess/precipes)
+Some of the software supporting running Common Workflow Language tools or workflows:
+
+* [cwltool (reference implementation)](https://github.com/common-workflow-language/cwltool),
+  [cwltool wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/cwltool-%28reference-implementation%29)
+* [Rabix](https://github.com/rabix/rabix),
+  [Rabix wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Rabix)
+* [Arvados](https://arvados.org),
+  [Arvados wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Arvados)
+* [Galaxy](https://github.com/common-workflow-language/Galaxy),
+  [Galaxy wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Galaxy)
+* [Parallel Recipes](https://github.com/yvdriess/precipes),
+  [Parallel Recipes wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Parallel-Recipes)
+* [Toil](https://github.com/BD2KGenomics/toil),
+  [Toil wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Toil)
+* [CancerCollaboratory](https://github.com/CancerCollaboratory),
+  [CancerCollaboratory wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/CancerCollaboratory)
+* [Airflow (SciDAP)](https://github.com/SciDAP/scidap),
+  [Airflow wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/SciDAP)
+* [cwl2script](https://github.com/common-workflow-language/cwl2script),
+  [cwl2script page](https://github.com/common-workflow-language/common-workflow-language/wiki/cwl2script)
 
 ## Examples
 
 [Github repository of example tools and workflows.](https://github.com/common-workflow-language/workflows)
 
 ## Development and testing
 
-The CWL effort is on Github:
+The [CWL project](https://github.com/common-workflow-language/common-workflow-language)
+is on Github.
 
-https://github.com/common-workflow-language/common-workflow-language
+[![GitHub
+stars](https://img.shields.io/github/stars/common-workflow-language/common-workflow-language.svg)](https://github.com/common-workflow-language/common-workflow-language/stargazers)
 
 There is a Jenkins server maintained by Curoverse that runs tests for the
 reference implementation, builds and uploads packages, and builds and uploads
@@ -62,6 +78,21 @@ and send a pull request!
 
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/common-workflow-language/common-workflow-language?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
+### Code of Conduct
+
+The CWL Project is dedicated to providing a harassment-free experience for
+everyone, regardless of gender, gender identity and expression, sexual
+orientation, disability, physical appearance, body size, age, race, or
+religion. We do not tolerate harassment of participants in any form.
+
+This code of conduct applies to all CWL Project spaces, including the Google
+Group, the Gitter chat room, the Google Hangouts chats, both online and off.
+Anyone who violates this code of conduct may be sanctioned or expelled from
+these spaces at the discretion of the leadership team.
+
+For more details, see our [Code of
+Conduct](https://github.com/common-workflow-language/common-workflow-language/blob/master/CODE_OF_CONDUCT.md)
+
 ## Participating Organizations
 
 * [Curoverse](http://curoverse.com)
@@ -73,6 +104,7 @@ and send a pull request!
 * [University of California Davis](http://ucdavis.edu)
 * [University of California Santa Cruz](https://cbse.soe.ucsc.edu/research/bioinfo)
 * [Harvard Chan School of Public Health](http://www.hsph.harvard.edu/)
+* [Cincinnati Children's Hospital Medical Center](http://www.cincinnatichildrens.org/)
 * [Broad Institute](https://www.broadinstitute.org)
 * [BioDatomics](http://www.biodatomics.com/)
 
@@ -96,3 +128,4 @@ and send a pull request!
 * Josh Randall <[email protected]>
 * Andrey Kartashov <[email protected]>
 * Dan Leehr <[email protected]>
+* Andrey Kartashov <[email protected]>

cwltool/schemas/draft-2/draft-2/node-engine.cwl

@@ -3,7 +3,7 @@
     requirements: [
       {
         class: DockerRequirement,
-        dockerImageId: commonworkflowlanguage/nodejs-engine
+        dockerPull: commonworkflowlanguage/nodejs-engine
       }
     ],
     engineCommand: cwlNodeEngine.js

cwltool/schemas/draft-3/conformance_test_draft-3.yaml

@@ -260,4 +260,27 @@
       size: 1111
   tool: draft-3/rename.cwl
   doc: |
-    Test CreateFileRequirement with expression in filename.
+    Test CreateFileRequirement with expression in filename.
+
+- job: draft-3/schemadef-job.json
+  output:
+    output:
+        path: output.txt
+        size: 12
+        class: File
+        checksum: "sha1$f12e6cfe70f3253f70b0dbde17c692e7fb0f1e5e"
+  tool: draft-3/schemadef-tool.cwl
+  doc: |
+    Test SchemaDefRequirement definition used in tool parameter
+
+
+- job: draft-3/schemadef-job.json
+  output:
+    output:
+        path: output.txt
+        size: 12
+        class: File
+        checksum: "sha1$f12e6cfe70f3253f70b0dbde17c692e7fb0f1e5e"
+  tool: draft-3/schemadef-wf.cwl
+  doc: |
+    Test SchemaDefRequirement definition used in workflow parameter

cwltool/schemas/draft-3/cwl-avro.yml

@@ -34,7 +34,7 @@
 
     * Luka Stojanovic <[email protected]>, Seven Bridges Genomics
     * John Chilton <[email protected]>, Galaxy Project, Pennsylvania State University
-    * Michael R. Crusoe <mcrusoe@msu.edu>, Michigan State University
+    * Michael R. Crusoe <crusoe@ucdavis.edu>, University of California, Davis
     * Hervé Ménager <[email protected]>, Institut Pasteur
     * Maxim Mikheev <[email protected]>, BioDatomics
     * Stian Soiland-Reyes [[email protected]](mailto:[email protected]), University of Manchester
@@ -1007,7 +1007,7 @@
         Defines the input parameters of the process.  The process is ready to
         run when all required input parameters are associated with concrete
         values.  Input parameters include a schema for each parameter which is
-        used to validate the input object.  It may also be used build a user
+        used to validate the input object.  It may also be used to build a user
         interface for constructing the input object.
     - name: outputs
       type:
@@ -1143,7 +1143,12 @@
 
         When a binding is part of the `CommandLineTool.arguments` field,
         the `valueFrom` field is required.
-
+    - name: shellQuote
+      type: ["null", boolean]
+      doc: |
+        If `ShellCommandRequirement` is in the requirements for the current command,
+        this controls whether the value is quoted on the command line (default is true).
+        Use `shellQuote: false` to inject metacharacters for operations such as pipes.
 
 - type: record
   name: CommandOutputBinding
@@ -1692,7 +1697,7 @@
 
     A "scatter" operation specifies that the associated workflow step or
     subworkflow should execute separately over a list of input elements.  Each
-    job making up a scatter operaution is independent and may be executed
+    job making up a scatter operation is independent and may be executed
     concurrently.
 
     The `scatter` field specifies one or more input parameters which will be
@@ -2044,3 +2049,15 @@
         the expression engine.  The semantics of this field are defined by the
         underlying expression engine.  Intended for uses such as providing
         function definitions that will be called from CWL expressions.
+
+- type: record
+  name: ShellCommandRequirement
+  extends: "#ProcessRequirement"
+  doc: |
+    Modify the behavior of CommandLineTool to generate a single string
+    containing a shell command line.  Each item in the argument list must be
+    joined into a string separated by single spaces and quoted to prevent
+    intepretation by the shell, unless `CommandLineBinding` for that argument
+    contains `shellQuote: false`.  If `shellQuote: false` is specified, the
+    argument is joined into the command string without quoting, which allows
+    the use of shell metacharacters such as `|` for pipes.

cwltool/schemas/draft-3/draft-3/schemadef-job.json

@@ -0,0 +1,6 @@
+{
+    "hello": {
+        "a": "hello",
+        "b": "world"
+    }
+}

cwltool/schemas/draft-3/draft-3/schemadef-tool.cwl

@@ -0,0 +1,26 @@
+#!/usr/bin/env cwl-runner
+class: CommandLineTool
+cwlVersion: "cwl:draft-3.dev1"
+requirements:
+  - "@import": node-engine.cwl
+  - "@import": schemadef-type.yml
+
+inputs:
+    - id: "#hello"
+      type: "schemadef-type.yml#HelloType"
+      inputBinding:
+        valueFrom:
+          engine: node-engine.cwl
+          script: |
+            {
+            return $self.a + "/" + $self.b;
+            }
+
+outputs:
+    - id: "#output"
+      type: File
+      outputBinding:
+        glob: output.txt
+
+stdout: output.txt
+baseCommand: echo

cwltool/schemas/draft-3/draft-3/schemadef-type.yml

@@ -0,0 +1,9 @@
+class: SchemaDefRequirement
+types:
+  - name: "#HelloType"
+    type: record
+    fields:
+      - name: a
+        type: string
+      - name: b
+        type: string