Merge pull request #7 from bmeg/generate

Working on streamlining process of creating a new workflow

Author: khillion

.gitignore

@@ -5,3 +5,4 @@ doc/build/
 build/
 cwlgen.egg*
 dist/
+*.pyc

cwlgen/__init__.py

@@ -16,19 +16,13 @@
 from .version import __version__
 
 from .utils import *
+from .elements import Parameter, CWL_VERSIONS
+from .workflow import Workflow, File
+from .import_cwl import parse_cwl
 
 logging.basicConfig(level=logging.INFO)
 LOGGER = logging.getLogger(__name__)
 
-#  Constant(s)  ------------------------------
-
-CWL_SHEBANG = "#!/usr/bin/env cwl-runner"
-CWL_VERSIONS = ['draft-2', 'draft-3.dev1', 'draft-3.dev2', 'draft-3.dev3',
-                'draft-3.dev4', 'draft-3.dev5', 'draft-3', 'draft-4.dev1',
-                'draft-4.dev2', 'draft-4.dev3', 'v1.0.dev4', 'v1.0', None]
-DEF_VERSION = 'v1.0'
-CWL_TYPE = ['null', 'boolean', 'int', 'long', 'float', 'double', 'string', 'File',
-            'Directory', None]
 
 #  Function(s)  ------------------------------
 
@@ -43,7 +37,7 @@ class CommandLineTool(object):
     __CLASS__ = 'CommandLineTool'
 
     def __init__(self, tool_id=None, base_command=None, label=None, doc=None,
-                 cwl_version=None, stdin=None, stderr=None, stdout=None):
+                 cwl_version=None, stdin=None, stderr=None, stdout=None, path=None):
         '''
         :param tool_id: unique identifier for this tool
         :type tool_id: STRING
@@ -87,6 +81,7 @@ def __init__(self, tool_id=None, base_command=None, label=None, doc=None,
         self.successCodes = []
         self.temporaryFailCodes = []
         self.permanentFailCodes = []
+        self._path = path
         self.namespaces = Namespaces()
 
     def export(self, outfile=None):
@@ -144,58 +139,6 @@ def export(self, outfile=None):
             out_write.close()
 
 
-class Parameter(object):
-    '''
-    A parameter (common field of Input and Output) for a CommandLineTool
-    '''
-
-    def __init__(self, param_id, label=None, secondary_files=None, param_format=None,
-                 streamable=False, doc=None, param_type=None):
-        '''
-        :param param_id: unique identifier for this parameter
-        :type param_id: STRING
-        :param label: short, human-readable label
-        :type label: STRING
-        :param secondary_files: If type is a file, describes files that must be
-                                included alongside the primary file(s)
-        :type secondary_files: STRING
-        :param param_format: If type is a file, uri to ontology of the format or exact format
-        :type param_format: STRING
-        :param streamable: If type is a file, true indicates that the file is read or written
-                           sequentially without seeking
-        :type streamable: BOOLEAN
-        :param doc: documentation
-        :type doc: STRING
-        :param param_type: type of data assigned to the parameter
-        :type param_type: STRING corresponding to CWLType
-        '''
-        if param_type not in CWL_TYPE:
-            LOGGER.warning("The type is incorrect for the parameter.")
-            param_type = None
-        self.id = param_id
-        self.label = label
-        self.secondaryFiles = secondary_files
-        self.format = param_format
-        self.streamable = streamable
-        self.doc = doc
-        self.type = param_type
-
-    def get_dict(self):
-        '''
-        Transform the object to a [DICT] to write CWL.
-
-        :return: dictionnary of the object
-        :rtype: DICT
-        '''
-        dict_param = {k: v for k, v in vars(self).items() if v is not None and v is not False}
-        if dict_param['type'] != 'File':
-            # Remove what is only for File
-            for key in ['format', 'secondaryFiles', 'streamable']:
-               try:
-                   del(dict_param[key])
-               except KeyError:
-                   pass
-        return dict_param
 
 
 class CommandInputParameter(Parameter):

cwlgen/elements.py

@@ -0,0 +1,66 @@
+
+
+#  Constant(s)  ------------------------------
+
+CWL_SHEBANG = "#!/usr/bin/env cwl-runner"
+CWL_VERSIONS = ['draft-2', 'draft-3.dev1', 'draft-3.dev2', 'draft-3.dev3',
+                'draft-3.dev4', 'draft-3.dev5', 'draft-3', 'draft-4.dev1',
+                'draft-4.dev2', 'draft-4.dev3', 'v1.0.dev4', 'v1.0', None]
+DEF_VERSION = 'v1.0'
+CWL_TYPE = ['null', 'boolean', 'int', 'long', 'float', 'double', 'string', 'File',
+            'Directory', None]
+
+
+
+class Parameter(object):
+    '''
+    Based class for parameters (common field of Input and Output) for CommandLineTool and Workflow
+    '''
+
+    def __init__(self, param_id, label=None, secondary_files=None, param_format=None,
+                 streamable=False, doc=None, param_type=None):
+        '''
+        :param param_id: unique identifier for this parameter
+        :type param_id: STRING
+        :param label: short, human-readable label
+        :type label: STRING
+        :param secondary_files: If type is a file, describes files that must be
+                                included alongside the primary file(s)
+        :type secondary_files: STRING
+        :param param_format: If type is a file, uri to ontology of the format or exact format
+        :type param_format: STRING
+        :param streamable: If type is a file, true indicates that the file is read or written
+                           sequentially without seeking
+        :type streamable: BOOLEAN
+        :param doc: documentation
+        :type doc: STRING
+        :param param_type: type of data assigned to the parameter
+        :type param_type: STRING corresponding to CWLType
+        '''
+        if param_type not in CWL_TYPE:
+            LOGGER.warning("The type is incorrect for the parameter.")
+            param_type = None
+        self.id = param_id
+        self.label = label
+        self.secondaryFiles = secondary_files
+        self.format = param_format
+        self.streamable = streamable
+        self.doc = doc
+        self.type = param_type
+
+    def get_dict(self):
+        '''
+        Transform the object to a [DICT] to write CWL.
+
+        :return: dictionnary of the object
+        :rtype: DICT
+        '''
+        dict_param = {k: v for k, v in vars(self).items() if v is not None and v is not False}
+        if dict_param['type'] != 'File':
+            # Remove what is only for File
+            for key in ['format', 'secondaryFiles', 'streamable']:
+               try:
+                   del(dict_param[key])
+               except KeyError:
+                   pass
+        return dict_param

cwlgen/import_cwl.py

@@ -195,6 +195,7 @@ def import_cwl(self, cwl_path):
                 getattr(self, '_load_{}'.format(key))(tool, element)
             except AttributeError:
                 logger.warning(key + " content is not processed (yet).")
+        tool._path = cwl_path
         return tool
 
 def dict_or_idlist_items(v):
@@ -674,6 +675,7 @@ def import_cwl(self, cwl_path):
                 getattr(self, '_load_{}'.format(key))(tool, element)
             except AttributeError:
                 logger.warning(key + " workflow content is not processed (yet).")
+        tool._path = cwl_path
         return tool
 
     def _load_id(self, tool, id_el):

cwlgen/workflow.py

@@ -3,10 +3,9 @@
 import ruamel.yaml
 import six
 
-from . import CWL_SHEBANG, Parameter
+from .elements import CWL_SHEBANG, Parameter
 from .utils import *
 
-
 class Workflow(object):
     '''
     Contain all informations to describe a CWL workflow.
@@ -17,6 +16,7 @@ def __init__(self):
         self.steps = []
         self.inputs = []
         self.outputs = []
+        self._path = None
 
     def export(self, outfile=None):
         """
@@ -57,6 +57,10 @@ def export(self, outfile=None):
             out_write.write(ruamel.yaml.dump(cwl_workflow))
             out_write.close()
 
+    def add(self, step_id, tool, params):
+        return StepRun(self, step_id, tool, params)
+
+
 class InputParameter(Parameter):
     def __init__(self, param_id, label=None, secondary_files=None, param_format=None,
                  streamable=False, doc=None, input_binding=None, default=None, param_type=None):
@@ -138,3 +142,64 @@ def __init__(self, param_id, outputSource, label=None, secondary_files=None, par
                            secondary_files=secondary_files, param_format=param_format,
                            streamable=streamable, doc=doc, param_type=param_type)
         self.outputSource = outputSource
+
+############################
+# Workflow contruction classes
+
+class File:
+    """
+    An abstract file reference used for generating workflows
+    """
+    def __init__(self, path):
+        self.path = path
+
+class Variable:
+    """
+    An output variable from a workflow step
+    """
+    def __init__(self, workflow, step, name):
+        self.step = step
+        self.name = name
+        self.workflow = workflow
+
+    def path(self):
+        return "%s/%s" % (self.step, self.name)
+
+    def store(self):
+        self.workflow.outputs.append(WorkflowOutputParameter(self.path().replace("/", "_"), outputSource=self.path(), param_type="File"))
+        return
+
+
+class StepRun:
+    """
+    Result of adding a step into a workflow
+    """
+    def __init__(self, workflow, id, tool, params):
+        self.tool = tool
+        self.workflow = workflow
+        self.id = id
+
+        step = WorkflowStep(id=id, run=tool._path)
+        workflow.steps.append(step)
+
+        for i, j in params.items():
+            if isinstance(j, six.string_types):
+                step.inputs.append(WorkflowStepInput(i, default=j))
+            elif isinstance(j, Variable):
+                step.inputs.append(WorkflowStepInput(i, src=j.path()))
+            elif isinstance(j, File):
+                self.workflow.inputs.append( InputParameter(i, param_type="File") )
+                step.inputs.append(WorkflowStepInput(i, src=i))
+
+        for o in tool.outputs:
+            step.outputs.append(o.id)
+
+    def store_all(self):
+        for i in self.tool.outputs:
+            Variable(self.workflow, self.id, i.id).store()
+
+    def __getitem__(self, key):
+        for i in self.tool.outputs:
+            if i.id == key:
+                return Variable(self.workflow, self.id, key)
+        raise KeyError

test/test_unit_workflow.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+
+'''
+Unit tests for cwlgen library
+'''
+
+#  Import  ------------------------------
+
+# General libraries
+import os
+import filecmp
+import unittest
+
+# External libraries
+import cwlgen
+
+#  Class(es)  ------------------------------
+
+class TestCommandLineTool(unittest.TestCase):
+
+
+    def test_build(self):
+
+        w = cwlgen.Workflow()
+        tool = cwlgen.parse_cwl("test/import_cwl.cwl")
+
+        f = cwlgen.File("input_file")
+
+        o1 = w.add('step-a', tool, {"INPUT1" : f})
+        o2 = w.add('step-b', tool, {"INPUT1" : o1['OUTPUT1']})
+        o2['OUTPUT1'].store()
+        w.export("test.cwl")