Merge pull request #54 from fact-project/submit_runlist

Submit runlist

Author: maxnoe

README.md

@@ -70,7 +70,11 @@ It sets up the necessary port forwarding and proxy to speak to the non-isdc worl
 
 ## Submit runs:
 
-Fire up erna_console and enter: 
+We provide a script `erna_submit_runlist` to submit a csv runlist into the
+automatic processing.
+
+
+Alternatively, you can use the erna_console and enter: 
 
 ```python
 files = (
@@ -82,9 +86,9 @@ files = (
 print("files.count():", files.count())
 
 # We only select Jar.id and Jar.version to not download the 20 MB binary blob
-jar = Jar.select(Jar.id, Jar.version).where(jar.version == '0.17.2').get()
+jar = Jar.select(Jar.id, Jar.version).where(Jar.version == '0.17.2').get()
 xml = XML.get(name='std_analysis', jar=jar)
 queue = Queue.get(name='fact_short')
 
-insert_jobs(files, xml=xml, jar=jar, queue=queue)
+insert_new_jobs(files, xml=xml, jar=jar, queue=queue)
 ```

erna/automatic_processing/database_utils.py

@@ -181,8 +181,13 @@ def insert_new_job(
 @requires_database_connection
 def insert_new_jobs(raw_data_files, jar, xml, queue, progress=True, **kwargs):
 
+    if isinstance(raw_data_files, list):
+        total = len(raw_data_files)
+    else:
+        total = raw_data_files.count()
+
     failed_files = []
-    for f in tqdm(raw_data_files, total=raw_data_files.count(), disable=not progress):
+    for f in tqdm(raw_data_files, total=total, disable=not progress):
         try:
             insert_new_job(f, jar=jar, xml=xml, queue=queue, **kwargs)
         except peewee.IntegrityError:

erna/scripts/submit_runlist.py

@@ -0,0 +1,65 @@
+import click
+import pandas as pd
+
+from ..automatic_processing.database_utils import insert_new_jobs
+from ..utils import load_config
+from datetime import date
+
+from ..automatic_processing.database import (
+    database,
+    RawDataFile,
+    Jar,
+    XML,
+    Queue,
+    Job,
+)
+
+
+@click.command()
+@click.argument('runlist')
+@click.argument('jar')
+@click.argument('xml')
+@click.option(
+    '-p', '--priority', default=5, type=int,
+    help='Priority of the jobs, lower value means more important'
+)
+@click.option(
+    '-q', '--queue', default='fact_short',
+    help='Name of the queue to use'
+)
+@click.option('--config', '-c', help='Path to the yaml config file')
+def main(runlist, jar, xml, priority, queue, config):
+    '''
+    Submit automatic processing jobs for a given runlist
+
+    Arguments
+
+    RUNLIST: csv file with columns `night, run_id`
+    JAR: version of the fact-tools jar
+    XML: Name of the xml file to use
+
+    Jar and XML must be uploaded to the processing db using erna_upload
+    '''
+    config = load_config(config)
+
+    database.init(**config['processing_database'])
+
+    jar = Jar.select(Jar.id, Jar.version).where(Jar.version == jar).get()
+    xml = XML.get(name=xml, jar=jar)
+    queue = Queue.get(name=queue)
+
+    runs = pd.read_csv(runlist)
+    runs['year'] = runs['night'] // 10000
+    runs['month'] = ((runs['night'] % 10000) // 100)
+    runs['day'] = (runs['night'] % 100)
+
+    files = [
+        RawDataFile.get(night=date(row.year, row.month, row.day), run_id=row.run_id)
+        for row in runs.itertuples()
+    ]
+
+    insert_new_jobs(files, xml=xml, jar=jar, queue=queue)
+
+
+if __name__ == '__main__':
+    main()

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name='erna',
-    version='0.6.0',
+    version='0.7.0',
     description='Easy RuN Access. Tools that help to do batch processing of FACT data',
     url='https://github.com/fact-project/erna',
     author='Kai Brügge, Jens Buss, Maximilian Nöthe',
@@ -60,6 +60,7 @@
             'erna_automatic_processing_executor = erna.automatic_processing.executor:main',
             'erna_automatic_processing = erna.automatic_processing.__main__:main',
             'erna_gather_fits = erna.scripts.gather_fits:main',
+            'erna_submit_runlist = erna.scripts.submit_runlist:main',
         ],
     },
     setup_requires=['pytest-runner'],