Merge pull request #68 from fact-project/new_conditions

New conditions

Author: jebuss

erna/scripts/fetch_fact_runs.py

@@ -10,6 +10,7 @@
 
 logger = logging.getLogger(__name__)
 
+from fact_conditions import create_condition_set
 
 @click.command()
 @click.argument('earliest_night')
@@ -18,7 +19,7 @@
 @click.option('--source',  help='Name of the source to analyze. e.g Crab', default='Crab')
 @click.option('--max_delta_t', default=30,  help='Maximum time difference (minutes) allowed between drs and data files.', type=click.INT)
 @click.option('--parts', default=1, help='Number of parts to split the .json file into. This is useful for submitting this to a cluster later on', type=click.INT)
-@click.option('--conditions',  help='Name of the data conditions as given in datacheck_conditions.py e.g std', default='std')
+@click.option('--conditions', '-c', help='Name of the data conditions as given in datacheck_conditions.py e.g @standard or "fParameter < 42 "', default=['@standard'], multiple=True)
 @click.password_option(help='password to read from the always awesome RunDB')
 def main(earliest_night, latest_night, data_dir, source,  max_delta_t, parts, password, conditions):
     '''  This script connects to the rundb and fetches all runs belonging to the specified source.
@@ -32,7 +33,9 @@ def main(earliest_night, latest_night, data_dir, source,  max_delta_t, parts, pa
 
     factdb = create_engine("mysql+pymysql://factread:{}@129.194.168.95/factdata".format(password))
 
-    data_conditions=dcc.conditions[conditions]
+    # create the set of conditions we want to use
+    data_conditions = create_condition_set(conditions)
+    
     mapping = erna.load(earliest_night, latest_night, data_dir,  source_name=source, timedelta_in_minutes=max_delta_t, factdb=factdb, data_conditions=data_conditions)
     if mapping.empty:
         logger.error('No entries matching the conditions could be found in the RunDB')

erna/scripts/process_fact_data.py

@@ -58,6 +58,7 @@ def make_jobs(jar, xml, aux_source_path, output_directory, df_mapping,  engine,
 
     return jobs
 
+from fact_conditions import create_condition_set
 
 @click.command()
 @click.argument('earliest_night')
@@ -75,7 +76,7 @@ def make_jobs(jar, xml, aux_source_path, output_directory, df_mapping,  engine,
 @click.option('--log_level', type=click.Choice(['INFO', 'DEBUG', 'WARN']), help='increase output verbosity', default='INFO')
 @click.option('--port', help='The port through which to communicate with the JobMonitor', default=12856, type=int)
 @click.option('--source',  help='Name of the source to analyze. e.g Crab', default='Crab')
-@click.option('--conditions',  help='Name of the data conditions as given in datacheck_conditions.py e.g standard', default='standard')
+@click.option('--conditions', '-c', help='Name of the data conditions as given in datacheck_conditions.py e.g @standard or "fParameter < 42 "', default=['@standard'], multiple=True)
 @click.option('--max_delta_t', default=30,  help='Maximum time difference (minutes) allowed between drs and data files.', type=click.INT)
 @click.option('--local', default=False,is_flag=True,   help='Flag indicating whether jobs should be executed localy .')
 @click.option('--local_output', default=False, is_flag=True,
@@ -114,7 +115,10 @@ def main(earliest_night, latest_night, data_dir, jar, xml, aux_source, out, queu
     os.makedirs(output_directory, exist_ok=True)
     logger.info("Writing output data  to {}".format(out))
     factdb = sqlalchemy.create_engine("mysql+pymysql://factread:{}@129.194.168.95/factdata".format(password))
-    data_conditions=dcc.conditions[conditions]
+
+    # create the set of conditions we want to use
+    data_conditions = create_condition_set(conditions)
+
     df_runs = erna.load(earliest_night, latest_night, data_dir, source_name=source, timedelta_in_minutes=max_delta_t, factdb=factdb, data_conditions=data_conditions)
 
     # check for missing data and fix possible wrong file extension (.fz->.gz)

erna/scripts/process_fact_data_qsub.py

@@ -46,7 +46,7 @@ def read_outputs_to_list(job_output_paths):
 
 
 
-
+from fact_conditions import create_condition_set
 
 @click.command()
 @click.argument('earliest_night')
@@ -66,7 +66,7 @@ def read_outputs_to_list(job_output_paths):
 @click.option('--log_level', type=click.Choice(['INFO', 'DEBUG', 'WARN']), help='increase output verbosity', default='INFO')
 @click.option('--port', help='The port through which to communicate with the JobMonitor', default=12856, type=int)
 @click.option('--source',  help='Name of the source to analyze. e.g Crab', default='Crab')
-@click.option('--conditions',  help='Name of the data conditions as given in datacheck_conditions.py e.g std', default='data')
+@click.option('--conditions', '-c', help='Name of the data conditions as given in datacheck_conditions.py e.g @standard or "fParameter < 42 "', default=['@standard'], multiple=True)
 @click.option('--max_delta_t', default=30,  help='Maximum time difference (minutes) allowed between drs and data files.', type=click.INT)
 @click.option('--local', default=False,is_flag=True,   help='Flag indicating whether jobs should be executed localy .')
 @click.option('--yes', help="Assume 'yes'if your asked to continue processing and start jobs", default=False, is_flag=True)
@@ -98,7 +98,10 @@ def main(earliest_night, latest_night, data_dir, jar, xml, aux_source, out, queu
     os.makedirs(output_directory, exist_ok=True)
 
     factdb = sqlalchemy.create_engine("mysql+pymysql://factread:{}@129.194.168.95/factdata".format(password))
-    data_conditions=dcc.conditions[conditions]
+    
+    # create the set of conditions we want to use
+    data_conditions = create_condition_set(conditions)
+    
     df_loaded = erna.load(earliest_night, latest_night, data_dir, source_name=source, timedelta_in_minutes=max_delta_t, factdb=factdb, data_conditions=data_conditions)
     df_loaded.to_hdf(out+".tmp", "loaded", mode="a")
 

erna/scripts/process_fact_mc.py

@@ -17,7 +17,7 @@
 logger = logging.getLogger(__name__)
 
 def make_jobs(jar, xml, data_paths, drs_paths,
-              engine, queue, vmem, num_jobs, walltime, output_path=None, filename_format="{basename}_{num}.json"):
+              engine, queue, vmem, num_jobs, walltime, output_path=None, local_output_extension="json"):
     jobs = []
 
     data_partitions = np.array_split(data_paths, num_jobs)
@@ -35,7 +35,7 @@ def make_jobs(jar, xml, data_paths, drs_paths,
         if output_path:
             # create the filenames for each single local run
             file_name, _ = path.splitext(path.basename(output_path))
-            file_name = create_filename_from_format(filename_format, file_name, num)
+            file_name = "{}_{}.{}".format(file_name, num, local_output_extension)
             out_path = path.dirname(output_path)
             run = [jar, xml, df, path.join(out_path, file_name)]
             stream_runner = stream_runner_local
@@ -81,12 +81,8 @@ def make_jobs(jar, xml, data_paths, drs_paths,
               show_default=True)
 @click.option('--mcdrs', type=click.Path(exists=True, dir_okay=False, file_okay=True, readable=True))
 @click.option('--mcwildcard', help="Gives the wildcard for searching the folder for files.", type=click.STRING, default='**/*_Events.fit*')
-@click.option('--local_output_format', default="{basename}_{num}.json", help="Give the file format for the local output funktionality."
-              + "%b will replace the out filename and %[1-9]n the given local number."
-              + "Default is: '{basename}_{num}.json'.Only works with option --local_output. ")
-@click.option('--yes', help="Assume 'yes'if your asked to continue processing and start jobs", default=False, is_flag=True)
-def main( jar, xml, out, mc_path, queue, walltime, engine, num_jobs, vmem, log_level, port, local, local_output, mcdrs, mcwildcard, local_output_format, yes):
-
+@click.option('--local_output_extension', default="json", help="Give the file format for the local output funktionality.")
+def main( jar, xml, out, mc_path, queue, walltime, engine, num_jobs, vmem, log_level, port, local, local_output, mcdrs, mcwildcard, local_output_extension):
     '''
     Script to execute fact-tools on MonteCarlo files. Use the MC_PATH argument to specifiy the folders containing the MC
     '''
@@ -142,7 +138,7 @@ def main( jar, xml, out, mc_path, queue, walltime, engine, num_jobs, vmem, log_l
         job_list = make_jobs(
                         jarpath, xmlpath, mc_paths_array,
                         drs_paths_array,  engine, queue,
-                        vmem, num_jobs, walltime, output_path=local_output_dir, filename_format=local_output_format
+                        vmem, num_jobs, walltime, output_path=local_output_dir, local_output_extension=local_output_extension
                         )
     else:
         job_list = make_jobs(

requirements.txt

@@ -0,0 +1,2 @@
+https://github.com/fact-project/fact_conditions/archive/v0.1.tar.gz
+https://github.com/mackaiver/gridmap/archive/a38271b35a96adbda0c0f3e62f3751b3c8c3982a.tar.gz

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name='erna',
-    version='0.8.2',
+    version='0.8.3',
     description='Easy RuN Access. Tools that help to do batch processing of FACT data',
     url='https://github.com/fact-project/erna',
     author='Kai Brügge, Jens Buss, Maximilian Nöthe',
@@ -41,6 +41,7 @@
         'xmltodict',
         'wrapt',
         # 'gridmap>=0.13.1', install from https://github.com/mackaiver/gridmap'
+        # 'fact_condition', install from https://github.com/fact-project/fact_conditions
     ],
     zip_safe=False,
     entry_points={