Feature/87 add status script

mcpartools/generator.py

@@ -146,6 +146,12 @@ def run(self):
         # make symlinks to external files found
         self.symlink_external_files()
 
+        # generate script merging info logs
+        self.generate_merge_logs_script()
+
+        # generate status script
+        self.generate_status_script()
+
         # store information about command line arguments, date, time, user and hostname into generatemc.log
         self.save_logs()
 
@@ -242,3 +248,15 @@ def save_logs(self):
         file_logger.info('Date and time: ' + time.strftime("%Y-%m-%d %H:%M:%S"))
         file_logger.info('username@hostname: ' + getpass.getuser() + '@' + socket.gethostname())
         file_logger.info('Current working directory: ' + os.getcwd())
+
+    def generate_merge_logs_script(self):
+        wspdir_name = 'workspace'
+        output_name = 'output'
+        wspdir_path = os.path.join(self.main_dir, wspdir_name)
+        collect_path = os.path.join(self.main_dir, output_name)
+        self.scheduler.write_merge_logs_script(wspdir_path, collect_path, self.main_dir)
+
+    def generate_status_script(self):
+        wspdir_name = 'workspace'
+        wspdir_path = os.path.join(self.main_dir, wspdir_name)
+        self.scheduler.write_status_script(self.main_dir, wspdir_path)

mcpartools/mcengine/data/collect.sh

@@ -3,13 +3,36 @@
 # Exit immediately if a simple command exits with a non-zero status.
 set -e
 
+START=$(date +%s)
+
 INPUT_WILDCARD={output_dir:s}/workspace/job_*/{wildcard:s}
 OUTPUT_DIRECTORY={output_dir:s}/output
+LOG_FILE=$OUTPUT_DIRECTORY/info.log
 
 # change working directory
 cd {output_dir:s}
 
 # make output folder
 mkdir -p $OUTPUT_DIRECTORY
 
-{collect_action:s}
+echo "###########################################################" > $LOG_FILE
+echo "################### COLLECT INFORMATION ###################" >> $LOG_FILE
+echo "###########################################################" >> $LOG_FILE
+echo "#" >> $LOG_FILE
+echo "# START           = `date +"%Y-%m-%d %H:%M:%S"`" >> $LOG_FILE
+echo "# END             =                   -" >> $LOG_FILE
+echo "# TIME IN SECONDS =                   -" >> $LOG_FILE
+echo "# STATUS          =                   1" >> $LOG_FILE
+echo "#" >> $LOG_FILE
+
+{collect_action:s}
+COLLECT_STATUS=$?
+
+let "EXECUTION_TIME = $(date +%s) - $START"
+
+#    end time is in line number 6
+sed -i "6s/.*/# END             = `date +"%Y-%m-%d %H:%M:%S"`/" $LOG_FILE
+#    collapsed time is in line number 7
+sed -i "7s/.*/# TIME IN SECONDS =`printf "%20d" $EXECUTION_TIME`/" $LOG_FILE
+#    status is in line number 8
+sed -i "8s/.*/# STATUS          =                   0/" $LOG_FILE

mcpartools/mcengine/data/run_fluka.sh

@@ -3,14 +3,46 @@
 # Exit immediately if a simple command exits with a non-zero status.
 set -e
 
+START=$(date +%s)
+
 # location of FLUKA binary file
 FLUKA_BIN={fluka_bin:s}
 
+# number of particles per job
+PARTICLE_NO={particle_no:d}
+
+# seed of RNG
+RNG_SEED={job_id:d}
+
+WORK_DIR={working_directory:s}
 # go to working directory
-cd {working_directory:s}
+cd $WORK_DIR
+
+LOG_FILE=$WORK_DIR"/info.log"
+
+echo "###########################################################" > $LOG_FILE
+echo "######### DETAILED INFORMATION ABOUT JOB `printf "%5d" $RNG_SEED` ############" >> $LOG_FILE
+echo "###########################################################" >> $LOG_FILE
+echo "#" >> $LOG_FILE
+echo "# START           = `date +"%Y-%m-%d %H:%M:%S"`" >> $LOG_FILE
+echo "# END             =                   -" >> $LOG_FILE
+echo "# TIME IN SECONDS =                   -" >> $LOG_FILE
+echo "# NO OF PARTICLES =`printf "%20d" $PARTICLE_NO`" >> $LOG_FILE
+echo "# STATUS          =                   -" >> $LOG_FILE
+echo "#" >> $LOG_FILE
 
 # run rfluka
 $FLUKA_BIN -N0 -M1 {engine_options:s} {input_basename:s}
+SIMULATION_STATUS=$?
+
+let "EXECUTION_TIME = $(date +%s) - $START"
+
+#    end time is in line number 6
+sed -i "6s/.*/# END             = `date +"%Y-%m-%d %H:%M:%S"`/" $LOG_FILE
+#    collapsed time is in line number 7
+sed -i "7s/.*/# TIME IN SECONDS =`printf "%20d" $EXECUTION_TIME`/" $LOG_FILE
+#    status is in line number 9
+sed -i "9s/.*/# STATUS          =`printf "%20d" $SIMULATION_STATUS`/" $LOG_FILE
 
 # each fluka run will save files with same name, in order to distinguish output from multiple runs
 # we rename output files, appending suffix with jobid to each of them

mcpartools/mcengine/data/run_shieldhit.sh

@@ -3,6 +3,9 @@
 # Exit immediately if a simple command exits with a non-zero status.
 set -e
 
+
+START=$(date +%s)
+
 # location of SHIELD-HIT12A binary file
 SHIELDHIT_BIN={shieldhit_bin:s}
 
@@ -21,9 +24,31 @@ GEO_FILE={geo_file:s}
 MAT_FILE={mat_file:s}
 DETECT_FILE={detect_file:s}
 
+LOG_FILE=$WORK_DIR"/info.log"
+
+echo "###########################################################" > $LOG_FILE
+echo "######### DETAILED INFORMATION ABOUT JOB `printf "%5d" $RNG_SEED` ############" >> $LOG_FILE
+echo "###########################################################" >> $LOG_FILE
+echo "#" >> $LOG_FILE
+echo "# START           = `date +"%Y-%m-%d %H:%M:%S"`" >> $LOG_FILE
+echo "# END             =                   -" >> $LOG_FILE
+echo "# TIME IN SECONDS =                   -" >> $LOG_FILE
+echo "# NO OF PARTICLES =`printf "%20d" $PARTICLE_NO`" >> $LOG_FILE
+echo "# STATUS          =                   -" >> $LOG_FILE
+echo "#" >> $LOG_FILE
+
 # go to working directory
 cd {working_directory:s}
 
 # execute simulation
 $SHIELDHIT_BIN --beamfile=$BEAM_FILE --geofile=$GEO_FILE --matfile=$MAT_FILE --detectfile=$DETECT_FILE -n $PARTICLE_NO -N $RNG_SEED {engine_options:s} $WORK_DIR
+SIMULATION_STATUS=$?
+
+let "EXECUTION_TIME = $(date +%s) - $START"
 
+#    end time is in line number 6
+sed -i "6s/.*/# END             = `date +"%Y-%m-%d %H:%M:%S"`/" $LOG_FILE
+#    collapsed time is in line number 7
+sed -i "7s/.*/# TIME IN SECONDS =`printf "%20d" $EXECUTION_TIME`/" $LOG_FILE
+#    status is in line number 9
+sed -i "9s/.*/# STATUS          =`printf "%20d" $SIMULATION_STATUS`/" $LOG_FILE

mcpartools/mcengine/fluka.py

@@ -32,6 +32,8 @@ def __init__(self, input_path, mc_run_script, collect_method, mc_engine_options)
 
         self.collect_script_content = resource_string(__name__, self.collect_script).decode('ascii')
 
+        self.particle_no = 1
+
     @property
     def input_files(self):
         # TODO check if additional files are needed
@@ -54,6 +56,7 @@ def randomize(self, new_seed):
         self.input_lines = result
 
     def set_particle_no(self, particle_no):
+        self.particle_no = particle_no
         result = []
         for l in self.input_lines:
             # TODO better discovery needed
@@ -82,6 +85,7 @@ def save_run_script(self, output_dir, jobid):
             engine_options=self.engine_options,
             working_directory=output_dir_abs_path,
             input_basename=input_base_name,
+            particle_no=self.particle_no,
             job_id=jobid)
         out_file_name = 'run.sh'
         out_file_path = os.path.join(output_dir, out_file_name)

mcpartools/scheduler/base.py

@@ -25,6 +25,8 @@ def __init__(self, scheduler_options):
 
     submit_script = 'submit.sh'
     main_run_script = 'main_run.sh'
+    merge_logs_script = 'merge_logs.sh'
+    status_script = 'status.sh'
 
     def submit_script_body(self, jobs_no, main_dir, workspace_dir):
         from pkg_resources import resource_string
@@ -51,6 +53,18 @@ def main_run_script_body(self, jobs_no, workspace_dir):
                                                           jobs_no=jobs_no)
         return self.main_run_script
 
+    def merge_logs_body(self, workspace_dir, collect_dir, main_dir):
+        from pkg_resources import resource_string
+        tpl = resource_string(__name__, self.merge_logs_script_template)
+        return tpl.decode("ascii").format(workspace_dir=workspace_dir,
+                                          collect_dir=collect_dir,
+                                          main_dir=main_dir)
+
+    def status_body(self, merge_script_path):
+        from pkg_resources import resource_string
+        tpl = resource_string(__name__, self.status_script_template)
+        return tpl.decode("ascii").format(merge_script_path=merge_script_path)
+
     def write_submit_script(self, main_dir, script_basename, jobs_no, workspace_dir):
         script_path = os.path.join(main_dir, script_basename)
         fd = open(script_path, 'w')
@@ -72,3 +86,29 @@ def write_main_run_script(self, jobs_no, output_dir):
         os.chmod(out_file_path, 0o750)
         logger.debug("Saved main run script: " + out_file_path)
         logger.debug("Output dir " + output_dir)
+
+    def write_merge_logs_script(self, workspace_dir, collect_dir, main_dir):
+        workspace_dir_abspath = os.path.abspath(workspace_dir)
+        collect_dir_abspath = os.path.abspath(collect_dir)
+        main_dir_abspath = os.path.abspath(main_dir)
+
+        out_file_path = os.path.join(workspace_dir_abspath, self.merge_logs_script)
+
+        fd = open(out_file_path, 'w')
+        fd.write(self.merge_logs_body(workspace_dir_abspath, collect_dir_abspath, main_dir_abspath))
+        fd.close()
+        os.chmod(out_file_path, 0o750)
+        logger.debug("Saved merge logs script: " + out_file_path)
+
+    def write_status_script(self, main_dir, workspace_dir):
+        main_dir_abspath = os.path.abspath(main_dir)
+        out_file_path = os.path.join(main_dir_abspath, self.status_script)
+
+        workspace_dir_abspath = os.path.abspath(workspace_dir)
+        merge_log_script = os.path.join(workspace_dir_abspath, self.merge_logs_script)
+
+        fd = open(out_file_path, 'w')
+        fd.write(self.status_body(merge_log_script))
+        fd.close()
+        os.chmod(out_file_path, 0o750)
+        logger.debug("Saved status script: " + out_file_path)

mcpartools/scheduler/common.py

@@ -16,6 +16,7 @@ def __init__(self):
     @classmethod
     def get_scheduler(cls, scheduler_options, log_location):
         file_logger = logging.getLogger('file_logger')
+
         try:
             srun_output = check_output(['srun --version'], shell=True)
             file_logger.info("srun version: {}".format(srun_output[:-1]))