Merge branch 'master' of github.com:sate-dev/sate-core

Author: joaks1

sate/__init__.py

@@ -40,6 +40,8 @@
 If configuration files are read in the order they occur as arguments (with values in later files replacing previously read values). Options specified in the command line are read last. Thus these values "overwrite" any settings from the configuration files.
 """
 
+TEMP_SEQ_ALIGNMENT_TAG = "seq_alignment.txt"
+TEMP_TREE_TAG = "tree.tre"
 __all__ = []
 
 import os

sate/filemgr.py

@@ -390,7 +390,7 @@ def get_input_source_directory(self):
             # single locus dataset: return directory nanme
             return os.path.dirname(os.path.abspath(options.input))
 
-    def get_abs_path_for_iter_output(self, iter_num, out_tag):
+    def get_abs_path_for_iter_output(self, iter_num, out_tag, allow_existing=False):
         """
         Returns an absolute path or None for the file for an iteration temporary
             file for iteration `iter_num` with the specificed `out_tag`
@@ -403,15 +403,16 @@ def get_abs_path_for_iter_output(self, iter_num, out_tag):
         """
         p = "iteration_" + str(iter_num) + '_' + out_tag
         o_path = self.output_prefix  + "_temp_" + p
-        if os.path.exists(o_path):
-            n = 1
-            while os.path.exists(o_path):
-                t_tag = "_temp%d_" % n
-                if n > 100:
-                    _LOG.warn('File %s exists iteration-specific output skipped!' % o_path)
-                    return None # don't create a huge # of numbered files
-                o_path = self.output_prefix + t_tag + p
-                n += 1
+        if not allow_existing:
+            if os.path.exists(o_path):
+                n = 1
+                while os.path.exists(o_path):
+                    t_tag = "_temp%d_" % n
+                    if n > 100:
+                        _LOG.warn('File %s exists iteration-specific output skipped!' % o_path)
+                        return None # don't create a huge # of numbered files
+                    o_path = self.output_prefix + t_tag + p
+                    n += 1
         return os.path.abspath(o_path)
 
     def get_abs_path_for_tag(self, out_tag):

sate/mainsate.py

@@ -40,6 +40,8 @@
 from sate.utility import IndentedHelpFormatterWithNL
 from sate.filemgr import open_with_intermediates
 from sate import filemgr
+from sate import TEMP_SEQ_ALIGNMENT_TAG, TEMP_TREE_TAG
+
 
 _RunningJobs = None
 
@@ -171,6 +173,9 @@ def finish_sate_execution(sate_team,
     # We must read the incoming tree in before we call the get_sequences_for_sate
     #   function that relabels that taxa in the dataset
     ######
+    alignment_as_tmp_filename_to_report = None
+    tree_as_tmp_filename_to_report = None
+    
     tree_file = options.treefile
     if tree_file:
         if not os.path.exists(tree_file):
@@ -183,6 +188,7 @@ def finish_sate_execution(sate_team,
             MESSENGER.send_warning('%d starting trees found in "%s". The first tree will be used.' % (len(tree_list), tree_file))
         starting_tree = tree_list[0]
         score = None
+        tree_as_tmp_filename_to_report = tree_file
 
     ############################################################################
     # This will relabel the taxa if they have problematic names
@@ -271,6 +277,8 @@ def finish_sate_execution(sate_team,
             jobq.put(job)
             score, starting_tree_str = job.get_results()
             _RunningJobs = None
+            alignment_as_tmp_filename_to_report = sate_products.get_abs_path_for_iter_output("initialsearch", TEMP_SEQ_ALIGNMENT_TAG, allow_existing=True)
+            tree_as_tmp_filename_to_report = sate_products.get_abs_path_for_iter_output("initialsearch", TEMP_TREE_TAG, allow_existing=True)
             if delete_tree_temps:
                 sate_team.temp_fs.remove_dir(init_tree_dir)
         _LOG.debug('We have the tree and whole_alignment, partitions...')
@@ -289,6 +297,8 @@ def finish_sate_execution(sate_team,
                         score=score,
                         **sate_config_dict)
         job.tree_str = starting_tree_str
+        job.curr_iter_align_tmp_filename = alignment_as_tmp_filename_to_report
+        job.curr_iter_tree_tmp_filename = tree_as_tmp_filename_to_report
         if score is not None:
             job.store_optimum_results(new_multilocus_dataset=multilocus_dataset,
                     new_tree_str=starting_tree_str,
@@ -302,6 +312,11 @@ def finish_sate_execution(sate_team,
             MESSENGER.send_info("Starting SATe algorithm on initial tree...")
             job.run(tmp_dir_par=temporaries_dir, sate_products=sate_products)
             _RunningJobs = None
+
+            if job.return_final_tree_and_alignment:
+                alignment_as_tmp_filename_to_report = job.curr_iter_align_tmp_filename
+            else:
+                alignment_as_tmp_filename_to_report = job.best_alignment_tmp_filename
 
             if user_config.commandline.raxml_search_after:
                 raxml_model = user_config.raxml.model.strip()
@@ -327,26 +342,31 @@ def finish_sate_execution(sate_team,
                 jobq.put(post_job)
                 post_score, post_tree = post_job.get_results()
                 _RunningJobs = None
+                tree_as_tmp_filename_to_report = sate_products.get_abs_path_for_iter_output("postraxtree", TEMP_TREE_TAG, allow_existing=True)
                 if delete_tree_temps:
                     sate_team.temp_fs.remove_dir(post_tree_dir)
                 job.tree_str = post_tree
                 job.score = post_score
                 if post_score > job.best_score:
                     job.best_tree_str = post_tree
                     job.best_score = post_score
+            else:
+                if job.return_final_tree_and_alignment:
+                    tree_as_tmp_filename_to_report = job.curr_iter_tree_tmp_filename
+                else:
+                    tree_as_tmp_filename_to_report = job.best_tree_tmp_filename
 
-        
 
         job.multilocus_dataset.restore_taxon_names()
         assert len(sate_products.alignment_streams) == len(job.multilocus_dataset)
         for i, alignment in enumerate(job.multilocus_dataset):
             alignment_stream = sate_products.alignment_streams[i]
-            MESSENGER.send_info("Writing final alignment to %s" % alignment_stream.name)
+            MESSENGER.send_info("Writing resulting alignment to %s" % alignment_stream.name)
             alignment.write(alignment_stream, file_format="FASTA")
             alignment_stream.close()
 
 
-        MESSENGER.send_info("Writing final tree to %s" % sate_products.tree_stream.name)
+        MESSENGER.send_info("Writing resulting tree to %s" % sate_products.tree_stream.name)
         tree_str = job.tree.compose_newick()
         sate_products.tree_stream.write("%s;\n" % tree_str)
 
@@ -357,13 +377,19 @@ def finish_sate_execution(sate_team,
         #        outtree_fn = os.path.join(seqdir, "combined_%s.tre" % options.job)
         #    else:
         #        outtree_fn = aln_filename + ".tre"
-        #MESSENGER.send_info("Writing final tree to %s" % outtree_fn)
+        #MESSENGER.send_info("Writing resulting tree to %s" % outtree_fn)
         #tree_str = job.tree.compose_newick()
         #sate_products.tree_stream.write("%s;\n" % tree_str)
 
 
-        MESSENGER.send_info("Writing final likelihood score to %s" % sate_products.score_stream.name)
+        MESSENGER.send_info("Writing resulting likelihood score to %s" % sate_products.score_stream.name)
         sate_products.score_stream.write("%s\n" % job.score)
+        
+        if alignment_as_tmp_filename_to_report is not None:
+            MESSENGER.send_info('The resulting alignment (with the names in a "safe" form) was first written as the file "%s"' % alignment_as_tmp_filename_to_report)
+        if tree_as_tmp_filename_to_report is not None:
+            MESSENGER.send_info('The resulting tree (with the names in a "safe" form) was first written as the file "%s"' % tree_as_tmp_filename_to_report)
+
     finally:
         for el in prev_signals:
             sig, prev_handler = el

sate/satejob.py

@@ -37,6 +37,8 @@
 from sate.utility import record_timestamp
 from sate.scheduler import jobq
 from sate.filemgr import  TempFS
+from sate import TEMP_SEQ_ALIGNMENT_TAG, TEMP_TREE_TAG
+
 
 class SateTeam (object):
     '''A blob for holding the appropriate merger, alignment, and tree_estimator tools
@@ -149,9 +151,6 @@ def _reset_current_run_settings(self):
         self.start_time = None
         self.current_iteration = 0
         self.last_improvement_time = None
-        #self.best_multilocus_dataset = None
-        #self.best_tree_str = self.get_tree_str()
-        #self.best_score = self.score
         self.num_iter_since_imp = 0
         self.is_stuck_in_blind = False
         self.switch_to_blind_iter = None
@@ -160,6 +159,11 @@ def _reset_current_run_settings(self):
         self._blindmode_trigger = None
         self._sate_alignment_job = None
         self._tree_build_job = None
+        self.curr_iter_align_tmp_filename = None
+        self.curr_iter_tree_tmp_filename = None
+        self.best_tree_tmp_filename = None
+        self.best_alignment_tmp_filename = None
+
 
     def _reset_jobs(self):
         self.tree_build_job = None
@@ -299,6 +303,8 @@ def store_optimum_results(self, new_multilocus_dataset, new_tree_str, new_score,
         self.best_score = new_score
         self.last_improvement_time = curr_timestamp
         self.num_iter_since_imp = 0
+        self.best_tree_tmp_filename = self.curr_iter_tree_tmp_filename
+        self.best_alignment_tmp_filename = self.curr_iter_align_tmp_filename
 
     def run(self, tmp_dir_par, sate_products=None):
         assert(os.path.exists(tmp_dir_par))
@@ -377,6 +383,11 @@ def run(self, tmp_dir_par, sate_products=None):
                                                                delete_temps=delete_iteration_temps,
                                                                sate_products=sate_products,
                                                                step_num=self.current_iteration)
+                prev_curr_align = self.curr_iter_align_tmp_filename
+                prev_curr_tree = self.curr_iter_tree_tmp_filename
+                self.curr_iter_align_tmp_filename = sate_products.get_abs_path_for_iter_output(self.current_iteration, TEMP_SEQ_ALIGNMENT_TAG, allow_existing=True)
+                self.curr_iter_tree_tmp_filename = sate_products.get_abs_path_for_iter_output(self.current_iteration, TEMP_TREE_TAG, allow_existing=True)
+
                 self.tree_build_job = tbj
                 jobq.put(tbj)
                 new_score, new_tree_str = tbj.get_results()
@@ -414,12 +425,18 @@ def run(self, tmp_dir_par, sate_products=None):
                     self.score = new_score
                     self.multilocus_dataset = new_multilocus_dataset
                     self.tree_str = new_tree_str
-                    self.status('realignment accepted.')
+                    if this_iter_score_improved:
+                        self.status('realignment accepted and score improved.')
+                    else:
+                        self.status('realignment accepted and despite the score not improving.')
                     # we do not want to continue to try different breaking strategies for this iteration so we break
                     self.status('current score: %s, best score: %s' % (self.score, self.best_score) )
                     break
                 else:
                     self.status('realignment NOT accepted.')
+                    self.curr_iter_align_tmp_filename = prev_curr_align
+                    self.curr_iter_tree_tmp_filename = prev_curr_tree 
+
                 break_strategy_index += 1
 
                 # self.status('current score: %s, best score: %s' % (self.score, self.best_score) )

sate/tools.py

@@ -29,6 +29,7 @@
 
 from alignment import Alignment
 from sate import get_logger, GLOBAL_DEBUG, SATE_SYSTEM_PATHS_CFGFILE, DEFAULT_MAX_MB
+from sate import TEMP_SEQ_ALIGNMENT_TAG, TEMP_TREE_TAG
 from sate.filemgr import open_with_intermediates
 from sate.scheduler import jobq, start_worker, DispatchableJob, FakeJob
 
@@ -78,7 +79,7 @@ def read_internal_alignment(fn,
 
 def copy_temp_tree(src_treef, sate_products, step_num):
     if (sate_products is not None) and (step_num is not None):
-        dest_treef = sate_products.get_abs_path_for_iter_output(step_num, 'tree.tre')
+        dest_treef = sate_products.get_abs_path_for_iter_output(step_num, TEMP_TREE_TAG)
         if dest_treef and os.path.exists(src_treef):
             if os.path.exists(dest_treef):
                 _LOG.warn('File "%s" exists. It will not be overwritten' % dest_treef)
@@ -539,7 +540,7 @@ def store_input(self, seqfn, **kwargs):
         if sate_products:
             step_num = kwargs.get('step_num')
             if step_num is not None:
-                i_concat_align = sate_products.get_abs_path_for_iter_output(step_num, 'seq_alignment.txt')
+                i_concat_align = sate_products.get_abs_path_for_iter_output(step_num, TEMP_SEQ_ALIGNMENT_TAG)
                 if i_concat_align and os.path.exists(seqfn):
                     if os.path.exists(i_concat_align):
                         _LOG.warn('File "%s" exists. It will not be overwritten' % i_concat_align)