Add support for specifying the temporary directory

benchmark

@@ -32,6 +32,7 @@ if __name__ == "__main__":
     'set_cpu_affinity'    : True,
     'set_gpu_affinity'    : True,
     'logdir'              : 'logs',                     # relative or absolute path, or None to disable storing the logs
+    'tmpdir'              : None,                       # temporary directory, or None to use a system dependent default temporary directory
     'keep'                : [ 'resources.json' ],       # additional output files to be kept, along with the logs
   }
 

multirun.py

@@ -154,6 +154,7 @@ def multiCmsRun(
     data = None,                    # a file-like object for storing performance measurements
     header = True,                  # write a header before the measurements
     warmup = True,                  # whether to run an extra warm-up job
+    tmpdir = None,                  # temporary directory, or None to use a system dependent default temporary directory (default: None)
     logdir = None,                  # a relative or absolute path where to store individual jobs' log files, or None
     keep = [],                      # additional output files to be kept
     verbose = False,                # whether to print extra messages
@@ -192,9 +193,14 @@ def multiCmsRun(
     reportEvery = cms.untracked.int32(1)
   )
 
+  # make sure the explicit temporary directory exists
+  if tmpdir is not None:
+      os.makedirs(tmpdir, exist_ok = True)
+      tmpdir = os.path.realpath(tmpdir)
+
   # make a full dump of the configuration, to make changes to the number of threads, streams, etc.
-  workdir = tempfile.mkdtemp(prefix = 'cmsRun')
-  config = open(os.path.join(workdir, 'process.py'), 'w')
+  workdir = tempfile.TemporaryDirectory(prefix = 'cmsRun', dir = tmpdir)
+  config = open(os.path.join(workdir.name, 'process.py'), 'w')
   config.write(process.dumpPython())
   config.close()
 
@@ -251,7 +257,7 @@ def multiCmsRun(
 
   if warmup:
     # warm up to cache the binaries, data and conditions
-    jobdir = os.path.join(workdir, "warmup")
+    jobdir = os.path.join(workdir.name, "warmup")
     os.mkdir(jobdir)
     # recreate logs' directory
     if logdir is not None:
@@ -309,7 +315,7 @@ def multiCmsRun(
       thislogdir = None
     # create work threads
     for job in range(jobs):
-      jobdir = os.path.join(workdir, "step%02d_part%02d" % (repeat, job))
+      jobdir = os.path.join(workdir.name, "step%02d_part%02d" % (repeat, job))
       os.mkdir(jobdir)
       job_threads[job] = singleCmsRun(config.name, jobdir, thislogdir, keep, verbose, cpu_assignment[job], gpu_assignment[job], *args)
 
@@ -346,7 +352,7 @@ def multiCmsRun(
 
     # if all jobs were successful, delete the temporary directories
     for job in range(jobs):
-      jobdir = os.path.join(workdir, "step%02d_part%02d" % (repeat, job))
+      jobdir = os.path.join(workdir.name, "step%02d_part%02d" % (repeat, job))
       shutil.rmtree(jobdir)
 
     reference_events = np.array(sorted(consistent_events, key = consistent_events.get, reverse = True)[0])
@@ -429,7 +435,7 @@ def multiCmsRun(
     sys.stdout.flush()
 
   # delete the temporary work dir
-  shutil.rmtree(workdir)
+  workdir.cleanup()
 
 
 def info():

scan

@@ -26,23 +26,24 @@ if __name__ == "__main__":
     'warmup'              : True,
     'events'              : 10100,
     'repeats'             : 3,
-    'jobs'                : 2,          # if None, set the number of jobs to fill all available CPUs
-    'threads'             : None,       # if None, overridden by the scan
-    'streams'             : None,       # if None, overridden by the scan
+    'jobs'                : 2,                          # if None, set the number of jobs to fill all available CPUs
+    'threads'             : None,                       # if None, overridden by the scan
+    'streams'             : None,                       # if None, overridden by the scan
     'gpus_per_job'        : 1,
-    'allow_hyperthreading': True,       # this determines the number and afifnity of the CPUs used by each job
+    'allow_hyperthreading': True,                       # this determines the number and afifnity of the CPUs used by each job
     'set_cpu_affinity'    : True,
     'set_gpu_affinity'    : True,
-    'logdir'              : None,       # relative or absolute path, or None to disable storing the logs
-    'keep'                : []          # list of files produced by the jobs to be kept (requires logdir)
+    'logdir'              : None,                       # relative or absolute path, or None to disable storing the logs
+    'tmpdir'              : None,                       # temporary directory, or None to use a system dependent default temporary directory
+    'keep'                : []                          # list of files produced by the jobs to be kept (requires logdir)
   }
 
   run_io_benchmark = True                               # measure the throughput for reading the input data
 
   # options specific to scan
-  steps = [ 6, 8, 10, 12 ]              # list, or None to make a linear scan from min_step to max_step
-  min_step  = 1                         # minimum is 1
-  max_step  = None                      # None to guess based on the number of available cores (or threads) and concurrent jobs
+  steps = [ 6, 8, 10, 12 ]                              # list, or None to make a linear scan from min_step to max_step
+  min_step  = 1                                         # minimum is 1
+  max_step  = None                                      # None to guess based on the number of available cores (or threads) and concurrent jobs
 
   events_extra_per_thread = 0                           # increase by this amount the number of events per thread
   events_limit = 0                                      # if not 0, limit the total number of events to be processed