Add admin option to enable/disable compression when archiving tasks #1123

BrainPortal/app/controllers/tasks_controller.rb

@@ -674,6 +674,7 @@ def update_multiple
   # [*Resume*] Release task from <tt>Suspended</tt> status (i.e. continue processing).
   # [*Terminate*] Kill the task, while maintaining its temporary files and its entry in the database.
   # [*Delete*] Kill the task, delete the temporary files and remove its entry in the database.
+  # [*Archive*] Archive the content of task work folder
   def operation
     @scope = scope_from_session('tasks#index')
 
@@ -707,12 +708,14 @@ def operation
     archive_dp_id   = params[:archive_dp_id].presence   # for 'archive as file' operation
     dup_bourreau_id = nil unless dup_bourreau_id && Bourreau.find_all_accessible_by_user(current_user).where(:id => dup_bourreau_id).exists?
     archive_dp_id   = nil unless archive_dp_id   && DataProvider.find_all_accessible_by_user(current_user).where(:id => archive_dp_id).exists?
+    nozip           = params[:nozip].presence  && current_user.has_role?(:admin_user) # for archiving without compression, admin only
 
     # This does the actual work and returns info about the
     # successes and failures.
     results = apply_operation(operation, tasklist,
       :dup_bourreau_id => dup_bourreau_id,
       :archive_dp_id   => archive_dp_id,
+      :nozip           => nozip
     )
 
     # Prepare counters for how many tasks affected.
@@ -748,6 +751,7 @@ def apply_operation(operation, taskids, options = {})
     # Some other parameters
     dup_bourreau_id = options[:dup_bourreau_id] # for 'duplicate' operation
     archive_dp_id   = options[:archive_dp_id]   # for 'archive as file' operation
+    nozip           = options[:nozip]           # for 'archive as file' and 'archive' operation
 
     # Prepare counters for how many tasks affected.
     skipped_list = {}
@@ -817,6 +821,9 @@ def apply_operation(operation, taskids, options = {})
             bac_klass = operation_to_bac[operation]
             if bac_klass
               bac = bac_klass.local_new(current_user.id, oktasks.map(&:id), bid, {})
+              # an non-copress option for archive or archive to file operations
+              bac.options[:nozip]                    = nozip           if nozip && operation =~ /^archive(_file)?$/
+              # other options and operations
               bac.options[:archive_data_provider_id] = archive_dp_id   if operation == 'archive_file'
               bac.options[:dup_bourreau_id]          = dup_bourreau_id if operation == 'duplicate'
               bac.options[:atwhat]                   = 'Setup'         if operation == 'restart_setup'

BrainPortal/app/models/background_activity/archive_task_workdir.rb

@@ -33,8 +33,9 @@ def process(item)
     super(item) # invokes the terminate code; will skip tasks that don't need to be terminated
     cbrain_task  = CbrainTask.where(:bourreau_id => CBRAIN::SelfRemoteResourceId).find(item)
     dest_dp_id   = self.options[:archive_data_provider_id] # can be nil
-    ok           = cbrain_task.archive_work_directory                              if dest_dp_id.blank?
-    ok           = cbrain_task.archive_work_directory_to_userfile(dest_dp_id.to_i) if dest_dp_id.present?
+    nozip        = self.options[:nozip]
+    ok           = cbrain_task.archive_work_directory(nozip)                              if dest_dp_id.blank?
+    ok           = cbrain_task.archive_work_directory_to_userfile(dest_dp_id.to_i, nozip) if dest_dp_id.present?
     return [ true,  "Archived" ] if   ok
     return [ false, "Skipped"  ] if ! ok
   end

BrainPortal/app/models/cluster_task.rb

@@ -1307,9 +1307,9 @@ def capture_job_out_err(run_number=nil,stdout_lim=2000,stderr_lim=2000)
   # Work Directory Archiving API
   ##################################################################
 
-  def in_situ_workdir_archive_file #:nodoc:
+  def in_situ_workdir_archive_file(nozip) #:nodoc:
     fn_id = self.fullname.gsub(/[^\w\-]+/,"_").sub(/\A_*/,"").sub(/_*$/,"")
-    "CbrainTask_Workdir_#{fn_id}.tar.gz" # note: also check in the TaskWorkdirArchive model
+    "CbrainTask_Workdir_#{fn_id}.tar#{'.gz' unless nozip}" # note: also check in the TaskWorkdirArchive model
   end
 
   # This method will create a .tar.gz file of the
@@ -1320,7 +1320,7 @@ def in_situ_workdir_archive_file #:nodoc:
   # in_situ_workdir_archive_file(). Restoring the
   # state of the workdir can be performed with
   # unarchive_work_directory().
-  def archive_work_directory
+  def archive_work_directory(nozip = false)
 
     # Keep updated_at value in order to reset it at the end of method
     updated_at_value = self.updated_at
@@ -1331,7 +1331,7 @@ def archive_work_directory
     cb_error "Tried to archive a task's work directory while in the wrong Rails app." unless
       self.bourreau_id == CBRAIN::SelfRemoteResourceId
 
-    tar_file      = self.in_situ_workdir_archive_file
+    tar_file      = self.in_situ_workdir_archive_file nozip
     temp_tar_file = "T_#{tar_file}"
     tar_capture   = "/tmp/tar.capture.#{Process.pid}.out"
 
@@ -1381,7 +1381,7 @@ def archive_work_directory
 
       system("chmod","-R","u+rwX",".") # uppercase X mode affects only directories
       status = with_stdout_stderr_capture(tar_capture) do
-        system("tar","-czf", temp_tar_file, "--exclude", "*#{temp_tar_file}", ".")
+        system("tar","-c#{'z' unless nozip }f", temp_tar_file, "--exclude", "*#{temp_tar_file}", ".")
         $? # a Process::Status object
       end
 
@@ -1455,9 +1455,8 @@ def unarchive_work_directory
     cb_error "Tried to unarchive a task's work directory while in the wrong Rails app." unless
       self.bourreau_id == CBRAIN::SelfRemoteResourceId
 
-    tar_file      = self.in_situ_workdir_archive_file
+    tar_file      = self.in_situ_workdir_archive_file false  # first assume tar file is gzip-compressed
     tar_capture   = "/tmp/tar.capture.#{Process.pid}.out"
-
     if self.cluster_workdir.blank?
       self.addlog("Cannot unarchive: no work directory configured.")
       return false
@@ -1472,9 +1471,10 @@ def unarchive_work_directory
     Dir.chdir(full) do
 
       if ! File.exists?(tar_file)
+        self.addlog("Cannot unarchive: tar archive #{tar_file} does not exist.")
         tar_file.sub!(/\.gz\z/,"") # try without the .gz
         if ! File.exists?(tar_file)
-          self.addlog("Cannot unarchive: tar archive does not exist.")
+          self.addlog("Cannot unarchive: tar archive #{tar_file} does not exist.")
           return false
         end
       end
@@ -1533,11 +1533,10 @@ def unarchive_work_directory
   # is the task's results_data_provider_id.
   # Restoring the state of the workdir can be performed
   # with unarchive_work_directory_from_userfile().
-  def archive_work_directory_to_userfile(dp_id = nil)
-    return false unless self.archive_work_directory
+  def archive_work_directory_to_userfile(dp_id = nil, nozip = nil)
+    return false unless self.archive_work_directory(nozip)
     file_id  = self.workdir_archive_userfile_id
     return true if file_id
-
     full=self.full_cluster_workdir
     if ! Dir.exists?(full)
       self.addlog("Cannot archive: work directory '#{full}' does not exist.")
@@ -1550,7 +1549,7 @@ def archive_work_directory_to_userfile(dp_id = nil)
       return false
     end
 
-    tar_file = self.in_situ_workdir_archive_file
+    tar_file = self.in_situ_workdir_archive_file(nozip)
 
     Dir.chdir(full) do
       if ! File.exists?(tar_file)
@@ -1595,7 +1594,6 @@ def archive_work_directory_to_userfile(dp_id = nil)
   # the method fetches it, and use its content
   # to recreate the task's work directory.
   def unarchive_work_directory_from_userfile
-    tar_file = self.in_situ_workdir_archive_file
 
     return false unless self.workdir_archived? && self.workdir_archive_userfile_id
 
@@ -1609,6 +1607,11 @@ def unarchive_work_directory_from_userfile
       return false
     end
 
+    # todo 'file' command can be used to reliably determine actual archive type, Alpine etc ...
+    # (save few custom, strip-down or minimalist OSs like Alpine)
+    nozip    = !taskarch_userfile.name.end_with?(".gz")  # archive is likely zipped if ends with gz
+    tar_file = self.in_situ_workdir_archive_file(nozip)
+
     self.addlog("Attempting to restore TaskWorkdirArchive.")
 
     taskarch_userfile.sync_to_cache

BrainPortal/app/views/tasks/_task_menu.html.erb

@@ -192,7 +192,13 @@
       <br/>
       Optional: when archiving <em>As File</em>, choose a destination Data Provider:<br/>
       <%= data_provider_select :archive_dp_id, { :selector => "" }, :include_blank => "" %>
+
+      <% if current_user.has_role?(:admin_user) %>
+        <br/><br/>
+        Do not compress while archiving <%= check_box_tag :nozip, "yes", false %>
+      <% end %>
     <p/>
+
   <% end %>
 
   <%= button_with_dropdown_menu("Filters", :content_id => 'tasks_filters') do %>