Hotfix/retry sample tracking (#59)

wow-such-code · sven1103 · web-flow · commit 9f554d76d36d · 2021-01-22T13:44:46.000+01:00
* retry sample tracking update for incoming data and log failure

Co-authored-by: Sven F &lt;sven.fillinger@qbic.uni-tuebingen.de&gt;
diff --git a/drop-boxes/register-IdXML-dropbox/register-IdXML-dropbox.py b/drop-boxes/register-IdXML-dropbox/register-IdXML-dropbox.py
@@ -7,6 +7,7 @@
 sys.path.append('/home-link/qeana10/bin/')
 
 import checksum
+import time
 import re
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
@@ -103,4 +104,16 @@ def process(transaction):
         transaction.moveFile(incomingPath, dataSet)
 
         #sample tracking section
-        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+        wait_seconds = 1
+        max_attempts = 3
+        for attempt in range(max_attempts):
+                try:
+                        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+                        break
+                except:
+                        print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1)
+                        if attempt < max_attempts -1:
+                                time.sleep(wait_seconds)
+                                continue
+                        else:
+                                raise
diff --git a/drop-boxes/register-archived-ms-raw/register-raw.py b/drop-boxes/register-archived-ms-raw/register-raw.py
@@ -8,6 +8,7 @@
 
 import checksum
 import re
+import time
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
 from java.io import File
@@ -104,4 +105,16 @@ def process(transaction):
         transaction.moveFile(incomingPath, dataSet)
 
         #sample tracking section
-        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+        wait_seconds = 1
+        max_attempts = 3
+        for attempt in range(max_attempts):
+                try:
+                        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+                        break
+                except:
+                        print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1)
+                        if attempt < max_attempts -1:
+                                time.sleep(wait_seconds)
+                                continue
+                        else:
+                                raise
diff --git a/drop-boxes/register-archived-ms-raw/register-raw.pyc b/drop-boxes/register-archived-ms-raw/register-raw.pyc
diff --git a/drop-boxes/register-bam-dropbox/register-bam.py b/drop-boxes/register-bam-dropbox/register-bam.py
@@ -8,6 +8,7 @@
 
 import checksum
 import re
+import time
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
 from java.io import File
@@ -136,4 +137,16 @@ def process(transaction):
         transaction.moveFile(incomingPath, dataSet)
 
         #sample tracking section
-        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+        wait_seconds = 1
+        max_attempts = 3
+        for attempt in range(max_attempts):
+                try:
+                        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+                        break
+                except:
+                        print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1)
+                        if attempt < max_attempts -1:
+                                time.sleep(wait_seconds)
+                                continue
+                        else:
+                                raise
diff --git a/drop-boxes/register-convert-ms-vendor-format/etl_msconvert.py b/drop-boxes/register-convert-ms-vendor-format/etl_msconvert.py
@@ -484,8 +484,19 @@ def __str__(self):
         return self.value
 
 def handleSampleTracking(barcode):
-    #sample tracking section
-    SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(barcode)
+    wait_seconds = 1
+    max_attempts = 3
+    for attempt in range(max_attempts):
+        try:
+            SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(barcode)
+            break
+        except:
+            print "Updating location for sample "+barcode+" failed on attempt "+str(attempt+1)
+            if attempt < max_attempts -1:
+                time.sleep(wait_seconds)
+                continue
+            else:
+                raise
 
 def createRawDataSet(transaction, incomingPath, sample, format, time_stamp):
     rawDataSet = transaction.createNewDataSet("Q_MS_RAW_DATA")
diff --git a/drop-boxes/register-fasta-dropbox/register-fasta.py b/drop-boxes/register-fasta-dropbox/register-fasta.py
@@ -8,6 +8,7 @@
 
 import checksum
 import re
+import time
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
 from java.io import File
@@ -152,4 +153,16 @@ def process(transaction):
         transaction.moveFile(new_folder, dataSet)
 
         # Updates the sample location of the extract sample
-        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+        wait_seconds = 1
+        max_attempts = 3
+        for attempt in range(max_attempts):
+                try:
+                        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+                        break
+                except:
+                        print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1)
+                        if attempt < max_attempts -1:
+                                time.sleep(wait_seconds)
+                                continue
+                        else:
+                                raise
diff --git a/drop-boxes/register-fastq-dropbox-with-metadata/register-fastq-with-metadata-dropbox.py b/drop-boxes/register-fastq-dropbox-with-metadata/register-fastq-with-metadata-dropbox.py
@@ -8,6 +8,7 @@
 
 import checksum
 import re
+import time
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
 from java.io import File
@@ -167,4 +168,16 @@ def process(transaction):
     transaction.moveFile(incomingPath, dataSet)
 
     #sample tracking section
-    SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+    wait_seconds = 1
+    max_attempts = 3
+    for attempt in range(max_attempts):
+        try:
+            SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+            break
+        except:
+            print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1)
+            if attempt < max_attempts -1:
+                time.sleep(wait_seconds)
+                continue
+            else:
+                raise
diff --git a/drop-boxes/register-fastq-dropbox/register-fastq-dropbox.py b/drop-boxes/register-fastq-dropbox/register-fastq-dropbox.py
@@ -8,6 +8,7 @@
 
 import checksum
 import re
+import time
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
 from java.io import File
@@ -128,4 +129,17 @@ def process(transaction):
                 nameFile.close()
         transaction.moveFile(incomingPath, dataSet)
 
-        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+        # sample tracking
+        wait_seconds = 1
+        max_attempts = 3
+        for attempt in range(max_attempts):
+                try:
+                        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+                        break
+                except:
+                        print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1)
+                        if attempt < max_attempts -1:
+                                time.sleep(wait_seconds)
+                                continue
+                        else:
+                                raise
diff --git a/drop-boxes/register-featureXML-dropbox/register-fXML-dropbox.py b/drop-boxes/register-featureXML-dropbox/register-fXML-dropbox.py
@@ -8,6 +8,7 @@
 
 import checksum
 import re
+import time
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
 from java.io import File
@@ -103,4 +104,16 @@ def process(transaction):
         transaction.moveFile(incomingPath, dataSet)
 
         #sample tracking section
-        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+        wait_seconds = 1
+        max_attempts = 3
+        for attempt in range(max_attempts):
+                try:
+                        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+                        break
+                except:
+                        print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1)
+                        if attempt < max_attempts -1:
+                                time.sleep(wait_seconds)
+                                continue
+                        else:
+                                raise
diff --git a/drop-boxes/register-hlatyping-dropbox/register-hlatyping.py b/drop-boxes/register-hlatyping-dropbox/register-hlatyping.py
@@ -7,6 +7,7 @@
 sys.path.append('/home-link/qeana10/bin/')
 
 import checksum
+import time
 import re
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
@@ -149,4 +150,16 @@ def process(transaction):
     transaction.moveFile(resultPath, dataSet)
 
     #sample tracking section
-    SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+    wait_seconds = 1
+    max_attempts = 3
+    for attempt in range(max_attempts):
+        try:
+            SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+            break
+        except:
+            print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1)
+            if attempt < max_attempts -1:
+                time.sleep(wait_seconds)
+                continue
+            else:
+                raise
diff --git a/drop-boxes/register-imgag-dropbox/register-imgag.py b/drop-boxes/register-imgag-dropbox/register-imgag.py
@@ -664,7 +664,19 @@ def process(transaction):
 
             transaction.moveFile(vcfFolder, vcfDataSet)
     else:
-            find_and_register_ngs_without_metadata(transaction, parentCodes)
+        find_and_register_ngs_without_metadata(transaction, parentCodes)
     for code in parentCodes:
-            #sample tracking section
-            SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code)
+        #sample tracking section
+        wait_seconds = 1
+        max_attempts = 3
+        for attempt in range(max_attempts):
+            try:
+                SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code)
+                break
+            except:
+                print "Updating location for sample "+code+" failed on attempt "+str(attempt+1)
+                if attempt < max_attempts -1:
+                    time.sleep(wait_seconds)
+                    continue
+                else:
+                    raise
diff --git a/drop-boxes/register-immunmonitoring-dropbox/register-immunmonitoring.py b/drop-boxes/register-immunmonitoring-dropbox/register-immunmonitoring.py
@@ -8,6 +8,7 @@
 
 import checksum
 import re
+import time
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
 from java.io import File
@@ -103,4 +104,16 @@ def process(transaction):
     transaction.moveFile(incomingPath, dataSet)
 
     #sample tracking section
-    SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+    wait_seconds = 1
+    max_attempts = 3
+    for attempt in range(max_attempts):
+        try:
+            SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+            break
+        except:
+            print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1)
+            if attempt < max_attempts -1:
+                time.sleep(wait_seconds)
+                continue
+            else:
+                raise
diff --git a/drop-boxes/register-microarrays-dropbox/register-cel-dropbox.py b/drop-boxes/register-microarrays-dropbox/register-cel-dropbox.py
@@ -8,6 +8,7 @@
 
 import checksum
 import re
+import time
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
 from java.io import File
@@ -192,4 +193,16 @@ def process(transaction):
                 os.remove(os.path.realpath(os.path.join(incomingPath,f)))
         #sample tracking section
         for code in trackingCodes:
-                SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code)
+                wait_seconds = 1
+                max_attempts = 3
+                for attempt in range(max_attempts):
+                        try:
+                                SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code)
+                                break
+                        except:
+                                print "Updating location for sample "+code+" failed on attempt "+str(attempt+1)
+                                if attempt < max_attempts -1:
+                                        time.sleep(wait_seconds)
+                                        continue
+                                else:
+                                        raise
diff --git a/drop-boxes/register-mtb-data-dropbox/register-mtb-data-dropbox.py b/drop-boxes/register-mtb-data-dropbox/register-mtb-data-dropbox.py
@@ -164,11 +164,22 @@
 #############################################################################
 
 def update_sample_location_to_qbic(sampleId):
-   """Calls the sample status service and updates the
-   location to QBiC and the status 'DATA AT QBiC'.
-   """
-   # Update sample location
-   SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(sampleId)
+    """Calls the sample status service and updates the
+    location to QBiC and the status 'DATA AT QBiC'.
+    """
+    wait_seconds = 1
+    max_attempts = 3
+    for attempt in range(max_attempts):
+        try:
+            SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(sampleId)
+            break
+        except:
+            print("Updating location for sample " + sampleId + " failed on attempt "+str(attempt+1))
+            if attempt < max_attempts -1:
+                time.sleep(wait_seconds)
+                continue
+            else:
+                raise
 
 
 def process(transaction):
diff --git a/drop-boxes/register-mzML-dropbox/register-mzml-dropbox.py b/drop-boxes/register-mzML-dropbox/register-mzml-dropbox.py
@@ -9,6 +9,7 @@
 import checksum
 import re
 import os
+import time
 import ch.systemsx.cisd.etlserver.registrator.api.v2
 from java.io import File
 from org.apache.commons.io import FileUtils
@@ -107,4 +108,16 @@ def process(transaction):
         transaction.moveFile(incomingPath, dataSet)
 
         #sample tracking section
-        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+        wait_seconds = 1
+        max_attempts = 3
+        for attempt in range(max_attempts):
+                try:
+                        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+                        break
+                except:
+                        print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1)
+                        if attempt < max_attempts -1:
+                                time.sleep(wait_seconds)
+                                continue
+                        else:
+                                raise
diff --git a/drop-boxes/register-nanopore-dropbox/register-nanopore.py b/drop-boxes/register-nanopore-dropbox/register-nanopore.py
@@ -8,6 +8,7 @@
 
 import checksum
 import re
+import time
 import os
 import shutil
 from datetime import datetime
@@ -291,7 +292,19 @@ def createSampleWithData(transaction, space, parentSampleCode, mapWithDataForSam
     transaction.moveFile(absLogPath, logDataSet)
 
     # Updates the sample location of the measured sample
-    SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentSampleCode)
+    wait_seconds = 1
+    max_attempts = 3
+    for attempt in range(max_attempts):
+        try:
+            SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentSampleCode)
+            break
+        except:
+            print "Updating location for sample "+parentSampleCode+" failed on attempt "+str(attempt+1)
+            if attempt < max_attempts -1:
+                time.sleep(wait_seconds)
+                continue
+            else:
+                 raise
 
 def process(transaction):
     """Main ETL routine entry point"""
diff --git a/drop-boxes/register-nmr-dropbox/register-nmr.py b/drop-boxes/register-nmr-dropbox/register-nmr.py
diff --git a/drop-boxes/register-peptidedata-dropbox/register-peptidedata.py b/drop-boxes/register-peptidedata-dropbox/register-peptidedata.py
diff --git a/drop-boxes/register-qcml-dropbox/register-qcml-dropbox.py b/drop-boxes/register-qcml-dropbox/register-qcml-dropbox.py
diff --git a/drop-boxes/register-qpcr-dropbox/register-qpcr-dropbox.py b/drop-boxes/register-qpcr-dropbox/register-qpcr-dropbox.py
diff --git a/drop-boxes/register-vcf-dropbox/register-vcf.py b/drop-boxes/register-vcf-dropbox/register-vcf.py
diff --git a/drop-boxes/register-wiff-dropbox/register-wiff-data-dropbox.py b/drop-boxes/register-wiff-dropbox/register-wiff-data-dropbox.py
