Merge pull request #318 from ooni/fastpath-sshfeeder-fixes

FP 0.29: S3 cache trim, sshfeeder fixes

Author: hellais

af/fastpath/debian/changelog

@@ -1,3 +1,10 @@
+fastpath (0.29) unstable; urgency=medium
+
+  * Trim S3 cache immediately
+  * Sshfeeder fixes
+
+ -- Federico Ceratto <[email protected]>  Mon, 25 May 2020 12:35:26 +0100
+
 fastpath (0.28) unstable; urgency=medium
 
   * Add file trimming safety

af/fastpath/fastpath/core.py

@@ -110,6 +110,8 @@ def setup():
                     help="Do not write measurement on disk")
     ap.add_argument("--no-write-to-db", action="store_true",
                     help="Do not insert measurement in database")
+    ap.add_argument("--keep-s3-cache", action="store_true",
+                    help="Keep files downloaded from S3 in the local cache")
     conf = ap.parse_args()
 
     if conf.devel or conf.stdout or no_journal_handler:
@@ -291,8 +293,12 @@ def match_fingerprints(measurement):
     zzfps = fingerprints["ZZ"]
     ccfps = fingerprints.get(msm_cc, {})
 
+    test_keys = measurement["test_keys"]
+    if test_keys is None:
+        return []
+
     matches = []
-    for req in measurement["test_keys"].get("requests", ()):
+    for req in test_keys.get("requests", ()):
         r = req.get("response", None)
         if r is None:
             continue

af/fastpath/fastpath/s3feeder.py

@@ -222,6 +222,12 @@ def stream_cans(conf, start_day: date, end_day: date) -> Generator[MsmtTup, None
             except Exception as e:
                 log.error(str(e), exc_info=True)
 
+            if not conf.keep_s3_cache:
+                try:
+                    can_f.unlink()
+                except FileNotFoundError:
+                    pass
+
         day += timedelta(days=1)
 
     if end_day:

af/fastpath/fastpath/sshfeeder.py

@@ -119,7 +119,7 @@ def scan_new_files(self):
             stdin, stdout, stderr = self.ssh.exec_command(find_cmd, timeout=10)
         except (paramiko.ssh_exception.SSHException, AttributeError):
             # handle: AttributeError: 'NoneType' object has no attribute 'open_session'
-            self.error("Broken SSH connection. Retrying.")
+            log.error("Broken SSH connection. Retrying.")
             # We retry only once here. On failure we'll try again on the new
             # call to scan_new_files
             self._tryconnect()