Trim S3 cache immediately

Author: Federico Ceratto

af/fastpath/debian/changelog

@@ -1,6 +1,7 @@
 fastpath (0.29) unstable; urgency=medium
 
-  * sshfeeder fixes
+  * Trim S3 cache immediately
+  * Sshfeeder fixes
 
  -- Federico Ceratto <[email protected]>  Mon, 25 May 2020 12:35:26 +0100
 

af/fastpath/fastpath/core.py

@@ -110,6 +110,8 @@ def setup():
                     help="Do not write measurement on disk")
     ap.add_argument("--no-write-to-db", action="store_true",
                     help="Do not insert measurement in database")
+    ap.add_argument("--keep-s3-cache", action="store_true",
+                    help="Keep files downloaded from S3 in the local cache")
     conf = ap.parse_args()
 
     if conf.devel or conf.stdout or no_journal_handler:

af/fastpath/fastpath/s3feeder.py

@@ -222,6 +222,12 @@ def stream_cans(conf, start_day: date, end_day: date) -> Generator[MsmtTup, None
             except Exception as e:
                 log.error(str(e), exc_info=True)
 
+            if not conf.keep_s3_cache:
+                try:
+                    can_f.unlink()
+                except FileNotFoundError:
+                    pass
+
         day += timedelta(days=1)
 
     if end_day: