From 6492625f57b18aadbb8739e5858f8412de83a619 Mon Sep 17 00:00:00 2001 From: Quan Pham Date: Tue, 14 Jan 2025 17:37:55 -0500 Subject: [PATCH] Refactored all `fetch_s3` functions into one utility function --- process_report/process_report.py | 27 +++------------------------ process_report/util.py | 7 +++++++ 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/process_report/process_report.py b/process_report/process_report.py index 5505b7f..66cda09 100644 --- a/process_report/process_report.py +++ b/process_report/process_report.py @@ -218,18 +218,18 @@ def main(): if args.old_pi_file: old_pi_file = args.old_pi_file else: - old_pi_file = fetch_s3_old_pi_file() + old_pi_file = util.fetch_s3(PI_S3_FILEPATH) if args.alias_file: alias_file = args.alias_file else: - alias_file = fetch_s3_alias_file() + alias_file = util.fetch_s3(ALIAS_S3_FILEPATH) alias_dict = load_alias(alias_file) if args.prepay_debits: prepay_debits_filepath = args.prepay_debits else: - prepay_debits_filepath = fetch_s3_prepay_debits() + prepay_debits_filepath = util.fetch_s3(PREPAY_DEBITS_S3_FILEPATH) prepay_credits, prepay_projects, prepay_info = load_prepay_csv( args.prepay_credits, args.prepay_projects, args.prepay_contacts @@ -424,27 +424,6 @@ def timed_projects(timed_projects_file, invoice_date): return dataframe[mask]["Project"].to_list() -def fetch_s3_alias_file(): - local_name = "alias.csv" - invoice_bucket = util.get_invoice_bucket() - invoice_bucket.download_file(ALIAS_S3_FILEPATH, local_name) - return local_name - - -def fetch_s3_old_pi_file(): - local_name = "PI.csv" - invoice_bucket = util.get_invoice_bucket() - invoice_bucket.download_file(PI_S3_FILEPATH, local_name) - return local_name - - -def fetch_s3_prepay_debits(): - local_name = "prepay_debits.csv" - invoice_bucket = util.get_invoice_bucket() - invoice_bucket.download_file(PREPAY_DEBITS_S3_FILEPATH, local_name) - return local_name - - def backup_to_s3_old_pi_file(old_pi_file): invoice_bucket = util.get_invoice_bucket() invoice_bucket.upload_file(old_pi_file, f"PIs/Archive/PI {get_iso8601_time()}.csv") diff --git a/process_report/util.py b/process_report/util.py index b583cd2..acc46fd 100644 --- a/process_report/util.py +++ b/process_report/util.py @@ -83,3 +83,10 @@ def process_and_export_invoices(invoice_list, upload_to_s3): if upload_to_s3: bucket = get_invoice_bucket() invoice.export_s3(bucket) + + +def fetch_s3(s3_filepath): + local_name = os.path.basename(s3_filepath) + invoice_bucket = get_invoice_bucket() + invoice_bucket.download_file(s3_filepath, local_name) + return local_name