Consolidate invoicing configurations

All invoicing configurations, such as output file names, input file
paths, etc, have been grouped into a Pydantic module in `settings.py`
file. There is no longer any CLI arguments, and all
`Processor`/`Invoice` subclasses will now get their unique
configurations from `settings.py`, instead of being passed through
`__init__()`. Test cases, especially the e2e, have been changed accordingly.

Loading of configuration files moved to `loader.py`. This separation
between settings and loading logic should make the code more maintainable.

With how we previously handled invoicing configurations,
`process_report.py` had an unyieldingly long list of configurations,
which also forced us to initialize each processor/invoice individually,
creating much clutter over time. This change is made to reduce clutter by
grouping all configurations into one file.

Author: QuanMPhm

.gitignore

@@ -1,4 +1,5 @@
 *.csv
 !process_report/tests/e2e/test_data/*.csv
+!process_report/tests/e2e/test_data/test_invoices/*.csv
 __pycache__/
 *.py[cod]

process_report/invoices/NERC_total_invoice.py

@@ -2,6 +2,7 @@
 
 import process_report.invoices.invoice as invoice
 import process_report.util as util
+from process_report.settings import invoice_settings
 
 
 @dataclass
@@ -12,6 +13,7 @@ class NERCTotalInvoice(invoice.Invoice):
     - NewPICreditProcessor
     """
 
+    name: str = invoice_settings.nerc_total_invoice_name
     export_columns_list = [
         invoice.INVOICE_DATE_FIELD,
         invoice.PROJECT_FIELD,

process_report/invoices/billable_invoice.py

@@ -1,8 +1,6 @@
 from dataclasses import dataclass
 
-import pandas
-import pyarrow
-
+from process_report.settings import invoice_settings
 from process_report.invoices import invoice
 
 
@@ -14,11 +12,7 @@ class BillableInvoice(invoice.Invoice):
     - NewPICreditProcessor
     """
 
-    PI_S3_FILEPATH = "PIs/PI.csv"
-
-    old_pi_filepath: str
-    updated_old_pi_df: pandas.DataFrame
-
+    name: str = invoice_settings.billable_invoice_name
     export_columns_list = [
         invoice.INVOICE_DATE_FIELD,
         invoice.PROJECT_FIELD,
@@ -45,20 +39,3 @@ def _prepare_export(self):
         self.export_data = self.data[
             self.data[invoice.IS_BILLABLE_FIELD] & ~self.data[invoice.MISSING_PI_FIELD]
         ]
-        self.updated_old_pi_df = self.updated_old_pi_df.astype(
-            {
-                invoice.PI_INITIAL_CREDITS: pandas.ArrowDtype(
-                    pyarrow.decimal128(21, 2)
-                ),
-                invoice.PI_1ST_USED: pandas.ArrowDtype(pyarrow.decimal128(21, 2)),
-                invoice.PI_2ND_USED: pandas.ArrowDtype(pyarrow.decimal128(21, 2)),
-            },
-        )
-
-    def export(self):
-        super().export()
-        self.updated_old_pi_df.to_csv(self.old_pi_filepath, index=False)
-
-    def export_s3(self, s3_bucket):
-        super().export_s3(s3_bucket)
-        s3_bucket.upload_file(self.old_pi_filepath, self.PI_S3_FILEPATH)

process_report/invoices/bu_internal_invoice.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 
 import process_report.invoices.invoice as invoice
+from process_report.settings import invoice_settings
 
 
 @dataclass
@@ -11,6 +12,7 @@ class BUInternalInvoice(invoice.Invoice):
     - NewPICreditProcessor
     """
 
+    name: str = invoice_settings.bu_internal_invoice_name
     export_columns_list = [
         invoice.INVOICE_DATE_FIELD,
         invoice.PI_FIELD,

process_report/invoices/invoice.py

@@ -64,9 +64,9 @@ class Invoice:
     export_columns_list = list()
     exported_columns_map = dict()
 
-    name: str
     invoice_month: str
     data: pandas.DataFrame
+    name: str = ""
     export_data = None
 
     def process(self):

process_report/invoices/lenovo_invoice.py

@@ -1,5 +1,6 @@
 from dataclasses import dataclass
 
+from process_report.settings import invoice_settings
 import process_report.invoices.invoice as invoice
 
 
@@ -12,6 +13,7 @@ class LenovoInvoice(invoice.Invoice):
         "BM GPUH100",
     ]
 
+    name: str = invoice_settings.lenovo_invoice_name
     export_columns_list = [
         invoice.INVOICE_DATE_FIELD,
         invoice.PROJECT_FIELD,

process_report/invoices/nonbillable_invoice.py

@@ -1,13 +1,12 @@
 from dataclasses import dataclass
 
+from process_report.settings import invoice_settings
 import process_report.invoices.invoice as invoice
 
 
 @dataclass
 class NonbillableInvoice(invoice.Invoice):
-    nonbillable_pis: list[str]
-    nonbillable_projects: list[str]
-
+    name: str = invoice_settings.nonbillable_invoice_name
     export_columns_list = [
         invoice.INVOICE_DATE_FIELD,
         invoice.PROJECT_FIELD,

process_report/invoices/ocp_test_invoice.py

@@ -1,10 +1,13 @@
 from dataclasses import dataclass
 
 import process_report.invoices.invoice as invoice
+from process_report.settings import invoice_settings
 
 
 @dataclass
 class OcpTestInvoice(invoice.Invoice):
+    name: str = invoice_settings.ocp_test_invoice_name
+
     export_columns_list = [
         invoice.INVOICE_DATE_FIELD,
         invoice.PROJECT_FIELD,

process_report/invoices/pi_specific_invoice.py

@@ -9,6 +9,7 @@
 from jinja2 import Environment, FileSystemLoader
 
 import process_report.invoices.invoice as invoice
+from process_report.settings import invoice_settings
 import process_report.util as util
 
 
@@ -43,6 +44,8 @@ class PIInvoice(invoice.Invoice):
         invoice.BALANCE_FIELD,
     ]
 
+    name: str = invoice_settings.pi_specific_folder_name
+
     export_columns_list = [
         invoice.INVOICE_DATE_FIELD,
         invoice.PROJECT_FIELD,

process_report/invoices/prepay_credits_snapshot.py

@@ -1,16 +1,25 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 
 import pandas
 
+from process_report.loader import loader
+from process_report.settings import invoice_settings
 from process_report import util
 from process_report.invoices import invoice
 
 
 @dataclass
 class PrepayCreditsSnapshot(invoice.Invoice):
-    prepay_credits: pandas.DataFrame
-    prepay_contacts: pandas.DataFrame
-
+    prepay_credits: pandas.DataFrame = field(
+        default_factory=lambda: loader.load_dataframe(
+            invoice_settings.prepay_credits_filepath
+        )
+    )
+    prepay_contacts: pandas.DataFrame = field(
+        default_factory=lambda: loader.load_dataframe(
+            invoice_settings.prepay_contacts_filepath
+        )
+    )
     export_columns_list = [
         invoice.PREPAY_MONTH_FIELD,
         invoice.PREPAY_GROUP_NAME_FIELD,