Implemented processor for prepayments

Prepayments have been implemented by a new processor, `PrepaymentProcessor`.
The full implementation for prepayments involved the following changes:
- Several new columns names have been added to `invoices/invoice.py`,
including column names for the prepay data files and for the exported
invoices
- Many invoices will now export with 4 new columns: `Prepaid Group Name`,
`Prepaid Group Institution`, `Prepaid Group Balance`, `Prepaid Group Used`
- 4 command line arguments have been added to `process_report.py`.
3 of them allows the user to pass in a local version of the prepay credits,
contacts, and projects file. The last one (`—prepay-debits`) allows passing
a local version of the prepay debits files, and defaults to fetching from s3 if not
provided
- A set of test cases have been added for `PrepaymentProcessor`

Since the implementation of this feature required a lot of logic decisions
(i.e What happens if a prepaid project is active, but has no
funds?), below is (hopefully) an exhaustive list of code logic decisions
that were made. These can also be inferred through the test cases.
- Prepay projects listed in `prepaid_projects.csv` are identified by their
project name, not project - allocation name
- Attempting to process past invoices (“backprocessing”) may result in
incorrect output due to the nature of the prepay debit ledger
- While backprocessing is not supported for past months, processing
the same invoice month twice will still return correct output. In this
case, the month’s debit entry may be will be overwritten
- Prepay balances can be used in the same month they are added.
- The time range in which prepay projects are considered “active” includes
their start and end date
- After processing of any given invoice month, debit entries for that
month will be added. I emphasize this for clarification. A debit entry
such as:

`2024-11,G1,1062.48`

Should be interpreted as:
In the period from 2024-11-01 to 2024-11-30, prepay group G1 spent $1062.48
As opposed to:
In the period from 2024-10-01 to 2024-10-31, …

- If prepay projects are “active” but their prepay group has $0 balance,
their prepay info (group name, contact email) is still included, but the
prepay balance will be displayed as $0 and the prepay used as an empty field

Author: QuanMPhm

process_report/invoices/NERC_total_invoice.py

@@ -30,7 +30,11 @@ class NERCTotalInvoice(invoice.Invoice):
         invoice.SU_HOURS_FIELD,
         invoice.SU_TYPE_FIELD,
         invoice.RATE_FIELD,
+        invoice.GROUP_NAME_FIELD,
+        invoice.GROUP_INSTITUTION_FIELD,
+        invoice.GROUP_BALANCE_FIELD,
         invoice.COST_FIELD,
+        invoice.GROUP_BALANCE_USED_FIELD,
         invoice.CREDIT_FIELD,
         invoice.CREDIT_CODE_FIELD,
         invoice.BALANCE_FIELD,

process_report/invoices/billable_invoice.py

@@ -31,7 +31,11 @@ class BillableInvoice(invoice.Invoice):
         invoice.SU_HOURS_FIELD,
         invoice.SU_TYPE_FIELD,
         invoice.RATE_FIELD,
+        invoice.GROUP_NAME_FIELD,
+        invoice.GROUP_INSTITUTION_FIELD,
+        invoice.GROUP_BALANCE_FIELD,
         invoice.COST_FIELD,
+        invoice.GROUP_BALANCE_USED_FIELD,
         invoice.CREDIT_FIELD,
         invoice.CREDIT_CODE_FIELD,
         invoice.BALANCE_FIELD,

process_report/invoices/bu_internal_invoice.py

@@ -15,7 +15,11 @@ class BUInternalInvoice(invoice.Invoice):
         invoice.INVOICE_DATE_FIELD,
         invoice.PI_FIELD,
         "Project",
+        invoice.GROUP_NAME_FIELD,
+        invoice.GROUP_INSTITUTION_FIELD,
+        invoice.GROUP_BALANCE_FIELD,
         invoice.COST_FIELD,
+        invoice.GROUP_BALANCE_USED_FIELD,
         invoice.CREDIT_FIELD,
         invoice.SUBSIDY_FIELD,
         invoice.PI_BALANCE_FIELD,

process_report/invoices/invoice.py

@@ -12,6 +12,18 @@
 PI_2ND_USED = "2nd Month Used"
 ###
 
+### Prepay files fields
+PREPAY_MONTH_FIELD = "Month"
+PREPAY_CREDIT_FIELD = "Credit"
+PREPAY_DEBIT_FIELD = "Debit"
+PREPAY_GROUP_NAME_FIELD = "Group Name"
+PREPAY_GROUP_CONTACT_FIELD = "Group Contact Email"
+PREPAY_MANAGED_FIELD = "MGHPCC Managed"
+PREPAY_PROJECT_FIELD = "Project"
+PREPAY_START_DATE_FIELD = "Start Date"
+PREPAY_END_DATE_FIELD = "End Date"
+###
+
 ### Invoice field names
 INVOICE_DATE_FIELD = "Invoice Month"
 PROJECT_FIELD = "Project - Allocation"
@@ -21,6 +33,10 @@
 INVOICE_ADDRESS_FIELD = "Invoice Address"
 INSTITUTION_FIELD = "Institution"
 INSTITUTION_ID_FIELD = "Institution - Specific Code"
+GROUP_NAME_FIELD = "Prepaid Group Name"
+GROUP_INSTITUTION_FIELD = "Prepaid Group Institution"
+GROUP_BALANCE_FIELD = "Prepaid Group Balance"
+GROUP_BALANCE_USED_FIELD = "Prepaid Group Used"
 SU_HOURS_FIELD = "SU Hours (GBhr or SUhr)"
 SU_TYPE_FIELD = "SU Type"
 SU_CHARGE_FIELD = "SU Charge"
@@ -38,6 +54,7 @@
 MISSING_PI_FIELD = "Missing PI"
 PI_BALANCE_FIELD = "PI Balance"
 PROJECT_NAME_FIELD = "Project"
+GROUP_MANAGED_FIELD = "MGHPCC Managed"
 ###
 
 

process_report/invoices/pi_specific_invoice.py

@@ -27,7 +27,11 @@ class PIInvoice(invoice.Invoice):
         invoice.SU_HOURS_FIELD,
         invoice.SU_TYPE_FIELD,
         invoice.RATE_FIELD,
+        invoice.GROUP_NAME_FIELD,
+        invoice.GROUP_INSTITUTION_FIELD,
+        invoice.GROUP_BALANCE_FIELD,
         invoice.COST_FIELD,
+        invoice.GROUP_BALANCE_USED_FIELD,
         invoice.CREDIT_FIELD,
         invoice.CREDIT_CODE_FIELD,
         invoice.BALANCE_FIELD,

process_report/process_report.py

@@ -23,6 +23,7 @@
     validate_billable_pi_processor,
     new_pi_credit_processor,
     bu_subsidy_processor,
+    prepayment_processor,
 )
 
 ### PI file field names
@@ -54,8 +55,8 @@
 ###
 
 PI_S3_FILEPATH = "PIs/PI.csv"
-
 ALIAS_S3_FILEPATH = "PIs/alias.csv"
+PREPAY_DEBITS_S3_FILEPATH = "Prepay/prepay_debits.csv"
 
 
 logger = logging.getLogger(__name__)
@@ -77,6 +78,14 @@ def load_alias(alias_file):
     return alias_dict
 
 
+def load_prepay_csv(prepay_credits_path, prepay_projects_path, prepay_contacts_path):
+    return (
+        pandas.read_csv(prepay_credits_path),
+        pandas.read_csv(prepay_projects_path),
+        pandas.read_csv(prepay_contacts_path),
+    )
+
+
 def get_iso8601_time():
     return datetime.datetime.now().strftime("%Y%m%dT%H%M%SZ")
 
@@ -121,6 +130,24 @@ def main():
         required=True,
         help="File containing list of projects that are non-billable within a specified duration",
     )
+    parser.add_argument(
+        "--prepay-credits",
+        required=False,
+        default="prepaid_credits.csv",
+        help="CSV listing all prepay group credits. Defaults to 'prepaid_credits.csv'",
+    )
+    parser.add_argument(
+        "--prepay-projects",
+        required=False,
+        default="prepaid_projects.csv",
+        help="CSV listing all prepay group projects. Defaults to 'prepaid_projects.csv'",
+    )
+    parser.add_argument(
+        "--prepay-contacts",
+        required=False,
+        default="prepaid_contacts.csv",
+        help="CSV listing all prepay group contact information. Defaults to 'prepaid_contacts.csv'",
+    )
 
     parser.add_argument(
         "--nonbillable-file",
@@ -168,6 +195,11 @@ def main():
         required=False,
         help="Name of alias file listing PIs with aliases (and their aliases). If not provided, defaults to fetching from S3",
     )
+    parser.add_argument(
+        "--prepay-debits",
+        required=False,
+        help="Name of csv file listing all prepay group debits. If not provided, defaults to fetching from S3",
+    )
     parser.add_argument(
         "--BU-subsidy-amount",
         required=True,
@@ -194,6 +226,15 @@ def main():
         alias_file = fetch_s3_alias_file()
     alias_dict = load_alias(alias_file)
 
+    if args.prepay_debits:
+        prepay_debits_filepath = args.prepay_debits
+    else:
+        prepay_debits_filepath = fetch_s3_prepay_debits()
+
+    prepay_credits, prepay_projects, prepay_info = load_prepay_csv(
+        args.prepay_credits, args.prepay_projects, args.prepay_contacts
+    )
+
     merged_dataframe = merge_csv(csv_files)
 
     pi = []
@@ -255,7 +296,19 @@ def main():
     )
     bu_subsidy_proc.process()
 
-    processed_data = bu_subsidy_proc.data
+    prepayment_proc = prepayment_processor.PrepaymentProcessor(
+        "",
+        invoice_month,
+        bu_subsidy_proc.data,
+        prepay_credits,
+        prepay_projects,
+        prepay_info,
+        prepay_debits_filepath,
+        args.upload_to_s3,
+    )
+    prepayment_proc.process()
+
+    processed_data = prepayment_proc.data
 
     ### Initialize invoices
 
@@ -385,6 +438,13 @@ def fetch_s3_old_pi_file():
     return local_name
 
 
+def fetch_s3_prepay_debits():
+    local_name = "prepay_debits.csv"
+    invoice_bucket = util.get_invoice_bucket()
+    invoice_bucket.download_file(PREPAY_DEBITS_S3_FILEPATH, local_name)
+    return local_name
+
+
 def backup_to_s3_old_pi_file(old_pi_file):
     invoice_bucket = util.get_invoice_bucket()
     invoice_bucket.upload_file(old_pi_file, f"PIs/Archive/PI {get_iso8601_time()}.csv")