From 3787e5aed4717bc8dfad3df1a6a2c46c68d373e1 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Mon, 15 Apr 2024 12:41:17 -0400 Subject: [PATCH 01/10] directly import PAC committees --- Makefile | 8 ++- .../management/commands/import_api_data.py | 22 +------ .../commands/import_candidate_api_data.py | 2 - .../commands/import_pac_api_data.py | 66 +++++++++++++++++++ 4 files changed, 73 insertions(+), 25 deletions(-) create mode 100644 camp_fin/management/commands/import_pac_api_data.py diff --git a/Makefile b/Makefile index de07243..279feca 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,9 @@ .PHONY : quarterly -quarterly: import/candidates import/CON_2020 import/EXP_2020 import/CON_2021 import/EXP_2021 import/CON_2022 import/EXP_2022 import/CON_2023 import/EXP_2023 import/CON_2024 import/EXP_2024 +quarterly: import/candidates import/pacs import/CON_2020 import/EXP_2020 import/CON_2021 import/EXP_2021 import/CON_2022 import/EXP_2022 import/CON_2023 import/EXP_2023 import/CON_2024 import/EXP_2024 .PHONY : nightly -nightly: import/candidates import/CON_2023 import/EXP_2023 import/CON_2024 import/EXP_2024 +nightly: import/candidates import/pacs import/CON_2023 import/EXP_2023 import/CON_2024 import/EXP_2024 import/% : _data/sorted/%.csv @@ -14,6 +14,10 @@ import/% : _data/sorted/%.csv import/candidates : _data/raw/candidate_committees.csv python manage.py import_candidate_api_data --file $< +import/pacs : _data/raw/pac_committees.csv + python manage.py import_pac_api_data --file $< + + _data/raw/candidate_committees.csv : wget --no-use-server-timestamps -O $@ "https://openness-project-nmid.s3.amazonaws.com/candidate_committees.csv" diff --git a/camp_fin/management/commands/import_api_data.py b/camp_fin/management/commands/import_api_data.py index a534fc1..c936347 100644 --- a/camp_fin/management/commands/import_api_data.py +++ b/camp_fin/management/commands/import_api_data.py @@ -198,24 +198,6 @@ def make_contributor(self, record): return contact - def make_pac(self, record, entity_type=None): - - entity_type, _ = models.EntityType.objects.get_or_create( - description=entity_type or record["Report Entity Type"] - ) - - entity, _ = models.Entity.objects.get_or_create( - user_id=record["OrgID"], entity_type=entity_type - ) - - pac, _ = models.PAC.objects.get_or_create( - name=record["Committee Name"], - entity=entity, - slug=f'{slugify(record["Committee Name"])}-{get_random_string(5)}', - ) - - return pac - def make_filing(self, record): if not any( @@ -235,8 +217,6 @@ def make_filing(self, record): raise ValueError if record["Report Entity Type"] == "Candidate": - # Create PAC associated with candidate - self.make_pac(record, entity_type="Political Committee") candidate = self._get_candidate(record) @@ -269,7 +249,7 @@ def make_filing(self, record): filing_kwargs = {"entity": candidate.entity} else: - pac = self.make_pac(record) + pac = models.PAC.get(entity__user_id=record["OrgID"]) filing_kwargs = {"entity": pac.entity} diff --git a/camp_fin/management/commands/import_candidate_api_data.py b/camp_fin/management/commands/import_candidate_api_data.py index eed9be4..95d7968 100644 --- a/camp_fin/management/commands/import_candidate_api_data.py +++ b/camp_fin/management/commands/import_candidate_api_data.py @@ -31,7 +31,6 @@ def handle(self, *args, **options): candidates_created = 0 candidates_linked = 0 - candidates_skipped = 0 models.Campaign.objects.filter(election_season__year__gte=2021).delete() @@ -105,5 +104,4 @@ def handle(self, *args, **options): self.stderr.write( f"Linked {candidates_linked} candidates with a campaign, " f"created {candidates_created} candidates" - f"skipped {candidates_skipped} candidates" ) diff --git a/camp_fin/management/commands/import_pac_api_data.py b/camp_fin/management/commands/import_pac_api_data.py new file mode 100644 index 0000000..a0dee6c --- /dev/null +++ b/camp_fin/management/commands/import_pac_api_data.py @@ -0,0 +1,66 @@ +import csv + +from django.core.management.base import BaseCommand +from django.utils.crypto import get_random_string +from django.utils.text import slugify +from tqdm import tqdm + +from camp_fin import models + + +class Command(BaseCommand): + help = """ + Import data from the New Mexico Campaign Finance System: + https://github.com/datamade/nmid-scrapers/pull/2 + + """ + + def add_arguments(self, parser): + parser.add_argument( + "--file", + dest="file", + help="Absolute path of CSV file to import", + required=True, + ) + + def handle(self, *args, **options): + with open(options["file"]) as f: + + reader = csv.DictReader(f) + + pacs_created = 0 + pacs_linked = 0 + pacs_skipped = 0 + + for record in tqdm(reader): + + if not record["StateID"]: + pacs_skipped += 1 + continue + + try: + models.PAC.objects.get(entity__user_id=record["StateID"]) + pacs_linked += 1 + + except models.PAC.DoesNotExist: + entity_type, _ = models.EntityType.objects.get_or_create( + description=record["CommitteeType"] + ) + entity = models.Entity.objects.create( + entity_type=entity_type, + user_id=record["StateID"], + ) + + models.PAC.objects.create( + name=record["CommitteeName"], + slug=f'{slugify(record["CommitteeName"])}-{get_random_string(5)}', + entity=entity, + ) + + pacs_created += 1 + + self.stderr.write( + f"found {pacs_linked} pacs, " + f"created {pacs_created} pacs, " + f"skipped {pacs_skipped} pacs" + ) From e60544b3d9d079dc5131e81532b3bd739b3e2d08 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Fri, 19 Apr 2024 12:28:44 -0400 Subject: [PATCH 02/10] load contributions with filings --- Makefile | 8 +- .../management/commands/aggregate_data.py | 4 +- .../management/commands/import_api_data.py | 149 ++++++------------ .../commands/import_candidate_filings.py | 120 ++++++++++++++ .../commands/import_pac_api_data.py | 69 +++++--- .../management/commands/import_pac_filings.py | 103 ++++++++++++ .../migrations/0082_auto_20240418_0737.py | 32 ++++ .../migrations/0083_auto_20240418_0905.py | 25 +++ .../migrations/0084_auto_20240418_1048.py | 19 +++ camp_fin/models.py | 27 ++-- 10 files changed, 411 insertions(+), 145 deletions(-) create mode 100644 camp_fin/management/commands/import_candidate_filings.py create mode 100644 camp_fin/management/commands/import_pac_filings.py create mode 100644 camp_fin/migrations/0082_auto_20240418_0737.py create mode 100644 camp_fin/migrations/0083_auto_20240418_0905.py create mode 100644 camp_fin/migrations/0084_auto_20240418_1048.py diff --git a/Makefile b/Makefile index 279feca..4430a3c 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,12 @@ import/% : _data/sorted/%.csv --year $(word 2, $(subst _, , $*)) \ --file $< +import/pac_filings : _data/raw/pac_committee_filings.csv + python manage.py import_pac_filings --file $< + +import/candidate_filings : _data/raw/candidate_committee_filings.csv + python manage.py import_candidate_filings --file $< + import/candidates : _data/raw/candidate_committees.csv python manage.py import_candidate_api_data --file $< @@ -22,7 +28,7 @@ _data/raw/candidate_committees.csv : wget --no-use-server-timestamps -O $@ "https://openness-project-nmid.s3.amazonaws.com/candidate_committees.csv" _data/sorted/%.csv : _data/raw/%.csv - xsv fixlengths $< | xsv sort -s OrgID,"Report Name" > $@ + xsv fixlengths $< | xsv sort -s OrgID,"Report Name","Start of Period","End of Period" > $@ _data/raw/CON_%.csv : wget --no-use-server-timestamps \ diff --git a/camp_fin/management/commands/aggregate_data.py b/camp_fin/management/commands/aggregate_data.py index 3f886db..a8a2a1c 100644 --- a/camp_fin/management/commands/aggregate_data.py +++ b/camp_fin/management/commands/aggregate_data.py @@ -98,7 +98,7 @@ def makeTransactionAggregates(self): JOIN camp_fin_filingperiod AS fp ON filing.filing_period_id = fp.id WHERE tt.contribution = FALSE - AND fp.filing_date >= '2010-01-01' + AND filing.filed_date >= '2010-01-01' GROUP BY filing.entity_id, date_trunc('{0}', e.received_date) UNION @@ -115,7 +115,7 @@ def makeTransactionAggregates(self): JOIN camp_fin_filingperiod AS fp ON filing.filing_period_id = fp.id WHERE ltt.description = 'Payment' - AND fp.filing_date >= '2010-01-01' + AND filing.filed_date >= '2010-01-01' GROUP BY filing.entity_id, date_trunc('{0}', lt.transaction_date) ) AS s GROUP BY entity_id, {0} diff --git a/camp_fin/management/commands/import_api_data.py b/camp_fin/management/commands/import_api_data.py index c936347..57ff818 100644 --- a/camp_fin/management/commands/import_api_data.py +++ b/camp_fin/management/commands/import_api_data.py @@ -2,7 +2,6 @@ import re from itertools import groupby -from dateutil.parser import ParserError, parse from django.core.management import call_command from django.core.management.base import BaseCommand from django.db.models import Sum @@ -12,6 +11,8 @@ from camp_fin import models +from .utils import parse_date + class Command(BaseCommand): help = """ @@ -56,15 +57,6 @@ def handle(self, *args, **options): self.total_filings(options["year"]) call_command("aggregate_data") - def parse_date(self, date_str): - try: - return parse(date_str).date() - except (ParserError, TypeError): - self.stderr.write( - self.style.ERROR(f"Could not parse date from string '{date_str}'") - ) - return None - def import_contributions(self, f): reader = csv.DictReader(f) @@ -75,7 +67,7 @@ def key_func(record): for i, record in enumerate(records): if i == 0: try: - filing = self.make_filing(record) + filing = self._get_filing(record) except ValueError: continue @@ -87,13 +79,10 @@ def key_func(record): contributor = self.make_contributor(record) - if record["Contribution Type"] == "Loans Received": - self.make_contribution(record, contributor, filing).save() - - elif record["Contribution Type"] == "Special Event": - self.make_contribution(record, contributor, filing).save() - - elif "Contribution" in record["Contribution Type"]: + if ( + record["Contribution Type"] in {"Loans Received", "Special Event"} + or "Contribution" in record["Contribution Type"] + ): self.make_contribution(record, contributor, filing).save() else: @@ -111,7 +100,7 @@ def key_func(record): for i, record in enumerate(records): if i == 0: try: - filing = self.make_filing(record) + filing = self._get_filing(record) except ValueError: continue @@ -198,85 +187,43 @@ def make_contributor(self, record): return contact - def make_filing(self, record): - - if not any( - ( - self.parse_date(record["Filed Date"]), - self.parse_date(record["End of Period"]), - ) - ): - raise ValueError - - if not record["Report Entity Type"]: - self.stderr.write( - self.style.ERROR( - f"Report entity type not provided. Skipping record {record}..." - ) - ) - raise ValueError - - if record["Report Entity Type"] == "Candidate": + def _get_filing(self, record): - candidate = self._get_candidate(record) + start_date = parse_date(record["Start of Period"]) + end_date = parse_date(record["End of Period"]) + if start_date is None or end_date is None: + raise ValueError("Record is missing 'Start of Period' or 'End of Period'") - transaction_date = ( - record["Transaction Date"] - if "Transaction Date" in record - else record["Expenditure Date"] - ) + state_id = record["OrgID"] - transaction_year = self.parse_date(transaction_date).year - - try: - campaign = models.Campaign.objects.get( - candidate=candidate, election_season__year=transaction_year - ) - except models.Campaign.DoesNotExist: - self.stderr.write( - f"Could not find campaign for {candidate} in {transaction_year}" - ) - - campaign = None - - if campaign and campaign.committee_name != record["Committee Name"]: - campaign.committee_name = record["Committee Name"] - campaign.save() - - if campaign: - filing_kwargs = {"entity": candidate.entity, "campaign": campaign} - else: - filing_kwargs = {"entity": candidate.entity} - - else: - pac = models.PAC.get(entity__user_id=record["OrgID"]) - - filing_kwargs = {"entity": pac.entity} - - filing_type, _ = models.FilingType.objects.get_or_create( - description=record["Report Name"][:24] + try: + entity = models.Entity.objects.get(user_id=state_id) + except models.Entity.DoesNotExist: + entity = models.PAC.objects.get(name=record["Committee Name"]).entity + entity.user_id = state_id + entity.save() + + # We want to associate the tranasctions with the final filing + # for a reporting period + filings = models.Filing.objects.filter( + filing_period__description=record["Report Name"], + filing_period__initial_date__year=start_date.year, + filing_period__end_date__year=end_date.year, + final=True, + entity=entity, ) - filing_period, _ = models.FilingPeriod.objects.get_or_create( - description=record["Report Name"], - filing_date=( - self.parse_date(record["Filed Date"]) - or self.parse_date(record["End of Period"]) - ), - initial_date=self.parse_date(record["Start of Period"]), - due_date=self.parse_date(record["End of Period"]), - allow_no_activity=False, - exclude_from_cascading=False, - email_sent_status=0, - filing_period_type=filing_type, - ) + try: + filing = filings.get() + except models.Filing.MultipleObjectsReturned: + import pdb - filing, _ = models.Filing.objects.get_or_create( - filing_period=filing_period, - date_closed=self.parse_date(record["End of Period"]), - final=True, - **filing_kwargs, - ) + pdb.set_trace() + except models.Filing.DoesNotExist: + import pdb + + pdb.set_trace() + raise ValueError return filing @@ -299,7 +246,7 @@ def make_contribution(self, record, contributor, filing): loan, _ = models.Loan.objects.get_or_create( amount=record["Transaction Amount"], - received_date=self.parse_date(record["Transaction Date"]), + received_date=parse_date(record["Transaction Date"]), check_number=record["Check Number"], status_id=0, contact=contributor, @@ -310,7 +257,7 @@ def make_contribution(self, record, contributor, filing): contribution = models.LoanTransaction( amount=record["Transaction Amount"], - transaction_date=self.parse_date(record["Transaction Date"]), + transaction_date=parse_date(record["Transaction Date"]), transaction_status_id=0, loan=loan, filing=filing, @@ -322,7 +269,7 @@ def make_contribution(self, record, contributor, filing): contribution = models.SpecialEvent( anonymous_contributions=record["Transaction Amount"], - event_date=self.parse_date(record["Transaction Date"]), + event_date=parse_date(record["Transaction Date"]), admission_price=0, attendance=0, total_admissions=0, @@ -342,7 +289,7 @@ def make_contribution(self, record, contributor, filing): contribution = models.Transaction( amount=record["Transaction Amount"], - received_date=self.parse_date(record["Transaction Date"]), + received_date=parse_date(record["Transaction Date"]), check_number=record["Check Number"], description=record["Description"][:74], contact=contributor, @@ -392,7 +339,7 @@ def make_contribution(self, record, contributor, filing): contribution = models.Transaction( amount=record["Expenditure Amount"], - received_date=self.parse_date(record["Expenditure Date"]), + received_date=parse_date(record["Expenditure Date"]), description=(record["Description"] or record["Expenditure Type"])[:74], full_name=payee_full_name, name_prefix=record["Payee Prefix"], @@ -410,7 +357,7 @@ def make_contribution(self, record, contributor, filing): def total_filings(self, year): for filing in models.Filing.objects.filter( - filing_period__filing_date__year=year + filing_period__end_date__year=year ).iterator(): contributions = filing.contributions().aggregate(total=Sum("amount")) expenditures = filing.expenditures().aggregate(total=Sum("amount")) @@ -420,12 +367,6 @@ def total_filings(self, year): filing.total_expenditures = expenditures["total"] or 0 filing.total_loans = loans["total"] or 0 - filing.closing_balance = filing.opening_balance or 0 + ( - filing.total_contributions - + filing.total_loans - - filing.total_expenditures - ) - filing.save() self.stdout.write(f"Totalled {filing}") diff --git a/camp_fin/management/commands/import_candidate_filings.py b/camp_fin/management/commands/import_candidate_filings.py new file mode 100644 index 0000000..e78e19c --- /dev/null +++ b/camp_fin/management/commands/import_candidate_filings.py @@ -0,0 +1,120 @@ +import csv +import re + +from django.core.management.base import BaseCommand +from tqdm import tqdm + +from camp_fin import models + +from .utils import convert_to_float, parse_date + + +class Command(BaseCommand): + help = """ + Import data from the New Mexico Campaign Finance System: + https://github.com/datamade/nmid-scrapers/pull/2 + + """ + + def add_arguments(self, parser): + parser.add_argument( + "--file", + dest="file", + help="Absolute path of CSV file to import", + required=True, + ) + + def handle(self, *args, **options): + with open(options["file"]) as f: + + reader = csv.DictReader(f) + + filings_created = 0 + filings_linked = 0 + + for record in tqdm(reader): + + # Duplicate Filings: + # https://login.cfis.sos.state.nm.us/#/exploreDetails/mP8cXhW3dUTpRJ9Yk07LpZP4048PFnxLXRUfdOLcQk01/14/120/119/2022 (Second Primary Report) # noqa + if record["ReportID"] in {"21785"}: + continue + + amended = record["Amended"] != "0" + + try: + filing = models.Filing.objects.get( + report_id=record["ReportID"], + report_version_id=record["ReportVersionID"], + ) + filings_linked += 1 + except models.Filing.DoesNotExist: + filing = self._create_filing(record) + filings_created += 1 + else: + if filing.final and amended: + # if we need to change the status of a filing from + # final to amended, we will delete the entire filing + # so as to delete any associated transactions + filing.delete() + filing = self._create_filing(record) + + if record["opening_balance"]: + filing.opening_balance = convert_to_float(record["opening_balance"]) + filing.closing_balance = convert_to_float(record["closing_balance"]) + + filing.save() + + self.stderr.write( + f"found {filings_linked} filings, " f"created {filings_created} filings, " + ) + + def _create_filing(self, record): + + filing_type, _ = models.FilingType.objects.get_or_create( + description=record["ReportName"][:24] + ) + + filing_period, _ = models.FilingPeriod.objects.get_or_create( + description=record["ReportName"], + initial_date=parse_date(record["FilingStartDate"]), + end_date=parse_date(record["FilingEndDate"]), + due_date=parse_date(record["FilingDueDate"]), + allow_no_activity=False, + exclude_from_cascading=False, + email_sent_status=0, + filing_period_type=filing_type, + ) + + entity = models.Entity.objects.get(user_id=record["StateID"]) + + url = f"https://login.cfis.sos.state.nm.us//ReportsOutput//{record['ReportFileName']}" + final = record["Amended"] == "0" or None + + filing = models.Filing.objects.create( + filing_period=filing_period, + filed_date=parse_date(record["FilingEndDate"]), + date_closed=parse_date(record["FilingEndDate"]), + final=final, + pdf_report=url or None, + entity=entity, + report_id=record["ReportID"], + report_version_id=record["ReportVersionID"], + ) + + try: + campaign = models.Campaign.objects.get( + election_season__year=re.match(r"\d{4}", record["ElectionYear"]).group( + 0 + ), + candidate__entity=entity, + office__description=record["OfficeName"], + district__name=record["District"], + county__name=record["Jurisdiction"] or None, + ) + except models.Campaign.DoesNotExist: + pass + else: + filing.campaign = campaign + filing.save() + + return filing diff --git a/camp_fin/management/commands/import_pac_api_data.py b/camp_fin/management/commands/import_pac_api_data.py index a0dee6c..a872454 100644 --- a/camp_fin/management/commands/import_pac_api_data.py +++ b/camp_fin/management/commands/import_pac_api_data.py @@ -34,30 +34,51 @@ def handle(self, *args, **options): for record in tqdm(reader): - if not record["StateID"]: - pacs_skipped += 1 - continue - - try: - models.PAC.objects.get(entity__user_id=record["StateID"]) - pacs_linked += 1 - - except models.PAC.DoesNotExist: - entity_type, _ = models.EntityType.objects.get_or_create( - description=record["CommitteeType"] - ) - entity = models.Entity.objects.create( - entity_type=entity_type, - user_id=record["StateID"], - ) - - models.PAC.objects.create( - name=record["CommitteeName"], - slug=f'{slugify(record["CommitteeName"])}-{get_random_string(5)}', - entity=entity, - ) - - pacs_created += 1 + state_id = record["StateID"] + + if state_id: + + try: + models.PAC.objects.get(entity__user_id=record["StateID"]) + pacs_linked += 1 + + except models.PAC.DoesNotExist: + entity_type, _ = models.EntityType.objects.get_or_create( + description=record["CommitteeType"] + ) + entity = models.Entity.objects.create( + entity_type=entity_type, + user_id=record["StateID"], + ) + + models.PAC.objects.create( + name=record["CommitteeName"], + slug=f'{slugify(record["CommitteeName"])}-{get_random_string(5)}', + entity=entity, + ) + + pacs_created += 1 + + else: + try: + models.PAC.objects.get(name=record["CommitteeName"]) + pacs_linked += 1 + + except models.PAC.DoesNotExist: + entity_type, _ = models.EntityType.objects.get_or_create( + description=record["CommitteeType"] + ) + entity = models.Entity.objects.create( + entity_type=entity_type, + ) + + models.PAC.objects.create( + name=record["CommitteeName"], + slug=f'{slugify(record["CommitteeName"])}-{get_random_string(5)}', + entity=entity, + ) + + pacs_created += 1 self.stderr.write( f"found {pacs_linked} pacs, " diff --git a/camp_fin/management/commands/import_pac_filings.py b/camp_fin/management/commands/import_pac_filings.py new file mode 100644 index 0000000..11bf095 --- /dev/null +++ b/camp_fin/management/commands/import_pac_filings.py @@ -0,0 +1,103 @@ +import csv + +from django.core.management.base import BaseCommand +from tqdm import tqdm + +from camp_fin import models + +from .utils import convert_to_float, parse_date + + +class Command(BaseCommand): + help = """ + Import data from the New Mexico Campaign Finance System: + https://github.com/datamade/nmid-scrapers/pull/2 + + """ + + def add_arguments(self, parser): + parser.add_argument( + "--file", + dest="file", + help="Absolute path of CSV file to import", + required=True, + ) + + def handle(self, *args, **options): + with open(options["file"]) as f: + + reader = csv.DictReader(f) + + filings_created = 0 + filings_linked = 0 + + for record in tqdm(reader): + + amended = record["Amended"] != "0" + + try: + filing = models.Filing.objects.get( + report_id=record["ReportID"], + report_version_id=record["ReportVersionID"], + ) + filings_linked += 1 + except models.Filing.DoesNotExist: + filing = self._create_filing(record) + filings_created += 1 + else: + if filing.final and amended: + # if we need to change the status of a filing from + # final to amended, we will delete the entire filing + # so as to delete any associated transactions + filing.delete() + filing = self._create_filing(record) + + if record["opening_balance"]: + filing.opening_balance = convert_to_float(record["opening_balance"]) + filing.closing_balance = convert_to_float(record["closing_balance"]) + + filing.save() + + self.stderr.write( + f"found {filings_linked} filings, " f"created {filings_created} filings, " + ) + + def _create_filing(self, record): + + filing_type, _ = models.FilingType.objects.get_or_create( + description=record["ReportName"][:24] + ) + + filing_period, _ = models.FilingPeriod.objects.get_or_create( + description=record["ReportName"], + initial_date=parse_date(record["FilingStartDate"]), + end_date=parse_date(record["FilingEndDate"]), + due_date=parse_date(record["FilingDueDate"]), + allow_no_activity=False, + exclude_from_cascading=False, + email_sent_status=0, + filing_period_type=filing_type, + ) + + state_id = record["StateID"] + if state_id: + entity = models.Entity.objects.get(user_id=state_id) + + else: + entity = models.PAC.objects.get(name=record["CommitteeName"]).entity + + url = f"https://login.cfis.sos.state.nm.us//ReportsOutput//{record['ReportFileName']}" + final = record["Amended"] == "0" or None + + filing = models.Filing.objects.create( + filing_period=filing_period, + filed_date=parse_date(record["FilingEndDate"]), + date_closed=parse_date(record["FilingEndDate"]), + entity=entity, + final=final, + pdf_report=url or None, + report_id=record["ReportID"], + report_version_id=record["ReportVersionID"], + ) + + return filing diff --git a/camp_fin/migrations/0082_auto_20240418_0737.py b/camp_fin/migrations/0082_auto_20240418_0737.py new file mode 100644 index 0000000..854f66b --- /dev/null +++ b/camp_fin/migrations/0082_auto_20240418_0737.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-04-18 13:37 +from __future__ import unicode_literals + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("camp_fin", "0081_auto_20231221_1042"), + ] + + operations = [ + migrations.RemoveField( + model_name="filingperiod", + name="filing_date", + ), + migrations.AddField( + model_name="filing", + name="filed_date", + field=models.DateTimeField(default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.AddField( + model_name="filingperiod", + name="end_date", + field=models.DateField(default=django.utils.timezone.now), + preserve_default=False, + ), + ] diff --git a/camp_fin/migrations/0083_auto_20240418_0905.py b/camp_fin/migrations/0083_auto_20240418_0905.py new file mode 100644 index 0000000..f6c1f85 --- /dev/null +++ b/camp_fin/migrations/0083_auto_20240418_0905.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-04-18 15:05 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("camp_fin", "0082_auto_20240418_0737"), + ] + + operations = [ + migrations.AddField( + model_name="filing", + name="report_id", + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name="filing", + name="report_version_id", + field=models.IntegerField(null=True), + ), + ] diff --git a/camp_fin/migrations/0084_auto_20240418_1048.py b/camp_fin/migrations/0084_auto_20240418_1048.py new file mode 100644 index 0000000..8f5a074 --- /dev/null +++ b/camp_fin/migrations/0084_auto_20240418_1048.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-04-18 16:48 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("camp_fin", "0083_auto_20240418_0905"), + ] + + operations = [ + migrations.RunSQL( + "create unique index camp_fin_filing_report_id_idx on camp_fin_filing (report_id) where final = True", + reverse_sql="drop index camp_fin_filing_report_id_idx", + ), + ] diff --git a/camp_fin/models.py b/camp_fin/models.py index 4751881..4bca6ff 100644 --- a/camp_fin/models.py +++ b/camp_fin/models.py @@ -1,8 +1,7 @@ from collections import namedtuple -from datetime import datetime, timedelta +from datetime import datetime from dateutil.rrule import MONTHLY, rrule -from django.conf import settings from django.db import connection, models from django.utils import timezone from django.utils.translation import ugettext as _ @@ -733,14 +732,18 @@ def __str__(self): class Filing(models.Model): + entity = models.ForeignKey("Entity", db_constraint=False) olddb_campaign_id = models.IntegerField(null=True) olddb_profile_id = models.IntegerField(null=True) + report_id = models.IntegerField(null=True) + report_version_id = models.IntegerField(null=True) filing_period = models.ForeignKey("FilingPeriod", db_constraint=False) status = models.ForeignKey("Status", db_constraint=False, null=True) date_added = models.DateTimeField(default=timezone.now) olddb_ethics_report_id = models.IntegerField(null=True) campaign = models.ForeignKey("Campaign", db_constraint=False, null=True) + filed_date = models.DateTimeField() date_closed = models.DateTimeField(null=True) date_last_amended = models.DateTimeField(null=True) opening_balance = models.FloatField(null=True) @@ -823,7 +826,6 @@ def expenditures(self, since=None): class FilingPeriod(models.Model): - filing_date = models.DateTimeField() due_date = models.DateTimeField() olddb_id = models.IntegerField(null=True) description = models.CharField(max_length=255, null=True) @@ -835,13 +837,14 @@ class FilingPeriod(models.Model): "RegularFilingPeriod", db_constraint=False, null=True ) initial_date = models.DateField() + end_date = models.DateField() email_sent_status = models.IntegerField() reminder_sent_status = models.IntegerField(blank=True, null=True) def __str__(self): return "{0}/{1} ({2})".format( - self.filing_date.month, - self.filing_date.year, + self.initial_date.month, + self.initial_date.year, self.filing_period_type.description, ) @@ -1063,10 +1066,6 @@ def stack_trends(trend): contributions_lookup = {r.month.date(): r.amount for r in contributions} expenditures_lookup = {r.month.date(): r.amount for r in expenditures} - all_months = list(contributions_lookup.keys()) + list( - expenditures_lookup.keys() - ) - start_month = datetime(int(since), 1, 1) try: @@ -1075,7 +1074,7 @@ def stack_trends(trend): .first() .filing_period.filing_date.date() ) - except: + except AttributeError: end_month = ( FilingPeriod.objects.order_by("-filing_date") .first() @@ -1720,10 +1719,10 @@ class LobbyistBundlingDisclosureContributor(models.Model): ) -###################################################################### -### Below here are normalized tables that we may or may not end up ### -### getting. Just stubbing them out in case we do ### -###################################################################### +################################################################## +# Below here are normalized tables that we may or may not end up # +# getting. Just stubbing them out in case we do # +################################################################## class RegularFilingPeriod(models.Model): From a282e5f1f99571c94f6adff20e31b4cd27edd9c5 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Fri, 19 Apr 2024 12:31:53 -0400 Subject: [PATCH 03/10] remove pdb points --- camp_fin/management/commands/import_api_data.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/camp_fin/management/commands/import_api_data.py b/camp_fin/management/commands/import_api_data.py index 57ff818..9019580 100644 --- a/camp_fin/management/commands/import_api_data.py +++ b/camp_fin/management/commands/import_api_data.py @@ -215,14 +215,7 @@ def _get_filing(self, record): try: filing = filings.get() - except models.Filing.MultipleObjectsReturned: - import pdb - - pdb.set_trace() except models.Filing.DoesNotExist: - import pdb - - pdb.set_trace() raise ValueError return filing From 7a4fbe54712b03fad3234c0f0f65eb667f2e8d59 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Fri, 19 Apr 2024 13:15:35 -0400 Subject: [PATCH 04/10] adjsut code for moved filing date --- camp_fin/base_views.py | 238 ++++++++++++++--------------- camp_fin/models.py | 27 ++-- camp_fin/templates/index.html | 2 +- camp_fin/tests/conftest.py | 7 +- camp_fin/tests/test_integration.py | 6 +- camp_fin/tests/test_unit.py | 41 +---- camp_fin/views.py | 4 +- 7 files changed, 140 insertions(+), 185 deletions(-) diff --git a/camp_fin/base_views.py b/camp_fin/base_views.py index b65d3d3..d7fdb46 100644 --- a/camp_fin/base_views.py +++ b/camp_fin/base_views.py @@ -5,21 +5,19 @@ from django.core.exceptions import ObjectDoesNotExist from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db import connection -from django.http import JsonResponse, StreamingHttpResponse +from django.http import StreamingHttpResponse from django.utils import timezone from django.utils.text import slugify from django.views.generic import DetailView, ListView, TemplateView -from rest_framework import filters, renderers, viewsets +from rest_framework import filters, viewsets from rest_framework.response import Response from camp_fin.api_parts import ( - DataTablesPagination, - SearchCSVRenderer, TopMoneySerializer, TransactionCSVRenderer, TransactionSerializer, ) -from camp_fin.models import PAC, Candidate, Entity, Lobbyist, Organization, Transaction +from camp_fin.models import PAC, Candidate, Lobbyist, Organization, Transaction from pages.models import Page TWENTY_TEN = timezone.make_aware(datetime(2010, 1, 1)) @@ -331,7 +329,7 @@ def transaction_query(self, entity_id=None, start_date=None, end_date=None): transaction.description, tt.description AS transaction_type, entity.*, - fp.filing_date, + f.filed_date, fp.description AS filing_name FROM camp_fin_transaction AS transaction JOIN camp_fin_transactiontype AS tt @@ -361,7 +359,7 @@ def transaction_query(self, entity_id=None, start_date=None, end_date=None): ) AS entity ON f.entity_id = entity.{subj_type}_entity_id WHERE tt.contribution = {contribution_bool} - """.format( + """.format( # noqa contribution_bool=contribution_bool, subj_type=subj_type ) @@ -373,12 +371,12 @@ def transaction_query(self, entity_id=None, start_date=None, end_date=None): if start_date: base_query += """ - AND fp.filing_date >= %s + AND f.filed_date >= %s """ if end_date: base_query += """ - AND fp.filing_date <= %s + AND f.filed_date <= %s """ base_query += """ORDER BY transaction.received_date""" @@ -430,46 +428,46 @@ class TopMoneyView(viewsets.ViewSet): def list(self, request): cursor = connection.cursor() - query = """ + query = """ SELECT * FROM ( - SELECT - DENSE_RANK() + SELECT + DENSE_RANK() OVER ( - PARTITION BY year + PARTITION BY year ORDER BY amount DESC ) AS rank, * FROM ( - SELECT + SELECT SUM(transaction.amount) AS amount, NULL AS latest_date, - transaction.name_prefix, - transaction.first_name, - transaction.middle_name, + transaction.name_prefix, + transaction.first_name, + transaction.middle_name, transaction.last_name, transaction.suffix, transaction.company_name, election_season.year, redact, ct.description - FROM camp_fin_transaction AS transaction - JOIN camp_fin_transactiontype AS tt - ON transaction.transaction_type_id = tt.id - JOIN camp_fin_filing AS f - ON transaction.filing_id = f.id - JOIN camp_fin_campaign AS c - ON f.campaign_id = c.id + FROM camp_fin_transaction AS transaction + JOIN camp_fin_transactiontype AS tt + ON transaction.transaction_type_id = tt.id + JOIN camp_fin_filing AS f + ON transaction.filing_id = f.id + JOIN camp_fin_campaign AS c + ON f.campaign_id = c.id JOIN camp_fin_electionseason AS election_season ON c.election_season_id = election_season.id LEFT JOIN camp_fin_contact AS contact ON transaction.contact_id = contact.id LEFT JOIN camp_fin_contacttype AS ct ON contact.contact_type_id = ct.id - WHERE tt.contribution = %s - AND year >= '2010' - GROUP BY - transaction.name_prefix, - transaction.first_name, - transaction.middle_name, + WHERE tt.contribution = %s + AND year >= '2010' + GROUP BY + transaction.name_prefix, + transaction.first_name, + transaction.middle_name, transaction.last_name, transaction.suffix, transaction.company_name, @@ -477,56 +475,56 @@ def list(self, request): redact, ct.description ORDER BY SUM(amount) DESC - ) AS ranked_list + ) AS ranked_list ORDER BY year DESC, amount DESC - ) AS election_groups + ) AS election_groups WHERE rank < 11 """ if self.request.GET.get("entity_type") == "pac": - query = """ + query = """ SELECT *, NULL AS year FROM ( - SELECT - DENSE_RANK() + SELECT + DENSE_RANK() OVER ( ORDER BY amount DESC ) AS rank, * FROM ( - SELECT - SUM(transaction.amount) AS amount, + SELECT + SUM(transaction.amount) AS amount, MAX(transaction.received_date) AS latest_date, - transaction.name_prefix, - transaction.first_name, - transaction.middle_name, + transaction.name_prefix, + transaction.first_name, + transaction.middle_name, transaction.last_name, transaction.suffix, transaction.company_name, transaction.redact, ct.description - FROM camp_fin_transaction AS transaction - JOIN camp_fin_transactiontype AS tt - ON transaction.transaction_type_id = tt.id - JOIN camp_fin_filing AS f - ON transaction.filing_id = f.id + FROM camp_fin_transaction AS transaction + JOIN camp_fin_transactiontype AS tt + ON transaction.transaction_type_id = tt.id + JOIN camp_fin_filing AS f + ON transaction.filing_id = f.id LEFT JOIN camp_fin_contact AS contact ON transaction.contact_id = contact.id LEFT JOIN camp_fin_contacttype AS ct ON contact.contact_type_id = ct.id - WHERE tt.contribution = %s + WHERE tt.contribution = %s AND transaction.received_date >= '2010-01-01' - GROUP BY - transaction.name_prefix, - transaction.first_name, - transaction.middle_name, + GROUP BY + transaction.name_prefix, + transaction.first_name, + transaction.middle_name, transaction.last_name, transaction.suffix, transaction.company_name, transaction.redact, ct.description ORDER BY SUM(amount) DESC - ) AS ranked_list + ) AS ranked_list ORDER BY amount DESC - ) AS election_groups + ) AS election_groups WHERE rank < 11 """ @@ -544,34 +542,34 @@ def list(self, request): def retrieve(self, request, pk=None): cursor = connection.cursor() - query = """ + query = """ SELECT * FROM ( - SELECT - DENSE_RANK() + SELECT + DENSE_RANK() OVER ( - PARTITION BY year + PARTITION BY year ORDER BY amount DESC ) AS rank, * FROM ( - SELECT - SUM(transaction.amount) AS amount, + SELECT + SUM(transaction.amount) AS amount, MAX(transaction.received_date) AS latest_date, - transaction.name_prefix, - transaction.first_name, - transaction.middle_name, + transaction.name_prefix, + transaction.first_name, + transaction.middle_name, transaction.last_name, transaction.suffix, transaction.company_name, election_season.year, transaction.redact, ct.description - FROM camp_fin_transaction AS transaction - JOIN camp_fin_transactiontype AS tt - ON transaction.transaction_type_id = tt.id - JOIN camp_fin_filing AS f - ON transaction.filing_id = f.id - JOIN camp_fin_campaign AS c - ON f.campaign_id = c.id + FROM camp_fin_transaction AS transaction + JOIN camp_fin_transactiontype AS tt + ON transaction.transaction_type_id = tt.id + JOIN camp_fin_filing AS f + ON transaction.filing_id = f.id + JOIN camp_fin_campaign AS c + ON f.campaign_id = c.id JOIN camp_fin_candidate as candidate ON c.candidate_id = candidate.id JOIN camp_fin_electionseason AS election_season @@ -580,13 +578,13 @@ def retrieve(self, request, pk=None): ON transaction.contact_id = contact.id LEFT JOIN camp_fin_contacttype AS ct ON contact.contact_type_id = ct.id - WHERE tt.contribution = %s + WHERE tt.contribution = %s AND candidate.id = %s - AND transaction.received_date >= '2010-01-01' - GROUP BY - transaction.name_prefix, - transaction.first_name, - transaction.middle_name, + AND transaction.received_date >= '2010-01-01' + GROUP BY + transaction.name_prefix, + transaction.first_name, + transaction.middle_name, transaction.last_name, transaction.suffix, transaction.company_name, @@ -594,58 +592,58 @@ def retrieve(self, request, pk=None): transaction.redact, ct.description ORDER BY SUM(amount) DESC - ) AS ranked_list + ) AS ranked_list ORDER BY year DESC, amount DESC - ) AS election_groups + ) AS election_groups WHERE rank < 11 """ if self.request.GET.get("entity_type") == "pac": - query = """ + query = """ SELECT *, NULL AS year FROM ( - SELECT - DENSE_RANK() + SELECT + DENSE_RANK() OVER ( ORDER BY amount DESC ) AS rank, * FROM ( - SELECT - SUM(transaction.amount) AS amount, + SELECT + SUM(transaction.amount) AS amount, MAX(transaction.received_date) AS latest_date, - transaction.name_prefix, - transaction.first_name, - transaction.middle_name, + transaction.name_prefix, + transaction.first_name, + transaction.middle_name, transaction.last_name, transaction.suffix, transaction.company_name, transaction.redact, ct.description - FROM camp_fin_transaction AS transaction - JOIN camp_fin_transactiontype AS tt - ON transaction.transaction_type_id = tt.id - JOIN camp_fin_filing AS f - ON transaction.filing_id = f.id + FROM camp_fin_transaction AS transaction + JOIN camp_fin_transactiontype AS tt + ON transaction.transaction_type_id = tt.id + JOIN camp_fin_filing AS f + ON transaction.filing_id = f.id JOIN camp_fin_pac AS pac ON f.entity_id = pac.entity_id LEFT JOIN camp_fin_contact AS contact ON transaction.contact_id = contact.id LEFT JOIN camp_fin_contacttype AS ct ON contact.contact_type_id = ct.id - WHERE tt.contribution = %s + WHERE tt.contribution = %s AND pac.id = %s - AND transaction.received_date >= '2010-01-01' - GROUP BY - transaction.name_prefix, - transaction.first_name, - transaction.middle_name, + AND transaction.received_date >= '2010-01-01' + GROUP BY + transaction.name_prefix, + transaction.first_name, + transaction.middle_name, transaction.last_name, transaction.suffix, transaction.company_name, transaction.redact, ct.description ORDER BY SUM(amount) DESC - ) AS ranked_list + ) AS ranked_list ORDER BY amount DESC - ) AS election_groups + ) AS election_groups WHERE rank < 11 """ @@ -668,33 +666,33 @@ def get_context_data(self, **kwargs): with connection.cursor() as cursor: # Top earners cursor.execute( - """ + """ SELECT * FROM ( - SELECT - dense_rank() OVER (ORDER BY new_funds DESC) AS rank, * + SELECT + dense_rank() OVER (ORDER BY new_funds DESC) AS rank, * FROM ( - SELECT - MAX(COALESCE(c.slug, p.slug)) AS slug, - MAX(COALESCE(c.full_name, p.name)) AS name, - SUM(t.amount) AS new_funds, - (array_agg(f.closing_balance ORDER BY f.id DESC))[1] AS current_funds, - CASE WHEN p.id IS NULL - THEN 'Candidate' - ELSE 'PAC' - END AS committee_type - FROM camp_fin_transaction AS t - JOIN camp_fin_transactiontype AS tt - ON t.transaction_type_id = tt.id - JOIN camp_fin_filing AS f - ON t.filing_id = f.id - LEFT JOIN camp_fin_pac AS p - ON f.entity_id = p.entity_id - LEFT JOIN camp_fin_candidate AS c - ON f.entity_id = c.entity_id - WHERE tt.contribution = TRUE + SELECT + MAX(COALESCE(c.slug, p.slug)) AS slug, + MAX(COALESCE(c.full_name, p.name)) AS name, + SUM(t.amount) AS new_funds, + (array_agg(f.closing_balance ORDER BY f.id DESC))[1] AS current_funds, + CASE WHEN p.id IS NULL + THEN 'Candidate' + ELSE 'PAC' + END AS committee_type + FROM camp_fin_transaction AS t + JOIN camp_fin_transactiontype AS tt + ON t.transaction_type_id = tt.id + JOIN camp_fin_filing AS f + ON t.filing_id = f.id + LEFT JOIN camp_fin_pac AS p + ON f.entity_id = p.entity_id + LEFT JOIN camp_fin_candidate AS c + ON f.entity_id = c.entity_id + WHERE tt.contribution = TRUE AND t.received_date >= (NOW() - INTERVAL '90 days') GROUP BY c.id, p.id - ) AS s + ) AS s WHERE name NOT ILIKE '%public election fund%' OR name NOT ILIKE '%department of finance%' ORDER BY new_funds DESC diff --git a/camp_fin/models.py b/camp_fin/models.py index 4bca6ff..ce65e34 100644 --- a/camp_fin/models.py +++ b/camp_fin/models.py @@ -148,7 +148,7 @@ def filings(self, since=None): if since: date = "{year}-01-01".format(year=since) - filings = filings.filter(filing_period__filing_date__gte=date) + filings = filings.filter(filed_date__gte=date) return filings @@ -971,7 +971,7 @@ def stack_trends(trend): SUM(f.closing_balance) AS closing_balance, SUM(f.opening_balance) AS opening_balance, SUM(f.total_debt_carried_forward) AS debt_carried_forward, - fp.filing_date, + f.filed_date, MIN(fp.initial_date) AS initial_date FROM camp_fin_filing AS f JOIN camp_fin_filingperiod AS fp @@ -979,9 +979,9 @@ def stack_trends(trend): WHERE f.entity_id = %s AND fp.exclude_from_cascading = FALSE AND fp.regular_filing_period_id IS NULL - AND fp.filing_date >= '{year}-01-01' - GROUP BY fp.filing_date - ORDER BY fp.filing_date + AND f.filed_date >= '{year}-01-01' + GROUP BY f.filed_date + ORDER BY f.filed_date """.format( year=since ) @@ -999,7 +999,7 @@ def stack_trends(trend): if summed_filings: for filing in summed_filings: - filing_date = filing.filing_date + filing_date = filing.filed_date date_array = [filing_date.year, filing_date.month, filing_date.day] debts = -1 * filing.total_unpaid_debts if filing.closing_balance: @@ -1068,18 +1068,9 @@ def stack_trends(trend): start_month = datetime(int(since), 1, 1) - try: - end_month = ( - self.filing_set.order_by("-filing_period__filing_date") - .first() - .filing_period.filing_date.date() - ) - except AttributeError: - end_month = ( - FilingPeriod.objects.order_by("-filing_date") - .first() - .filing_date.date() - ) + end_month = ( + self.filing_set.order_by("-filed_date").first().filed_date.date() + ) for month in rrule(freq=MONTHLY, dtstart=start_month, until=end_month): replacements = {"month": month.month - 1} diff --git a/camp_fin/templates/index.html b/camp_fin/templates/index.html index 57fee90..07e230f 100644 --- a/camp_fin/templates/index.html +++ b/camp_fin/templates/index.html @@ -110,7 +110,7 @@

Top Committees

{{ forloop.counter }} {{ pac.name }} - {{ filing.filing_period.filing_date | date:'M j, Y'}} + {{ filing.filed_date | date:'M j, Y'}} {{ filing.closing_balance|format_money_short }} diff --git a/camp_fin/tests/conftest.py b/camp_fin/tests/conftest.py index 91d4d47..0392484 100644 --- a/camp_fin/tests/conftest.py +++ b/camp_fin/tests/conftest.py @@ -16,7 +16,6 @@ FilingPeriod, FilingType, Loan, - LoanTransactionType, Lobbyist, Office, OfficeType, @@ -142,7 +141,6 @@ def races(cls): filing_type = FilingType.objects.create(description="type") cls.filing_period = FilingPeriod.objects.create( - filing_date=datetime.datetime.now(pytz.utc), due_date=datetime.datetime.now(pytz.utc), allow_no_activity=True, filing_period_type=filing_type, @@ -153,6 +151,7 @@ def races(cls): ) cls.first_filing = Filing.objects.create( + filed_date=datetime.datetime.now(pytz.utc), entity=cls.first_entity, campaign=cls.first_campaign, filing_period=cls.filing_period, @@ -183,8 +182,6 @@ def races(cls): filing=cls.first_filing, ) - loan_type = LoanTransactionType.objects.create(description="Payment") - cls.loan = Loan.objects.create( status=status, date_added=datetime.datetime.now(pytz.utc), @@ -264,7 +261,6 @@ def races(cls): ) cls.filtered_filing_period = FilingPeriod.objects.create( - filing_date=two_years_ago, due_date=two_years_ago, allow_no_activity=True, filing_period_type=filing_type, @@ -275,6 +271,7 @@ def races(cls): ) cls.filtered_filing = Filing.objects.create( + filed_date=two_years_ago, entity=cls.second_entity, campaign=cls.second_campaign, filing_period=cls.filtered_filing_period, diff --git a/camp_fin/tests/test_integration.py b/camp_fin/tests/test_integration.py index 6378208..5683275 100644 --- a/camp_fin/tests/test_integration.py +++ b/camp_fin/tests/test_integration.py @@ -1,5 +1,3 @@ -from django.urls import reverse - from camp_fin.tests.conftest import DatabaseTestCase @@ -51,7 +49,7 @@ def test_campaign_funds_raised(self): ) def test_campaign_funds_raised_since_date(self): - year = str(self.filing_period.filing_date.year) + year = str(self.filed_date.year) total_funds = self.second_contribution.amount self.assertEqual(self.second_campaign.funds_raised(since=year), total_funds) @@ -62,7 +60,7 @@ def test_campaign_expenditures(self): ) def test_campaign_expenditures_since_date(self): - year = str(self.filing_period.filing_date.year) + year = str(self.filed_date.year) total_expenditures = self.second_expenditure.amount self.assertEqual( self.second_campaign.expenditures(since=year), total_expenditures diff --git a/camp_fin/tests/test_unit.py b/camp_fin/tests/test_unit.py index da8f495..6717ebd 100644 --- a/camp_fin/tests/test_unit.py +++ b/camp_fin/tests/test_unit.py @@ -1,43 +1,14 @@ from django.contrib.auth.models import User -from django.core.management import call_command from django.db.utils import IntegrityError from django.http import HttpRequest, QueryDict from django.test import TestCase -from django.urls import resolve, reverse +from django.urls import reverse from camp_fin.base_views import TransactionDownloadViewSet from camp_fin.decorators import check_date_params -from camp_fin.models import ( - Campaign, - Candidate, - County, - District, - Division, - ElectionSeason, - Entity, - Filing, - FilingPeriod, - FilingType, - Loan, - LoanTransaction, - LoanTransactionType, - Office, - OfficeType, - PoliticalParty, - Race, - Status, - Transaction, - TransactionType, -) +from camp_fin.models import OfficeType, Race from camp_fin.templatetags.helpers import format_years -from camp_fin.tests.conftest import DatabaseTestCase, StatelessTestCase -from camp_fin.views import ( - LobbyistDetail, - LobbyistList, - LobbyistTransactionList, - RaceDetail, - RacesView, -) +from camp_fin.tests.conftest import StatelessTestCase class TestRace(StatelessTestCase): @@ -47,7 +18,7 @@ class TestRace(StatelessTestCase): def test_race_unique_constraint(self): with self.assertRaises(IntegrityError): - race = Race.objects.create( + Race.objects.create( division=self.division, district=self.district, office=self.office, @@ -125,7 +96,7 @@ def test_campaign_filings(self): self.assertEqual(set(campaign.filings()), set(filing)) def test_campaign_filings_since_date(self): - year = str(self.filing_period.filing_date.year) + year = str(self.filed_date.year) self.assertNotIn(self.filtered_filing, self.second_campaign.filings(since=year)) def test_campaign_is_winner(self): @@ -214,7 +185,7 @@ def test_race_change_list_loads(self): def test_race_edit_loads(self): url = reverse("admin:camp_fin_race_change", args=(self.race.id,)) edit_page = self.client.get(url, follow=True) - html = edit_page.content.decode("utf-8") + edit_page.content.decode("utf-8") self.assertEqual(edit_page.status_code, 200) diff --git a/camp_fin/views.py b/camp_fin/views.py index b576543..31a9f63 100644 --- a/camp_fin/views.py +++ b/camp_fin/views.py @@ -622,7 +622,7 @@ def get_queryset(self, **kwargs): campaign.division_id, office.description AS office_name, filing.closing_balance, - COALESCE(period.filing_date, filing.date_added) AS filing_date + COALESCE(filing.filed_date, filing.date_added) AS filing_date FROM camp_fin_candidate AS candidate JOIN camp_fin_filing AS filing USING(entity_id) @@ -698,7 +698,7 @@ def get_queryset(self, **kwargs): SELECT DISTINCT ON (pac.id) pac.*, filing.closing_balance, - COALESCE(period.filing_date, filing.date_added) AS filing_date + COALESCE(filing.filed_date, filing.date_added) AS filing_date FROM camp_fin_pac AS pac JOIN camp_fin_filing AS filing USING(entity_id) From 75ecdd3f1614f1471ecb5f2c28320ef8f6f72dc6 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Fri, 19 Apr 2024 13:22:54 -0400 Subject: [PATCH 05/10] clearer names for management commands --- Makefile | 14 +++++++++----- ..._candidate_api_data.py => import_candidates.py} | 0 ...rt_pac_api_data.py => import_pac_committees.py} | 0 .../{import_api_data.py => import_transactions.py} | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) rename camp_fin/management/commands/{import_candidate_api_data.py => import_candidates.py} (100%) rename camp_fin/management/commands/{import_pac_api_data.py => import_pac_committees.py} (100%) rename camp_fin/management/commands/{import_api_data.py => import_transactions.py} (99%) diff --git a/Makefile b/Makefile index 4430a3c..99674e3 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ nightly: import/candidates import/pacs import/CON_2023 import/EXP_2023 import/CO import/% : _data/sorted/%.csv - python manage.py import_api_data --transaction-type $(word 1, $(subst _, , $*)) \ + python manage.py import_transactions --transaction-type $(word 1, $(subst _, , $*)) \ --year $(word 2, $(subst _, , $*)) \ --file $< @@ -18,14 +18,18 @@ import/candidate_filings : _data/raw/candidate_committee_filings.csv python manage.py import_candidate_filings --file $< import/candidates : _data/raw/candidate_committees.csv - python manage.py import_candidate_api_data --file $< + python manage.py import_candidates --file $< import/pacs : _data/raw/pac_committees.csv - python manage.py import_pac_api_data --file $< + python manage.py import_pac_committees --file $< -_data/raw/candidate_committees.csv : - wget --no-use-server-timestamps -O $@ "https://openness-project-nmid.s3.amazonaws.com/candidate_committees.csv" +_data/raw/%_committees.csv : + wget --no-use-server-timestamps -O $@ "https://openness-project-nmid.s3.amazonaws.com/$*_committees.csv" + +_data/raw/%_committee_filings.csv : + wget --no-use-server-timestamps -O $@ "https://openness-project-nmid.s3.amazonaws.com/$*_committees.csv" + _data/sorted/%.csv : _data/raw/%.csv xsv fixlengths $< | xsv sort -s OrgID,"Report Name","Start of Period","End of Period" > $@ diff --git a/camp_fin/management/commands/import_candidate_api_data.py b/camp_fin/management/commands/import_candidates.py similarity index 100% rename from camp_fin/management/commands/import_candidate_api_data.py rename to camp_fin/management/commands/import_candidates.py diff --git a/camp_fin/management/commands/import_pac_api_data.py b/camp_fin/management/commands/import_pac_committees.py similarity index 100% rename from camp_fin/management/commands/import_pac_api_data.py rename to camp_fin/management/commands/import_pac_committees.py diff --git a/camp_fin/management/commands/import_api_data.py b/camp_fin/management/commands/import_transactions.py similarity index 99% rename from camp_fin/management/commands/import_api_data.py rename to camp_fin/management/commands/import_transactions.py index 9019580..643f263 100644 --- a/camp_fin/management/commands/import_api_data.py +++ b/camp_fin/management/commands/import_transactions.py @@ -203,7 +203,7 @@ def _get_filing(self, record): entity.user_id = state_id entity.save() - # We want to associate the tranasctions with the final filing + # We want to associate the transactions with the final filing # for a reporting period filings = models.Filing.objects.filter( filing_period__description=record["Report Name"], From 4b972102734f14f597585e923cc6560016c4e408 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Fri, 19 Apr 2024 13:24:19 -0400 Subject: [PATCH 06/10] utils file --- camp_fin/management/commands/utils.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 camp_fin/management/commands/utils.py diff --git a/camp_fin/management/commands/utils.py b/camp_fin/management/commands/utils.py new file mode 100644 index 0000000..b300790 --- /dev/null +++ b/camp_fin/management/commands/utils.py @@ -0,0 +1,20 @@ +import re + +from dateutil.parser import ParserError, parse +from dateutil.tz import gettz + + +def convert_to_float(value): + value = re.sub("[^0-9.]", "", value) + if value.startswith("("): + return -float(value) + else: + return float(value) + + +def parse_date(date_str): + try: + mountain_tz = gettz("America/Denver") + return parse(date_str).replace(tzinfo=mountain_tz) + except (ParserError, TypeError): + return None From 533edbf5ce6f87f0a87fd54e561013fd0a68a3a2 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Fri, 19 Apr 2024 13:26:12 -0400 Subject: [PATCH 07/10] used filed date for totalling --- camp_fin/management/commands/import_transactions.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/camp_fin/management/commands/import_transactions.py b/camp_fin/management/commands/import_transactions.py index 643f263..e69a5dc 100644 --- a/camp_fin/management/commands/import_transactions.py +++ b/camp_fin/management/commands/import_transactions.py @@ -349,9 +349,7 @@ def make_contribution(self, record, contributor, filing): return contribution def total_filings(self, year): - for filing in models.Filing.objects.filter( - filing_period__end_date__year=year - ).iterator(): + for filing in models.Filing.objects.filter(filed_date__year=year).iterator(): contributions = filing.contributions().aggregate(total=Sum("amount")) expenditures = filing.expenditures().aggregate(total=Sum("amount")) loans = filing.loans().aggregate(total=Sum("amount")) From 9eed797e4756fb1ef85ecb40c3f84f200b66fdf4 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Fri, 19 Apr 2024 13:37:59 -0400 Subject: [PATCH 08/10] adjust conftes --- camp_fin/tests/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/camp_fin/tests/conftest.py b/camp_fin/tests/conftest.py index 0392484..343cf57 100644 --- a/camp_fin/tests/conftest.py +++ b/camp_fin/tests/conftest.py @@ -146,6 +146,7 @@ def races(cls): filing_period_type=filing_type, exclude_from_cascading=True, initial_date=datetime.datetime.now(pytz.utc), + end_date=datetime.datetime.now(pytz.utc), email_sent_status=0, reminder_sent_status=0, ) @@ -266,6 +267,7 @@ def races(cls): filing_period_type=filing_type, exclude_from_cascading=True, initial_date=two_years_ago, + end_date=two_years_ago, email_sent_status=0, reminder_sent_status=0, ) From e9d5645bab4143cd45dfb84cf08049d7f9553e6e Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Fri, 19 Apr 2024 13:48:58 -0400 Subject: [PATCH 09/10] adjust tests --- camp_fin/tests/conftest.py | 8 +++++--- camp_fin/tests/test_integration.py | 4 ++-- camp_fin/tests/test_unit.py | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/camp_fin/tests/conftest.py b/camp_fin/tests/conftest.py index 343cf57..2dcce6f 100644 --- a/camp_fin/tests/conftest.py +++ b/camp_fin/tests/conftest.py @@ -200,6 +200,7 @@ def races(cls): ) cls.second_filing = Filing.objects.create( + filed_date=datetime.datetime.now(pytz.utc), entity=cls.second_entity, campaign=cls.second_campaign, filing_period=cls.filing_period, @@ -231,6 +232,7 @@ def races(cls): ) cls.third_filing = Filing.objects.create( + filed_date=datetime.datetime.now(pytz.utc), entity=cls.third_entity, campaign=cls.third_campaign, filing_period=cls.filing_period, @@ -356,9 +358,9 @@ def lobbyists(cls): class StatelessTestCase(TestCase, FakeTestData): """ - Test class that does not commit changes to the database. Inherits from TestCase so that - every test runs in a rolled-back transaction. - """ + Test class that does not commit changes to the database. Inherits from TestCase so that + every test runs in a rolled-back transaction. + g""" @classmethod def setUpTestData(cls): diff --git a/camp_fin/tests/test_integration.py b/camp_fin/tests/test_integration.py index 5683275..8fd8ab9 100644 --- a/camp_fin/tests/test_integration.py +++ b/camp_fin/tests/test_integration.py @@ -49,7 +49,7 @@ def test_campaign_funds_raised(self): ) def test_campaign_funds_raised_since_date(self): - year = str(self.filed_date.year) + year = str(self.filing_period.end_date.year) total_funds = self.second_contribution.amount self.assertEqual(self.second_campaign.funds_raised(since=year), total_funds) @@ -60,7 +60,7 @@ def test_campaign_expenditures(self): ) def test_campaign_expenditures_since_date(self): - year = str(self.filed_date.year) + year = str(self.filing_period.end_date.year) total_expenditures = self.second_expenditure.amount self.assertEqual( self.second_campaign.expenditures(since=year), total_expenditures diff --git a/camp_fin/tests/test_unit.py b/camp_fin/tests/test_unit.py index 6717ebd..9d13caf 100644 --- a/camp_fin/tests/test_unit.py +++ b/camp_fin/tests/test_unit.py @@ -96,7 +96,7 @@ def test_campaign_filings(self): self.assertEqual(set(campaign.filings()), set(filing)) def test_campaign_filings_since_date(self): - year = str(self.filed_date.year) + year = str(self.filing_period.end_date.year) self.assertNotIn(self.filtered_filing, self.second_campaign.filings(since=year)) def test_campaign_is_winner(self): From 9a004cbe39466a92af9f8311db6e7969fad41e4d Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Fri, 19 Apr 2024 13:50:45 -0400 Subject: [PATCH 10/10] typo --- camp_fin/tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/camp_fin/tests/conftest.py b/camp_fin/tests/conftest.py index 2dcce6f..72480a8 100644 --- a/camp_fin/tests/conftest.py +++ b/camp_fin/tests/conftest.py @@ -358,9 +358,9 @@ def lobbyists(cls): class StatelessTestCase(TestCase, FakeTestData): """ - Test class that does not commit changes to the database. Inherits from TestCase so that - every test runs in a rolled-back transaction. - g""" + Test class that does not commit changes to the database. Inherits from TestCase so that + every test runs in a rolled-back transaction. + """ @classmethod def setUpTestData(cls):