Include same-day data in pre-generated CSV exports

If a request for a CSV export includes the current day, stop the
pre-generated export at midnight and switch to a live export for
the current day.

perllib/FixMyStreet/App/Controller/Dashboard.pm

@@ -181,12 +181,12 @@ sub index : Path : Args(0) {
     my $reporting = $c->forward('construct_rs_filter', [ $c->get_param('updates') ]);
 
     if ( my $export = $c->get_param('export') ) {
-        $reporting->csv_parameters;
         if ($export == 1) {
             # Existing method, generate and serve
             if ($reporting->premade_csv_exists) {
                 $reporting->filter_premade_csv_http($c);
             } else {
+                $reporting->csv_parameters;
                 $reporting->generate_csv_http($c);
             }
         } elsif ($export == 2) {

perllib/FixMyStreet/Cobrand/TfL.pm

@@ -452,10 +452,6 @@ sub dashboard_export_problems_add_columns {
 
     my @contacts = $csv->body->contacts->order_by('category')->all;
     my %extra_columns;
-    if (@{$csv->category}) {
-        my %picked_cats = map { $_ => 1} @{$csv->category};
-        @contacts = grep { $picked_cats{$_->category} } @contacts;
-    }
     foreach my $contact (@contacts) {
         foreach (@{$contact->get_metadata_for_storage}) {
             next if $_->{code} eq 'safety_critical';

perllib/FixMyStreet/Reporting.pm

@@ -20,7 +20,7 @@ has body => ( is => 'ro', isa => Maybe[InstanceOf['FixMyStreet::DB::Result::Body
 has wards => ( is => 'ro', isa => ArrayRef[Int], default => sub { [] } );
 has category => ( is => 'ro', isa => ArrayRef[Str], default => sub { [] } );
 has state => ( is => 'ro', isa => Maybe[Str] );
-has start_date => ( is => 'ro',
+has start_date => ( is => 'rwp',
     isa => Str,
     default => sub {
         my $days30 = DateTime->now(time_zone => FixMyStreet->time_zone || FixMyStreet->local_time_zone)->subtract(days => 30);
@@ -268,10 +268,10 @@ Generates a CSV output to a file handler provided
 =cut
 
 sub generate_csv {
-    my ($self, $handle) = @_;
+    my ($self, $handle, $exclude_header) = @_;
 
     my $csv = Text::CSV->new({ binary => 1, eol => "\n" });
-    $csv->print($handle, $self->csv_headers);
+    $csv->print($handle, $self->csv_headers) unless $exclude_header;
 
     my $fixed_states = FixMyStreet::DB::Result::Problem->fixed_states;
     my $closed_states = FixMyStreet::DB::Result::Problem->closed_states;
@@ -460,9 +460,17 @@ sub filter_premade_csv {
         $state_column = 'DBState';
     }
 
+    my $add_on_today = 0;
+    my $today = DateTime->today(time_zone => FixMyStreet->time_zone || FixMyStreet->local_time_zone);
+    my $end_date = $self->end_date;
+    if (!$end_date || $end_date ge $today->strftime('%Y-%m-%d')) {
+        $add_on_today = 1;
+        $end_date = $today->subtract(days => 1)->strftime('%Y-%m-%d');
+    }
+
     my $range = FixMyStreet::DateRange->new(
         start_date => $self->start_date,
-        end_date => $self->end_date,
+        end_date => $end_date,
         formatter => FixMyStreet::DB->schema->storage->datetime_parser,
     );
 
@@ -511,6 +519,14 @@ sub filter_premade_csv {
         # Areas and Roles, that were only included for the filtering above
         $csv->print($handle, [ (@{$row}{@$arr})[$first_column..@$arr-$last_column-1] ]);
     }
+
+    if ($add_on_today) {
+        # Add in any information from today the 'live' way
+        $self->_set_start_date($today->strftime('%Y-%m-%d'));
+        $self->construct_rs_filter;
+        $self->csv_parameters;
+        $self->generate_csv($handle, 1);
+    }
 }
 
 # Outputs relevant CSV HTTP headers, and then streams the CSV

t/cobrand/tfl.t

@@ -662,7 +662,7 @@ subtest 'Dashboard CSV extra columns' => sub {
     $mech->content_contains(',12345,,no,[email protected],,', "Bike number added to csv");
     $mech->content_contains('"Council User",,,98756', "Stop code added to csv for all categories report");
     $mech->get_ok('/dashboard?export=1&category=Bus+stops');
-    $mech->content_contains('"Council User",,98756', "Stop code added to csv for bus stop category report");
+    $mech->content_contains('"Council User",,,98756', "Stop code added to csv for bus stop category report");
 
     $report->set_extra_fields({ name => 'leaning', value => 'Yes' }, { name => 'safety_critical', value => 'yes' },
         { name => 'stop_code', value => '98756' }, { name => 'Question', value => '12345' });