- Updated test script to use todays date instead of hardcoded date

- Added test to make sure we also add flag entries for deletions
- Fixed some typos in export script

Author: dklawren

Diff for: extensions/BMO/bin/export_bmo_etl.pl

@@ -18,6 +18,7 @@
 use Bugzilla::Flag;
 use Bugzilla::Group;
 use Bugzilla::User;
+use Bugzilla::Extension::Review::FlagStateActivity;
 
 use HTTP::Headers;
 use HTTP::Request;
@@ -93,7 +94,7 @@
 process_flags();
 process_flag_state_activity();
 process_tracking_flags();
-proccess_keywords();
+process_keywords();
 process_see_also();
 process_mentors();
 process_dependencies();
@@ -382,7 +383,7 @@ sub process_flag_state_activity {
         };
 
         # Store a new copy of the data for use later
-        store_cache($obj->id, $table_name, $obj->modification_date, $data);
+        store_cache($obj->id, $table_name, $obj->flag_when, $data);
       }
 
       push @flags, $data;
@@ -441,7 +442,7 @@ sub process_tracking_flags {
   }
 }
 
-sub proccess_keywords {
+sub process_keywords {
   my $table_name = 'keywords';
   my $rows       = $dbh->selectall_arrayref(
     'SELECT bug_id, keyworddefs.name AS name

Diff for: extensions/BMO/t/bmo/bmo_etl.t

@@ -19,9 +19,10 @@ BEGIN {
 }
 
 use Capture::Tiny qw(capture);
-use QA::Util      qw(get_config);
-use MIME::Base64  qw(encode_base64 decode_base64);
-use Mojo::JSON    qw(false);
+use DateTime;
+use QA::Util     qw(get_config);
+use MIME::Base64 qw(encode_base64 decode_base64);
+use Mojo::JSON   qw(false);
 use Test::Mojo;
 use Test::More;
 
@@ -30,6 +31,7 @@ my $admin_login      = $config->{admin_user_login};
 my $admin_api_key    = $config->{admin_user_api_key};
 my $editbugs_api_key = $config->{editbugs_user_api_key};
 my $url              = Bugzilla->localconfig->urlbase;
+my $snapshot_date    = DateTime->now()->strftime('%Y-%m-%d');
 
 my $t = Test::Mojo->new();
 
@@ -60,9 +62,13 @@ $t->post_ok($url
 my $bug_id_1 = $t->tx->res->json->{id};
 
 # Clear the needinfo which will add an X entry in flag_state_activity
+my $needinfo_update = {
+  comment => {body => 'This is my comment'},
+  flags   => [{name => 'needinfo', status => 'X'}],
+};
 $t->put_ok($url
     . "rest/bug/$bug_id_1" => {'X-Bugzilla-API-Key' => $editbugs_api_key} =>
-    json => {comment => {body => 'This is my comment'}})->status_is(200);
+    json                   => $needinfo_update)->status_is(200);
 
 ### Section 2: Create a new dependent bug
 
@@ -114,19 +120,19 @@ my ($attach_id) = keys %{$t->tx->res->json->{attachments}};
 
 my @cmd = (
   './extensions/BMO/bin/export_bmo_etl.pl',
-  '--verbose', '--test', '--snapshot-date', '2000-01-01'
+  '--verbose', '--test', '--snapshot-date', $snapshot_date,
 );
 
 my ($output, $error, $rv) = capture { system @cmd; };
 ok(!$rv, 'Data exported to test files without error');
-ok(glob(bz_locations()->{'datadir'} . '/2000-01-01-bugs-*.json'),
+ok(glob(bz_locations()->{'datadir'} . '/' . $snapshot_date . '-bugs-*.json'),
   'Export test files exist');
 
 ### Section 5: Export data to BigQuery test instance
 
 @cmd = (
-  './extensions/BMO/bin/export_bmo_etl.pl', '--verbose',
-  '--snapshot-date',                        '2000-01-01'
+  './extensions/BMO/bin/export_bmo_etl.pl',
+  '--verbose', '--snapshot-date', $snapshot_date,
 );
 
 ($output, $error, $rv) = capture { system @cmd; };
@@ -135,7 +141,10 @@ ok(!$rv, 'Data exported to BigQuery test instance without error');
 ### Section 6: Retrieve data from BigQuery instance and verify
 
 my $query = {
-  query => 'SELECT summary FROM test.bugzilla.bugs WHERE id = ' . $bug_id_1 . ';',
+  query => 'SELECT summary FROM test.bugzilla.bugs WHERE id = '
+    . $bug_id_1
+    . ' AND snapshot_date = \''
+    . $snapshot_date . '\';',
   useLegacySql => false
 };
 $t->post_ok(
@@ -144,7 +153,9 @@ $t->post_ok(
 
 $query = {
   query => 'SELECT description FROM test.bugzilla.attachments WHERE id = '
-    . $attach_id . ';',
+    . $attach_id
+    . ' AND snapshot_date = \''
+    . $snapshot_date . '\';',
   useLegacySql => false
 };
 $t->post_ok(
@@ -154,7 +165,9 @@ $t->post_ok(
 $query = {
   query =>
     'SELECT depends_on_id FROM test.bugzilla.bug_dependencies WHERE bug_id = '
-    . $bug_id_2 . ';',
+    . $bug_id_2
+    . ' AND snapshot_date = \''
+    . $snapshot_date . '\';',
   useLegacySql => false
 };
 $t->post_ok(
@@ -163,7 +176,9 @@ $t->post_ok(
 
 $query = {
   query => 'SELECT bug_id FROM test.bugzilla.flags WHERE bug_id = '
-    . $bug_id_1 . " AND name = 'needinfo';",
+    . $bug_id_1
+    . ' AND name = \'needinfo\' AND value = \'X\' AND snapshot_date = \''
+    . $snapshot_date . '\'',
   useLegacySql => false
 };
 $t->post_ok(
@@ -173,8 +188,10 @@ $t->post_ok(
 $query = {
   query => 'SELECT bug_id FROM test.bugzilla.flags WHERE bug_id = '
     . $bug_id_1
-    . " AND name = 'review' AND attachment_id = "
-    . $attach_id . ';',
+    . ' AND name = \'review\' AND attachment_id = '
+    . $attach_id
+    . ' AND snapshot_date = \''
+    . $snapshot_date . '\';',
   useLegacySql => false
 };
 
@@ -185,8 +202,8 @@ $t->post_ok(
 ### Section 7: Exporting again on the same day (with the same snapshot date) will cause the script to exit
 
 @cmd = (
-  './extensions/BMO/bin/export_bmo_etl.pl', '--verbose',
-  '--snapshot-date',                        '2000-01-01',
+  './extensions/BMO/bin/export_bmo_etl.pl',
+  '--verbose', '--snapshot-date', $snapshot_date,
 );
 
 ($output, $error, $rv) = capture { system @cmd; };