add counting of number of page edits

Author: carlinmack

nsdb/parse.py

@@ -96,12 +96,16 @@ def parseNonTargetNamespace(
 
     detector = mwreverts.Detector()
 
+    pageEdits = 0
+
     for revision in tqdm.tqdm(
         page, desc=title, unit=" edits", smoothing=0, disable=parallel
     ):
         if not revision.user:
             continue
 
+        pageEdits = pageEdits + 1
+
         # Check if not None as there is a user 0, Larry Sanger
         if revision.user.id is not None:
             userId = revision.user.id
@@ -175,6 +179,11 @@ def parseNonTargetNamespace(
                 ),
             )
 
+    
+    query = """UPDATE page (number_of_edits)
+                VALUES (%s) 
+                WHERE title=%s;"""
+    cursor.execute(query, (pageEdits, title))
 
 def parseTargetNamespace(page, title: str, namespace: str, cursor, parallel: str):
     """Extracts features from each revision of a page into a database
@@ -200,13 +209,17 @@ def parseTargetNamespace(page, title: str, namespace: str, cursor, parallel: str
 
     detector = mwreverts.Detector()
 
+    pageEdits = 0
+
     ## Extract page features from each revision
     for revision in tqdm.tqdm(
         page, desc=title, unit=" edits", smoothing=0, disable=parallel
     ):
         if not revision.user:
             continue
 
+        pageEdits = pageEdits + 1
+
         # Check if not None as there is a user 0, Larry Sanger
         if revision.user.id is not None:
             userId = revision.user.id
@@ -366,6 +379,11 @@ def parseTargetNamespace(page, title: str, namespace: str, cursor, parallel: str
         ## Insert page features into database
         cursor.execute(query, editTuple)
 
+    query = """UPDATE page (number_of_edits)
+                VALUES (%s) 
+                WHERE title=%s;"""
+    cursor.execute(query, (pageEdits, title))
+
 
 def getDiff(old: str, new: str, parallel: str) -> Tuple[str, str]:
     """Returns the diff between two edits using wdiff

sql/schema.sql

@@ -42,6 +42,7 @@ CREATE TABLE page (
     namespace smallint NOT NULL,
     title varchar(255) binary NOT NULL,
     file_name varchar(85) NOT NULL,
+    number_of_edits int unsigned NOT NULL DEFAULT '0',
     PRIMARY KEY (page_id)
 );