Merge pull request #269 from opensanctions/wd-dates-filter

Handle imprecise historical dates in Wikidata better

Author: pudo

nomenklatura/wikidata/props.py

@@ -64,7 +64,6 @@
     # "Q392651": "role.spy",
     "Q14886050": "crime.terror",
     "Q16533": "role.judge",
-    "Q17276321": "role.pep",  # member of the state duma
     "Q189290": "mil",  # military officer
     "Q47064": "mil",  # military personnel
 }

nomenklatura/wikidata/value.py

@@ -13,6 +13,7 @@
 
 
 log = logging.getLogger(__name__)
+MIN_DATE = "1001"
 PRECISION = {
     11: Precision.DAY,
     10: Precision.MONTH,
@@ -31,6 +32,13 @@ def snak_value_to_string(
             return LangText(None)
         time = raw_time.strip("+")
         prec_id = cast(int, value.get("precision"))
+        # cf. https://www.wikidata.org/wiki/Help:Dates#Precision
+        if prec_id >= 9:
+            if time < "1900":
+                # Hacky, but set all old dates to the minimum date so persons
+                # with historical birth dates are filtered out.
+                return LangText(MIN_DATE, original=raw_time)
+            return LangText(None, original=raw_time)
         prec = PRECISION.get(prec_id, Precision.DAY)
         time = time[: prec.value]
 
@@ -40,7 +48,7 @@ def snak_value_to_string(
             time = time[:4]
 
         # Date limit in FtM. These will be removed by the death filter:
-        time = max("1001", time)
+        time = max(MIN_DATE, time)
         return LangText(time, original=raw_time)
     elif value_type == "wikibase-entityid":
         qid = value.get("id")

tests/conftest.py

@@ -4,6 +4,8 @@
 from sqlalchemy import MetaData
 import yaml
 import pytest
+from urllib.error import HTTPError
+from urllib.request import Request, urlopen
 from pathlib import Path
 from tempfile import mkdtemp
 from normality import slugify_text
@@ -18,6 +20,10 @@
 from nomenklatura.cache import Cache
 
 FIXTURES_PATH = Path(__file__).parent.joinpath("fixtures/")
+FIXTURE_FETCH_HEADERS = {
+    "User-Agent": "followthemoney.tech/nomenklatura (https://github.com/opensanctions/nomenklatura)",
+    "Accept": "application/json",
+}
 settings.TESTING = True
 
 
@@ -119,9 +125,12 @@ def wd_read_response(request, context):
     assert file_name is not None, "Invalid Wikidata URL: %s" % request.url
     path = FIXTURES_PATH / f"wikidata/{file_name}.json"
     if not path.exists():
-        import urllib.request
-
-        data = json.load(urllib.request.urlopen(request.url))
+        try:
+            req = Request(request.url, headers=FIXTURE_FETCH_HEADERS)
+            data = json.load(urlopen(req))
+        except HTTPError as exc:
+            print("URL", request.url, "failed:", exc, exc.read())
+            raise
         for _, value in data["entities"].items():
             value.pop("sitelinks", None)
             for sect in ["labels", "aliases", "descriptions"]:

tests/fixtures/wikidata/api_php_action_wbgetentities_format_json_ids_p22_props_labels.json

@@ -0,0 +1 @@
+{"entities": {"P22": {"type": "property", "datatype": "wikibase-item", "id": "P22", "labels": {"en": {"language": "en", "value": "father"}}}}, "success": 1}

tests/fixtures/wikidata/api_php_action_wbgetentities_format_json_ids_p25_props_labels.json

@@ -0,0 +1 @@
+{"entities": {"P25": {"type": "property", "datatype": "wikibase-item", "id": "P25", "labels": {"en": {"language": "en", "value": "mother"}}}}, "success": 1}