fix: refined (thinned out) extraction of variants (#219)

This removes the "xrefs", "citations", "history_records", and "conditions" fields from the classifications of the records written out in the "data extract-vars" command.

Author: holtgrewe

clinvar_data/extract_vars.py

@@ -9,8 +9,8 @@
 from google.protobuf.json_format import MessageToJson, ParseDict
 import tqdm
 
-from clinvar_data.pbs.clinvar_public import Allele, VariationArchive
-from clinvar_data.pbs.clinvar_public_pb2 import ClassifiedRecord
+from clinvar_data.pbs.clinvar_public import Allele, ClassifiedRecord, VariationArchive
+from clinvar_data.pbs.clinvar_public_pb2 import AggregateClassificationSet
 from clinvar_data.pbs.extracted_vars import (
     ExtractedRcvRecord,
     ExtractedVcvRecord,
@@ -72,6 +72,27 @@ def from_string_value(cls, string_value: str) -> VariationType.ValueType:
         return cls.CONVERT.get(string_value.lower(), VariationType.VARIATION_TYPE_OTHER)
 
 
+def thin_out_aggregate_classification_set(
+    classifications: AggregateClassificationSet | None,
+) -> AggregateClassificationSet | None:
+    """Thin out the aggregate classifications set for extracted variants."""
+    if classifications is None:
+        return None
+    else:
+        result = AggregateClassificationSet()
+        result.CopyFrom(classifications)
+        if result.HasField("germline_classification"):
+            for key in ("xrefs", "citations", "history_records", "conditions"):
+                result.germline_classification.ClearField(key)
+        for somatic_clinical_impacts in result.somatic_clinical_impacts:
+            for key in ("xrefs", "citations", "history_records", "conditions"):
+                somatic_clinical_impacts.ClearField(key)
+        if result.HasField("oncogenicity_classification"):
+            for key in ("xrefs", "citations", "history_records", "conditions"):
+                result.oncogenicity_classification.ClearField(key)
+        return result
+
+
 def run(path_input: str, output_dir: str, gzip_output: bool):
     """Execute the variant extraction."""
     os.makedirs(output_dir, exist_ok=True)
@@ -107,11 +128,12 @@ def run(path_input: str, output_dir: str, gzip_output: bool):
             )
             rcvs: list[ExtractedRcvRecord] = [
                 ExtractedRcvRecord(
-                    title=rcva.title,
                     accession=VersionedAccession(
                         accession=rcva.accession,
                         version=rcva.version,
                     ),
+                    title=rcva.title,
+                    classifications=rcva.rcv_classifications,
                 )
                 for rcva in classified_record.rcv_list.rcv_accessions
             ]
@@ -128,7 +150,9 @@ def run(path_input: str, output_dir: str, gzip_output: bool):
                         rcvs=rcvs,
                         name=name,
                         variation_type=variation_type,
-                        classifications=classified_record.classifications,
+                        classifications=(
+                            thin_out_aggregate_classification_set(classified_record.classifications)
+                        ),
                         sequence_location=sequence_location,
                         hgnc_ids=hgnc_ids,
                     )

clinvar_data/pbs/class_by_freq_pb2.py

@@ -2537,7 +2537,10 @@ class Indication(google.protobuf.message.Message):
     COMMENTS_FIELD_NUMBER: builtins.int
     TYPE_FIELD_NUMBER: builtins.int
     type: global___Indication.Type.ValueType
-    """attributes"""
+    """attributes
+
+    The type of indication.
+    """
     @property
     def traits(
         self,

clinvar_data/pbs/clinvar_public_pb2.py

@@ -123,21 +123,39 @@ class ExtractedRcvRecord(google.protobuf.message.Message):
 
     ACCESSION_FIELD_NUMBER: builtins.int
     TITLE_FIELD_NUMBER: builtins.int
+    CLASSIFICATIONS_FIELD_NUMBER: builtins.int
     title: builtins.str
     """Title of RCV."""
     @property
     def accession(self) -> global___VersionedAccession:
         """The accession."""
 
+    @property
+    def classifications(
+        self,
+    ) -> clinvar_data.pbs.clinvar_public_pb2.RcvAccession.RcvClassifications:
+        """Classifications (thinned out)."""
+
     def __init__(
         self,
         *,
         accession: global___VersionedAccession | None = ...,
         title: builtins.str = ...,
+        classifications: (
+            clinvar_data.pbs.clinvar_public_pb2.RcvAccession.RcvClassifications | None
+        ) = ...,
     ) -> None: ...
-    def HasField(self, field_name: typing.Literal["accession", b"accession"]) -> builtins.bool: ...
+    def HasField(
+        self,
+        field_name: typing.Literal[
+            "accession", b"accession", "classifications", b"classifications"
+        ],
+    ) -> builtins.bool: ...
     def ClearField(
-        self, field_name: typing.Literal["accession", b"accession", "title", b"title"]
+        self,
+        field_name: typing.Literal[
+            "accession", b"accession", "classifications", b"classifications", "title", b"title"
+        ],
     ) -> None: ...
 
 global___ExtractedRcvRecord = ExtractedRcvRecord
@@ -173,7 +191,7 @@ class ExtractedVcvRecord(google.protobuf.message.Message):
 
     @property
     def classifications(self) -> clinvar_data.pbs.clinvar_public_pb2.AggregateClassificationSet:
-        """Classifications."""
+        """Classifications (thinned out)."""
 
     @property
     def sequence_location(self) -> clinvar_data.pbs.clinvar_public_pb2.Location.SequenceLocation:

clinvar_data/pbs/clinvar_public_pb2.pyi

@@ -52,6 +52,8 @@ message ExtractedRcvRecord {
   VersionedAccession accession = 1;
   // Title of RCV.
   string title = 2;
+  // Classifications (thinned out).
+  clinvar_data.pbs.clinvar_public.RcvAccession.RcvClassifications classifications = 3;
 }
 
 // Protocol buffer for storing essential information of one VCV.
@@ -64,7 +66,7 @@ message ExtractedVcvRecord {
   string name = 3;
   // The type of the variant.
   VariationType variation_type = 4;
-  // Classifications.
+  // Classifications (thinned out).
   clinvar_data.pbs.clinvar_public.AggregateClassificationSet classifications = 5;
   // The sequence location on one reference.
   clinvar_data.pbs.clinvar_public.Location.SequenceLocation sequence_location = 6;

clinvar_data/pbs/extracted_vars_pb2.py

@@ -1 +1 @@
-{"accession": {"accession": "VCV000978270", "version": 1}, "rcvs": [{"accession": {"accession": "RCV001256675", "version": 1}, "title": "NM_003937.3(KYNU):c.326G>C (p.Trp109Ser) AND Catel-Manzke syndrome"}], "name": "NM_003937.3(KYNU):c.326G>C (p.Trp109Ser)", "variationType": "VARIATION_TYPE_SNV", "classifications": {"germlineClassification": {"reviewStatus": "AGGREGATE_GERMLINE_REVIEW_STATUS_CRITERIA_PROVIDED_SINGLE_SUBMITTER", "description": "Pathogenic", "conditions": [{"traits": [{"names": [{"value": "Hyperphalangy-clinodactyly of index finger with Pierre Robin syndrome", "type": "Alternate"}, {"value": "Pierre Robin syndrome with hyperphalangy and clinodactyly", "type": "Alternate"}, {"value": "Palatodigital syndrome Catel-Manzke type", "type": "Alternate"}, {"value": "Index finger anomaly with Pierre Robin syndrome", "type": "Alternate"}, {"value": "Catel-Manzke syndrome", "type": "Preferred", "xrefs": [{"db": "MONDO", "id": "MONDO:0014507"}]}, {"value": "MICROGNATHIA DIGITAL SYNDROME", "type": "Alternate", "xrefs": [{"db": "OMIM", "id": "616145", "type": "MIM"}]}], "symbols": [{"value": "CATMANS", "type": "Alternate", "xrefs": [{"db": "OMIM", "id": "616145", "type": "MIM"}]}], "attributes": [{"attribute": {"base": {"integerValue": "28"}, "type": "GARD id"}, "xrefs": [{"db": "Office of Rare Diseases", "id": "28"}]}], "xrefs": [{"db": "Orphanet", "id": "1388"}, {"db": "MedGen", "id": "C1844887"}, {"db": "MONDO", "id": "MONDO:0014507"}, {"db": "OMIM", "id": "616145", "type": "MIM"}]}], "type": "TYPE_DISEASE", "id": "20503", "contributesToAggregateClassification": true}], "dateLastEvaluated": "2012-01-07T00:00:00Z", "dateCreated": "2020-09-27T00:00:00Z", "mostRecentSubmission": "2020-09-27T00:00:00Z", "numberOfSubmitters": 1, "numberOfSubmissions": 1}}, "sequenceLocation": {"assembly": "GRCh37", "chr": "CHROMOSOME_2", "accession": "NC_000002.11", "start": 143685263, "stop": 143685263, "displayStart": 143685263, "displayStop": 143685263, "variantLength": 1, "referenceAllele": "G", "alternateAllele": "C", "positionVcf": 143685263, "referenceAlleleVcf": "G", "alternateAlleleVcf": "C"}, "hgncIds": ["HGNC:6469"]}
+{"accession": {"accession": "VCV000978270", "version": 1}, "rcvs": [{"accession": {"accession": "RCV001256675", "version": 1}, "title": "NM_003937.3(KYNU):c.326G>C (p.Trp109Ser) AND Catel-Manzke syndrome", "classifications": {"germlineClassification": {"reviewStatus": "AGGREGATE_GERMLINE_REVIEW_STATUS_CRITERIA_PROVIDED_SINGLE_SUBMITTER", "description": {"value": "Pathogenic", "dateLastEvaluated": "2012-01-07T00:00:00Z", "submissionCount": 1}}}}], "name": "NM_003937.3(KYNU):c.326G>C (p.Trp109Ser)", "variationType": "VARIATION_TYPE_SNV", "classifications": {"germlineClassification": {"reviewStatus": "AGGREGATE_GERMLINE_REVIEW_STATUS_CRITERIA_PROVIDED_SINGLE_SUBMITTER", "description": "Pathogenic", "dateLastEvaluated": "2012-01-07T00:00:00Z", "dateCreated": "2020-09-27T00:00:00Z", "mostRecentSubmission": "2020-09-27T00:00:00Z", "numberOfSubmitters": 1, "numberOfSubmissions": 1}}, "sequenceLocation": {"assembly": "GRCh37", "chr": "CHROMOSOME_2", "accession": "NC_000002.11", "start": 143685263, "stop": 143685263, "displayStart": 143685263, "displayStop": 143685263, "variantLength": 1, "referenceAllele": "G", "alternateAllele": "C", "positionVcf": 143685263, "referenceAlleleVcf": "G", "alternateAlleleVcf": "C"}, "hgncIds": ["HGNC:6469"]}