extraction-to-storage.map.yaml

# extraction-to-storage.map.yaml
#
# How a neuroimaging-study-extraction record becomes a neuroimaging-study-storage record.
#
# THE MAP IS AN IDENTITY MAP
# --------------------------
# The extraction schema is generated from the storage schema by gen_extraction_schema.py,
# so the two have the same classes with the same slots in the same nesting. A storage
# field is therefore populated from the extraction field of the same name on the same
# class, and the mapper needs no instruction to do it. What the mapper does uniformly:
#
#   unwrap        Every extracted value arrives as an ExtractedValue. Take `.value` and
#                 keep `.evidence` and `.extraction_status` in the audit log. An
#                 extraction_status of not_reported leaves the storage field unset.
#   local_id      `id` on a storage class is `local_id` on its extraction counterpart.
#                 Mint the storage identifier, record the local_id in the audit log, and
#                 resolve every reference to that class through the same table.
#
# There is no normalize step. There used to be: extraction flattened every vocabulary to
# text, and 16 tables holding 316 synonyms were the only route from a paper's wording to a
# permissible value. The extraction schema now carries the vocabularies themselves, with
# each field's range copied from storage exactly -- closed where storage is closed, and
# `any_of: [<Enum>, string]` where storage left an escape hatch. The incoming value
# is already a value storage accepts, in both cases, and the tables had nothing left to do.
# They were deleted rather than kept as a fallback; `git log` has them if a fallback turns
# out to be wanted.
#
# Only two things need writing down, and they are the two sections of this file.
#
# derivations
#   Storage fields no extraction field feeds, because code fills them: a bibliographic
#   fact an API states, an identifier the pipeline mints, a value computed from an
#   ingestion artifact. They are exactly the storage fields marked
#   `in_subset: [deterministic]`, and check_extraction_to_storage_map.py asserts that
#   correspondence in both directions.
#
# free_text_normalizations
#   The five fields storage keeps as `range: string` -- `Group.age_unit`,
#   `ModelEstimation.stage` and friends. There is no vocabulary to project and nothing
#   makes their wording consistent, so a table still earns its place. It normalizes for
#   queryability rather than for validity: an unmatched value is already storable.
#
# Nothing else. A shape difference between the schemas would have to be written here too,
# but there are none: extraction-deviations.yaml is where one would be declared, and it
# is empty. If it stops being empty, this file grows a matching entry.

id: https://neurostuff.org/schemas/extraction-to-storage-map
name: extraction-to-storage-map
title: Neuroimaging Study Extraction-to-Storage Transformation Map
version: 0.5.0
source_schema: https://neurostuff.org/schemas/neuroimaging-study-extraction
target_schema: https://neurostuff.org/schemas/neuroimaging-study-storage

# Storage fields with no extraction counterpart. One entry per field marked
# `in_subset: [deterministic]` in the storage schema.
derivations:
  Acquisition.id:
    transform: generate
  Analysis.id:
    transform: generate
  Analysis.mirror_of:
    transform: derive
    source_representation: sibling_field
    source_field: Analysis.effect
    notes: >-
      Set by the builder on the half of a sign-split contrast it rebuilt: same cells,
      directions flipped, naming the analysis it reverses. Not extracted, because the
      paper's prose describes one direction and the other is arithmetic.
  Analysis.statistical_maps:
    transform: api_lookup
    notes: >-
      The statistical maps NeuroVault holds for this study, matched to the analysis at
      ingestion. Not extracted: a paper that shares maps says so in a data-availability
      sentence, and the sentence is not the map.
  Arm.id:
    transform: generate
  Assessment.id:
    transform: generate
  Condition.id:
    transform: generate
  ConnectivityEdge.directionality:
    transform: derive
    source_representation: sibling_field
    source_field: ConnectivityDetails.connectivity_method
    notes: >-
      The method decides whether an edge is directed, and no wording of the paper can override
      it: only a generative or precedence-based model supports a claim about which region
      influences which. This is therefore a lookup on the containing analysis's `connectivity_method`
      rather than something a reader is asked for. A method written as free text -- one the
      vocabulary has no value for -- leaves the field unset, since there is nothing to look up;
      that is the one case where an edge's directionality is genuinely unknown to the record.
    value_map:
      dcm: directed
      granger: directed
      ppi: undirected
      gppi: undirected
      seed_based: undirected
      coherence: undirected
      structural_covariance: undirected
      multivariate: undirected
  Device.id:
    transform: generate
  Region.id:
    transform: generate
  Group.id:
    transform: generate
    notes: UUID assigned by mapper; extraction local_id stored in audit log.
  InferenceSettings.id:
    transform: generate
  Measure.id:
    transform: generate
  ModelEstimation.id:
    transform: generate
  ModelTerm.id:
    transform: generate
  Preprocessing.id:
    transform: generate
  Study.authors:
    transform: api_lookup
    notes: 'Author list for the record''s PMID, from the E-utilities API, in published order. Not
      extracted: a bibliographic fact the API states is not one a model should be asked to read off
      the page.'
  Study.doi:
    transform: api_lookup
    notes: 'DOI from the record''s identifiers, resolved at ingest. Not extracted: a bibliographic
      fact the API states is not one a model should be asked to read off the page.'
  Study.id:
    transform: generate
    notes: Assigned by the ingestion pipeline; typically DOI-derived (doi:10.../record) or a UUID.
  Study.journal:
    transform: api_lookup
    notes: 'Journal title for the record''s PMID, from the E-utilities API. Not extracted: a bibliographic
      fact the API states is not one a model should be asked to read off the page.'
  Study.publication_year:
    transform: api_lookup
    notes: 'Publication year for the record''s PMID, from the E-utilities API. Not extracted: a bibliographic
      fact the API states is not one a model should be asked to read off the page.'
  Study.language:
    transform: api_lookup
    notes: 'Article languages for the record''s PMID, from the same E-utilities call that fills
      `study_type`. Not extracted: what language an article is indexed in is a bibliographic fact
      the API states, and the text a reader is handed is in one language whatever that fact says.'
  Study.study_type:
    transform: api_lookup
    notes: 'PubMed publication types for the record''s PMID, from the E-utilities API, stored verbatim
      and multivalued. Not extracted: the LLM would be guessing a bibliographic fact the API states,
      and the value gates extraction rather than following it -- a record whose types include `Meta-Analysis`
      is skipped before any text is read.'
  Study.title:
    transform: api_lookup
    notes: 'Article title for the record''s PMID, from the E-utilities API. Not extracted: a bibliographic
      fact the API states is not one a model should be asked to read off the page.'
  Group.is_healthy:
    transform: derive
    source_representation: sibling_field
    source_field: Group.medical_condition
    notes: >-
      Set by `pondie.normalization.is_healthy` from `medical_condition`, so the flag cannot
      contradict the field it summarises. Asked directly, a model answers with the source's
      wording: 168 groups came back healthy beside a real diagnosis.
  Group.other_characteristics:
    transform: derive
    source_representation: sibling_field
    source_field: Group.population_characteristics
    notes: >-
      The characteristics left over once the vocabulary-backed ones are projected out, kept
      as free text so nothing a cohort was described by is dropped.
  Table.coordinate_count:
    transform: derive
    source_representation: normalized_table
    notes: Count coordinate rows or points in the normalized table representation. Set zero when the
      table has no coordinates.
  Table.coordinate_space:
    transform: derive
    notes: 'Normalized from the coordinate space the table parser recovered, not extracted. Per
      table, and independent of Analysis.coordinate_space: fill it wherever the parse yields a
      space. Where every table behind an analysis agrees, the same parse also supplies that
      analysis''s space; the analysis field is authoritative if the two ever differ.'
  Table.id:
    transform: generate
  Table.source_path:
    transform: generate
    notes: Set by the ingestion pipeline to the repository path of the raw table file.
  Task.id:
    transform: generate
  Timepoint.id:
    transform: generate

# Source wording -> a consistent value, for the five fields storage keeps as free text.
# Keyed by field because there is no vocabulary to key on, and checked against nothing for
# the same reason: an unmatched value is already storable.
free_text_normalizations:
  Analysis.coordinate_space:
    MNI: MNI152
    MNI space: MNI152
    MNI152: MNI152
    MNI152NLin2009cAsym: MNI152
    MNI305: MNI305
    Montreal Neurological Institute: MNI152
    TAL: Talairach
    Talairach: Talairach
    Talairach space: Talairach
    fsaverage: fsaverage
    native space: native
    subject native: native
  Group.age_unit:
    gestational week: gestational_weeks
    gestational weeks: gestational_weeks
    gw: gestational_weeks
    mo: months
    month: months
    mos: months
    week: weeks
    wk: weeks
    wks: weeks
    y: years
    year: years
    yr: years
    yrs: years
  Group.diagnostic_system:
    DSM 5: DSM-5
    DSM-IV: DSM-IV
    DSM-IV-TR: DSM-IV-TR
    DSM-V: DSM-5
    DSM5: DSM-5
    ICD-10: ICD-10
    ICD-11: ICD-11
    ICD-9: ICD-9
    RDoC: RDoC
  ModelEstimation.stage:
    1st level: run
    2nd level: group
    first level: run
    first-level: run
    group: group
    group level: group
    group-level: group
    individual: subject
    population: group
    random effects: group
    run: run
    second level: group
    second-level: group
    session: session
    single-subject: subject
    subject: subject
    within-subject: subject
  Task.design_type:
    ER: event-related
    block: block
    block design: block
    blocked: block
    continuous: continuous
    event related: event-related
    event-related design: event-related
    mixed block-event: mixed
    mixed block/event: mixed
    mixed design: mixed
    naturalistic: naturalistic
    resting state: resting-state
    resting-state: resting-state