datahub-oai-to-blacklight-solr-nl.fix

# Catmandu Fix script
#
# Convert data from the VKC Datahub to the VKC Arthub.
#
# This Fix script will take LIDO XML records as an input and transform
# them into JSON objects that can be uploaded into an Apache Solr store.
#
# The Solr schema has been optimised for Project Blacklight (http://projectblacklight.org).
# The JSON objects should conform to the schema.
#
# See: https://github.com/VlaamseKunstcollectie/Arthub-Frontend/tree/master/jetty/solr
# 
# Usage:
#
#  $ catmandu convert OAI --url "http://endpoint/oai/" --metadataPrefix oai_lido 
#    --handler lido to JSON --pretty 1 --fix datahub-oai-to-blacklight-solr.fix > out.json
#
# In order to import data into the Solr core, create a pipeline and use the Datahub::Factory:
#
# see: 
#
# arthub.ini:
#
#    [Indexer]
#    plugin = Solr
#
#    [plugin_indexer_Solr]
#    file_name = '/tmp/bulk.json'
#    request_handler = 'http://datahub.box:8983/solr/blacklight-core/update/json'
#
# Usage:
#
#  $ catmandu convert OAI --url http://datahub.box/oai --handler lido 
#    to JSON --fix datahub-oai-to-blacklight-solr.fix > /tmp/bulk.json
#  $ dhconveyor index -p ../pipelines/arthub.ini -v


### Mapping between LIDO elements and Solr fields

# Mapping
# --------------------------------------------------------------------------
# LIDO                                          Solr
#                                                   id
# lido:lidoRecID
#   type: purl                                      data_pid
# lido:objectPublishedID                            work_pid
# lido:objectWorkType
#   lido:term
#       pref: preferred                             artwork_subtype_display
#       pref: alternate                             artwork_subtype
# lido:classification
#   type: object-category
#       lido:term
#           pref: preferred                         artwork_type_display
#           pref: alternate                         artwork_type  
# lido:titleSet                                     
#   pref: preferred                                 title_display
# lido:workID
#   type: object-number                             object_number
# lido:objectDescriptionSet                         description
# lido:objectMeasurementsSet
#   lido:measurementsSet
#       lido:extentMeasurements
#           geheel                                  dimensions
#
#                                                   creator
# lido:eventType
#   lido:term
#       production
#           lido:eventActor
#               lido:nameActorSet
#                   pref: preferred                 creator_display
#                   pref: alternate                 creator_index
#               lido:roleActor                      creator_role
#               lido:attributionQualifierActor      
#           lido:eventDate
#               lido:displayDate                    production_date
#           lido:periodName
#               lido:term
#                   lang: nl                        period
#           lido:termMaterialsTech
#               type: material
#                   lido:term
#                       pref: preferred             material_display
#                       pref: alternate             material
# lido:objectRelationWrap
#   lido:subject
#       type: content-motif-general
#           lido:subjectConcept
#               lido:term
#                   pref: preferred                 artwork_category_display
#                   pref: alternate                 artwork_category
# lido:recordWrap          
#   lido:recordSource
#       lido:legalBodyName                          repository
# lido:resourceWrap
#   lido:resourceSet
#       lido:resourceID                             representation_pid
#       lido:resourceRepresentation
#   lido:resourceSet
#       lido:resourceID                             manifest_url
#       lido:resourceType
#           lido:term
#               IIIF Manifest
#   lido:resourceSet
#       lido:resourceID                             thumbnail_url
#       lido:resourceType
#           lido:term
#               thumbnail
#
#                                                   publish_image          


## DescriptiveMetadata

# Copy field to filter on language

do list(path:_metadata.descriptiveMetadata, var:dm)
    if all_match('dm.lang', 'nl')
        copy_field('dm', 'filteredDescriptiveMetadata')
    end
end


## AdministrativeMetadata

# Copy field to filter on language

do list(path:_metadata.administrativeMetadata, var:am)
    if all_match('am.lang', 'nl')
        copy_field('am', 'filteredAdministrativeMetadata')
    end
end


## Title

# Get preferred nl title from LIDO if exists

do list(path:filteredDescriptiveMetadata.objectIdentificationWrap.titleWrap.titleSet.0.appellationValue, var:el)
    if all_match('el.pref', 'preferred')
        if all_match('el.lang', 'nl')
            if all_match('el._', '.*\S.*')
                copy_field('el._', "title_display")
            end
        end
    end
end

# If no preferred nl title exists, try to get any title from LIDO

unless all_match(title_display, '.*\S.*')
    do list(path:filteredDescriptiveMetadata.objectIdentificationWrap.titleWrap.titleSet.0.appellationValue, var:el)
        if all_match('el.pref', 'preferred')
            if all_match('el._', '.*\S.*')
                copy_field('el._', "title_display")
            end
        end
    end
end

# If no preferred title exists, try to get any title from LIDO

unless all_match(title_display, '.*\S.*')
    do list(path:filteredDescriptiveMetadata.objectIdentificationWrap.titleWrap.titleSet.0.appellationValue, var:el)
        if all_match('el._', '.*\S.*')
            copy_field('el._', "title_display")
        end
    end
end

# If no title exists, use n/a (not available)

unless all_match('title_display', '.*\S.*')
    add_field('title_display', 'n/a')
end


## Creator
 
set_array(creator)
set_array(creator_dod)
set_array(creator_role)
set_array(creator_display)
set_array(creator_index)

# Use the event 'production'

unless is_array('filteredDescriptiveMetadata.eventWrap.eventSet')
    move_field('filteredDescriptiveMetadata.eventWrap.eventSet', tmp)
    set_array('filteredDescriptiveMetadata.eventWrap.eventSet')
    move_field(tmp,'filteredDescriptiveMetadata.eventWrap.eventSet.$append')
end

do list(path:'filteredDescriptiveMetadata.eventWrap.eventSet', var:e)

    do list(path:e.event.eventType.term, var:p)
    
        if all_match(p._, 'production')

            # Get the name of the creator from LIDO

            do list(path:e.event.eventActor, var:c)
                set_array(tmp_creator_display)
                set_array(tmp_creator_index)

                # Use the name as established by the museum for the detail page (preferred name)
                # Use the alternate name in the search results if exists, else use the preferred name
                # If no creator name exists, use n/a (not available)

                do list(path:c.actorInRole.actor.nameActorSet.0.appellationValue, var:d)
                    if all_match('d.pref', 'preferred')
                        if all_match(d._, '.*\S.*')
                            copy_field(d._, tmp_creator_display.$append)
                            copy_field(d._, tmp_creator_index.$append)
                        else
                            add_field(creator_display_na, 'n/a')
                            copy_field(creator_display_na, tmp_creator_display.$append)
                            copy_field(creator_display_na, tmp_creator_index.$append)
                            remove_field(creator_display_na)
                        end
                    end
                end

                do list(path:c.actorInRole.actor.nameActorSet.0.appellationValue, var:d)
                    if all_match('d.pref', 'alternate')
                        if all_match(d._, '.*\S.*')
                            copy_field(d._, tmp_creator_index.$last)
                        end
                    end                
                end

                # Add VIAF alternate names from CREATORS_UTF8.csv

                # The original LIDO data doesn't contain those. It only contains the
                # data sanctioned by the institutions, not the actual VIAF data itself.

                copy_field(tmp_creator_display.$last, tmp_creator_concord)
                downcase(tmp_creator_concord)
                lookup_in_store(tmp_creator_concord, DBI, data_source: "dbi:SQLite:/tmp/import.CREATORS_UTF8.sqlite")

                # We store the VIAF alternate names in creator search field. 
                # This allows users to search on name variants of the creator.
                # If no VIAF alternate names are found, use n/a (not available)

                if all_match(tmp_creator_concord.viaf_alternate, '.*\S.*')
                    copy_field(tmp_creator_concord.viaf_alternate, creator.$append)
                else
                    add_field(tmp_creator, 'n/a')
                    copy_field(tmp_creator, creator.$append)
                    remove_field(tmp_creator)
                end

                remove_field(tmp_creator_concord)

                # Add LIDO date of death of the creator
                # If no date of death exists, use n/a (not available)
                # We use the date of death later on to establish copyright.
                # Thererfor we don't need the date of birth of the creator.

                if all_match(c.actorInRole.actor.vitalDatesActor.latestDate._, '.*\S.*')
                    copy_field(c.actorInRole.actor.vitalDatesActor.latestDate._, creator_dod.$append)
                else
                    add_field(tmp_creator_dod, 'n/a')
                    copy_field(tmp_creator_dod, creator_dod.$append)
                    remove_field(tmp_creator_dod)
                end

                # LIDO qualifier (copy after, attributed to ...)

                if all_match('c.actorInRole.attributionQualifierActor.$last._', '.*\S.*')
                    if all_match('c.actorInRole.attributionQualifierActor.$last._', '.*\S.*')
                        
                        # add the attribution to the creator_display and creator_index

                        copy_field(c.actorInRole.attributionQualifierActor.$last._, tmp_creator_display.$prepend)
                        copy_field(c.actorInRole.attributionQualifierActor.$last._, tmp_creator_index.$prepend)

                        # We use the date of death later on to establish copyright.
                        #
                        # In case the work is tentatively "attributed to" an artist, we don't take
                        # the date of death into account.
                        #
                        # Why? Because we can't reliable establish that the work was truly created
                        # by the attributed artist. i.e. The qualifier "copy after" implies
                        # that this work was created by someone else. "copy after Anthony van Dyck" 
                        # might have us erroneously infer that the work was created by Anthony van Dyck
                        # while it may have actually been created in the 20th century.
                        #
                        # Instead we'll check the production date for these works.

                        set_field(check_production_date, 'true')

                    end
                end

                # Creator role (Publisher, Painter,...)
                # If no creator role exists, use n/a (not available)

                if all_match('c.actorInRole.roleActor.$last.term.$last._', '.*\S.*')
                    copy_field(c.actorInRole.roleActor.$last.term.$last._, creator_role.$append)
                else
                    add_field(tmp_creator_role, 'n/a')
                    copy_field(tmp_creator_role, creator_role.$append)
                    remove_field(tmp_creator_role)
                end

                # Flatten creator_display & creator_index

                join_field(tmp_creator_display, " ")
                move_field(tmp_creator_display, creator_display.$append)

                join_field(tmp_creator_index, " ")
                move_field(tmp_creator_index, creator_index.$append)    
            end

        end

    end

end

# If the fields for creator are empty, use n/a (not available)
   
unless all_match(creator.$first, '.*\S.*')
    set_field(creator.$first, 'n/a')
end

unless all_match(creator_dod.$first, '.*\S.*')
    set_field(creator_dod.$first, 'n/a')
end
   
unless all_match(creator_role.$first, '.*\S.*')
    set_field(creator_role.$first, 'n/a')
end

unless all_match(creator_display.$first, '.*\S.*')
    set_field(creator_display.$first, 'n/a')
end

unless all_match(creator_index.$first, '.*\S.*')
    set_field(creator_index.$first, 'n/a')
end


## Production date

set_array(production_date)

# Use the event 'production'

do list(path:'filteredDescriptiveMetadata.eventWrap.eventSet', var:e)

    do list(path:e.event.eventType.term, var:p)

        if all_match(p._, 'production')

            # Get production date from LIDO
            # If no production date exists, use n/a (not available)

            do list(path:e.event.eventDate.displayDate, var:dat)
                if all_match(dat._, '.*\S.*')
                    copy_field(dat._, production_date.$append)
                else
                    add_field(tmp_production_date, 'n/a')
                    copy_field(tmp_production_date, production_date.$append)
                    remove_field(tmp_production_date)
                end
            end

        end

    end

end

# If the field for production date is empty, use n/a (not available)

unless all_match(production_date.$first, '.*\S.*')
    set_field(production_date.$first, 'n/a')
end


## Publish image

# This algorithm establishes wether or not a digital representation
# may be published based on copyright rules.
#
# General rule: date of death artist < current date - 70 years
# 
# We check if the LIDO date of death of the longest living contributor to the
# work is more than 70 years ago.
#
# In case the work is tentatively "attributed to" an artist, we check the
# production date, that is, the most recent asserted date instead.
# 
# When asserting copyright via the production date of the work, we use
# this rule: production date < current date - 150 years.
#
# Assuming a max life expectancy of 90 years, we rule out all works 
# created in the past 150 years (70 after presumable death + 80 years).
# This would rule out the edge case of a creator who died in his 90's, exactly
# 70 years ago and made his first known works in his teenage years.
#
# The algorithm is a two step process:
#
#  1. Clean up the date formatting of fields we're going to check against.
#     They all need to be of the ISO format YYYY. This requires some regex fu.
#  2. Check against the dates according to the algorithm.
#
# This field is what ultimately determines whether an image can be published
# or not. The default assumption is 'false'. We need to prove that we are
# allowed to publish the digital image.

add_field(publish_image, "false")

# This is a 'sentinel' variable. As soon as the copyright has been 
# determined (true), we bail out of the algorithm.

add_field(determined, "false")

# Step 1: Clean out the date formatting

set_field(production_date_copyright)
set_array(pretty_dod)

copy_field(filteredDescriptiveMetadata.eventWrap.eventSet.0.event.eventDate, dates)

if all_match(check_production_date, 'true')

    if all_match(dates.date.latestDate._, '.*\S.*')
        if any_match(dates.date.latestDate._, '(\d\d\d\d)-(\d\d)-(\d\d)')
            parse_text(dates.date.latestDate._, '(\d\d\d\d)-(\d\d)-(\d\d)')
            if is_number(dates.date.latestDate._.$first)
                copy_field(dates.date.latestDate._.$first, production_date_copyright)
            end
        else 
            if any_match(dates.date.latestDate._, '(\d\d\d\d)-(\d\d)')
                parse_text(dates.date.latestDate._, '(\d\d\d\d)-(\d\d)')
                if is_number(dates.date.latestDate._.$first)
                    copy_field(dates.date.latestDate._.$first, production_date_copyright)
                end
            else
                if any_match(dates.date.latestDate._, '(\d\d\d\d)')
                    if is_number(dates.date.latestDate._)
                        copy_field(dates.date.latestDate._, production_date_copyright)
                    end
                else
                    set_field(determined, "true")
                end
            end
        end
    end

else

    do list(path: 'creator_dod', var: 'dod')
        copy_field(dod, tmp_dod)
        if any_match(tmp_dod, '(\d\d\d\d)-(\d\d)-(\d\d)')
            parse_text(tmp_dod, '(\d\d\d\d)-(\d\d)-(\d\d)')
            if is_number(tmp_dod.$first)
                copy_field(tmp_dod.$first, pretty_dod.$append)
            end
        else
            if any_match(tmp_dod, '(\d\d\d\d)-(\d\d)')
                parse_text(tmp_dod, '(\d\d\d\d)-(\d\d)')
                if is_number(tmp_dod.$first)
                    copy_field(tmp_dod.$first, pretty_dod.$append)
                end
            else
                if any_match(tmp_dod, '(\d\d\d\d)')
                    if is_number(tmp_dod)
                        copy_field(tmp_dod, pretty_dod.$append)
                    end
                else
                    set_field(determined, "true")
                end
            end
        end
        remove_field(tmp_dod)
    end

end

# Step 2: Actual determination of the copyright

if all_match(check_production_date, 'true')

    if all_equal(determined, "false")
        if all_match(production_date_copyright, ".*\S.*")
            if less_than(production_date_copyright, 1870)
                set_field(publish_image, "true")
            end
            set_field(determined, "true")
        end
    end

    remove_field(check_production_date)

else

    if all_equal(determined, "false")
        sort_field(pretty_dod, numeric: 1)
        if all_match(pretty_dod.$last, ".*\S.*")
            if less_than(pretty_dod.$last, 1948)
                set_field(publish_image, "true")
            end
            set_field(determined, "true")
        end
    end

end

# Done. Remove fields. We don't need them anymore.

remove_field(creator_dod)
remove_field(pretty_dod)
remove_field(production_date_copyright)
remove_field(dates)
remove_field(determined)


## Period

set_array(period)

# Use the event 'production'

do list(path:'filteredDescriptiveMetadata.eventWrap.eventSet', var: e)
    
    do list(path:e.event.eventType.term, var:p)

        if all_match(p._, 'production')

            # Get period from LIDO
            # If no period exists, use n/a (not available)

            do list(path:e.event.periodName, var:per)

                # Use NL field

                if all_match('per.term.*.lang', 'nl')
                    if all_match('per.term.*._', '.*\S.*')
                        copy_field('per.term.*._', period.$append)
                    else
                        add_field(tmp_period, 'n/a')
                        copy_field(tmp_period, period.$append)
                        remove_field(tmp_period)
                    end
                end
            end

        end

    end

end

# If the field for period is empty, use n/a (not available)

unless all_match(period.$first, '.*\S.*')
    set_field(period.$first, 'n/a')
end

sort_field(period)


## Repository

# Get repository from LIDO if exists (repository = data provider / source of record)
# If no repository exists, use n/a (not available)

if all_match('filteredAdministrativeMetadata.recordWrap.recordSource.0.legalBodyName.0.appellationValue.0._', '.*\S.*')
    copy_field('filteredAdministrativeMetadata.recordWrap.recordSource.0.legalBodyName.0.appellationValue.0._', 'repository')
else
    add_field('repository', 'n/a')
end


## Material

set_array(material_display)
set_array(material)

# Use the event 'production'

do list(path:'filteredDescriptiveMetadata.eventWrap.eventSet', var:e)
    
    do list(path:e.event.eventType.term, var:p)

        if all_match(p._, 'production')

            # Get materials from LIDO           

            do list(path:e.event.eventMaterialsTech, var:mat)

                # Use the preferred term for the detail page
                # Use the alternate term in the search results / facet if exists, else use the preferred term
                # If no material exists, use n/a (not available)

                if all_match(mat.materialsTech.termMaterialsTech.0.type, 'material')

                    do list(path: 'mat.materialsTech.termMaterialsTech.0.term', var:prefMaterial)
                        if all_match('prefMaterial.pref', 'preferred')
                            if all_match(prefMaterial._, '.*\S.*')
                                copy_field(prefMaterial._, material_display.$append)
                                copy_field(prefMaterial._, material.$append)
                            else
                                add_field(tmp_material, 'n/a')
                                copy_field(tmp_material, material_display.$append)
                                copy_field(tmp_material, material.$append)
                                remove_field(tmp_material)
                            end
                        end
                    end

                    do list(path: 'mat.materialsTech.termMaterialsTech.0.term', var:prefMaterial)
                        if all_match('prefMaterial.pref', 'alternate')
                            if all_match(prefMaterial._, '.*\S.*')
                                copy_field(prefMaterial._, material.$last)
                            end
                        end
                    end

                end

            end

        end

    end

end

# If the fields for material are empty, use n/a (not available)

unless all_match(material_display.$first, '.*\S.*')
    set_field(material_display.$first, 'n/a')
end

unless all_match(material.$first, '.*\S.*')
    set_field(material.$first, 'n/a')
end


## Dimensions

set_array(dimensions)

# Use extent 'geheel' (used in AdLib) or 'Dagmaat' (used in TMS)

do list(path:filteredDescriptiveMetadata.objectIdentificationWrap.objectMeasurementsWrap.objectMeasurementsSet, var:el2)

    if all_match(el2.objectMeasurements.extentMeasurements.0._, 'geheel')

        # Get measurements from LIDO
        # If no measurement exist, use n/a (not available)

        do list(path:el2.objectMeasurements.measurementsSet, var:el)
            if all_match('el.measurementType.0._', 'hoogte')
                if all_match(el.measurementValue._, '.*\S.*')
                    copy_field(el.measurementValue._, dimensionsh.$append)
                else
                    add_field(tmp_dimensionsh, 'n/a')
                    copy_field(tmp_dimensionsh, dimensionsh.$append)
                    remove_field(tmp_dimensionsh)
                end
                if all_match(el.measurementUnit.0._, '.*\S.*')
                    copy_field(el.measurementUnit.0._, dimensionsh.$append)
                else
                    add_field(tmp_dimensionsh, 'n/a')
                    copy_field(tmp_dimensionsh, dimensionsh.$append)
                    remove_field(tmp_dimensionsh)
                end
                join_field(dimensionsh, ' ')
                if all_match(dimensionsh, 'n/a n/a')
                    remove_field(dimensionsh)
                else
                    prepend(dimensionsh, 'h ')
                    move_field(dimensionsh, dimensions.$append)
                end
            end
            if all_match('el.measurementType.0._', 'breedte')
                if all_match(el.measurementValue._, '.*\S.*')
                    copy_field(el.measurementValue._, dimensionsw.$append)
                else
                    add_field(tmp_dimensionsw, 'n/a')
                    copy_field(tmp_dimensionsw, dimensionsw.$append)
                    remove_field(tmp_dimensionsw)                    
                end
                if all_match(el.measurementUnit.0._, '.*\S.*')
                    copy_field(el.measurementUnit.0._, dimensionsw.$append)
                else
                    add_field(tmp_dimensionsw, 'n/a')
                    copy_field(tmp_dimensionsw, dimensionsw.$append)
                    remove_field(tmp_dimensionsw) 
                end
                join_field(dimensionsw, ' ')
                if all_match(dimensionsw, 'n/a n/a')
                    remove_field(dimensionsw)
                else
                    prepend(dimensionsw, 'b ')
                    move_field(dimensionsw, dimensions.$append)
                end
            end
            if all_match('el.measurementType.0._', 'diepte')
                if all_match(el.measurementValue._, '.*\S.*')
                    copy_field(el.measurementValue._, dimensionsd.$append)
                else
                    add_field(tmp_dimensionsd, 'n/a')
                    copy_field(tmp_dimensionsd, dimensionsd.$append)
                    remove_field(tmp_dimensionsd)                    
                end
                if all_match(el.measurementUnit.0._, '.*\S.*')
                    copy_field(el.measurementUnit.0._, dimensionsd.$append)
                else
                    add_field(tmp_dimensionsd, 'n/a')
                    copy_field(tmp_dimensionsd, dimensionsd.$append)
                    remove_field(tmp_dimensionsd) 
                end
                join_field(dimensionsd, ' ')
                if all_match(dimensionsd, 'n/a n/a')
                    remove_field(dimensionsd)
                else
                    prepend(dimensionsd, 'd ')
                    move_field(dimensionsd, dimensions.$append)
                end
            end
            if all_match('el.measurementType.0._', 'diameter')
                if all_match(el.measurementValue._, '.*\S.*')
                    copy_field(el.measurementValue._, dimensionsr.$append)
                else
                    add_field(tmp_dimensionsr, 'n/a')
                    copy_field(tmp_dimensionsr, dimensionsr.$append)
                    remove_field(tmp_dimensionsr) 
                end
                if all_match(el.measurementUnit.0._, '.*\S.*')
                    copy_field(el.measurementUnit.0._, dimensionsr.$append)
                else
                    add_field(tmp_dimensionsr, 'n/a')
                    copy_field(tmp_dimensionsr, dimensionsr.$append)
                    remove_field(tmp_dimensionsr) 
                end
                join_field(dimensionsr, ' ')
                if all_match(dimensionsr, 'n/a n/a')
                    remove_field(dimensionsr)
                else
                    append(dimensionsr, ' (diameter)')
                    move_field(dimensionsr, dimensions.$append)
                end
            end

        end

    else

        if all_match(el2.objectMeasurements.extentMeasurements.0._, 'Dagmaat')

            if all_match(el2.displayObjectMeasurements.0._, '.*\S.*')

                if all_match(dimensions.$first, '.*\S.*')
                    unless in(el2.displayObjectMeasurements.0._, dimensions.$last)
                        copy_field(el2.displayObjectMeasurements.0._, dimensions.$append)
                    end
                else
                    copy_field(el2.displayObjectMeasurements.0._, dimensions.$append)
                end

            end

        end

    end

end

# If the field for dimensions is empty, use n/a (not available)

unless all_match(dimensions.$first, '.*\S.*')
    set_field(dimensions.$first, 'n/a')
end


## Category

set_array(artwork_category_display)
set_array(artwork_category)

# Use type 'content-motif-general'

do list(path: 'filteredDescriptiveMetadata.objectRelationWrap.subjectWrap.subjectSet', var:c)

    if all_match(c.subject.type, 'content-motif-general')

        # Get content motif general from LIDO
        # Use the preferred term for the detail page
        # Use the alternate term in the search results / facet if exists, else use the preferred term
        # If no content motif general exists, use n/a (not available)

        do list(path: 'c.subject.subjectConcept', var:subjectconcept)

            do list(path: 'subjectconcept.term', var:subject)
                if all_match('subject.pref', 'preferred')
                    if all_match(subject._, '.*\S.*')
                        copy_field(subject._, artwork_category_display.$append)
                        copy_field(subject._, artwork_category.$append)
                    else
                        add_field(tmp_artwork_category_display, 'n/a')
                        copy_field(tmp_artwork_category_display, artwork_category_display.$append)
                        copy_field(tmp_artwork_category_display, artwork_category.$append)
                        remove_field(tmp_artwork_category_display)
                    end
                end
            end

            do list(path: 'subjectconcept.term', var:subject)
                if all_match('subject.pref', 'alternate')
                    if all_match(subject._, '.*\S.*')
                        copy_field(subject._, artwork_category.$last)
                    end
                end
            end

        end

    end

end

# If the fields for artwork category are empty, use n/a (not available)

unless all_match(artwork_category_display.$first, '.*\S.*')
    set_field(artwork_category_display.$first, 'n/a')
end

unless all_match(artwork_category.$first, '.*\S.*')
    set_field(artwork_category.$first, 'n/a')
end


## Artwork_type

set_array(artwork_type_display)
set_array(artwork_type)

# Use type 'object-category'

do list(path: 'filteredDescriptiveMetadata.objectClassificationWrap.classificationWrap.classification', var:c)
    
    if all_match(c.type, 'object-category')
        
        # Get artwork type from LIDO
        # Use the preferred term for the detail page
        # Use the alternate term in the search results / facet if exists, else use the preferred term
        # If no artwork type exists, use n/a (not available)

        do list(path: 'c.term', var:classification)
            if all_match('classification.pref', 'preferred')
                if all_match(classification._, '.*\S.*')
                    copy_field(classification._, artwork_type_display.$append)
                    copy_field(classification._, artwork_type.$append)
                else
                    add_field(tmp_artwork_type_display, 'n/a')
                    copy_field(tmp_artwork_type_display, artwork_type_display.$append)
                    copy_field(tmp_artwork_type_display, artwork_type.$append)
                    remove_field(tmp_artwork_type_display)
                end
            end
        end

        do list(path: 'c.term', var:classification)
            if all_match('classification.pref', 'alternate')
                if all_match(classification._, '.*\S.*')
                    copy_field(classification._, artwork_type.$last)
                end
            end
        end

    end

end

# If the fields for artwork type are empty, use n/a (not available)

unless all_match(artwork_type_display.$first, '.*\S.*')
    set_field(artwork_type_display.$first, 'n/a')
end

unless all_match(artwork_type.$first, '.*\S.*')
    set_field(artwork_type.$first, 'n/a')
end


## Artwork_subtype

set_array(artwork_subtype_display)
set_array(artwork_subtype)

# Get artwork subtype from LIDO
# Use the preferred term for the detail page
# Use the alternate term in the search results / facet if exists, else use the preferred term
# If no artwork subtype exists, use n/a (not available)

do list(path: 'filteredDescriptiveMetadata.objectClassificationWrap.objectWorkTypeWrap.objectWorkType', var:t)
    
    do list(path: 't.term', var:type)
        if all_match('type.pref', 'preferred')
            if all_match(type._, '.*\S.*')
                copy_field(type._, artwork_subtype_display.$append)
                copy_field(type._, artwork_subtype.$append)
            else
                add_field(tmp_artwork_subtype_display, 'n/a')
                copy_field(tmp_artwork_subtype_display, artwork_subtype_display.$append)
                copy_field(tmp_artwork_subtype_display, artwork_subtype.$append)
                remove_field(tmp_artwork_subtype_display)
            end
        end
    end

    do list(path: 't.term', var:type)
        if all_match('type.pref', 'alternate')
            if all_match(type._, '.*\S.*')
                copy_field(type._, artwork_subtype.$last)
            end
        end
    end

end

# If the fields for artwork subtype are empty, use n/a (not available)

unless all_match(artwork_subtype_display.$first, '.*\S.*')
    set_field(artwork_subtype_display.$first, 'n/a')
end

unless all_match(artwork_subtype.$first, '.*\S.*')
    set_field(artwork_subtype.$first, 'n/a')
end


## Object_number

if all_match('filteredDescriptiveMetadata.objectIdentificationWrap.repositoryWrap.repositorySet.0.workID.0.type', 'object-number')
    if all_match('filteredDescriptiveMetadata.objectIdentificationWrap.repositoryWrap.repositorySet.0.workID.0._', '.*\S.*')
        copy_field('filteredDescriptiveMetadata.objectIdentificationWrap.repositoryWrap.repositorySet.0.workID.0._', 'object_number')
    else
        add_field('object_number', 'n/a')
    end
end

unless exists(object_number)
    add_field(object_number, 'n/a')
end


## Description

if all_match('filteredDescriptiveMetadata.objectIdentificationWrap.objectDescriptionWrap.objectDescriptionSet.0.descriptiveNoteValue.0._', '.*\S.*')
    if all_match('filteredDescriptiveMetadata.objectIdentificationWrap.objectDescriptionWrap.objectDescriptionSet.0.descriptiveNoteValue.0._', '.*\S.*')
        copy_field('filteredDescriptiveMetadata.objectIdentificationWrap.objectDescriptionWrap.objectDescriptionSet.0.descriptiveNoteValue.0._', 'description')
    else
        add_field('description', 'n/a')
    end
end

unless exists(description)
    add_field(description, 'n/a')
end


## Iconclass

set_array(iconclass)

do list(path:_metadata.descriptiveMetadata.0.objectRelationWrap.subjectWrap.subjectSet, var:ic)

    # Use source 'Iconclass'

    if all_match(ic.subject.subjectConcept.0.conceptID.0.source, 'Iconclass')

        # Get Iconclass ID and term from LIDO

        if all_match(ic.subject.subjectConcept.0.conceptID.0._, '.*\S.*')
            copy_field(ic.subject.subjectConcept.0.conceptID.0._, iconclassid.$append)
            replace_all(iconclassid.$last, '.*iconclass\.org/', '')
        end

        do list(path:ic.subject.subjectConcept.0.term, var:ic2)

            if all_match('ic2.lang', 'nl')

                copy_field(iconclassid.$last, iconclasstmp.$append)

                if all_match(ic2._, '.*\S.*')
                    copy_field(ic2._, iconclasstmp.$append)
                end

                join_field(iconclasstmp, ' ')
                move_field(iconclasstmp, iconclass.$append)

            end

        end

        remove_field('iconclassid')

    end

end

# If the field for iconclass is empty, use n/a (not available)

unless all_match(iconclass.$first, '.*\S.*')
    set_field(iconclass.$first, 'n/a')
end


## Work_pid

if all_match('_metadata.objectPublishedID.0._', '.*\S.*')
    copy_field('_metadata.objectPublishedID.0._', 'work_pid')
else
    add_field('work_pid', 'n/a')
end

unless exists(work_pid)
    add_field(work_pid, 'n/a')
end


## Data_pid

do list(path: '_metadata.lidoRecID', var:lidoRecID)
    if all_match('lidoRecID.type', 'purl')
        if all_match(lidoRecID._, '.*\S.*')
            copy_field(lidoRecID._, data_pid)
        else
            add_field(data_pid, 'n/a')
        end
    end

    # The field 'raw' is used in process.pl to map the entire LIDO XML to a Solr field called 'xml' and is removed there at the end
    # See: https://github.com/VlaamseKunstcollectie/Datahub-Fixes/blob/master/scripts/process.pl

    if all_match('lidoRecID.type', 'urn')
        copy_field(lidoRecID._, raw)
    end
end

unless exists(data_pid)
    add_field(data_pid, 'n/a')
end


## Resource sets

unless is_array('filteredAdministrativeMetadata.resourceWrap.resourceSet')
    move_field('filteredAdministrativeMetadata.resourceWrap.resourceSet', tmp)
    set_array('filteredAdministrativeMetadata.resourceWrap.resourceSet')
    move_field(tmp,'filteredAdministrativeMetadata.resourceWrap.resourceSet.$append')
end


## Representation_pid

do list(path:'filteredAdministrativeMetadata.resourceWrap.resourceSet', var:r)
    if all_match('r.resourceID.label', 'representationPID')
        if all_match('r.resourceID._', '.*\S.*')
            copy_field('r.resourceID._', 'representation_pid')
        end
    end
end

unless exists(representation_pid)
    add_field(representation_pid, 'n/a')
end


## Manifest_url

#do list(path:'filteredAdministrativeMetadata.resourceWrap.resourceSet', var:r)
    
#    do list(path:'r.resourceType.term', var:m)
#        if all_match(m._, 'IIIF Manifest')
#            if all_match('r.resourceID._', '.*\S.*')
#                copy_field('r.resourceID._', 'manifest_url')
#            end
#        end
#    end

#end

#unless exists(manifest_url)
#    add_field(manifest_url, 'n/a')
#end


## Thumbnail_url

#do list(path:'filteredAdministrativeMetadata.resourceWrap.resourceSet', var:r)
    
#    do list(path:'r.resourceType.term', var:t)
#        if all_match(t._, 'thumbnail')
#            if all_match('r.resourceID._', '.*\S.*')
#                copy_field('r.resourceID._', 'thumbnail_url')
#            end
#        end
#    end

#end

#unless exists(thumbnail_url)
#    add_field(thumbnail_url, 'n/a')
#end


## Language

add_field('language', 'Dutch')

## ID

# The ID in Solr is based on the data_pid. The data_pid is converted to a string
# which can be safely used as an identifier in Project Blacklight. The format of
# the ID field looks like this:
#
#   <domain>:<identifier>
#   ex. kmksa:254
#   ex. collectievlaamsegemeenschap:837
#
# Note: the .tld is stripped from the domainname because the . (dot) breaks the
# route matching algoritm.

unless all_match(data_pid, 'n/a')
    copy_field('data_pid', 'id')
    parse_text('id', '.*://([A-Za-z0-9\-\.]+)/collection/work/data/(.*)')
    replace_all('id.0', '([^.]*)\.[^.]{2,3}(?:\.[^.]{2,3})?$', '$1')
    join_field(id, ':')
    prepend('id', 'nl:')
end

unless exists(id)
    add_field(id, 'n/a')
end


# Retain only those records that have a workPid
unless all_match('work_pid', 'http://*')
    reject()
end

# Remove all original fields and only retain the ones we set
remove_field('_id')
remove_field('_identifier')
remove_field('_status')
remove_field('_setSpec')
remove_field('_about')
remove_field('_datestamp')
remove_field('_metadata')
remove_field('_resumption')
remove_field('_resumptionToken')

# Remove the fields that filter on language
remove_field('filteredDescriptiveMetadata')
remove_field('filteredAdministrativeMetadata')