Skip to content

Commit

Permalink
AtlasOfLivingAustralia/fieldcapture#3432
Browse files Browse the repository at this point in the history 
 - added expression evaluator to generate value of an attribute in a record
 - added test cases for checking event core archive and expression evaluation
  • Loading branch information
@temi
temi committed Feb 6, 2025
1 parent 6cb56ff commit 69cd8c1
Show file tree
Hide file tree
Showing 17 changed files with 616 additions and 193 deletions.
6 changes: 3 additions & 3 deletions grails-app/conf/application.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ environments {
mongodb {
host = "localhost"
port = "27017"
databaseName = "ecodata"
databaseName = "ecodata-test-server"
}
}
}
Expand Down Expand Up @@ -1657,7 +1657,7 @@ if (!darwinCore.termsGroupedByClass) {
multimedia = document
return [
"eventID" : params?.activity?.activityId,
"occurrenceID": record?.outputSpeciesId ,
"occurrenceID": record?.occurrenceID ,
"type" : multimedia?.type,
"identifier" : identifier,
"format" : multimedia?.contentType,
Expand Down Expand Up @@ -1690,7 +1690,7 @@ if (!darwinCore.termsGroupedByClass) {
"name" : "occurrenceID",
"namespace": "dwc",
"code" : { record, params ->
record?.outputSpeciesId
record?.occurrenceID ?: record?.outputSpeciesId
}
],
[
Expand Down
28 changes: 26 additions & 2 deletions grails-app/controllers/au/org/ala/ecodata/HarvestController.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package au.org.ala.ecodata

import grails.converters.JSON
import org.apache.http.HttpStatus
import org.apache.http.entity.ContentType

import java.text.SimpleDateFormat;

Expand Down Expand Up @@ -70,9 +71,11 @@ class HarvestController {
* @param sort = asc | desc | default:asc
* @param lastUpdated = date | dd/MM/yyyy | default:null
* @param status = active | deleted | default:active
*
* @deprecated ALA's records harvester will use getDarwinCoreArchiveForProject once Events system is setup.
* To access it use archiveURL property from {@link HarvestController#listHarvestDataResource}.
*/
def listRecordsForDataResourceId (){
@Deprecated
def listRecordsForDataResourceId () {
def result = [], error, project
Date lastUpdated = null
try {
Expand Down Expand Up @@ -134,4 +137,25 @@ class HarvestController {
response.setContentType("application/json")
render result as JSON
}

/**
* Get Darwin Core Archive for a project that has ala harvest enabled.
* @param projectId
* @return
* At the moment, you need to add their IP address to whitelist.
*/
def getDarwinCoreArchiveForProject (String projectId) {
if (projectId) {
Project project = Project.findByProjectId(projectId)
if(project?.alaHarvest) {
// Simulate BioCollect as the hostname calling this method. This is done to get the correct URL for
// documents.
DocumentHostInterceptor.documentHostUrlPrefix.set(grailsApplication.config.getProperty("biocollect.baseURL"))
recordService.getDarwinCoreArchiveForProject(response.outputStream, project)
} else
response status: HttpStatus.SC_NOT_FOUND, text: [error: "project not found or ala harvest flag is switched off"] as JSON, contentType: ContentType.APPLICATION_JSON
} else {
response status: HttpStatus.SC_BAD_REQUEST, text: [error: "projectId is required"] as JSON, contentType: ContentType.APPLICATION_JSON
}
}
}
148 changes: 1 addition & 147 deletions grails-app/controllers/au/org/ala/ecodata/RecordController.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,9 @@ package au.org.ala.ecodata
import grails.converters.JSON
import groovy.json.JsonSlurper
import org.apache.commons.codec.binary.Base64
import org.apache.http.HttpStatus
import org.apache.http.entity.ContentType
import org.springframework.web.multipart.MultipartFile
import org.springframework.web.multipart.MultipartHttpServletRequest
import java.text.SimpleDateFormat

import static au.org.ala.ecodata.Status.DELETED
import static javax.servlet.http.HttpServletResponse.*
/**
Expand All @@ -28,131 +26,6 @@ class RecordController {

def index() {}

/**
* List of supported data resource id available for harvesting.
* Note: Data Provider must be BioCollect or MERIT
*
* @param max = number
* @param offset = number
* @param order = lastUpdated
* @param sort = asc | desc
*
*/
def listHarvestDataResource() {
def result, error
try {
if (params.max && !params.max?.isNumber()) {
error = "Invalid parameter max"
} else if (params.offset && !params.offset?.isNumber()) {
error = "Invalid parameter offset"
} else if (params.sort && params.sort != "asc" && params.sort != "desc") {
error = "Invalid parameter sort"
} else if (params.order && params.order != "lastUpdated") {
error = "Invalid parameter order (Expected: lastUpdated)"
}

if (!error) {
def pagination = [
max : params.max ?: 10,
offset: params.offset ?: 0,
order : params.order ?: 'asc',
sort : params.sort ?: 'lastUpdated'
]

result = projectService.listProjectForAlaHarvesting(pagination)

} else {
response.status = HttpStatus.SC_BAD_REQUEST
result = [status: 'error', error: error]
}

} catch (Exception ex) {
response.status = HttpStatus.SC_INTERNAL_SERVER_ERROR
result << [status: 'error', error: "Unexpected error."]
}

response.setContentType("application/json")
render result as JSON
}

/**
* List records associated with the given data resource id
* Data Provider must be BioCollect or MERIT
* @param id dataResourceId
* @param max = number
* @param offset = number
* @param order = lastUpdated
* @param sort = asc | desc | default:asc
* @param lastUpdated = date | dd/MM/yyyy | default:null
* @param status = active | deleted | default:active
* @deprecated ALA's records harvester will use getDarwinCoreArchiveForProject once Events system is setup.
* To access it use archiveURL property from {@link RecordController#listHarvestDataResource}.
*
*/
@Deprecated
def listRecordsForDataResourceId (){
def result = [], error, project
Date lastUpdated = null
try {
if(!params.id) {
error = "Invalid data resource id"
} else if (params.max && !params.max.isNumber()) {
error = "Invalid max parameter vaue"
} else if (params.offset && !params.offset.isNumber()) {
error = "Invalid offset parameter vaue"
} else if (params.order && params.order != "asc" && params.order != "desc") {
error = "Invalid order parameter value (expected: asc, desc)"
} else if (params.sort && params.sort != "lastUpdated") {
error = "Invalid sort parameter value (expected: lastUpdated)"
} else if (params.status && params.status != "active" && params.status != "deleted") {
error = "Invalid status parameter value (expected: active or deleted)"
} else if(params.id){
project = projectService.getByDataResourceId(params.id, 'active', 'basic')
error = !project ? 'No valid project found for the given data resource id' : !project.alaHarvest ? "Harvest disabled for data resource id - ${params.id}" : ''
}

if (params.lastUpdated) {
try{
def df = new SimpleDateFormat("dd/MM/yyyy")
lastUpdated = df.parse(params.lastUpdated)
} catch (Exception ex) {
error = "Invalid lastUpdated format (Expected date format - Example: dd/MM/yyyy"
}
}

if (!error && project) {
def args = [
max : params.max ?: 10,
offset : params.offset ?: 0,
order : params.order ?: 'asc',
sort : params.sort ?: 'lastUpdated',
status : params.status ?: 'active',
projectId: project.projectId
]

List<String> restrictedProjectActivities = projectActivityService.listRestrictedProjectActivityIds(null, params.id)
log.debug("Retrieving results...")
result = recordService.listByProjectId(args, lastUpdated, restrictedProjectActivities)
result?.list?.each {
it.projectName = project?.name
it.license = recordService.getLicense(it)
}
} else {
response.status = HttpStatus.SC_BAD_REQUEST
log.error(error.toString())
result = [status: 'error', error: error]
}

} catch (Exception ex) {
response.status = HttpStatus.SC_INTERNAL_SERVER_ERROR
log.error(ex.toString())
result << [status: 'error', error: "Unexpected error."]
}

response.setContentType("application/json")
render result as JSON
}


/**
* Exports all active Records with lat/lng coords into a .csv suitable for use by the Biocache to create occurrence records.
Expand Down Expand Up @@ -552,25 +425,6 @@ class RecordController {
}
}

/**
* Get Darwin Core Archive for a project that has ala harvest enabled.
* @param projectId
* @return
*/
def getDarwinCoreArchiveForProject (String projectId) {
if (projectId) {
Project project = Project.findByProjectId(projectId)
if(project?.alaHarvest) {
// Simulate BioCollect as the hostname calling this method. This is done to get the correct URL for
// documents.
DocumentHostInterceptor.documentHostUrlPrefix.set(grailsApplication.config.getProperty("biocollect.baseURL"))
recordService.getDarwinCoreArchiveForProject(response.outputStream, project)
} else
response status: HttpStatus.SC_NOT_FOUND, text: [error: "project not found or ala harvest flag is switched off"] as JSON, contentType: ContentType.APPLICATION_JSON
} else {
response status: HttpStatus.SC_BAD_REQUEST, text: [error: "projectId is required"] as JSON, contentType: ContentType.APPLICATION_JSON
}
}

private def setResponseHeadersForRecord(response, record) {
response.addHeader("content-location", grailsApplication.config.getProperty('grails.serverURL') + "/record/" + record.occurrenceID)
Expand Down
2 changes: 1 addition & 1 deletion grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ class UrlMappings {
"/ws/project/getDataCollectionWhiteList"(controller: "project"){ action = [GET:"getDataCollectionWhiteList"] }
"/ws/project/getBiocollectFacets"(controller: "project"){ action = [GET:"getBiocollectFacets"] }
"/ws/project/getDefaultFacets"(controller: "project", action: "getDefaultFacets")
"/ws/project/$projectId/archive"(controller: "record", action: "getDarwinCoreArchiveForProject")
"/ws/project/$projectId/archive"(controller: "harvest", action: "getDarwinCoreArchiveForProject")
"/ws/project/$projectId/dataSet/$dataSetId/records"(controller: "project", action: "fetchDataSetRecords")
"/ws/project/findStateAndElectorateForProject"(controller: "project", action: "findStateAndElectorateForProject")
"/ws/admin/initiateSpeciesRematch"(controller: "admin", action: "initiateSpeciesRematch")
Expand Down
10 changes: 6 additions & 4 deletions grails-app/services/au/org/ala/ecodata/RecordService.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1147,6 +1147,7 @@ class RecordService {
void getDarwinCoreArchiveForProject(outputStream, Project project) {
Map result
Map<String,List> headersByDwcClass = [:].withDefault {[]}
Map dwcGroups = grailsApplication.config.getProperty("darwinCore.termsGroupedByClass", Map)
new ZipOutputStream(outputStream).withStream { zip ->
try {
Long start = System.currentTimeMillis(), end
Expand All @@ -1162,7 +1163,7 @@ class RecordService {
zip.putNextEntry(new ZipEntry("${dwcClass}.csv"))
CSVWriter csvWriter = new CSVWriter(new OutputStreamWriter(zip))
List headers = getHeadersFromRows(rows)
List defaultOrder = getHeaderOrder(grailsApplication.config.darwinCore.termsGroupedByClass[dwcClass])
List defaultOrder = getHeaderOrder(dwcGroups[dwcClass])
headers = reorderHeaders(headers, defaultOrder)
headersByDwcClass[dwcClass] = headers
csvWriter.writeNext(headers as String[])
Expand Down Expand Up @@ -1383,12 +1384,12 @@ class RecordService {
}

Map getAttributeConfig(String dwcClass, String attribute) {
Map groups = grailsApplication.config.darwinCore.termsGroupedByClass
Map groups = grailsApplication.config.getProperty("darwinCore.termsGroupedByClass", Map)
groups[dwcClass].find { it.name == attribute }
}

Map darwinCoreTermsGroupedByClass() {
Map groups = grailsApplication.config.darwinCore.termsGroupedByClass
Map groups = grailsApplication.config.getProperty("darwinCore.termsGroupedByClass", Map)
Map<String, List> trimmedGroups = [:]
groups.each { String key, List values ->
trimmedGroups[key] = values?.collect {
Expand Down Expand Up @@ -1534,7 +1535,8 @@ class RecordService {
*/
Map convertProjectActivityToEvent (pActivity, project) {
String dwcClass = DWC_EVENT
List configs = grailsApplication.config.darwinCore.projectActivityToDwC[dwcClass]
Map paDWC = grailsApplication.config.getProperty("darwinCore.projectActivityToDwC", Map)
List configs = paDWC[dwcClass]
Map<String, Map> result = [:].withDefault { [:] }
configs.each { config ->
transformToEventCoreTerm(config, pActivity, [project: project, pActivity: pActivity, recordService: this], result, dwcClass)
Expand Down
Loading

0 comments on commit 69cd8c1

Please sign in to comment.