From 90415fbd641b04b493525096eba63cf015beb713 Mon Sep 17 00:00:00 2001 From: Andy Date: Mon, 26 Aug 2024 18:17:15 +0200 Subject: [PATCH 01/12] 1571: add mapping, store data in tables --- .../src/bp-modules/stores/StoresCSVInput.tsx | 16 +++++++ .../stores/StoresImportController.tsx | 4 +- .../stores/__tests__/StoreCSVInput.test.tsx | 42 +++++++++++++++++++ .../stores/importAcceptingStores.graphql | 4 +- .../repos/AcceptingStoresRepository.kt | 40 ++++++++++++++++++ .../importer/nuernberg/steps/DownloadCsv.kt | 4 +- .../backend/stores/utils/MapCsvToStore.kt | 21 ++++++++++ .../AcceptingStoresMutationService.kt | 14 +++++-- .../schema/types/CSVAcceptingStore.kt | 1 + specs/backend-api.graphql | 2 +- 10 files changed, 139 insertions(+), 9 deletions(-) create mode 100644 backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/utils/MapCsvToStore.kt diff --git a/administration/src/bp-modules/stores/StoresCSVInput.tsx b/administration/src/bp-modules/stores/StoresCSVInput.tsx index 876de21c5..28620304c 100644 --- a/administration/src/bp-modules/stores/StoresCSVInput.tsx +++ b/administration/src/bp-modules/stores/StoresCSVInput.tsx @@ -7,6 +7,7 @@ import { StoreFieldConfig } from '../../project-configs/getProjectConfig' import { useAppToaster } from '../AppToaster' import FileInputStateIcon, { FileInputStateType } from '../FileInputStateIcon' import { AcceptingStoreEntry } from './AcceptingStoreEntry' +import { StoreData } from './StoresImportController' import StoresRequirementsText from './StoresRequirementsText' const StoreImportInputContainer = styled.div` @@ -40,6 +41,16 @@ const lineToStoreEntry = (line: string[], headers: string[], fields: StoreFieldC return new AcceptingStoreEntry(storeData, fields) } +const hasStoreDuplicates = (stores: AcceptingStoreEntry[]) => { + return ( + new Set( + stores.map(({ data }: { data: StoreData }) => + JSON.stringify([data['name'], data['street'], data['houseNumber'], data['postalCode'], data['location']]) + ) + ).size < stores.length + ) +} + const StoresCsvInput = ({ setAcceptingStores, fields }: StoresCsvInputProps): ReactElement => { const [inputState, setInputState] = useState('idle') const fileInput = useRef(null) @@ -102,6 +113,11 @@ const StoresCsvInput = ({ setAcceptingStores, fields }: StoresCsvInputProps): Re } const acceptingStores = lines.map((line: string[]) => lineToStoreEntry(line, csvHeader, fields)) + if (hasStoreDuplicates(acceptingStores)) { + showInputError(`Die CSV enthält doppelte Einträge.`) + return + } + setAcceptingStores(acceptingStores) setInputState('idle') }, diff --git a/administration/src/bp-modules/stores/StoresImportController.tsx b/administration/src/bp-modules/stores/StoresImportController.tsx index 722edbb7e..ce5ef5a5b 100644 --- a/administration/src/bp-modules/stores/StoresImportController.tsx +++ b/administration/src/bp-modules/stores/StoresImportController.tsx @@ -37,6 +37,7 @@ export type StoreData = { [key: string]: string } const StoresImport = ({ fields }: StoreImportProps): ReactElement => { + const { projectId } = useContext(ProjectConfigContext) const navigate = useNavigate() const appToaster = useAppToaster() const [acceptingStores, setAcceptingStores] = useState([]) @@ -59,7 +60,8 @@ const StoresImport = ({ fields }: StoreImportProps): ReactElement => { } } - const onImportStores = () => importStores({ variables: { stores: acceptingStores.map(store => store.data) } }) + const onImportStores = () => + importStores({ variables: { stores: acceptingStores.map(store => store.data), project: projectId } }) return ( <> diff --git a/administration/src/bp-modules/stores/__tests__/StoreCSVInput.test.tsx b/administration/src/bp-modules/stores/__tests__/StoreCSVInput.test.tsx index f64951036..0167e1d26 100644 --- a/administration/src/bp-modules/stores/__tests__/StoreCSVInput.test.tsx +++ b/administration/src/bp-modules/stores/__tests__/StoreCSVInput.test.tsx @@ -177,4 +177,46 @@ describe('StoreCSVInput', () => { expect(toaster).toHaveBeenCalledWith({ intent: 'danger', message: error }) expect(setAcceptingStores).not.toHaveBeenCalled() }) + + it(`should fail if the csv includes duplicated stores`, async () => { + const error = `Die CSV enthält doppelte Einträge.` + const csv = '' + const toaster = jest.spyOn(OverlayToaster.prototype, 'show') + mocked(parse).mockReturnValueOnce([ + fieldNames, + [ + 'Test store', + 'Teststr.', + '10', + '90408', + 'Nürnberg', + '12.700', + '11.0765467', + '0911/123456', + 'info@test.de', + 'https://www.test.de/kontakt/', + '20% Ermäßigung für Erwachsene', + '20% discount for adults', + '17', + ], + [ + 'Test store', + 'Teststr.', + '10', + '90408', + 'Nürnberg', + '12.700', + '11.0765467', + '0911/123456', + 'info@test.de', + 'https://www.test.de/kontakt/', + '20% Ermäßigung für Erwachsene', + '20% discount for adults', + '17', + ], + ]) + await waitFor(async () => await renderAndSubmitStoreInput(csv)) + expect(toaster).toHaveBeenCalledWith({ intent: 'danger', message: error }) + expect(setAcceptingStores).not.toHaveBeenCalled() + }) }) diff --git a/administration/src/graphql/stores/importAcceptingStores.graphql b/administration/src/graphql/stores/importAcceptingStores.graphql index 92eec4c1e..61d163436 100644 --- a/administration/src/graphql/stores/importAcceptingStores.graphql +++ b/administration/src/graphql/stores/importAcceptingStores.graphql @@ -1,3 +1,3 @@ -mutation importAcceptingStores($stores: [CSVAcceptingStoreInput!]!) { - success: importAcceptingStores(stores: $stores) +mutation importAcceptingStores($stores: [CSVAcceptingStoreInput!]!, $project: String!) { + success: importAcceptingStores(stores: $stores, project: $project) } diff --git a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/database/repos/AcceptingStoresRepository.kt b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/database/repos/AcceptingStoresRepository.kt index 8b9154d15..6d282df32 100644 --- a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/database/repos/AcceptingStoresRepository.kt +++ b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/database/repos/AcceptingStoresRepository.kt @@ -2,13 +2,20 @@ package app.ehrenamtskarte.backend.stores.database.repos import app.ehrenamtskarte.backend.common.database.sortByKeys import app.ehrenamtskarte.backend.projects.database.Projects +import app.ehrenamtskarte.backend.stores.COUNTRY_CODE import app.ehrenamtskarte.backend.stores.database.AcceptingStoreEntity import app.ehrenamtskarte.backend.stores.database.AcceptingStores +import app.ehrenamtskarte.backend.stores.database.AddressEntity import app.ehrenamtskarte.backend.stores.database.Addresses +import app.ehrenamtskarte.backend.stores.database.Categories +import app.ehrenamtskarte.backend.stores.database.ContactEntity import app.ehrenamtskarte.backend.stores.database.Contacts +import app.ehrenamtskarte.backend.stores.database.PhysicalStoreEntity import app.ehrenamtskarte.backend.stores.database.PhysicalStores +import app.ehrenamtskarte.backend.stores.importer.common.types.AcceptingStore import app.ehrenamtskarte.backend.stores.webservice.schema.types.Coordinates import net.postgis.jdbc.geometry.Point +import org.jetbrains.exposed.dao.id.EntityID import org.jetbrains.exposed.sql.ComparisonOp import org.jetbrains.exposed.sql.CustomFunction import org.jetbrains.exposed.sql.DoubleColumnType @@ -76,6 +83,39 @@ object AcceptingStoresRepository { .limit(limit, offset) } + fun deleteAllStoresByProject(projectId: Int) { + // TODO + } + + fun createStores(acceptingStores: List, project: EntityID) { + for (acceptingStore in acceptingStores) { + val address = AddressEntity.new { + street = acceptingStore.streetWithHouseNumber + postalCode = acceptingStore.postalCode!! + location = acceptingStore.location + countryCode = COUNTRY_CODE + } + val contact = ContactEntity.new { + email = acceptingStore.email + telephone = acceptingStore.telephone + website = acceptingStore.website + } + val storeEntity = AcceptingStoreEntity.new { + name = acceptingStore.name + description = acceptingStore.discount + contactId = contact.id + categoryId = EntityID(acceptingStore.categoryId, Categories) + regionId = null // TODO #538: For now the region is always null + projectId = project + } + PhysicalStoreEntity.new { + storeId = storeEntity.id + addressId = address.id + coordinates = Point(acceptingStore.longitude!!, acceptingStore.latitude!!) + } + } + } + fun deleteStores(acceptingStoreIds: Iterable) { val contactsDelete = (AcceptingStores innerJoin Contacts).slice(Contacts.id) diff --git a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/importer/nuernberg/steps/DownloadCsv.kt b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/importer/nuernberg/steps/DownloadCsv.kt index be8b58ef0..80bbcb4a1 100644 --- a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/importer/nuernberg/steps/DownloadCsv.kt +++ b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/importer/nuernberg/steps/DownloadCsv.kt @@ -25,14 +25,14 @@ class DownloadCsv(config: ImportConfig, private val logger: Logger) : ) val parser = CSVFormat.RFC4180.builder().setHeader().setSkipHeaderRecord(true).build().parse(inputCSV) - return getCSVAcceptingStores(parser) + return getCSVAcceptingStoreDeprecateds(parser) } catch (e: Exception) { logger.error("Unknown exception while downloading data from csv", e) throw e } } - private fun getCSVAcceptingStores(parser: CSVParser): List { + private fun getCSVAcceptingStoreDeprecateds(parser: CSVParser): List { val stores: MutableList = arrayListOf() parser.records.forEach { record -> val name = record.get("Name") diff --git a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/utils/MapCsvToStore.kt b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/utils/MapCsvToStore.kt new file mode 100644 index 000000000..b771e711e --- /dev/null +++ b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/utils/MapCsvToStore.kt @@ -0,0 +1,21 @@ +package app.ehrenamtskarte.backend.stores.utils + +import app.ehrenamtskarte.backend.stores.COUNTRY_CODE +import app.ehrenamtskarte.backend.stores.importer.common.types.AcceptingStore +import app.ehrenamtskarte.backend.stores.webservice.schema.types.CSVAcceptingStore + +fun mapCsvToStore(csvStore: CSVAcceptingStore): AcceptingStore { + return AcceptingStore( + csvStore.name.clean()!!, COUNTRY_CODE, csvStore.location.clean()!!, csvStore.postalCode.clean()!!, csvStore.street.clean()!!, csvStore.houseNumber.clean()!!, "", csvStore.longitude!!.toDouble(), csvStore.latitude!!.toDouble(), csvStore.categoryId!!.toInt(), csvStore.email, csvStore.telephone, csvStore.homepage, csvStore.discountDE.orEmpty() + "\n\n" + csvStore.discountEN.orEmpty(), null, null + ) +} + +fun String?.clean(removeSubsequentWhitespaces: Boolean = true): String? { + val trimmed = this?.trim() + if (removeSubsequentWhitespaces) { + if (trimmed != null) { + return trimmed.replace(Regex("""\s{2,}"""), " ") + } + } + return trimmed +} diff --git a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/webservice/AcceptingStoresMutationService.kt b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/webservice/AcceptingStoresMutationService.kt index bb44e65a8..0374ab9e8 100644 --- a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/webservice/AcceptingStoresMutationService.kt +++ b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/webservice/AcceptingStoresMutationService.kt @@ -1,5 +1,10 @@ package app.ehrenamtskarte.backend.stores.webservice +import app.ehrenamtskarte.backend.exception.service.ProjectNotFoundException +import app.ehrenamtskarte.backend.projects.database.ProjectEntity +import app.ehrenamtskarte.backend.projects.database.Projects +import app.ehrenamtskarte.backend.stores.database.repos.AcceptingStoresRepository +import app.ehrenamtskarte.backend.stores.utils.mapCsvToStore import app.ehrenamtskarte.backend.stores.webservice.schema.types.CSVAcceptingStore import com.expediagroup.graphql.generator.annotations.GraphQLDescription import org.jetbrains.exposed.sql.transactions.transaction @@ -7,10 +12,13 @@ import org.jetbrains.exposed.sql.transactions.transaction @Suppress("unused") class AcceptingStoresMutationService { @GraphQLDescription("Import accepting stores via csv") - fun importAcceptingStores(stores: List): Boolean { + fun importAcceptingStores(stores: List, project: String): Boolean { transaction { - // TODO #1571 store the store data in the database - print(stores) + val projectEntity = + ProjectEntity.find { Projects.project eq project }.firstOrNull() ?: throw ProjectNotFoundException( + project + ) + AcceptingStoresRepository.createStores(stores.map { store -> mapCsvToStore(store) }, projectEntity.id) } return true } diff --git a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/webservice/schema/types/CSVAcceptingStore.kt b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/webservice/schema/types/CSVAcceptingStore.kt index 7f75eeefe..288285523 100644 --- a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/webservice/schema/types/CSVAcceptingStore.kt +++ b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/webservice/schema/types/CSVAcceptingStore.kt @@ -1,5 +1,6 @@ package app.ehrenamtskarte.backend.stores.webservice.schema.types +// TODO 1591 remove this class data class CSVAcceptingStore( var name: String?, var street: String?, diff --git a/specs/backend-api.graphql b/specs/backend-api.graphql index cdecf5727..441e991c9 100644 --- a/specs/backend-api.graphql +++ b/specs/backend-api.graphql @@ -139,7 +139,7 @@ type Mutation { "Edits an existing administrator" editAdministrator(adminId: Int!, newEmail: String!, newRegionId: Int, newRole: Role!, project: String!): Boolean! "Import accepting stores via csv" - importAcceptingStores(stores: [CSVAcceptingStoreInput!]!): Boolean! + importAcceptingStores(project: String!, stores: [CSVAcceptingStoreInput!]!): Boolean! "Reset the administrator's password" resetPassword(email: String!, newPassword: String!, passwordResetKey: String!, project: String!): Boolean! "Sends a confirmation mail to the user when the card creation was successful" From 3de4630b09d2db4824af623246f5dc9c66c7fe71 Mon Sep 17 00:00:00 2001 From: Andy Date: Tue, 27 Aug 2024 13:21:29 +0200 Subject: [PATCH 02/12] 1571: refactoring cleanup and store functions, add result model to backend and frontend --- .../src/bp-modules/stores/StoresButtonBar.tsx | 26 ++++++- .../stores/StoresImportController.tsx | 19 ++++- .../bp-modules/stores/StoresImportResult.tsx | 31 ++++++++ .../stores/importAcceptingStores.graphql | 6 +- .../repos/AcceptingStoresRepository.kt | 75 ++++++++++++------- .../stores/importer/common/steps/Store.kt | 59 +-------------- .../importer/nuernberg/steps/MapFromCsv.kt | 10 +-- .../backend/stores/utils/MapCsvToStore.kt | 12 +-- .../backend/stores/utils/StoreFieldCleaner.kt | 14 ++++ .../AcceptingStoresMutationService.kt | 36 ++++++++- .../schema/types/StoreImportResultModel.kt | 7 ++ specs/backend-api.graphql | 8 +- 12 files changed, 184 insertions(+), 119 deletions(-) create mode 100644 administration/src/bp-modules/stores/StoresImportResult.tsx create mode 100644 backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/utils/StoreFieldCleaner.kt create mode 100644 backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/webservice/schema/types/StoreImportResultModel.kt diff --git a/administration/src/bp-modules/stores/StoresButtonBar.tsx b/administration/src/bp-modules/stores/StoresButtonBar.tsx index fb82f2550..f0dd06209 100644 --- a/administration/src/bp-modules/stores/StoresButtonBar.tsx +++ b/administration/src/bp-modules/stores/StoresButtonBar.tsx @@ -1,5 +1,5 @@ -import { Button, Tooltip } from '@blueprintjs/core' -import React, { ReactElement } from 'react' +import {Alert, Button, Tooltip } from '@blueprintjs/core' +import React, {ReactElement, useState} from 'react' import ButtonBar from '../ButtonBar' import { AcceptingStoreEntry } from './AcceptingStoreEntry' @@ -8,6 +8,7 @@ type UploadStoresButtonBarProps = { goBack: () => void acceptingStores: AcceptingStoreEntry[] importStores: () => void + loading: boolean } const getToolTipMessage = (hasNoAcceptingStores: boolean, hasInvalidStores: boolean): string => { @@ -20,9 +21,15 @@ const getToolTipMessage = (hasNoAcceptingStores: boolean, hasInvalidStores: bool return 'Importiere Akzeptanzpartner' } -const StoresButtonBar = ({ goBack, acceptingStores, importStores }: UploadStoresButtonBarProps): ReactElement => { +const StoresButtonBar = ({ goBack, acceptingStores, importStores, loading }: UploadStoresButtonBarProps): ReactElement => { const hasInvalidStores = !acceptingStores.every(store => store.isValid()) const hasNoAcceptingStores = acceptingStores.length === 0 + const [uploadDialog, setUploadDialog] = useState(false) + + const confirmImportDialog = () => { + importStores() + setUploadDialog(false) + } return ( @@ -32,10 +39,21 @@ const StoresButtonBar = ({ goBack, acceptingStores, importStores }: UploadStores icon='upload' text='Import Stores' intent='success' - onClick={importStores} + onClick={() => setUploadDialog(true)} disabled={hasNoAcceptingStores || hasInvalidStores} /> + setUploadDialog(false)} + onConfirm={confirmImportDialog}> +

Achtung: Akzeptanzpartner, welche aktuell in der Datenbank gespeichert, aber nicht in der Tabelle vorhanden sind, werden gelöscht!

+ ) } diff --git a/administration/src/bp-modules/stores/StoresImportController.tsx b/administration/src/bp-modules/stores/StoresImportController.tsx index ce5ef5a5b..855c551bd 100644 --- a/administration/src/bp-modules/stores/StoresImportController.tsx +++ b/administration/src/bp-modules/stores/StoresImportController.tsx @@ -11,6 +11,7 @@ import { useAppToaster } from '../AppToaster' import { AcceptingStoreEntry } from './AcceptingStoreEntry' import StoresButtonBar from './StoresButtonBar' import StoresCSVInput from './StoresCSVInput' +import StoresImportResult from './StoresImportResult' import StoresTable from './StoresTable' const StoresImportController = (): ReactElement => { @@ -41,9 +42,19 @@ const StoresImport = ({ fields }: StoreImportProps): ReactElement => { const navigate = useNavigate() const appToaster = useAppToaster() const [acceptingStores, setAcceptingStores] = useState([]) - const [importStores] = useImportAcceptingStoresMutation({ - onCompleted: () => { - appToaster?.show({ intent: 'success', message: 'Ihre Akzeptanzpartner wurden importiert.' }) + const [importStores,{loading}] = useImportAcceptingStoresMutation({ + onCompleted: ({ result }) => { + appToaster?.show({ + intent: 'none', + timeout: 0, + message: ( + + ), + }) setAcceptingStores([]) }, onError: error => { @@ -70,7 +81,7 @@ const StoresImport = ({ fields }: StoreImportProps): ReactElement => { ) : ( )} - + ) } diff --git a/administration/src/bp-modules/stores/StoresImportResult.tsx b/administration/src/bp-modules/stores/StoresImportResult.tsx new file mode 100644 index 000000000..b7ff33b6e --- /dev/null +++ b/administration/src/bp-modules/stores/StoresImportResult.tsx @@ -0,0 +1,31 @@ +import { H4 } from '@blueprintjs/core' +import React, { ReactElement } from 'react' +import styled from 'styled-components' + +const Container = styled.div` + display: flex; + flex-direction: column; +` + +type StoreImportResultProps = { + storesCreated: number + storesDeleted: number + storesUntouched: number +} + +const StoresImportResult = ({ + storesDeleted, + storesUntouched, + storesCreated, +}: StoreImportResultProps): ReactElement => { + return ( + +

Der Import der Akzeptanzpartner war erfolgreich!

+ Akzeptanzstellen erstellt: {storesCreated} + Akzeptanzstellen gelöscht: {storesDeleted} + Akzeptanzstellen unverändert: {storesUntouched} + + ) +} + +export default StoresImportResult diff --git a/administration/src/graphql/stores/importAcceptingStores.graphql b/administration/src/graphql/stores/importAcceptingStores.graphql index 61d163436..8c7e7c6e2 100644 --- a/administration/src/graphql/stores/importAcceptingStores.graphql +++ b/administration/src/graphql/stores/importAcceptingStores.graphql @@ -1,3 +1,7 @@ mutation importAcceptingStores($stores: [CSVAcceptingStoreInput!]!, $project: String!) { - success: importAcceptingStores(stores: $stores, project: $project) + result: importAcceptingStores(stores: $stores, project: $project) { + storesCreated + storesDeleted + storesUntouched + } } diff --git a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/database/repos/AcceptingStoresRepository.kt b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/database/repos/AcceptingStoresRepository.kt index 6d282df32..56ddea111 100644 --- a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/database/repos/AcceptingStoresRepository.kt +++ b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/database/repos/AcceptingStoresRepository.kt @@ -1,6 +1,7 @@ package app.ehrenamtskarte.backend.stores.database.repos import app.ehrenamtskarte.backend.common.database.sortByKeys +import app.ehrenamtskarte.backend.projects.database.ProjectEntity import app.ehrenamtskarte.backend.projects.database.Projects import app.ehrenamtskarte.backend.stores.COUNTRY_CODE import app.ehrenamtskarte.backend.stores.database.AcceptingStoreEntity @@ -83,36 +84,54 @@ object AcceptingStoresRepository { .limit(limit, offset) } - fun deleteAllStoresByProject(projectId: Int) { - // TODO + fun determineRemovableAcceptingStoreId(acceptingStore: AcceptingStore, project: ProjectEntity): Int? { + return AcceptingStores.innerJoin(PhysicalStores).innerJoin(Addresses).innerJoin(Contacts) + .slice(AcceptingStores.id).select { + (Addresses.street eq acceptingStore.streetWithHouseNumber) and + (Addresses.postalCode eq acceptingStore.postalCode!!) and + (Addresses.location eq acceptingStore.location) and + (Addresses.countryCode eq acceptingStore.countryCode) and + (Contacts.email eq acceptingStore.email) and + (Contacts.telephone eq acceptingStore.telephone) and + (Contacts.website eq acceptingStore.website) and + (AcceptingStores.name eq acceptingStore.name) and + (AcceptingStores.description eq acceptingStore.discount) and + (AcceptingStores.categoryId eq acceptingStore.categoryId) and + (AcceptingStores.regionId.isNull()) and // TODO #538: For now the region is always null + (AcceptingStores.projectId eq project.id) and + ( + PhysicalStores.coordinates eq Point( + acceptingStore.longitude!!, + acceptingStore.latitude!! + ) + ) + }.firstOrNull()?.let { it[AcceptingStores.id].value } } - fun createStores(acceptingStores: List, project: EntityID) { - for (acceptingStore in acceptingStores) { - val address = AddressEntity.new { - street = acceptingStore.streetWithHouseNumber - postalCode = acceptingStore.postalCode!! - location = acceptingStore.location - countryCode = COUNTRY_CODE - } - val contact = ContactEntity.new { - email = acceptingStore.email - telephone = acceptingStore.telephone - website = acceptingStore.website - } - val storeEntity = AcceptingStoreEntity.new { - name = acceptingStore.name - description = acceptingStore.discount - contactId = contact.id - categoryId = EntityID(acceptingStore.categoryId, Categories) - regionId = null // TODO #538: For now the region is always null - projectId = project - } - PhysicalStoreEntity.new { - storeId = storeEntity.id - addressId = address.id - coordinates = Point(acceptingStore.longitude!!, acceptingStore.latitude!!) - } + fun createStore(acceptingStore: AcceptingStore, project: ProjectEntity) { + val address = AddressEntity.new { + street = acceptingStore.streetWithHouseNumber + postalCode = acceptingStore.postalCode!! + location = acceptingStore.location + countryCode = COUNTRY_CODE + } + val contact = ContactEntity.new { + email = acceptingStore.email + telephone = acceptingStore.telephone + website = acceptingStore.website + } + val storeEntity = AcceptingStoreEntity.new { + name = acceptingStore.name + description = acceptingStore.discount + contactId = contact.id + categoryId = EntityID(acceptingStore.categoryId, Categories) + regionId = null // TODO #538: For now the region is always null + projectId = project.id + } + PhysicalStoreEntity.new { + storeId = storeEntity.id + addressId = address.id + coordinates = Point(acceptingStore.longitude!!, acceptingStore.latitude!!) } } diff --git a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/importer/common/steps/Store.kt b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/importer/common/steps/Store.kt index 248521827..996223223 100644 --- a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/importer/common/steps/Store.kt +++ b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/importer/common/steps/Store.kt @@ -2,22 +2,11 @@ package app.ehrenamtskarte.backend.stores.importer.common.steps import app.ehrenamtskarte.backend.projects.database.ProjectEntity import app.ehrenamtskarte.backend.projects.database.Projects -import app.ehrenamtskarte.backend.stores.database.AcceptingStoreEntity import app.ehrenamtskarte.backend.stores.database.AcceptingStores -import app.ehrenamtskarte.backend.stores.database.AddressEntity -import app.ehrenamtskarte.backend.stores.database.Addresses -import app.ehrenamtskarte.backend.stores.database.Categories -import app.ehrenamtskarte.backend.stores.database.ContactEntity -import app.ehrenamtskarte.backend.stores.database.Contacts -import app.ehrenamtskarte.backend.stores.database.PhysicalStoreEntity -import app.ehrenamtskarte.backend.stores.database.PhysicalStores import app.ehrenamtskarte.backend.stores.database.repos.AcceptingStoresRepository import app.ehrenamtskarte.backend.stores.importer.ImportConfig import app.ehrenamtskarte.backend.stores.importer.PipelineStep import app.ehrenamtskarte.backend.stores.importer.common.types.AcceptingStore -import net.postgis.jdbc.geometry.Point -import org.jetbrains.exposed.dao.id.EntityID -import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.transactions.transaction import org.slf4j.Logger @@ -44,58 +33,14 @@ class Store(config: ImportConfig, private val logger: Logger) : // If an exact duplicate is found in the DB, we do not recreate it and instead // remove the id from `acceptingStoreIdsToRemove`. val idInDb: Int? = - AcceptingStores.innerJoin(PhysicalStores).innerJoin(Addresses).innerJoin(Contacts) - .slice(AcceptingStores.id).select { - (Addresses.street eq acceptingStore.streetWithHouseNumber) and - (Addresses.postalCode eq acceptingStore.postalCode!!) and - (Addresses.location eq acceptingStore.location) and - (Addresses.countryCode eq acceptingStore.countryCode) and - (Contacts.email eq acceptingStore.email) and - (Contacts.telephone eq acceptingStore.telephone) and - (Contacts.website eq acceptingStore.website) and - (AcceptingStores.name eq acceptingStore.name) and - (AcceptingStores.description eq acceptingStore.discount) and - (AcceptingStores.categoryId eq acceptingStore.categoryId) and - (AcceptingStores.regionId.isNull()) and // TODO #538: For now the region is always null - (AcceptingStores.projectId eq project.id) and - ( - PhysicalStores.coordinates eq Point( - acceptingStore.longitude!!, - acceptingStore.latitude!! - ) - ) - }.firstOrNull()?.let { it[AcceptingStores.id].value } + AcceptingStoresRepository.determineRemovableAcceptingStoreId(acceptingStore, project) if (idInDb != null) { acceptingStoreIdsToRemove.remove(idInDb) numStoresUntouched += 1 continue } - val address = AddressEntity.new { - street = acceptingStore.streetWithHouseNumber - postalCode = acceptingStore.postalCode!! - location = acceptingStore.location - countryCode = acceptingStore.countryCode - } - val contact = ContactEntity.new { - email = acceptingStore.email - telephone = acceptingStore.telephone - website = acceptingStore.website - } - val storeEntity = AcceptingStoreEntity.new { - name = acceptingStore.name - description = acceptingStore.discount - contactId = contact.id - categoryId = EntityID(acceptingStore.categoryId, Categories) - regionId = null // TODO #538: For now the region is always null - projectId = project.id - } - PhysicalStoreEntity.new { - storeId = storeEntity.id - addressId = address.id - coordinates = Point(acceptingStore.longitude!!, acceptingStore.latitude!!) - } - + AcceptingStoresRepository.createStore(acceptingStore, project) numStoresCreated += 1 } diff --git a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/importer/nuernberg/steps/MapFromCsv.kt b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/importer/nuernberg/steps/MapFromCsv.kt index c8897b08a..ef20c625d 100644 --- a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/importer/nuernberg/steps/MapFromCsv.kt +++ b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/importer/nuernberg/steps/MapFromCsv.kt @@ -5,7 +5,7 @@ import app.ehrenamtskarte.backend.stores.importer.ImportConfig import app.ehrenamtskarte.backend.stores.importer.PipelineStep import app.ehrenamtskarte.backend.stores.importer.common.types.AcceptingStore import app.ehrenamtskarte.backend.stores.importer.nuernberg.constants.categoryMapping -import app.ehrenamtskarte.backend.stores.importer.replaceNa +import app.ehrenamtskarte.backend.stores.utils.clean import app.ehrenamtskarte.backend.stores.webservice.schema.types.CSVAcceptingStore import org.slf4j.Logger @@ -53,12 +53,4 @@ class MapFromCsv(config: ImportConfig, private val logger: Logger) : null } } - - private fun String?.clean(removeSubsequentWhitespaces: Boolean = true): String? { - val trimmed = this?.replaceNa()?.trim() - if (removeSubsequentWhitespaces) { - return trimmed?.replace(Regex("""\s{2,}"""), " ") - } - return trimmed - } } diff --git a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/utils/MapCsvToStore.kt b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/utils/MapCsvToStore.kt index b771e711e..a02ea3e0e 100644 --- a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/utils/MapCsvToStore.kt +++ b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/utils/MapCsvToStore.kt @@ -6,16 +6,6 @@ import app.ehrenamtskarte.backend.stores.webservice.schema.types.CSVAcceptingSto fun mapCsvToStore(csvStore: CSVAcceptingStore): AcceptingStore { return AcceptingStore( - csvStore.name.clean()!!, COUNTRY_CODE, csvStore.location.clean()!!, csvStore.postalCode.clean()!!, csvStore.street.clean()!!, csvStore.houseNumber.clean()!!, "", csvStore.longitude!!.toDouble(), csvStore.latitude!!.toDouble(), csvStore.categoryId!!.toInt(), csvStore.email, csvStore.telephone, csvStore.homepage, csvStore.discountDE.orEmpty() + "\n\n" + csvStore.discountEN.orEmpty(), null, null + csvStore.name.clean()!!, COUNTRY_CODE, csvStore.location.clean()!!, csvStore.postalCode.clean()!!, csvStore.street.clean()!!, csvStore.houseNumber.clean()!!, "", csvStore.longitude!!.toDouble(), csvStore.latitude!!.toDouble(), csvStore.categoryId!!.toInt(), csvStore.email.clean()!!, csvStore.telephone.clean()!!, csvStore.homepage.clean()!!, csvStore.discountDE.orEmpty() + "\n\n" + csvStore.discountEN.orEmpty(), null, null ) } - -fun String?.clean(removeSubsequentWhitespaces: Boolean = true): String? { - val trimmed = this?.trim() - if (removeSubsequentWhitespaces) { - if (trimmed != null) { - return trimmed.replace(Regex("""\s{2,}"""), " ") - } - } - return trimmed -} diff --git a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/utils/StoreFieldCleaner.kt b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/utils/StoreFieldCleaner.kt new file mode 100644 index 000000000..22adf14b4 --- /dev/null +++ b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/utils/StoreFieldCleaner.kt @@ -0,0 +1,14 @@ +package app.ehrenamtskarte.backend.stores.utils + +/** Returns null if string can't be trimmed f.e. empty string + * Removes subsequent whitespaces + * */ +fun String?.clean(removeSubsequentWhitespaces: Boolean = true): String? { + val trimmed = this?.trim() + if (removeSubsequentWhitespaces) { + if (trimmed != null) { + return trimmed.replace(Regex("""\s{2,}"""), " ") + } + } + return trimmed +} diff --git a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/webservice/AcceptingStoresMutationService.kt b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/webservice/AcceptingStoresMutationService.kt index 0374ab9e8..65391d034 100644 --- a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/webservice/AcceptingStoresMutationService.kt +++ b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/webservice/AcceptingStoresMutationService.kt @@ -3,23 +3,51 @@ package app.ehrenamtskarte.backend.stores.webservice import app.ehrenamtskarte.backend.exception.service.ProjectNotFoundException import app.ehrenamtskarte.backend.projects.database.ProjectEntity import app.ehrenamtskarte.backend.projects.database.Projects +import app.ehrenamtskarte.backend.stores.database.AcceptingStores import app.ehrenamtskarte.backend.stores.database.repos.AcceptingStoresRepository import app.ehrenamtskarte.backend.stores.utils.mapCsvToStore import app.ehrenamtskarte.backend.stores.webservice.schema.types.CSVAcceptingStore +import app.ehrenamtskarte.backend.stores.webservice.schema.types.StoreImportResultModel import com.expediagroup.graphql.generator.annotations.GraphQLDescription +import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.transactions.transaction @Suppress("unused") class AcceptingStoresMutationService { @GraphQLDescription("Import accepting stores via csv") - fun importAcceptingStores(stores: List, project: String): Boolean { - transaction { + fun importAcceptingStores(stores: List, project: String): StoreImportResultModel { + return transaction t@{ val projectEntity = ProjectEntity.find { Projects.project eq project }.firstOrNull() ?: throw ProjectNotFoundException( project ) - AcceptingStoresRepository.createStores(stores.map { store -> mapCsvToStore(store) }, projectEntity.id) + try { + var numStoresCreated = 0 + var numStoresUntouched = 0 + val acceptingStoreIdsToRemove = + AcceptingStores.slice(AcceptingStores.id).select { AcceptingStores.projectId eq projectEntity.id } + .map { it[AcceptingStores.id].value }.toMutableSet() + + for (acceptingStore in stores) { + // If an exact duplicate is found in the DB, we do not recreate it and instead + // remove the id from `acceptingStoreIdsToRemove`. + val existingStoreId: Int? = + AcceptingStoresRepository.determineRemovableAcceptingStoreId(mapCsvToStore(acceptingStore), projectEntity) + if (existingStoreId != null) { + acceptingStoreIdsToRemove.remove(existingStoreId) + numStoresUntouched += 1 + continue + } + AcceptingStoresRepository.createStore(mapCsvToStore(acceptingStore), projectEntity) + numStoresCreated += 1 + } + AcceptingStoresRepository.deleteStores(acceptingStoreIdsToRemove) + return@t StoreImportResultModel(numStoresCreated, acceptingStoreIdsToRemove.size, numStoresUntouched) + } catch (e: Exception) { + // TODO add error handling + rollback() + throw e + } } - return true } } diff --git a/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/webservice/schema/types/StoreImportResultModel.kt b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/webservice/schema/types/StoreImportResultModel.kt new file mode 100644 index 000000000..dd2100b8f --- /dev/null +++ b/backend/src/main/kotlin/app/ehrenamtskarte/backend/stores/webservice/schema/types/StoreImportResultModel.kt @@ -0,0 +1,7 @@ +package app.ehrenamtskarte.backend.stores.webservice.schema.types + +data class StoreImportResultModel( + val storesCreated: Int, + val storesDeleted: Int, + val storesUntouched: Int +) diff --git a/specs/backend-api.graphql b/specs/backend-api.graphql index 441e991c9..fad5aaf0a 100644 --- a/specs/backend-api.graphql +++ b/specs/backend-api.graphql @@ -139,7 +139,7 @@ type Mutation { "Edits an existing administrator" editAdministrator(adminId: Int!, newEmail: String!, newRegionId: Int, newRole: Role!, project: String!): Boolean! "Import accepting stores via csv" - importAcceptingStores(project: String!, stores: [CSVAcceptingStoreInput!]!): Boolean! + importAcceptingStores(project: String!, stores: [CSVAcceptingStoreInput!]!): StoreImportResultModel! "Reset the administrator's password" resetPassword(email: String!, newPassword: String!, passwordResetKey: String!, project: String!): Boolean! "Sends a confirmation mail to the user when the card creation was successful" @@ -250,6 +250,12 @@ type StaticVerificationCodeResult { codeBase64: String! } +type StoreImportResultModel { + storesCreated: Int! + storesDeleted: Int! + storesUntouched: Int! +} + enum ActivationState { did_not_overwrite_existing failed From 12deb102b3302f4138445922ea5db59e230360c6 Mon Sep 17 00:00:00 2001 From: Andy Date: Wed, 28 Aug 2024 15:54:32 +0200 Subject: [PATCH 03/12] 1571: add error handling, add tests for backend functions and frontend components, add dryRun option --- administration/jest.setup.ts | 4 ++ .../cards/CreateCardsButtonBar.test.tsx | 3 - .../cards/ImportCardsInput.test.tsx | 4 -- .../cards/hooks/useCardGenerator.test.tsx | 3 - .../src/bp-modules/stores/StoresButtonBar.tsx | 45 +++++++++------ .../bp-modules/stores/StoresImportAlert.tsx | 41 +++++++++++++ .../stores/StoresImportController.tsx | 15 ++++- .../bp-modules/stores/StoresImportResult.tsx | 26 +++++++-- .../__tests__/AcceptanceStoreEntry.test.ts | 3 - .../stores/__tests__/StoreCSVInput.test.tsx | 3 - .../__tests__/StoreImportAlert.test.tsx | 32 +++++++++++ .../__tests__/StoreImportResult.test.tsx | 43 ++++++++++++++ .../stores/__tests__/StoresButtonBar.test.tsx | 57 +++++++++++++++---- administration/src/errors/DefaultErrorMap.tsx | 4 ++ .../stores/importAcceptingStores.graphql | 4 +- .../exceptions/DatabaseIOException.kt | 6 ++ .../webservice/schema/GraphQLExceptionCode.kt | 1 + .../AcceptingStoresMutationService.kt | 35 ++++++++++-- .../backend/stores/MapCsvToStoreTest.kt | 32 +++++++++++ .../backend/stores/StoreFieldCleanerTest.kt | 19 +++++++ specs/backend-api.graphql | 3 +- 21 files changed, 323 insertions(+), 60 deletions(-) create mode 100644 administration/src/bp-modules/stores/StoresImportAlert.tsx create mode 100644 administration/src/bp-modules/stores/__tests__/StoreImportAlert.test.tsx create mode 100644 administration/src/bp-modules/stores/__tests__/StoreImportResult.test.tsx create mode 100644 backend/src/main/kotlin/app/ehrenamtskarte/backend/exception/webservice/exceptions/DatabaseIOException.kt create mode 100644 backend/src/test/kotlin/app/ehrenamtskarte/backend/stores/MapCsvToStoreTest.kt create mode 100644 backend/src/test/kotlin/app/ehrenamtskarte/backend/stores/StoreFieldCleanerTest.kt diff --git a/administration/jest.setup.ts b/administration/jest.setup.ts index 4717d980a..bb68f92f2 100644 --- a/administration/jest.setup.ts +++ b/administration/jest.setup.ts @@ -6,3 +6,7 @@ Object.assign(global, { TextDecoder, TextEncoder }) Object.assign(window, { isSecureContext: true }) Object.assign(window.crypto, { subtle }) Object.assign(window.URL, { createObjectURL: jest.fn() }) + +jest.mock('csv-stringify/browser/esm/sync', () => ({ + stringify: jest.fn(), +})) diff --git a/administration/src/bp-modules/cards/CreateCardsButtonBar.test.tsx b/administration/src/bp-modules/cards/CreateCardsButtonBar.test.tsx index f58c91f46..4a5d8bb34 100644 --- a/administration/src/bp-modules/cards/CreateCardsButtonBar.test.tsx +++ b/administration/src/bp-modules/cards/CreateCardsButtonBar.test.tsx @@ -8,9 +8,6 @@ import bayernConfig from '../../project-configs/bayern/config' import CreateCardsButtonBar from './CreateCardsButtonBar' jest.useFakeTimers() -jest.mock('csv-stringify/browser/esm/sync', () => ({ - stringify: jest.fn(), -})) const wrapper = ({ children }: { children: ReactElement }) => {children} diff --git a/administration/src/bp-modules/cards/ImportCardsInput.test.tsx b/administration/src/bp-modules/cards/ImportCardsInput.test.tsx index 6e7865b47..e45252056 100644 --- a/administration/src/bp-modules/cards/ImportCardsInput.test.tsx +++ b/administration/src/bp-modules/cards/ImportCardsInput.test.tsx @@ -13,10 +13,6 @@ import { AppToasterProvider } from '../AppToaster' import { getHeaders } from './ImportCardsController' import ImportCardsInput, { ENTRY_LIMIT } from './ImportCardsInput' -jest.mock('csv-stringify/browser/esm/sync', () => ({ - stringify: jest.fn(), -})) - jest.mock('../../Router', () => ({})) const wrapper = ({ children }: { children: ReactElement }) => ( diff --git a/administration/src/bp-modules/cards/hooks/useCardGenerator.test.tsx b/administration/src/bp-modules/cards/hooks/useCardGenerator.test.tsx index dfa3d8a1e..3f69064d9 100644 --- a/administration/src/bp-modules/cards/hooks/useCardGenerator.test.tsx +++ b/administration/src/bp-modules/cards/hooks/useCardGenerator.test.tsx @@ -35,9 +35,6 @@ jest.mock('../../../cards/createCards', () => ({ __esModule: true, default: jest.fn(), })) -jest.mock('csv-stringify/browser/esm/sync', () => ({ - stringify: jest.fn(), -})) jest.mock('../../../cards/deleteCards') jest.mock('../../../util/downloadDataUri') diff --git a/administration/src/bp-modules/stores/StoresButtonBar.tsx b/administration/src/bp-modules/stores/StoresButtonBar.tsx index f0dd06209..006a4d513 100644 --- a/administration/src/bp-modules/stores/StoresButtonBar.tsx +++ b/administration/src/bp-modules/stores/StoresButtonBar.tsx @@ -1,14 +1,17 @@ -import {Alert, Button, Tooltip } from '@blueprintjs/core' -import React, {ReactElement, useState} from 'react' +import { Alert, Button, Tooltip } from '@blueprintjs/core' +import React, { ReactElement, useState } from 'react' import ButtonBar from '../ButtonBar' import { AcceptingStoreEntry } from './AcceptingStoreEntry' +import StoresImportAlert from './StoresImportAlert' type UploadStoresButtonBarProps = { goBack: () => void acceptingStores: AcceptingStoreEntry[] importStores: () => void loading: boolean + dryRun: boolean + setDryRun: (value: boolean) => void } const getToolTipMessage = (hasNoAcceptingStores: boolean, hasInvalidStores: boolean): string => { @@ -21,11 +24,17 @@ const getToolTipMessage = (hasNoAcceptingStores: boolean, hasInvalidStores: bool return 'Importiere Akzeptanzpartner' } -const StoresButtonBar = ({ goBack, acceptingStores, importStores, loading }: UploadStoresButtonBarProps): ReactElement => { +const StoresButtonBar = ({ + goBack, + acceptingStores, + importStores, + loading, + dryRun, + setDryRun, +}: UploadStoresButtonBarProps): ReactElement => { const hasInvalidStores = !acceptingStores.every(store => store.isValid()) const hasNoAcceptingStores = acceptingStores.length === 0 const [uploadDialog, setUploadDialog] = useState(false) - const confirmImportDialog = () => { importStores() setUploadDialog(false) @@ -34,7 +43,11 @@ const StoresButtonBar = ({ goBack, acceptingStores, importStores, loading }: Upl return (