From 8287861b6875ea8a86e2ad8c3aba60c4dccb9676 Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Fri, 9 Aug 2024 11:33:10 +0200 Subject: [PATCH 01/10] fix(server): update status and sub_status --- .../scripts/import-lovac/housings/housing-processor.ts | 2 +- .../source-housings/source-housing-command.ts | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/server/src/scripts/import-lovac/housings/housing-processor.ts b/server/src/scripts/import-lovac/housings/housing-processor.ts index 14cf1a892..7b3d01e55 100644 --- a/server/src/scripts/import-lovac/housings/housing-processor.ts +++ b/server/src/scripts/import-lovac/housings/housing-processor.ts @@ -15,7 +15,7 @@ export interface ProcessorOptions extends ReporterOptions { housingRepository: { update( id: Pick, - housing: Partial + housing: Pick ): Promise; }; housingEventRepository: { diff --git a/server/src/scripts/import-lovac/source-housings/source-housing-command.ts b/server/src/scripts/import-lovac/source-housings/source-housing-command.ts index 996136e20..e804faf76 100644 --- a/server/src/scripts/import-lovac/source-housings/source-housing-command.ts +++ b/server/src/scripts/import-lovac/source-housings/source-housing-command.ts @@ -141,13 +141,11 @@ export function createSourceHousingCommand() { abortEarly: options.abortEarly, reporter: housingReporter, housingRepository: { - async update( - { geoCode, id }: Pick, - housing: Partial - ): Promise { + async update({ geoCode, id }, housing): Promise { if (!options.dryRun) { await Housing().where({ geo_code: geoCode, id }).update({ - data_file_years: housing.dataFileYears, + status: housing.status, + sub_status: housing.subStatus, occupancy: housing.occupancy }); } From 5c8db576de81456913ba7cc5a9a7b7f9eac6bef8 Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Fri, 9 Aug 2024 12:24:02 +0200 Subject: [PATCH 02/10] fix(server): find housing by localId and geoCode because geoCode cannot be derived from localId --- .../src/scripts/import-lovac/history/history-processor.ts | 2 +- server/src/scripts/import-lovac/history/history.ts | 1 + .../source-housing-owner-processor.ts | 4 ++-- .../source-housing-owners/source-housing-owner.ts | 2 ++ .../test/source-housing-owner.test.ts | 1 + .../import-lovac/source-housings/source-housing-command.ts | 6 ++++-- .../source-housings/source-housing-processor.ts | 7 +++++-- 7 files changed, 16 insertions(+), 7 deletions(-) diff --git a/server/src/scripts/import-lovac/history/history-processor.ts b/server/src/scripts/import-lovac/history/history-processor.ts index 15db81837..85b1edf37 100644 --- a/server/src/scripts/import-lovac/history/history-processor.ts +++ b/server/src/scripts/import-lovac/history/history-processor.ts @@ -24,7 +24,7 @@ export function historyProcessor(opts: ProcessorOptions) { if (dataFileYears.length > 0) { await housingRepository.update( { - geoCode: chunk.ff_idlocal.substring(0, 5), + geoCode: chunk.geo_code, localId: chunk.ff_idlocal }, { dataFileYears } diff --git a/server/src/scripts/import-lovac/history/history.ts b/server/src/scripts/import-lovac/history/history.ts index 1e29bb2b5..be85fb6f0 100644 --- a/server/src/scripts/import-lovac/history/history.ts +++ b/server/src/scripts/import-lovac/history/history.ts @@ -1,4 +1,5 @@ export interface History { + geo_code: string; ff_idlocal: string; files_years: string[]; } diff --git a/server/src/scripts/import-lovac/source-housing-owners/source-housing-owner-processor.ts b/server/src/scripts/import-lovac/source-housing-owners/source-housing-owner-processor.ts index 864654989..1c0067704 100644 --- a/server/src/scripts/import-lovac/source-housing-owners/source-housing-owner-processor.ts +++ b/server/src/scripts/import-lovac/source-housing-owners/source-housing-owner-processor.ts @@ -18,7 +18,7 @@ const logger = createLogger('sourceHousingOwnerProcessor'); export interface ProcessorOptions extends ReporterOptions { auth: UserApi; housingRepository: { - findOne(localId: string): Promise; + findOne(geoCode: string, localId: string): Promise; }; housingEventRepository: { insert(event: HousingEventApi): Promise; @@ -51,7 +51,7 @@ export function createSourceHousingOwnerProcessor(opts: ProcessorOptions) { const [departmentalOwner, housing] = await Promise.all([ ownerRepository.findOne(chunk.idpersonne), - housingRepository.findOne(chunk.local_id) + housingRepository.findOne(chunk.geo_code, chunk.local_id) ]); if (!departmentalOwner) { throw new OwnerMissingError(chunk.idpersonne); diff --git a/server/src/scripts/import-lovac/source-housing-owners/source-housing-owner.ts b/server/src/scripts/import-lovac/source-housing-owners/source-housing-owner.ts index cbf77416f..e9b220828 100644 --- a/server/src/scripts/import-lovac/source-housing-owners/source-housing-owner.ts +++ b/server/src/scripts/import-lovac/source-housing-owners/source-housing-owner.ts @@ -3,6 +3,7 @@ import { number, object, ObjectSchema, string } from 'yup'; import { POSITIVE_RANKS, PositiveRank } from '~/models/HousingOwnerApi'; export interface SourceHousingOwner { + geo_code: string; local_id: string; idpersonne: string; idprocpte: string; @@ -13,6 +14,7 @@ export interface SourceHousingOwner { export const sourceHousingOwnerSchema: ObjectSchema = object({ + geo_code: string().required('geo_code is required').length(5), local_id: string().required('local_id is required').length(12), idpersonne: string().required('idpersonne is required').length(8), idprocpte: string().required('idprocpte is required').length(11), diff --git a/server/src/scripts/import-lovac/source-housing-owners/test/source-housing-owner.test.ts b/server/src/scripts/import-lovac/source-housing-owners/test/source-housing-owner.test.ts index 08e2c9682..08d6f08cc 100644 --- a/server/src/scripts/import-lovac/source-housing-owners/test/source-housing-owner.test.ts +++ b/server/src/scripts/import-lovac/source-housing-owners/test/source-housing-owner.test.ts @@ -9,6 +9,7 @@ import { PositiveRank } from '~/models/HousingOwnerApi'; describe('SourceHousingOwner', () => { describe('sourceHousingOwnerSchema', () => { test.prop({ + geo_code: fc.string({ minLength: 5, maxLength: 5 }), local_id: fc.string({ minLength: 12, maxLength: 12 }), idpersonne: fc.string({ minLength: 8, maxLength: 8 }), idprocpte: fc.string({ minLength: 11, maxLength: 11 }), diff --git a/server/src/scripts/import-lovac/source-housings/source-housing-command.ts b/server/src/scripts/import-lovac/source-housings/source-housing-command.ts index e804faf76..aeda11c84 100644 --- a/server/src/scripts/import-lovac/source-housings/source-housing-command.ts +++ b/server/src/scripts/import-lovac/source-housings/source-housing-command.ts @@ -76,8 +76,10 @@ export function createSourceHousingCommand() { } }, housingRepository: { - findOne(localId: string): Promise { - const geoCode = localId.substring(0, 5); + findOne( + geoCode: string, + localId: string + ): Promise { return housingRepository.findOne({ localId, geoCode diff --git a/server/src/scripts/import-lovac/source-housings/source-housing-processor.ts b/server/src/scripts/import-lovac/source-housings/source-housing-processor.ts index 94507a521..e9da1a1b0 100644 --- a/server/src/scripts/import-lovac/source-housings/source-housing-processor.ts +++ b/server/src/scripts/import-lovac/source-housings/source-housing-processor.ts @@ -29,7 +29,7 @@ export interface ProcessorOptions extends ReporterOptions { find(id: HousingId): Promise>; }; housingRepository: { - findOne(localId: string): Promise; + findOne(geoCode: string, localId: string): Promise; insert(housing: HousingApi): Promise; update( id: Pick, @@ -53,7 +53,10 @@ export function createSourceHousingProcessor(opts: ProcessorOptions) { try { logger.debug('Processing source housing...', { chunk }); - const existingHousing = await housingRepository.findOne(chunk.local_id); + const existingHousing = await housingRepository.findOne( + chunk.geo_code, + chunk.local_id + ); if (!existingHousing) { const housing: HousingApi = { id: uuidv4(), From d61bc22e22d7fede0a3b6000773d05ffba149d16 Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Fri, 9 Aug 2024 14:28:37 +0200 Subject: [PATCH 03/10] fix(server): fix build error --- server/src/scripts/import-lovac/infra/fixtures.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/scripts/import-lovac/infra/fixtures.ts b/server/src/scripts/import-lovac/infra/fixtures.ts index 692af981e..1f0369c69 100644 --- a/server/src/scripts/import-lovac/infra/fixtures.ts +++ b/server/src/scripts/import-lovac/infra/fixtures.ts @@ -61,6 +61,7 @@ export function genSourceHousingOwner( sourceOwner: SourceOwner ): SourceHousingOwner { return { + geo_code: sourceHousing.geo_code, local_id: sourceHousing.local_id, idpersonne: sourceOwner.idpersonne, idprocpte: faker.string.alphanumeric(11), From c8bb420a06fdce41f10855e1faebeb3dec458b83 Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Fri, 9 Aug 2024 14:42:26 +0200 Subject: [PATCH 04/10] fix(server): set cadastral classification as nullable --- .../import-lovac/source-housings/source-housing-processor.ts | 3 ++- .../scripts/import-lovac/source-housings/source-housing.ts | 5 +++-- .../import-lovac/source-housings/test/source-housing.test.ts | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/server/src/scripts/import-lovac/source-housings/source-housing-processor.ts b/server/src/scripts/import-lovac/source-housings/source-housing-processor.ts index e9da1a1b0..d7754585b 100644 --- a/server/src/scripts/import-lovac/source-housings/source-housing-processor.ts +++ b/server/src/scripts/import-lovac/source-housings/source-housing-processor.ts @@ -67,7 +67,8 @@ export function createSourceHousingProcessor(opts: ProcessorOptions) { geoCode: chunk.geo_code, longitude: chunk.dgfip_longitude ?? undefined, latitude: chunk.dgfip_latitude ?? undefined, - cadastralClassification: chunk.cadastral_classification, + cadastralClassification: + chunk.cadastral_classification ?? undefined, uncomfortable: chunk.uncomfortable, vacancyStartYear: chunk.vacancy_start_year, housingKind: chunk.housing_kind, diff --git a/server/src/scripts/import-lovac/source-housings/source-housing.ts b/server/src/scripts/import-lovac/source-housings/source-housing.ts index 89af809e2..35124a9ea 100644 --- a/server/src/scripts/import-lovac/source-housings/source-housing.ts +++ b/server/src/scripts/import-lovac/source-housings/source-housing.ts @@ -24,7 +24,7 @@ export interface SourceHousing { rooms_count: number; building_year: number | null; uncomfortable: boolean; - cadastral_classification: number; + cadastral_classification: number | null; beneficiary_count: number; taxed: boolean; vacancy_start_year: number; @@ -85,7 +85,8 @@ export const sourceHousingSchema: ObjectSchema = object({ .max(new Date().getUTCFullYear()), uncomfortable: boolean().required('uncomfortable is required'), cadastral_classification: number() - .required('cadastral_classification is required') + .defined('cadastral_classification must be defined') + .nullable() .min(0), beneficiary_count: number() .integer('beneficiary_count must be an integer') diff --git a/server/src/scripts/import-lovac/source-housings/test/source-housing.test.ts b/server/src/scripts/import-lovac/source-housings/test/source-housing.test.ts index 723c371ec..ee306ab16 100644 --- a/server/src/scripts/import-lovac/source-housings/test/source-housing.test.ts +++ b/server/src/scripts/import-lovac/source-housings/test/source-housing.test.ts @@ -34,7 +34,7 @@ describe('SourceHousing', () => { fc.integer({ min: 1, max: new Date().getUTCFullYear() }) ), uncomfortable: fc.boolean(), - cadastral_classification: fc.integer({ min: 0 }), + cadastral_classification: fc.option(fc.integer({ min: 0 })), beneficiary_count: fc.integer({ min: 1 }), taxed: fc.boolean(), vacancy_start_year: fc.integer({ From 96ad754281862349d229dd0e0e1bffac3912ebbb Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Fri, 9 Aug 2024 17:00:32 +0200 Subject: [PATCH 05/10] fix(server): fix history import based on the new given LOVAC file --- .../scripts/import-lovac/history/history-file-repository.ts | 4 +--- server/src/scripts/import-lovac/history/history-processor.ts | 4 ++-- server/src/scripts/import-lovac/history/history.ts | 4 ++-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/server/src/scripts/import-lovac/history/history-file-repository.ts b/server/src/scripts/import-lovac/history/history-file-repository.ts index 5ee9eab65..32703b9dc 100644 --- a/server/src/scripts/import-lovac/history/history-file-repository.ts +++ b/server/src/scripts/import-lovac/history/history-file-repository.ts @@ -32,9 +32,7 @@ function filterHistory( return new TransformStream(); } - return filter((history) => - departments.includes(history.ff_idlocal.substring(0, 2)) - ); + return filter((history) => departments.includes(history.geo_code)); } function createHistoryFileRepository(file: string): SourceRepository { diff --git a/server/src/scripts/import-lovac/history/history-processor.ts b/server/src/scripts/import-lovac/history/history-processor.ts index 85b1edf37..584292df7 100644 --- a/server/src/scripts/import-lovac/history/history-processor.ts +++ b/server/src/scripts/import-lovac/history/history-processor.ts @@ -20,12 +20,12 @@ export function historyProcessor(opts: ProcessorOptions) { return new WritableStream({ async write(chunk) { try { - const dataFileYears: string[] = normalize(chunk.files_years); + const dataFileYears: string[] = normalize(chunk.file_years); if (dataFileYears.length > 0) { await housingRepository.update( { geoCode: chunk.geo_code, - localId: chunk.ff_idlocal + localId: chunk.local_id }, { dataFileYears } ); diff --git a/server/src/scripts/import-lovac/history/history.ts b/server/src/scripts/import-lovac/history/history.ts index be85fb6f0..d9baa05e9 100644 --- a/server/src/scripts/import-lovac/history/history.ts +++ b/server/src/scripts/import-lovac/history/history.ts @@ -1,5 +1,5 @@ export interface History { geo_code: string; - ff_idlocal: string; - files_years: string[]; + local_id: string; + file_years: string[]; } From ef8d7d1b47909259254a8b615745840ea198db7f Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Sat, 10 Aug 2024 19:07:38 +0200 Subject: [PATCH 06/10] refactor(server): move normalizeDataFileYears to the housing model --- server/src/models/HousingApi.ts | 5 +++ server/src/models/test/HousingApi.test.ts | 31 ++++++++++++++++- .../import-lovac/history/history-processor.ts | 16 +++------ .../history/test/history-processor.test.ts | 33 ------------------- 4 files changed, 39 insertions(+), 46 deletions(-) delete mode 100644 server/src/scripts/import-lovac/history/test/history-processor.test.ts diff --git a/server/src/models/HousingApi.ts b/server/src/models/HousingApi.ts index 7c0f5f8c4..73bacac41 100644 --- a/server/src/models/HousingApi.ts +++ b/server/src/models/HousingApi.ts @@ -1,3 +1,4 @@ +import fp from 'lodash/fp'; import { assert, MarkRequired } from 'ts-essentials'; import { HousingSource } from '@zerologementvacant/shared'; @@ -202,3 +203,7 @@ export function isSupervised( return false; } + +export function normalizeDataFileYears(dataFileYears: string[]): string[] { + return fp.pipe(fp.sortBy(fp.identity), fp.sortedUniq)(dataFileYears); +} diff --git a/server/src/models/test/HousingApi.test.ts b/server/src/models/test/HousingApi.test.ts index 69c05c33f..e6eab65a1 100644 --- a/server/src/models/test/HousingApi.test.ts +++ b/server/src/models/test/HousingApi.test.ts @@ -1,7 +1,8 @@ import { getBuildingLocation, HousingApi, - isSupervised + isSupervised, + normalizeDataFileYears } from '~/models/HousingApi'; import { HousingStatusApi } from '~/models/HousingStatusApi'; import { @@ -87,4 +88,32 @@ describe('HousingApi', () => { }); }); }); + + describe('normalizeDataFileYears', () => { + it('should sort data file years', () => { + const actual = normalizeDataFileYears([ + 'lovac-2020', + 'ff-2024', + 'lovac-2022', + 'lovac-2021' + ]); + + expect(actual).toStrictEqual([ + 'ff-2024', + 'lovac-2020', + 'lovac-2021', + 'lovac-2022' + ]); + }); + + it('should filter duplicates', () => { + const actual = normalizeDataFileYears([ + 'lovac-2021', + 'lovac-2022', + 'lovac-2022' + ]); + + expect(actual).toStrictEqual(['lovac-2021', 'lovac-2022']); + }); + }); }); diff --git a/server/src/scripts/import-lovac/history/history-processor.ts b/server/src/scripts/import-lovac/history/history-processor.ts index 584292df7..0203b46b4 100644 --- a/server/src/scripts/import-lovac/history/history-processor.ts +++ b/server/src/scripts/import-lovac/history/history-processor.ts @@ -1,9 +1,8 @@ -import fp from 'lodash/fp'; import { WritableStream } from 'node:stream/web'; import { ReporterError, ReporterOptions } from '~/scripts/import-lovac/infra'; import { History } from '~/scripts/import-lovac/history/history'; -import { HousingApi } from '~/models/HousingApi'; +import { HousingApi, normalizeDataFileYears } from '~/models/HousingApi'; interface ProcessorOptions extends ReporterOptions { housingRepository: { @@ -20,7 +19,9 @@ export function historyProcessor(opts: ProcessorOptions) { return new WritableStream({ async write(chunk) { try { - const dataFileYears: string[] = normalize(chunk.file_years); + const dataFileYears: string[] = normalizeDataFileYears( + chunk.file_years + ); if (dataFileYears.length > 0) { await housingRepository.update( { @@ -47,12 +48,3 @@ export function historyProcessor(opts: ProcessorOptions) { } }); } - -export function normalize(dataFileYears: string[]): string[] { - return fp.pipe( - fp.sortBy(fp.identity), - fp.sortedUniq, - // "lovac-2024" should be added later to the array by the `housings` command - fp.filter((dataFileYear) => dataFileYear !== 'lovac-2024') - )(dataFileYears); -} diff --git a/server/src/scripts/import-lovac/history/test/history-processor.test.ts b/server/src/scripts/import-lovac/history/test/history-processor.test.ts deleted file mode 100644 index 15c4f0a49..000000000 --- a/server/src/scripts/import-lovac/history/test/history-processor.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { normalize } from '~/scripts/import-lovac/history/history-processor'; - -describe('History processor', () => { - describe('normalize', () => { - it('should sort data file years', () => { - const actual = normalize([ - 'lovac-2020', - 'ff-2024', - 'lovac-2022', - 'lovac-2021' - ]); - - expect(actual).toStrictEqual([ - 'ff-2024', - 'lovac-2020', - 'lovac-2021', - 'lovac-2022' - ]); - }); - - it('should filter "lovac-2024"', () => { - const actual = normalize(['lovac-2021', 'lovac-2022', 'lovac-2024']); - - expect(actual).toStrictEqual(['lovac-2021', 'lovac-2022']); - }); - - it('should filter duplicates', () => { - const actual = normalize(['lovac-2021', 'lovac-2022', 'lovac-2022']); - - expect(actual).toStrictEqual(['lovac-2021', 'lovac-2022']); - }); - }); -}); From 5c01e1f3744d0eed6b064d8726194c7cae1a38aa Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Sat, 10 Aug 2024 19:09:28 +0200 Subject: [PATCH 07/10] refactor(server): simplify source housing processor --- .../source-housing-processor.ts | 93 ++++++++----------- 1 file changed, 37 insertions(+), 56 deletions(-) diff --git a/server/src/scripts/import-lovac/source-housings/source-housing-processor.ts b/server/src/scripts/import-lovac/source-housings/source-housing-processor.ts index d7754585b..153c56d29 100644 --- a/server/src/scripts/import-lovac/source-housings/source-housing-processor.ts +++ b/server/src/scripts/import-lovac/source-housings/source-housing-processor.ts @@ -9,6 +9,7 @@ import { HousingApi, HousingId, isSupervised, + normalizeDataFileYears, OccupancyKindApi, OwnershipKindsApi } from '~/models/HousingApi'; @@ -16,6 +17,7 @@ import { HousingStatusApi } from '~/models/HousingStatusApi'; import { HousingEventApi } from '~/models/EventApi'; import { AddressApi } from '~/models/AddressApi'; import { UserApi } from '~/models/UserApi'; +import { compact } from '~/utils/object'; const logger = createLogger('sourceHousingProcessor'); @@ -105,67 +107,46 @@ export function createSourceHousingProcessor(opts: ProcessorOptions) { return; } - if (existingHousing) { - const existingEvents = await housingEventRepository.find({ - id: existingHousing.id, - geoCode: existingHousing.geoCode - }); + // The housing exists + const existingEvents = await housingEventRepository.find({ + id: existingHousing.id, + geoCode: existingHousing.geoCode + }); - if (existingHousing.occupancy !== OccupancyKindApi.Vacant) { - if (!isSupervised(existingHousing, existingEvents)) { - const occupancy = OccupancyKindApi.Vacant; - const dataFileYears = [ - ...existingHousing.dataFileYears, - 'lovac-2024' - ]; - await housingRepository.update( - { id: existingHousing.id, geoCode: existingHousing.geoCode }, - { dataFileYears, occupancy } - ); - await housingEventRepository.insert({ - id: uuidv4(), - name: 'Changement de statut d’occupation', - kind: 'Update', - category: 'Followup', - section: 'Situation', - conflict: false, - old: existingHousing, - new: { ...existingHousing, dataFileYears, occupancy }, - createdBy: auth.id, - createdAt: new Date(), - housingGeoCode: existingHousing.geoCode, - housingId: existingHousing.id - }); - reporter.passed(chunk); - return; - } else { - await housingRepository.update( - { id: existingHousing.id, geoCode: existingHousing.geoCode }, - { - dataFileYears: [ - ...existingHousing.dataFileYears, - 'lovac-2024' - ] - } - ); - reporter.passed(chunk); - return; - } - } + const dataFileYears = normalizeDataFileYears( + existingHousing.dataFileYears.concat('lovac-2024') + ); + let occupancy: OccupancyKindApi | undefined; + let event: HousingEventApi | undefined; - if (existingHousing.occupancy === OccupancyKindApi.Vacant) { - await housingRepository.update( - { id: existingHousing.id, geoCode: existingHousing.geoCode }, - { - dataFileYears: [...existingHousing.dataFileYears, 'lovac-2024'] - } - ); - reporter.passed(chunk); - return; + if (existingHousing.occupancy !== OccupancyKindApi.Vacant) { + if (!isSupervised(existingHousing, existingEvents)) { + occupancy = OccupancyKindApi.Vacant; + event = { + id: uuidv4(), + name: 'Changement de statut d’occupation', + kind: 'Update', + category: 'Followup', + section: 'Situation', + conflict: false, + old: existingHousing, + new: { ...existingHousing, dataFileYears, occupancy }, + createdBy: auth.id, + createdAt: new Date(), + housingGeoCode: existingHousing.geoCode, + housingId: existingHousing.id + }; } } - reporter.skipped(chunk); + await Promise.all([ + housingRepository.update( + { id: existingHousing.id, geoCode: existingHousing.geoCode }, + compact({ dataFileYears, occupancy }) + ), + event ? housingEventRepository.insert(event) : Promise.resolve() + ]); + reporter.passed(chunk); } catch (error) { reporter.failed( chunk, From ba5ea32e161d4881e6985917ca0375a9b39a0316 Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Sat, 10 Aug 2024 19:14:10 +0200 Subject: [PATCH 08/10] fix(server): fix housing event insertion order; make db calls concurrent --- .../housings/housing-processor.ts | 85 ++++++++++--------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/server/src/scripts/import-lovac/housings/housing-processor.ts b/server/src/scripts/import-lovac/housings/housing-processor.ts index 7b3d01e55..fe00d0219 100644 --- a/server/src/scripts/import-lovac/housings/housing-processor.ts +++ b/server/src/scripts/import-lovac/housings/housing-processor.ts @@ -40,47 +40,50 @@ export function createHousingProcessor(opts: ProcessorOptions) { if (!chunk.dataFileYears.includes('lovac-2024')) { if (chunk.occupancy === OccupancyKindApi.Vacant) { if (!isInProgress(chunk) && !isCompleted(chunk)) { - await housingRepository.update( - { geoCode: chunk.geoCode, id: chunk.id }, - { - occupancy: OccupancyKindApi.Unknown, - status: HousingStatusApi.Completed, - subStatus: 'Sortie de la vacance' - } - ); - const now = new Date(); - await housingEventRepository.insert({ - id: uuidv4(), - name: 'Changement de statut d’occupation', - kind: 'Update', - category: 'Followup', - section: 'Situation', - conflict: false, - old: chunk, - new: { ...chunk, occupancy: OccupancyKindApi.Unknown }, - createdAt: now, - createdBy: auth.id, - housingId: chunk.id, - housingGeoCode: chunk.geoCode - }); - await housingEventRepository.insert({ - id: uuidv4(), - name: 'Changement de statut de suivi', - kind: 'Update', - category: 'Followup', - section: 'Situation', - conflict: false, - old: chunk, - new: { - ...chunk, - status: HousingStatusApi.Completed, - subStatus: 'Sortie de la vacance' - }, - createdAt: now, - createdBy: auth.id, - housingId: chunk.id, - housingGeoCode: chunk.geoCode - }); + await Promise.all([ + housingRepository.update( + { geoCode: chunk.geoCode, id: chunk.id }, + { + occupancy: OccupancyKindApi.Unknown, + status: HousingStatusApi.Completed, + subStatus: 'Sortie de la vacance' + } + ), + housingEventRepository.insert({ + id: uuidv4(), + name: 'Changement de statut d’occupation', + kind: 'Update', + category: 'Followup', + section: 'Situation', + conflict: false, + old: chunk, + new: { ...chunk, occupancy: OccupancyKindApi.Unknown }, + createdAt: new Date(), + createdBy: auth.id, + housingId: chunk.id, + housingGeoCode: chunk.geoCode + }), + housingEventRepository.insert({ + id: uuidv4(), + name: 'Changement de statut de suivi', + kind: 'Update', + category: 'Followup', + section: 'Situation', + conflict: false, + // This event should come after the above one + old: { ...chunk, occupancy: OccupancyKindApi.Unknown }, + new: { + ...chunk, + occupancy: OccupancyKindApi.Unknown, + status: HousingStatusApi.Completed, + subStatus: 'Sortie de la vacance' + }, + createdAt: new Date(), + createdBy: auth.id, + housingId: chunk.id, + housingGeoCode: chunk.geoCode + }) + ]); reporter.passed(chunk); return; From 52acb08c25a4a69615c16e2f9243222ea5dc91f1 Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Sat, 10 Aug 2024 19:18:22 +0200 Subject: [PATCH 09/10] fix(server): remove undefined values before updating housing --- .../source-housings/source-housing-command.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/server/src/scripts/import-lovac/source-housings/source-housing-command.ts b/server/src/scripts/import-lovac/source-housings/source-housing-command.ts index aeda11c84..1dd1d86a2 100644 --- a/server/src/scripts/import-lovac/source-housings/source-housing-command.ts +++ b/server/src/scripts/import-lovac/source-housings/source-housing-command.ts @@ -19,6 +19,7 @@ import { HousingEventApi } from '~/models/EventApi'; import userRepository from '~/repositories/userRepository'; import config from '~/infra/config'; import UserMissingError from '~/errors/userMissingError'; +import { compact } from '~/utils/object'; const logger = createLogger('sourceHousingCommand'); @@ -97,10 +98,14 @@ export function createSourceHousingCommand() { housing: Partial ): Promise { if (!options.dryRun) { - await Housing().where({ geo_code: geoCode, id }).update({ - data_file_years: housing.dataFileYears, - occupancy: housing.occupancy - }); + await Housing() + .where({ geo_code: geoCode, id }) + .update( + compact({ + data_file_years: housing.dataFileYears, + occupancy: housing.occupancy + }) + ); } } }, From 26bf440030837c99d233b114f30ca16fa8ddf548 Mon Sep 17 00:00:00 2001 From: Andrea Gueugnaut Date: Mon, 12 Aug 2024 11:37:08 +0200 Subject: [PATCH 10/10] test(server): fix failing test --- .../import-lovac/housings/test/housing-processor.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/scripts/import-lovac/housings/test/housing-processor.test.ts b/server/src/scripts/import-lovac/housings/test/housing-processor.test.ts index 062cbcac7..cd7c7664e 100644 --- a/server/src/scripts/import-lovac/housings/test/housing-processor.test.ts +++ b/server/src/scripts/import-lovac/housings/test/housing-processor.test.ts @@ -226,9 +226,10 @@ describe('Housing processor', () => { category: 'Followup', section: 'Situation', conflict: false, - old: housing, + old: { ...housing, occupancy: OccupancyKindApi.Unknown }, new: { ...housing, + occupancy: OccupancyKindApi.Unknown, status: HousingStatusApi.Completed, subStatus: 'Sortie de la vacance' },