diff --git a/server/src/models/HousingApi.ts b/server/src/models/HousingApi.ts index 7c0f5f8c4..73bacac41 100644 --- a/server/src/models/HousingApi.ts +++ b/server/src/models/HousingApi.ts @@ -1,3 +1,4 @@ +import fp from 'lodash/fp'; import { assert, MarkRequired } from 'ts-essentials'; import { HousingSource } from '@zerologementvacant/shared'; @@ -202,3 +203,7 @@ export function isSupervised( return false; } + +export function normalizeDataFileYears(dataFileYears: string[]): string[] { + return fp.pipe(fp.sortBy(fp.identity), fp.sortedUniq)(dataFileYears); +} diff --git a/server/src/models/test/HousingApi.test.ts b/server/src/models/test/HousingApi.test.ts index 69c05c33f..e6eab65a1 100644 --- a/server/src/models/test/HousingApi.test.ts +++ b/server/src/models/test/HousingApi.test.ts @@ -1,7 +1,8 @@ import { getBuildingLocation, HousingApi, - isSupervised + isSupervised, + normalizeDataFileYears } from '~/models/HousingApi'; import { HousingStatusApi } from '~/models/HousingStatusApi'; import { @@ -87,4 +88,32 @@ describe('HousingApi', () => { }); }); }); + + describe('normalizeDataFileYears', () => { + it('should sort data file years', () => { + const actual = normalizeDataFileYears([ + 'lovac-2020', + 'ff-2024', + 'lovac-2022', + 'lovac-2021' + ]); + + expect(actual).toStrictEqual([ + 'ff-2024', + 'lovac-2020', + 'lovac-2021', + 'lovac-2022' + ]); + }); + + it('should filter duplicates', () => { + const actual = normalizeDataFileYears([ + 'lovac-2021', + 'lovac-2022', + 'lovac-2022' + ]); + + expect(actual).toStrictEqual(['lovac-2021', 'lovac-2022']); + }); + }); }); diff --git a/server/src/scripts/import-lovac/history/history-file-repository.ts b/server/src/scripts/import-lovac/history/history-file-repository.ts index 5ee9eab65..32703b9dc 100644 --- a/server/src/scripts/import-lovac/history/history-file-repository.ts +++ b/server/src/scripts/import-lovac/history/history-file-repository.ts @@ -32,9 +32,7 @@ function filterHistory( return new TransformStream(); } - return filter((history) => - departments.includes(history.ff_idlocal.substring(0, 2)) - ); + return filter((history) => departments.includes(history.geo_code)); } function createHistoryFileRepository(file: string): SourceRepository { diff --git a/server/src/scripts/import-lovac/history/history-processor.ts b/server/src/scripts/import-lovac/history/history-processor.ts index 15db81837..0203b46b4 100644 --- a/server/src/scripts/import-lovac/history/history-processor.ts +++ b/server/src/scripts/import-lovac/history/history-processor.ts @@ -1,9 +1,8 @@ -import fp from 'lodash/fp'; import { WritableStream } from 'node:stream/web'; import { ReporterError, ReporterOptions } from '~/scripts/import-lovac/infra'; import { History } from '~/scripts/import-lovac/history/history'; -import { HousingApi } from '~/models/HousingApi'; +import { HousingApi, normalizeDataFileYears } from '~/models/HousingApi'; interface ProcessorOptions extends ReporterOptions { housingRepository: { @@ -20,12 +19,14 @@ export function historyProcessor(opts: ProcessorOptions) { return new WritableStream({ async write(chunk) { try { - const dataFileYears: string[] = normalize(chunk.files_years); + const dataFileYears: string[] = normalizeDataFileYears( + chunk.file_years + ); if (dataFileYears.length > 0) { await housingRepository.update( { - geoCode: chunk.ff_idlocal.substring(0, 5), - localId: chunk.ff_idlocal + geoCode: chunk.geo_code, + localId: chunk.local_id }, { dataFileYears } ); @@ -47,12 +48,3 @@ export function historyProcessor(opts: ProcessorOptions) { } }); } - -export function normalize(dataFileYears: string[]): string[] { - return fp.pipe( - fp.sortBy(fp.identity), - fp.sortedUniq, - // "lovac-2024" should be added later to the array by the `housings` command - fp.filter((dataFileYear) => dataFileYear !== 'lovac-2024') - )(dataFileYears); -} diff --git a/server/src/scripts/import-lovac/history/history.ts b/server/src/scripts/import-lovac/history/history.ts index 1e29bb2b5..d9baa05e9 100644 --- a/server/src/scripts/import-lovac/history/history.ts +++ b/server/src/scripts/import-lovac/history/history.ts @@ -1,4 +1,5 @@ export interface History { - ff_idlocal: string; - files_years: string[]; + geo_code: string; + local_id: string; + file_years: string[]; } diff --git a/server/src/scripts/import-lovac/history/test/history-processor.test.ts b/server/src/scripts/import-lovac/history/test/history-processor.test.ts deleted file mode 100644 index 15c4f0a49..000000000 --- a/server/src/scripts/import-lovac/history/test/history-processor.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { normalize } from '~/scripts/import-lovac/history/history-processor'; - -describe('History processor', () => { - describe('normalize', () => { - it('should sort data file years', () => { - const actual = normalize([ - 'lovac-2020', - 'ff-2024', - 'lovac-2022', - 'lovac-2021' - ]); - - expect(actual).toStrictEqual([ - 'ff-2024', - 'lovac-2020', - 'lovac-2021', - 'lovac-2022' - ]); - }); - - it('should filter "lovac-2024"', () => { - const actual = normalize(['lovac-2021', 'lovac-2022', 'lovac-2024']); - - expect(actual).toStrictEqual(['lovac-2021', 'lovac-2022']); - }); - - it('should filter duplicates', () => { - const actual = normalize(['lovac-2021', 'lovac-2022', 'lovac-2022']); - - expect(actual).toStrictEqual(['lovac-2021', 'lovac-2022']); - }); - }); -}); diff --git a/server/src/scripts/import-lovac/housings/housing-processor.ts b/server/src/scripts/import-lovac/housings/housing-processor.ts index 14cf1a892..fe00d0219 100644 --- a/server/src/scripts/import-lovac/housings/housing-processor.ts +++ b/server/src/scripts/import-lovac/housings/housing-processor.ts @@ -15,7 +15,7 @@ export interface ProcessorOptions extends ReporterOptions { housingRepository: { update( id: Pick, - housing: Partial + housing: Pick ): Promise; }; housingEventRepository: { @@ -40,47 +40,50 @@ export function createHousingProcessor(opts: ProcessorOptions) { if (!chunk.dataFileYears.includes('lovac-2024')) { if (chunk.occupancy === OccupancyKindApi.Vacant) { if (!isInProgress(chunk) && !isCompleted(chunk)) { - await housingRepository.update( - { geoCode: chunk.geoCode, id: chunk.id }, - { - occupancy: OccupancyKindApi.Unknown, - status: HousingStatusApi.Completed, - subStatus: 'Sortie de la vacance' - } - ); - const now = new Date(); - await housingEventRepository.insert({ - id: uuidv4(), - name: 'Changement de statut d’occupation', - kind: 'Update', - category: 'Followup', - section: 'Situation', - conflict: false, - old: chunk, - new: { ...chunk, occupancy: OccupancyKindApi.Unknown }, - createdAt: now, - createdBy: auth.id, - housingId: chunk.id, - housingGeoCode: chunk.geoCode - }); - await housingEventRepository.insert({ - id: uuidv4(), - name: 'Changement de statut de suivi', - kind: 'Update', - category: 'Followup', - section: 'Situation', - conflict: false, - old: chunk, - new: { - ...chunk, - status: HousingStatusApi.Completed, - subStatus: 'Sortie de la vacance' - }, - createdAt: now, - createdBy: auth.id, - housingId: chunk.id, - housingGeoCode: chunk.geoCode - }); + await Promise.all([ + housingRepository.update( + { geoCode: chunk.geoCode, id: chunk.id }, + { + occupancy: OccupancyKindApi.Unknown, + status: HousingStatusApi.Completed, + subStatus: 'Sortie de la vacance' + } + ), + housingEventRepository.insert({ + id: uuidv4(), + name: 'Changement de statut d’occupation', + kind: 'Update', + category: 'Followup', + section: 'Situation', + conflict: false, + old: chunk, + new: { ...chunk, occupancy: OccupancyKindApi.Unknown }, + createdAt: new Date(), + createdBy: auth.id, + housingId: chunk.id, + housingGeoCode: chunk.geoCode + }), + housingEventRepository.insert({ + id: uuidv4(), + name: 'Changement de statut de suivi', + kind: 'Update', + category: 'Followup', + section: 'Situation', + conflict: false, + // This event should come after the above one + old: { ...chunk, occupancy: OccupancyKindApi.Unknown }, + new: { + ...chunk, + occupancy: OccupancyKindApi.Unknown, + status: HousingStatusApi.Completed, + subStatus: 'Sortie de la vacance' + }, + createdAt: new Date(), + createdBy: auth.id, + housingId: chunk.id, + housingGeoCode: chunk.geoCode + }) + ]); reporter.passed(chunk); return; diff --git a/server/src/scripts/import-lovac/housings/test/housing-processor.test.ts b/server/src/scripts/import-lovac/housings/test/housing-processor.test.ts index 062cbcac7..cd7c7664e 100644 --- a/server/src/scripts/import-lovac/housings/test/housing-processor.test.ts +++ b/server/src/scripts/import-lovac/housings/test/housing-processor.test.ts @@ -226,9 +226,10 @@ describe('Housing processor', () => { category: 'Followup', section: 'Situation', conflict: false, - old: housing, + old: { ...housing, occupancy: OccupancyKindApi.Unknown }, new: { ...housing, + occupancy: OccupancyKindApi.Unknown, status: HousingStatusApi.Completed, subStatus: 'Sortie de la vacance' }, diff --git a/server/src/scripts/import-lovac/infra/fixtures.ts b/server/src/scripts/import-lovac/infra/fixtures.ts index 692af981e..1f0369c69 100644 --- a/server/src/scripts/import-lovac/infra/fixtures.ts +++ b/server/src/scripts/import-lovac/infra/fixtures.ts @@ -61,6 +61,7 @@ export function genSourceHousingOwner( sourceOwner: SourceOwner ): SourceHousingOwner { return { + geo_code: sourceHousing.geo_code, local_id: sourceHousing.local_id, idpersonne: sourceOwner.idpersonne, idprocpte: faker.string.alphanumeric(11), diff --git a/server/src/scripts/import-lovac/source-housing-owners/source-housing-owner-processor.ts b/server/src/scripts/import-lovac/source-housing-owners/source-housing-owner-processor.ts index 864654989..1c0067704 100644 --- a/server/src/scripts/import-lovac/source-housing-owners/source-housing-owner-processor.ts +++ b/server/src/scripts/import-lovac/source-housing-owners/source-housing-owner-processor.ts @@ -18,7 +18,7 @@ const logger = createLogger('sourceHousingOwnerProcessor'); export interface ProcessorOptions extends ReporterOptions { auth: UserApi; housingRepository: { - findOne(localId: string): Promise; + findOne(geoCode: string, localId: string): Promise; }; housingEventRepository: { insert(event: HousingEventApi): Promise; @@ -51,7 +51,7 @@ export function createSourceHousingOwnerProcessor(opts: ProcessorOptions) { const [departmentalOwner, housing] = await Promise.all([ ownerRepository.findOne(chunk.idpersonne), - housingRepository.findOne(chunk.local_id) + housingRepository.findOne(chunk.geo_code, chunk.local_id) ]); if (!departmentalOwner) { throw new OwnerMissingError(chunk.idpersonne); diff --git a/server/src/scripts/import-lovac/source-housing-owners/source-housing-owner.ts b/server/src/scripts/import-lovac/source-housing-owners/source-housing-owner.ts index cbf77416f..e9b220828 100644 --- a/server/src/scripts/import-lovac/source-housing-owners/source-housing-owner.ts +++ b/server/src/scripts/import-lovac/source-housing-owners/source-housing-owner.ts @@ -3,6 +3,7 @@ import { number, object, ObjectSchema, string } from 'yup'; import { POSITIVE_RANKS, PositiveRank } from '~/models/HousingOwnerApi'; export interface SourceHousingOwner { + geo_code: string; local_id: string; idpersonne: string; idprocpte: string; @@ -13,6 +14,7 @@ export interface SourceHousingOwner { export const sourceHousingOwnerSchema: ObjectSchema = object({ + geo_code: string().required('geo_code is required').length(5), local_id: string().required('local_id is required').length(12), idpersonne: string().required('idpersonne is required').length(8), idprocpte: string().required('idprocpte is required').length(11), diff --git a/server/src/scripts/import-lovac/source-housing-owners/test/source-housing-owner.test.ts b/server/src/scripts/import-lovac/source-housing-owners/test/source-housing-owner.test.ts index 08e2c9682..08d6f08cc 100644 --- a/server/src/scripts/import-lovac/source-housing-owners/test/source-housing-owner.test.ts +++ b/server/src/scripts/import-lovac/source-housing-owners/test/source-housing-owner.test.ts @@ -9,6 +9,7 @@ import { PositiveRank } from '~/models/HousingOwnerApi'; describe('SourceHousingOwner', () => { describe('sourceHousingOwnerSchema', () => { test.prop({ + geo_code: fc.string({ minLength: 5, maxLength: 5 }), local_id: fc.string({ minLength: 12, maxLength: 12 }), idpersonne: fc.string({ minLength: 8, maxLength: 8 }), idprocpte: fc.string({ minLength: 11, maxLength: 11 }), diff --git a/server/src/scripts/import-lovac/source-housings/source-housing-command.ts b/server/src/scripts/import-lovac/source-housings/source-housing-command.ts index 996136e20..1dd1d86a2 100644 --- a/server/src/scripts/import-lovac/source-housings/source-housing-command.ts +++ b/server/src/scripts/import-lovac/source-housings/source-housing-command.ts @@ -19,6 +19,7 @@ import { HousingEventApi } from '~/models/EventApi'; import userRepository from '~/repositories/userRepository'; import config from '~/infra/config'; import UserMissingError from '~/errors/userMissingError'; +import { compact } from '~/utils/object'; const logger = createLogger('sourceHousingCommand'); @@ -76,8 +77,10 @@ export function createSourceHousingCommand() { } }, housingRepository: { - findOne(localId: string): Promise { - const geoCode = localId.substring(0, 5); + findOne( + geoCode: string, + localId: string + ): Promise { return housingRepository.findOne({ localId, geoCode @@ -95,10 +98,14 @@ export function createSourceHousingCommand() { housing: Partial ): Promise { if (!options.dryRun) { - await Housing().where({ geo_code: geoCode, id }).update({ - data_file_years: housing.dataFileYears, - occupancy: housing.occupancy - }); + await Housing() + .where({ geo_code: geoCode, id }) + .update( + compact({ + data_file_years: housing.dataFileYears, + occupancy: housing.occupancy + }) + ); } } }, @@ -141,13 +148,11 @@ export function createSourceHousingCommand() { abortEarly: options.abortEarly, reporter: housingReporter, housingRepository: { - async update( - { geoCode, id }: Pick, - housing: Partial - ): Promise { + async update({ geoCode, id }, housing): Promise { if (!options.dryRun) { await Housing().where({ geo_code: geoCode, id }).update({ - data_file_years: housing.dataFileYears, + status: housing.status, + sub_status: housing.subStatus, occupancy: housing.occupancy }); } diff --git a/server/src/scripts/import-lovac/source-housings/source-housing-processor.ts b/server/src/scripts/import-lovac/source-housings/source-housing-processor.ts index 94507a521..153c56d29 100644 --- a/server/src/scripts/import-lovac/source-housings/source-housing-processor.ts +++ b/server/src/scripts/import-lovac/source-housings/source-housing-processor.ts @@ -9,6 +9,7 @@ import { HousingApi, HousingId, isSupervised, + normalizeDataFileYears, OccupancyKindApi, OwnershipKindsApi } from '~/models/HousingApi'; @@ -16,6 +17,7 @@ import { HousingStatusApi } from '~/models/HousingStatusApi'; import { HousingEventApi } from '~/models/EventApi'; import { AddressApi } from '~/models/AddressApi'; import { UserApi } from '~/models/UserApi'; +import { compact } from '~/utils/object'; const logger = createLogger('sourceHousingProcessor'); @@ -29,7 +31,7 @@ export interface ProcessorOptions extends ReporterOptions { find(id: HousingId): Promise>; }; housingRepository: { - findOne(localId: string): Promise; + findOne(geoCode: string, localId: string): Promise; insert(housing: HousingApi): Promise; update( id: Pick, @@ -53,7 +55,10 @@ export function createSourceHousingProcessor(opts: ProcessorOptions) { try { logger.debug('Processing source housing...', { chunk }); - const existingHousing = await housingRepository.findOne(chunk.local_id); + const existingHousing = await housingRepository.findOne( + chunk.geo_code, + chunk.local_id + ); if (!existingHousing) { const housing: HousingApi = { id: uuidv4(), @@ -64,7 +69,8 @@ export function createSourceHousingProcessor(opts: ProcessorOptions) { geoCode: chunk.geo_code, longitude: chunk.dgfip_longitude ?? undefined, latitude: chunk.dgfip_latitude ?? undefined, - cadastralClassification: chunk.cadastral_classification, + cadastralClassification: + chunk.cadastral_classification ?? undefined, uncomfortable: chunk.uncomfortable, vacancyStartYear: chunk.vacancy_start_year, housingKind: chunk.housing_kind, @@ -101,67 +107,46 @@ export function createSourceHousingProcessor(opts: ProcessorOptions) { return; } - if (existingHousing) { - const existingEvents = await housingEventRepository.find({ - id: existingHousing.id, - geoCode: existingHousing.geoCode - }); + // The housing exists + const existingEvents = await housingEventRepository.find({ + id: existingHousing.id, + geoCode: existingHousing.geoCode + }); - if (existingHousing.occupancy !== OccupancyKindApi.Vacant) { - if (!isSupervised(existingHousing, existingEvents)) { - const occupancy = OccupancyKindApi.Vacant; - const dataFileYears = [ - ...existingHousing.dataFileYears, - 'lovac-2024' - ]; - await housingRepository.update( - { id: existingHousing.id, geoCode: existingHousing.geoCode }, - { dataFileYears, occupancy } - ); - await housingEventRepository.insert({ - id: uuidv4(), - name: 'Changement de statut d’occupation', - kind: 'Update', - category: 'Followup', - section: 'Situation', - conflict: false, - old: existingHousing, - new: { ...existingHousing, dataFileYears, occupancy }, - createdBy: auth.id, - createdAt: new Date(), - housingGeoCode: existingHousing.geoCode, - housingId: existingHousing.id - }); - reporter.passed(chunk); - return; - } else { - await housingRepository.update( - { id: existingHousing.id, geoCode: existingHousing.geoCode }, - { - dataFileYears: [ - ...existingHousing.dataFileYears, - 'lovac-2024' - ] - } - ); - reporter.passed(chunk); - return; - } - } + const dataFileYears = normalizeDataFileYears( + existingHousing.dataFileYears.concat('lovac-2024') + ); + let occupancy: OccupancyKindApi | undefined; + let event: HousingEventApi | undefined; - if (existingHousing.occupancy === OccupancyKindApi.Vacant) { - await housingRepository.update( - { id: existingHousing.id, geoCode: existingHousing.geoCode }, - { - dataFileYears: [...existingHousing.dataFileYears, 'lovac-2024'] - } - ); - reporter.passed(chunk); - return; + if (existingHousing.occupancy !== OccupancyKindApi.Vacant) { + if (!isSupervised(existingHousing, existingEvents)) { + occupancy = OccupancyKindApi.Vacant; + event = { + id: uuidv4(), + name: 'Changement de statut d’occupation', + kind: 'Update', + category: 'Followup', + section: 'Situation', + conflict: false, + old: existingHousing, + new: { ...existingHousing, dataFileYears, occupancy }, + createdBy: auth.id, + createdAt: new Date(), + housingGeoCode: existingHousing.geoCode, + housingId: existingHousing.id + }; } } - reporter.skipped(chunk); + await Promise.all([ + housingRepository.update( + { id: existingHousing.id, geoCode: existingHousing.geoCode }, + compact({ dataFileYears, occupancy }) + ), + event ? housingEventRepository.insert(event) : Promise.resolve() + ]); + reporter.passed(chunk); } catch (error) { reporter.failed( chunk, diff --git a/server/src/scripts/import-lovac/source-housings/source-housing.ts b/server/src/scripts/import-lovac/source-housings/source-housing.ts index 89af809e2..35124a9ea 100644 --- a/server/src/scripts/import-lovac/source-housings/source-housing.ts +++ b/server/src/scripts/import-lovac/source-housings/source-housing.ts @@ -24,7 +24,7 @@ export interface SourceHousing { rooms_count: number; building_year: number | null; uncomfortable: boolean; - cadastral_classification: number; + cadastral_classification: number | null; beneficiary_count: number; taxed: boolean; vacancy_start_year: number; @@ -85,7 +85,8 @@ export const sourceHousingSchema: ObjectSchema = object({ .max(new Date().getUTCFullYear()), uncomfortable: boolean().required('uncomfortable is required'), cadastral_classification: number() - .required('cadastral_classification is required') + .defined('cadastral_classification must be defined') + .nullable() .min(0), beneficiary_count: number() .integer('beneficiary_count must be an integer') diff --git a/server/src/scripts/import-lovac/source-housings/test/source-housing.test.ts b/server/src/scripts/import-lovac/source-housings/test/source-housing.test.ts index 723c371ec..ee306ab16 100644 --- a/server/src/scripts/import-lovac/source-housings/test/source-housing.test.ts +++ b/server/src/scripts/import-lovac/source-housings/test/source-housing.test.ts @@ -34,7 +34,7 @@ describe('SourceHousing', () => { fc.integer({ min: 1, max: new Date().getUTCFullYear() }) ), uncomfortable: fc.boolean(), - cadastral_classification: fc.integer({ min: 0 }), + cadastral_classification: fc.option(fc.integer({ min: 0 })), beneficiary_count: fc.integer({ min: 1 }), taxed: fc.boolean(), vacancy_start_year: fc.integer({