Merge pull request #854 from MTES-MCT/fix-source-housing-status-import

Fix housing import

server/src/models/HousingApi.ts

@@ -1,3 +1,4 @@
+import fp from 'lodash/fp';
 import { assert, MarkRequired } from 'ts-essentials';
 
 import { HousingSource } from '@zerologementvacant/shared';
@@ -202,3 +203,7 @@ export function isSupervised(
 
   return false;
 }
+
+export function normalizeDataFileYears(dataFileYears: string[]): string[] {
+  return fp.pipe(fp.sortBy<string>(fp.identity), fp.sortedUniq)(dataFileYears);
+}

server/src/models/test/HousingApi.test.ts

@@ -1,7 +1,8 @@
 import {
   getBuildingLocation,
   HousingApi,
-  isSupervised
+  isSupervised,
+  normalizeDataFileYears
 } from '~/models/HousingApi';
 import { HousingStatusApi } from '~/models/HousingStatusApi';
 import {
@@ -87,4 +88,32 @@ describe('HousingApi', () => {
       });
     });
   });
+
+  describe('normalizeDataFileYears', () => {
+    it('should sort data file years', () => {
+      const actual = normalizeDataFileYears([
+        'lovac-2020',
+        'ff-2024',
+        'lovac-2022',
+        'lovac-2021'
+      ]);
+
+      expect(actual).toStrictEqual([
+        'ff-2024',
+        'lovac-2020',
+        'lovac-2021',
+        'lovac-2022'
+      ]);
+    });
+
+    it('should filter duplicates', () => {
+      const actual = normalizeDataFileYears([
+        'lovac-2021',
+        'lovac-2022',
+        'lovac-2022'
+      ]);
+
+      expect(actual).toStrictEqual(['lovac-2021', 'lovac-2022']);
+    });
+  });
 });

server/src/scripts/import-lovac/history/history-file-repository.ts

@@ -32,9 +32,7 @@ function filterHistory(
     return new TransformStream();
   }
 
-  return filter((history) =>
-    departments.includes(history.ff_idlocal.substring(0, 2))
-  );
+  return filter((history) => departments.includes(history.geo_code));
 }
 
 function createHistoryFileRepository(file: string): SourceRepository<History> {

server/src/scripts/import-lovac/history/history-processor.ts

@@ -1,9 +1,8 @@
-import fp from 'lodash/fp';
 import { WritableStream } from 'node:stream/web';
 
 import { ReporterError, ReporterOptions } from '~/scripts/import-lovac/infra';
 import { History } from '~/scripts/import-lovac/history/history';
-import { HousingApi } from '~/models/HousingApi';
+import { HousingApi, normalizeDataFileYears } from '~/models/HousingApi';
 
 interface ProcessorOptions extends ReporterOptions<History> {
   housingRepository: {
@@ -20,12 +19,14 @@ export function historyProcessor(opts: ProcessorOptions) {
   return new WritableStream<History>({
     async write(chunk) {
       try {
-        const dataFileYears: string[] = normalize(chunk.files_years);
+        const dataFileYears: string[] = normalizeDataFileYears(
+          chunk.file_years
+        );
         if (dataFileYears.length > 0) {
           await housingRepository.update(
             {
-              geoCode: chunk.ff_idlocal.substring(0, 5),
-              localId: chunk.ff_idlocal
+              geoCode: chunk.geo_code,
+              localId: chunk.local_id
             },
             { dataFileYears }
           );
@@ -47,12 +48,3 @@ export function historyProcessor(opts: ProcessorOptions) {
     }
   });
 }
-
-export function normalize(dataFileYears: string[]): string[] {
-  return fp.pipe(
-    fp.sortBy<string>(fp.identity),
-    fp.sortedUniq,
-    // "lovac-2024" should be added later to the array by the `housings` command
-    fp.filter((dataFileYear) => dataFileYear !== 'lovac-2024')
-  )(dataFileYears);
-}

server/src/scripts/import-lovac/history/history.ts

@@ -1,4 +1,5 @@
 export interface History {
-  ff_idlocal: string;
-  files_years: string[];
+  geo_code: string;
+  local_id: string;
+  file_years: string[];
 }

server/src/scripts/import-lovac/housings/housing-processor.ts

@@ -15,7 +15,7 @@ export interface ProcessorOptions extends ReporterOptions<HousingApi> {
   housingRepository: {
     update(
       id: Pick<HousingApi, 'geoCode' | 'id'>,
-      housing: Partial<HousingApi>
+      housing: Pick<HousingApi, 'occupancy' | 'status' | 'subStatus'>
     ): Promise<void>;
   };
   housingEventRepository: {
@@ -40,47 +40,50 @@ export function createHousingProcessor(opts: ProcessorOptions) {
         if (!chunk.dataFileYears.includes('lovac-2024')) {
           if (chunk.occupancy === OccupancyKindApi.Vacant) {
             if (!isInProgress(chunk) && !isCompleted(chunk)) {
-              await housingRepository.update(
-                { geoCode: chunk.geoCode, id: chunk.id },
-                {
-                  occupancy: OccupancyKindApi.Unknown,
-                  status: HousingStatusApi.Completed,
-                  subStatus: 'Sortie de la vacance'
-                }
-              );
-              const now = new Date();
-              await housingEventRepository.insert({
-                id: uuidv4(),
-                name: 'Changement de statut d’occupation',
-                kind: 'Update',
-                category: 'Followup',
-                section: 'Situation',
-                conflict: false,
-                old: chunk,
-                new: { ...chunk, occupancy: OccupancyKindApi.Unknown },
-                createdAt: now,
-                createdBy: auth.id,
-                housingId: chunk.id,
-                housingGeoCode: chunk.geoCode
-              });
-              await housingEventRepository.insert({
-                id: uuidv4(),
-                name: 'Changement de statut de suivi',
-                kind: 'Update',
-                category: 'Followup',
-                section: 'Situation',
-                conflict: false,
-                old: chunk,
-                new: {
-                  ...chunk,
-                  status: HousingStatusApi.Completed,
-                  subStatus: 'Sortie de la vacance'
-                },
-                createdAt: now,
-                createdBy: auth.id,
-                housingId: chunk.id,
-                housingGeoCode: chunk.geoCode
-              });
+              await Promise.all([
+                housingRepository.update(
+                  { geoCode: chunk.geoCode, id: chunk.id },
+                  {
+                    occupancy: OccupancyKindApi.Unknown,
+                    status: HousingStatusApi.Completed,
+                    subStatus: 'Sortie de la vacance'
+                  }
+                ),
+                housingEventRepository.insert({
+                  id: uuidv4(),
+                  name: 'Changement de statut d’occupation',
+                  kind: 'Update',
+                  category: 'Followup',
+                  section: 'Situation',
+                  conflict: false,
+                  old: chunk,
+                  new: { ...chunk, occupancy: OccupancyKindApi.Unknown },
+                  createdAt: new Date(),
+                  createdBy: auth.id,
+                  housingId: chunk.id,
+                  housingGeoCode: chunk.geoCode
+                }),
+                housingEventRepository.insert({
+                  id: uuidv4(),
+                  name: 'Changement de statut de suivi',
+                  kind: 'Update',
+                  category: 'Followup',
+                  section: 'Situation',
+                  conflict: false,
+                  // This event should come after the above one
+                  old: { ...chunk, occupancy: OccupancyKindApi.Unknown },
+                  new: {
+                    ...chunk,
+                    occupancy: OccupancyKindApi.Unknown,
+                    status: HousingStatusApi.Completed,
+                    subStatus: 'Sortie de la vacance'
+                  },
+                  createdAt: new Date(),
+                  createdBy: auth.id,
+                  housingId: chunk.id,
+                  housingGeoCode: chunk.geoCode
+                })
+              ]);
 
               reporter.passed(chunk);
               return;

server/src/scripts/import-lovac/housings/test/housing-processor.test.ts

@@ -226,9 +226,10 @@ describe('Housing processor', () => {
             category: 'Followup',
             section: 'Situation',
             conflict: false,
-            old: housing,
+            old: { ...housing, occupancy: OccupancyKindApi.Unknown },
             new: {
               ...housing,
+              occupancy: OccupancyKindApi.Unknown,
               status: HousingStatusApi.Completed,
               subStatus: 'Sortie de la vacance'
             },

server/src/scripts/import-lovac/infra/fixtures.ts

@@ -61,6 +61,7 @@ export function genSourceHousingOwner(
   sourceOwner: SourceOwner
 ): SourceHousingOwner {
   return {
+    geo_code: sourceHousing.geo_code,
     local_id: sourceHousing.local_id,
     idpersonne: sourceOwner.idpersonne,
     idprocpte: faker.string.alphanumeric(11),

server/src/scripts/import-lovac/source-housing-owners/source-housing-owner-processor.ts

@@ -18,7 +18,7 @@ const logger = createLogger('sourceHousingOwnerProcessor');
 export interface ProcessorOptions extends ReporterOptions<SourceHousingOwner> {
   auth: UserApi;
   housingRepository: {
-    findOne(localId: string): Promise<HousingApi | null>;
+    findOne(geoCode: string, localId: string): Promise<HousingApi | null>;
   };
   housingEventRepository: {
     insert(event: HousingEventApi): Promise<void>;
@@ -51,7 +51,7 @@ export function createSourceHousingOwnerProcessor(opts: ProcessorOptions) {
 
         const [departmentalOwner, housing] = await Promise.all([
           ownerRepository.findOne(chunk.idpersonne),
-          housingRepository.findOne(chunk.local_id)
+          housingRepository.findOne(chunk.geo_code, chunk.local_id)
         ]);
         if (!departmentalOwner) {
           throw new OwnerMissingError(chunk.idpersonne);

server/src/scripts/import-lovac/source-housing-owners/source-housing-owner.ts

@@ -3,6 +3,7 @@ import { number, object, ObjectSchema, string } from 'yup';
 import { POSITIVE_RANKS, PositiveRank } from '~/models/HousingOwnerApi';
 
 export interface SourceHousingOwner {
+  geo_code: string;
   local_id: string;
   idpersonne: string;
   idprocpte: string;
@@ -13,6 +14,7 @@ export interface SourceHousingOwner {
 
 export const sourceHousingOwnerSchema: ObjectSchema<SourceHousingOwner> =
   object({
+    geo_code: string().required('geo_code is required').length(5),
     local_id: string().required('local_id is required').length(12),
     idpersonne: string().required('idpersonne is required').length(8),
     idprocpte: string().required('idprocpte is required').length(11),

server/src/scripts/import-lovac/source-housing-owners/test/source-housing-owner.test.ts

@@ -9,6 +9,7 @@ import { PositiveRank } from '~/models/HousingOwnerApi';
 describe('SourceHousingOwner', () => {
   describe('sourceHousingOwnerSchema', () => {
     test.prop<SourceHousingOwner>({
+      geo_code: fc.string({ minLength: 5, maxLength: 5 }),
       local_id: fc.string({ minLength: 12, maxLength: 12 }),
       idpersonne: fc.string({ minLength: 8, maxLength: 8 }),
       idprocpte: fc.string({ minLength: 11, maxLength: 11 }),

server/src/scripts/import-lovac/source-housings/source-housing-command.ts

@@ -19,6 +19,7 @@ import { HousingEventApi } from '~/models/EventApi';
 import userRepository from '~/repositories/userRepository';
 import config from '~/infra/config';
 import UserMissingError from '~/errors/userMissingError';
+import { compact } from '~/utils/object';
 
 const logger = createLogger('sourceHousingCommand');
 
@@ -76,8 +77,10 @@ export function createSourceHousingCommand() {
               }
             },
             housingRepository: {
-              findOne(localId: string): Promise<HousingApi | null> {
-                const geoCode = localId.substring(0, 5);
+              findOne(
+                geoCode: string,
+                localId: string
+              ): Promise<HousingApi | null> {
                 return housingRepository.findOne({
                   localId,
                   geoCode
@@ -95,10 +98,14 @@ export function createSourceHousingCommand() {
                 housing: Partial<HousingApi>
               ): Promise<void> {
                 if (!options.dryRun) {
-                  await Housing().where({ geo_code: geoCode, id }).update({
-                    data_file_years: housing.dataFileYears,
-                    occupancy: housing.occupancy
-                  });
+                  await Housing()
+                    .where({ geo_code: geoCode, id })
+                    .update(
+                      compact({
+                        data_file_years: housing.dataFileYears,
+                        occupancy: housing.occupancy
+                      })
+                    );
                 }
               }
             },
@@ -141,13 +148,11 @@ export function createSourceHousingCommand() {
             abortEarly: options.abortEarly,
             reporter: housingReporter,
             housingRepository: {
-              async update(
-                { geoCode, id }: Pick<HousingApi, 'geoCode' | 'id'>,
-                housing: Partial<HousingApi>
-              ): Promise<void> {
+              async update({ geoCode, id }, housing): Promise<void> {
                 if (!options.dryRun) {
                   await Housing().where({ geo_code: geoCode, id }).update({
-                    data_file_years: housing.dataFileYears,
+                    status: housing.status,
+                    sub_status: housing.subStatus,
                     occupancy: housing.occupancy
                   });
                 }