Skip to content

Commit 0e24382

Browse files
pix-service-auto-mergepix-service-auto-merge
authored
[TECH] Ajout de la connexion à une BDD supplémentaire pour MaDDo
 #11517
2 parents 7be0163 + 3846068 commit 0e24382

File tree

55 files changed

+539
-573
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+539
-573
lines changed

‎.circleci/config.yml

+32-2
Original file line numberDiff line numberDiff line change
@@ -404,13 +404,20 @@ jobs:
404404
environment:
405405
NODE_ENV: test
406406
TEST_DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration_datamart
407+
- run:
408+
name: Prepare datawarehouse
409+
command: npm run datawarehouse:prepare
410+
environment:
411+
NODE_ENV: test
412+
TEST_DATAWAREHOUSE_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration_datawarehouse
407413
- run:
408414
name: Test
409415
command: npm run test:api:integration
410416
environment:
411417
NODE_ENV: test
412418
TEST_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration
413419
TEST_DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration_datamart
420+
TEST_DATAWAREHOUSE_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration_datawarehouse
414421
TEST_REDIS_URL: redis://localhost:6379
415422
TEST_IMPORT_STORAGE_ENDPOINT: http://localhost:9090
416423
TEST_IMPORT_STORAGE_BUCKET_NAME: pix-import-test
@@ -440,7 +447,13 @@ jobs:
440447
command: npm run datamart:prepare
441448
environment:
442449
NODE_ENV: test
443-
TEST_DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration_datamart
450+
TEST_DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_acceptance_datamart
451+
- run:
452+
name: Prepare datawarehouse
453+
command: npm run datawarehouse:prepare
454+
environment:
455+
NODE_ENV: test
456+
TEST_DATAWAREHOUSE_DATABASE_URL: postgres://circleci@localhost:5432/circleci_acceptance_datawarehouse
444457
- run:
445458
name: Test
446459
command: |
@@ -450,7 +463,8 @@ jobs:
450463
environment:
451464
NODE_ENV: test
452465
TEST_DATABASE_URL: postgres://circleci@localhost:5432/circleci_acceptance
453-
TEST_DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration_datamart
466+
TEST_DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_acceptance_datamart
467+
TEST_DATAWAREHOUSE_DATABASE_URL: postgres://circleci@localhost:5432/circleci_acceptance_datawarehouse
454468
TEST_REDIS_URL: redis://localhost:6379
455469
TEST_IMPORT_STORAGE_ENDPOINT: http://localhost:9090
456470
TEST_IMPORT_STORAGE_BUCKET_NAME: pix-import-test
@@ -831,6 +845,7 @@ jobs:
831845
working_directory: ~/pix/api
832846
environment:
833847
DATABASE_URL: 'postgres://circleci@localhost:5432/circleci'
848+
DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_datamart
834849
REDIS_URL: 'redis://localhost:6379'
835850
background: true
836851
command: npm start
@@ -858,6 +873,13 @@ jobs:
858873
working_directory: ~/pix/api
859874
command: npm run cache:refresh
860875

876+
- run:
877+
name: Prepare datamart
878+
command: npm run datamart:prepare
879+
working_directory: ~/pix/api
880+
environment:
881+
DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_datamart
882+
861883
- run:
862884
name: Run tests
863885
environment:
@@ -917,6 +939,7 @@ jobs:
917939
working_directory: ~/pix/api
918940
environment:
919941
DATABASE_URL: 'postgres://circleci@localhost:5432/circleci'
942+
DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_datamart
920943
REDIS_URL: 'redis://localhost:6379'
921944
background: true
922945
command: npm start
@@ -944,6 +967,13 @@ jobs:
944967
working_directory: ~/pix/api
945968
command: npm run cache:refresh
946969

970+
- run:
971+
name: Prepare datamart
972+
command: npm run datamart:prepare
973+
working_directory: ~/pix/api
974+
environment:
975+
DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_datamart
976+
947977
- run:
948978
name: Run tests
949979
environment:

‎api/datamart/datamart-builder/datamart-builder.js

+5-9
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
1-
import { config } from '../../src/shared/config.js';
21
import { datamartBuffer } from './datamart-buffer.js';
32
import { factory } from './factory/index.js';
43

5-
const datamartDbSchema = config.parcoursup.databaseSchema;
6-
74
/**
85
* @class DatamartBuilder
96
* @property {Factory} factory
@@ -46,12 +43,11 @@ class DatamartBuilder {
4643
async clean() {
4744
let rawQuery = '';
4845

49-
[
50-
`${datamartDbSchema}.data_export_parcoursup_certif_result`,
51-
`${datamartDbSchema}.data_export_parcoursup_certif_result_code_validation`,
52-
].forEach((tableName) => {
53-
rawQuery += `DELETE FROM ${tableName};`;
54-
});
46+
['data_export_parcoursup_certif_result', 'data_export_parcoursup_certif_result_code_validation'].forEach(
47+
(tableName) => {
48+
rawQuery += `DELETE FROM ${tableName};`;
49+
},
50+
);
5551

5652
try {
5753
await this.knex.raw(rawQuery);

‎api/datamart/datamart-builder/factory/build-certification-result-code-validation.js

+1-4
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
1-
import { config } from '../../../src/shared/config.js';
21
import { datamartBuffer } from '../datamart-buffer.js';
32

4-
const datamartDbSchema = config.parcoursup.databaseSchema;
5-
63
const buildCertificationResultCodeValidation = function ({
74
verificationCode,
85
lastName,
@@ -31,7 +28,7 @@ const buildCertificationResultCodeValidation = function ({
3128
};
3229

3330
datamartBuffer.pushInsertable({
34-
tableName: `${datamartDbSchema}.data_export_parcoursup_certif_result_code_validation`,
31+
tableName: 'data_export_parcoursup_certif_result_code_validation',
3532
values,
3633
});
3734
};

‎api/datamart/datamart-builder/factory/build-certification-result.js

+1-4
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
1-
import { config } from '../../../src/shared/config.js';
21
import { datamartBuffer } from '../datamart-buffer.js';
32

4-
const datamartDbSchema = config.parcoursup.databaseSchema;
5-
63
const buildCertificationResult = function ({
74
nationalStudentId,
85
organizationUai,
@@ -33,7 +30,7 @@ const buildCertificationResult = function ({
3330
};
3431

3532
datamartBuffer.pushInsertable({
36-
tableName: `${datamartDbSchema}.data_export_parcoursup_certif_result`,
33+
tableName: 'data_export_parcoursup_certif_result',
3734
values,
3835
});
3936

‎api/datamart/knex-database-connection.js

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
import { DatabaseConnection } from '../db/database-connection.js';
2+
import { databaseConnections } from '../db/database-connections.js';
3+
import { config } from '../src/shared/config.js';
4+
import datamartKnexConfigs from './knexfile.js';
5+
6+
const { environment } = config;
7+
8+
const databaseConnection = new DatabaseConnection(datamartKnexConfigs[environment]);
9+
const { knex } = databaseConnection;
10+
11+
databaseConnections.addConnection(databaseConnection);
12+
13+
async function disconnect() {
14+
await databaseConnections.disconnect();
15+
}
16+
17+
export { databaseConnection, disconnect, knex };

‎api/datamart/knexfile.js

+5-2
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
11
import * as url from 'node:url';
2-
const __dirname = url.fileURLToPath(new URL('.', import.meta.url));
2+
33
import * as dotenv from 'dotenv';
44

55
import { buildPostgresEnvironment } from '../db/utils/build-postgres-environment.js';
6+
7+
const __dirname = url.fileURLToPath(new URL('.', import.meta.url));
8+
69
dotenv.config({ path: `${__dirname}/../.env` });
710

811
const baseConfiguration = {
12+
name: 'datamart',
913
migrationsDirectory: './migrations/',
1014
seedsDirectory: './seeds/',
1115
databaseUrl: process.env.DATAMART_DATABASE_URL,
12-
databaseSchema: process.env.DATAMART_DATABASE_SCHEMA || 'public',
1316
};
1417

1518
const environments = {

‎api/datamart/migrations/20241224080125-create-certification-results-table.js

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,14 @@
1-
const SCHEMA_NAME = process.env.DATAMART_DATABASE_SCHEMA || 'public';
21
const TABLE_NAME = 'data_export_parcoursup_certif_result';
32

43
const up = async function (knex) {
5-
await knex.schema.withSchema(SCHEMA_NAME).createTable(TABLE_NAME, function (table) {
4+
await knex.schema.createTable(TABLE_NAME, function (table) {
65
table.string('national_student_id');
76
table.index('national_student_id');
87
});
98
};
109

1110
const down = async function (knex) {
12-
await knex.schema.withSchema(SCHEMA_NAME).dropTable(TABLE_NAME);
11+
await knex.schema.dropTable(TABLE_NAME);
1312
};
1413

1514
export { down, up };

‎api/datamart/migrations/20241230135120-add-certification-results-columns.js

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
1-
const SCHEMA_NAME = process.env.DATAMART_DATABASE_SCHEMA || 'public';
21
const TABLE_NAME = 'data_export_parcoursup_certif_result';
32

43
const up = async function (knex) {
5-
await knex.schema.withSchema(SCHEMA_NAME).table(TABLE_NAME, function (table) {
4+
await knex.schema.table(TABLE_NAME, function (table) {
65
table.string('organization_uai');
76
table.string('last_name');
87
table.string('first_name');
@@ -17,7 +16,7 @@ const up = async function (knex) {
1716
};
1817

1918
const down = async function (knex) {
20-
await knex.schema.withSchema(SCHEMA_NAME).table(TABLE_NAME, function (table) {
19+
await knex.schema.table(TABLE_NAME, function (table) {
2120
table.dropColumn('organization_uai');
2221
table.dropColumn('last_name');
2322
table.dropColumn('first_name');

‎api/datamart/migrations/20250103134809-create-data-export-parcoursup-certif-result-code-validation-table.js

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
1-
const SCHEMA_NAME = process.env.DATAMART_DATABASE_SCHEMA || 'public';
21
const TABLE_NAME = 'data_export_parcoursup_certif_result_code_validation';
32

43
const up = async function (knex) {
5-
await knex.schema.withSchema(SCHEMA_NAME).createTable(TABLE_NAME, function (table) {
4+
await knex.schema.createTable(TABLE_NAME, function (table) {
65
table.string('certification_code_verification');
76
table.string('last_name');
87
table.string('first_name');
@@ -17,7 +16,7 @@ const up = async function (knex) {
1716
};
1817

1918
const down = async function (knex) {
20-
await knex.schema.withSchema(SCHEMA_NAME).dropTable(TABLE_NAME);
19+
await knex.schema.dropTable(TABLE_NAME);
2120
};
2221

2322
export { down, up };

‎api/datamart/migrations/20250107082936-add-competence-infos.js

+10-15
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,22 @@
1-
const SCHEMA_NAME = process.env.DATAMART_DATABASE_SCHEMA || 'public';
21
const COLUMN_NAMES = ['competence_name', 'competence_code', 'area_name'];
32

43
const up = async function (knex) {
5-
await knex.schema
6-
.withSchema(SCHEMA_NAME)
7-
.table('data_export_parcoursup_certif_result_code_validation', function (table) {
8-
COLUMN_NAMES.forEach((columnName) => table.string(columnName));
9-
table.dropColumn('competence_id');
10-
});
11-
await knex.schema.withSchema(SCHEMA_NAME).table('data_export_parcoursup_certif_result', function (table) {
4+
await knex.schema.table('data_export_parcoursup_certif_result_code_validation', function (table) {
5+
COLUMN_NAMES.forEach((columnName) => table.string(columnName));
6+
table.dropColumn('competence_id');
7+
});
8+
await knex.schema.table('data_export_parcoursup_certif_result', function (table) {
129
COLUMN_NAMES.forEach((columnName) => table.string(columnName));
1310
table.dropColumn('competence_id');
1411
});
1512
};
1613

1714
const down = async function (knex) {
18-
await knex.schema
19-
.withSchema(SCHEMA_NAME)
20-
.table('data_export_parcoursup_certif_result_code_validation', function (table) {
21-
COLUMN_NAMES.forEach((columnName) => table.dropColumn(columnName));
22-
table.string('competence_id');
23-
});
24-
await knex.schema.withSchema(SCHEMA_NAME).table('data_export_parcoursup_certif_result', function (table) {
15+
await knex.schema.table('data_export_parcoursup_certif_result_code_validation', function (table) {
16+
COLUMN_NAMES.forEach((columnName) => table.dropColumn(columnName));
17+
table.string('competence_id');
18+
});
19+
await knex.schema.table('data_export_parcoursup_certif_result', function (table) {
2520
COLUMN_NAMES.forEach((columnName) => table.dropColumn(columnName));
2621
table.string('competence_id');
2722
});

‎api/datamart/migrations/20250109114100-add-missing-column.js

+12-18
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,21 @@
1-
const SCHEMA_NAME = process.env.DATAMART_DATABASE_SCHEMA || 'public';
2-
31
const up = async function (knex) {
4-
await knex.schema
5-
.withSchema(SCHEMA_NAME)
6-
.table('data_export_parcoursup_certif_result_code_validation', function (table) {
7-
table.string('certification_courses_id');
8-
table.string('national_student_id');
9-
table.string('organization_uai');
10-
});
11-
await knex.schema.withSchema(SCHEMA_NAME).table('data_export_parcoursup_certif_result', function (table) {
2+
await knex.schema.table('data_export_parcoursup_certif_result_code_validation', function (table) {
3+
table.string('certification_courses_id');
4+
table.string('national_student_id');
5+
table.string('organization_uai');
6+
});
7+
await knex.schema.table('data_export_parcoursup_certif_result', function (table) {
128
table.string('certification_courses_id');
139
});
1410
};
1511

1612
const down = async function (knex) {
17-
await knex.schema
18-
.withSchema(SCHEMA_NAME)
19-
.table('data_export_parcoursup_certif_result_code_validation', function (table) {
20-
table.dropColumn('certification_courses_id');
21-
table.dropColumn('national_student_id');
22-
table.dropColumn('organization_uai');
23-
});
24-
await knex.schema.withSchema(SCHEMA_NAME).table('data_export_parcoursup_certif_result', function (table) {
13+
await knex.schema.table('data_export_parcoursup_certif_result_code_validation', function (table) {
14+
table.dropColumn('certification_courses_id');
15+
table.dropColumn('national_student_id');
16+
table.dropColumn('organization_uai');
17+
});
18+
await knex.schema.table('data_export_parcoursup_certif_result', function (table) {
2519
table.dropColumn('certification_courses_id');
2620
});
2721
};

‎api/datamart/migrations/20250221162500-add-collation.js

+7-32
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,21 @@
1-
const SCHEMA_NAME = process.env.DATAMART_DATABASE_SCHEMA || 'public';
2-
31
const up = async function (knex) {
4-
await knex.schema.raw('drop collation if exists :schema:."parcoursup_case_accent_punctuation_insensitive"', {
5-
schema: SCHEMA_NAME,
6-
});
2+
await knex.schema.raw('drop collation if exists "parcoursup_case_accent_punctuation_insensitive"');
73
await knex.schema.raw(
8-
'create collation :schema:."parcoursup_case_accent_punctuation_insensitive" (provider = icu, locale = "und-u-ka-shifted-ks-level1-kv-punct", deterministic = false);',
9-
{
10-
schema: SCHEMA_NAME,
11-
},
4+
'create collation "parcoursup_case_accent_punctuation_insensitive" (provider = icu, locale = "und-u-ka-shifted-ks-level1-kv-punct", deterministic = false);',
125
);
136

147
await knex.schema.raw(
15-
'ALTER TABLE :schema:."data_export_parcoursup_certif_result" alter column first_name type varchar(255) COLLATE :schema:.parcoursup_case_accent_punctuation_insensitive;',
16-
{
17-
schema: SCHEMA_NAME,
18-
},
8+
'ALTER TABLE "data_export_parcoursup_certif_result" alter column first_name type varchar(255) COLLATE parcoursup_case_accent_punctuation_insensitive;',
199
);
2010
await knex.schema.raw(
21-
'ALTER TABLE :schema:.data_export_parcoursup_certif_result alter column last_name type varchar(255) COLLATE :schema:.parcoursup_case_accent_punctuation_insensitive;',
22-
{
23-
schema: SCHEMA_NAME,
24-
},
11+
'ALTER TABLE data_export_parcoursup_certif_result alter column last_name type varchar(255) COLLATE parcoursup_case_accent_punctuation_insensitive;',
2512
);
2613
};
2714

2815
const down = async function (knex) {
29-
await knex.schema.raw(
30-
'ALTER TABLE :schema:.data_export_parcoursup_certif_result alter column last_name type varchar(255);',
31-
{
32-
schema: SCHEMA_NAME,
33-
},
34-
);
35-
await knex.schema.raw(
36-
'ALTER TABLE :schema:.data_export_parcoursup_certif_result alter column first_name type varchar(255);',
37-
{
38-
schema: SCHEMA_NAME,
39-
},
40-
);
41-
await knex.schema.raw('drop collation if exists :schema:."parcoursup_case_accent_punctuation_insensitive"', {
42-
schema: SCHEMA_NAME,
43-
});
16+
await knex.schema.raw('ALTER TABLE data_export_parcoursup_certif_result alter column last_name type varchar(255);');
17+
await knex.schema.raw('ALTER TABLE data_export_parcoursup_certif_result alter column first_name type varchar(255);');
18+
await knex.schema.raw('drop collation if exists "parcoursup_case_accent_punctuation_insensitive";');
4419
};
4520

4621
export { down, up };

0 commit comments

Comments
 (0)