Skip to content

Commit 70dca9c

Browse files
yuchem2yuchem2
authored
fix: add error handling for prodive non-blocking while scheduler data fetching (#279)
1 parent b42b2f4 commit 70dca9c

3 files changed

Lines changed: 76 additions & 63 deletions

File tree

‎apps/server/src/services/seoul/databatch.ts‎

Lines changed: 37 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -18,39 +18,44 @@ async function saveSeoulData(data: Array<ICulturalEvent>, today: Date): Promise<
1818
const bulkOps = []
1919

2020
for (const event of data) {
21-
const startDate = toZonedTime(event.STRTDATE, 'Asia/Seoul')
22-
const endDate = parseEndDate(event.END_DATE)
23-
if (startDate < today && endDate < today) {
24-
continue
25-
}
21+
try {
22+
const startDate = toZonedTime(event.STRTDATE, 'Asia/Seoul')
23+
const endDate = parseEndDate(event.END_DATE)
24+
if (startDate < today && endDate < today) {
25+
continue
26+
}
2627

27-
const registeredAt = toZonedTime(event.RGSTDATE, 'Asia/Seoul')
28-
bulkOps.push({
29-
updateOne: {
30-
filter: {
31-
name: event.TITLE,
32-
registeredAt: registeredAt,
33-
},
34-
update: {
35-
name: event.TITLE,
36-
address: `서울시 ${event.GUNAME} ${event.PLACE}`,
37-
location: {
38-
type: 'Point',
39-
coordinates: [parseFloat(event.LAT), parseFloat(event.LOT)], // API 값이 위도 경도 값이 바껴있음
28+
const registeredAt = toZonedTime(event.RGSTDATE, 'Asia/Seoul')
29+
bulkOps.push({
30+
updateOne: {
31+
filter: {
32+
name: event.TITLE,
33+
registeredAt: registeredAt,
4034
},
41-
startDate: startDate,
42-
endDate: endDate,
43-
// TODO: change url to hyperlink
44-
description: `프로그램 소개: ${event.PROGRAM}\n${event.ETC_DESC && `기타내용: ${event.ETC_DESC}\n`}${event.PLAYER && `공연자: ${event.PLAYER}\n`}\n상세정보보기: ${event.HMPG_ADDR}\n주최기관: ${event.ORG_NAME}(${event.ORG_LINK})`,
45-
photo: [event.MAIN_IMG],
46-
cost: event.IS_FREE === '무료' ? '무료' : event.USE_FEE,
47-
category: [categoryMapTitleToCode[event.CODENAME] || CategoryCode.OTHERS],
48-
targetAudience: [event.USE_TRGT],
49-
registeredAt: registeredAt,
35+
update: {
36+
name: event.TITLE,
37+
address: `서울시 ${event.GUNAME} ${event.PLACE}`,
38+
location: {
39+
type: 'Point',
40+
coordinates: [parseFloat(event.LAT), parseFloat(event.LOT)], // API 값이 위도 경도 값이 바껴있음
41+
},
42+
startDate: startDate,
43+
endDate: endDate,
44+
// TODO: change url to hyperlink
45+
description: `프로그램 소개: ${event.PROGRAM}\n${event.ETC_DESC && `기타내용: ${event.ETC_DESC}\n`}${event.PLAYER && `공연자: ${event.PLAYER}\n`}\n상세정보보기: ${event.HMPG_ADDR}\n주최기관: ${event.ORG_NAME}(${event.ORG_LINK})`,
46+
photo: [event.MAIN_IMG],
47+
cost: event.IS_FREE === '무료' ? '무료' : event.USE_FEE,
48+
category: [categoryMapTitleToCode[event.CODENAME] || CategoryCode.OTHERS],
49+
targetAudience: [event.USE_TRGT],
50+
registeredAt: registeredAt,
51+
},
52+
upsert: true,
5053
},
51-
upsert: true,
52-
},
53-
})
54+
})
55+
} catch (error) {
56+
// 원천 데이터 필드 누락 혹은 잘못된 정보로 인한 오류 시 해당 데이터 패스
57+
logger.warn('Skipping a seoul event due to a processing error', { error, event })
58+
}
5459
}
5560
try {
5661
if (bulkOps.length > 0) {
@@ -73,8 +78,9 @@ export async function fetchAndSaveSeoulData(): Promise<void> {
7378
const end = Math.min((index + 1) * BATCH_SIZE, dataCount)
7479
const result = await getSeoulData(st, end)
7580
await saveSeoulData(result.culturalEventInfo.row, today)
76-
await new Promise(resolve => setTimeout(resolve, 500)) // 각 요청마다 500ms 간격 추가 (동시 요청 불가능)
81+
await new Promise(resolve => setTimeout(resolve, 1000)) // 각 요청마다 1000ms 간격 추가 (동시 요청 불가능)
7782
if (index === 0) {
83+
if (!result.culturalEventInfo.list_total_count) break
7884
dataCount = result.culturalEventInfo.list_total_count
7985
}
8086
index++

‎apps/server/src/services/tourApi/databatch.ts‎

Lines changed: 39 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ function getCopyrightText(cpyrhDivCd: string): string {
2424
}
2525

2626
function makeDescription(eventDescription: ITourDataInfo[]) {
27+
if (!eventDescription) return ''
28+
2729
const description = []
2830
eventDescription.sort((a, b) => parseInt(a.serialnum) - parseInt(b.serialnum))
2931
eventDescription.forEach(item => description.push(`${item.infoname}: ${item.infotext}`))
@@ -33,42 +35,48 @@ function makeDescription(eventDescription: ITourDataInfo[]) {
3335
async function saveTourData(data: ITourFestivalData[], today: Date) {
3436
const bulkOps = []
3537
for (const event of data) {
36-
const startDate = toZonedTime(parseDate(event.eventstartdate), 'Asia/Seoul')
37-
const endDate = parseEndDate(event.eventenddate)
38-
if (startDate < today && endDate < today) continue
38+
try {
39+
const startDate = toZonedTime(parseDate(event.eventstartdate), 'Asia/Seoul')
40+
const endDate = parseEndDate(event.eventenddate)
41+
if (startDate < today && endDate < today) continue
3942

40-
const [eventDetail, eventDescription] = await Promise.all([getTourDataDetail(event.contentid), getTourFestivalDataInfo(event.contentid)])
43+
const [eventDetail, eventDescription] = await Promise.all([getTourDataDetail(event.contentid), getTourFestivalDataInfo(event.contentid)])
4144

42-
const registeredAt = toZonedTime(parse(event.createdtime, 'yyyyMMddHHmmss', new Date()), 'Asia/Seoul')
43-
const images = []
44-
if (event.firstimage) images.push(event.firstimage)
45-
else if (event.firstimage2) images.push(event.firstimage2)
45+
const registeredAt = toZonedTime(parse(event.createdtime, 'yyyyMMddHHmmss', new Date()), 'Asia/Seoul')
46+
const images = []
47+
if (event.firstimage) images.push(event.firstimage)
48+
else if (event.firstimage2) images.push(event.firstimage2)
4649

47-
bulkOps.push({
48-
updateOne: {
49-
filter: {
50-
name: event.title,
51-
registeredAt: registeredAt,
52-
},
53-
update: {
54-
name: event.title,
55-
address: `${event.addr1}${event.addr2 === '' ? '' : `(${event.addr2})`}`,
56-
location: {
57-
type: 'Point',
58-
coordinates: [parseFloat(event.mapx), parseFloat(event.mapy)],
50+
bulkOps.push({
51+
updateOne: {
52+
filter: {
53+
name: event.title,
54+
registeredAt: registeredAt,
5955
},
60-
startDate: startDate,
61-
endDate: endDate,
62-
description: `${makeDescription(eventDescription.response.body.items.item)}\n${getCopyrightText(event.cpyrhDivCd)}`,
63-
photo: images,
64-
cost: eventDetail.response.body.items.item.usetimefestival || '',
65-
category: [categoryMapTourCodeToCode[event.lclsSystm3] || CategoryCode.OTHERS],
66-
targetAudience: [eventDetail.response.body.items.item.agelimit],
67-
registeredAt: registeredAt,
56+
update: {
57+
name: event.title,
58+
address: `${event.addr1}${event.addr2 === '' ? '' : `(${event.addr2})`}`,
59+
location: {
60+
type: 'Point',
61+
coordinates: [parseFloat(event.mapx), parseFloat(event.mapy)],
62+
},
63+
startDate: startDate,
64+
endDate: endDate,
65+
description: `${makeDescription(eventDescription.response.body.items.item)}\n${getCopyrightText(event.cpyrhDivCd)}`,
66+
photo: images,
67+
cost: eventDetail.response.body.items.item?.usetimefestival || '',
68+
category: [categoryMapTourCodeToCode[event.lclsSystm3] || CategoryCode.OTHERS],
69+
targetAudience: eventDetail.response.body.items.item?.agelimit
70+
? [String(eventDetail.response.body.items.item?.agelimit)]
71+
: [],
72+
registeredAt: registeredAt,
73+
},
74+
upsert: true,
6875
},
69-
upsert: true,
70-
},
71-
})
76+
})
77+
} catch (error) {
78+
logger.warn('Skipping a tour event due to a processing error', { error, event })
79+
}
7280
}
7381
try {
7482
if (bulkOps.length > 0) {

‎apps/server/src/services/tourApi/tour.data.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ async function fetchTourApi(endPoint: string, params: URLSearchParams) {
2626
})
2727

2828
if (!res.ok) throw new TourDataServerError()
29-
console.log(res)
3029

3130
const data = await res.json()
3231

0 commit comments

Comments
 (0)