From 00962d0db597a9864005bdb04f2926b262f6c9e0 Mon Sep 17 00:00:00 2001
From: Anis K <anis01000001@gmail.com>
Date: Sat, 8 Feb 2025 23:26:58 +0100
Subject: [PATCH] feat(scrape): Add data engineer position to scraping

---
 docker-compose.yml   | 10 ++++++++++
 scripts/load_cron.sh |  5 ++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 1be7954..5fc4378 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -45,6 +45,16 @@ services:
     volumes:
       - ./src:/app/src
     command: --query "data scientist"
+  de-script:
+    build: .
+    container_name: jma-ds-script
+    env_file: .env
+    depends_on:
+      db:
+        condition: service_healthy
+    volumes:
+      - ./src:/app/src
+    command: --query "data engineer"
   update-deleted-script:
     build: .
     container_name: jma-db-update
diff --git a/scripts/load_cron.sh b/scripts/load_cron.sh
index 8a9af28..29f979f 100755
--- a/scripts/load_cron.sh
+++ b/scripts/load_cron.sh
@@ -4,6 +4,7 @@
 DIR_PATH=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 DA_MAIN_SCRIPT_FILE="$DIR_PATH/data_collection.sh --container da-script"
 DS_MAIN_SCRIPT_FILE="$DIR_PATH/data_collection.sh --container ds-script"
+DE_MAIN_SCRIPT_FILE="$DIR_PATH/data_collection.sh --container de-script"
 UPDATE_DELETED_SCRIPT_FILE="$DIR_PATH/update_deleted.sh"
 CLEAN_DATA_SCRIPT_FILE="$DIR_PATH/clean_data.sh"
 TEMP_CRON="/tmp/crontab_temp"
@@ -17,7 +18,9 @@ echo "0 1 * * * $DA_MAIN_SCRIPT_FILE" >> "$TEMP_CRON" # Every day at 01:00
 
 echo "0 3 * * * $DS_MAIN_SCRIPT_FILE" >> "$TEMP_CRON" # Every day at 03:00
 
-echo "0 5 * * * $UPDATE_DELETED_SCRIPT_FILE" >> "$TEMP_CRON" # Every day at 05:00
+echo "0 5 * * * $DE_MAIN_SCRIPT_FILE" >> "$TEMP_CRON" # Every day at 05:00
+
+echo "0 7 * * * $UPDATE_DELETED_SCRIPT_FILE" >> "$TEMP_CRON" # Every day at 07:00
 
 
 crontab "$TEMP_CRON"