4

.gitignore

@@ -13,6 +13,6 @@ docker-compose.override.yml
 logs/
 mage-ai.db
 mage_data/
+mlruns
 secrets/
 titanic_clean.csv
-mlruns

Dockerfile

@@ -1,7 +1,7 @@
 FROM mageai/mageai:alpha
 
 ARG PROJECT_NAME=mlops
-ARG MAGE_CODE_PATH=/home/mage_code
+ARG MAGE_CODE_PATH=/home/src
 ARG USER_CODE_PATH=${MAGE_CODE_PATH}/${PROJECT_NAME}
 
 WORKDIR ${MAGE_CODE_PATH}

docker-compose.yml

@@ -8,7 +8,9 @@ services:
     ports:
       - 6789:6789
     volumes:
-      - .:/$MAGE_CODE_PATH
+      - .:/$MAGE_CODE_PATH # /home/src by default
+      # Store the data output on local machine to easily debug
+      - ~/.mage_data:/$MAGE_CODE_PATH/mage_data # /home/src/mage_data by default
     restart: on-failure:5
     networks:
       - app-network

mlops/settings.yaml

@@ -3,5 +3,5 @@ projects:
   unit_1_data_preparation: {}
   unit_2_training: {}
   unit_3_observability: {}
-  unit_4_running: {}
+  unit_4_triggering: {}
   unit_5_deploying: {}

mlops/unit_2_training/pipelines/xgboost_training/metadata.yaml

@@ -1,87 +1,92 @@
 blocks:
-- all_upstream_blocks_executed: true
-  color: null
-  configuration:
-    global_data_product:
-      uuid: training_set
-  downstream_blocks: []
-  executor_config: null
-  executor_type: local_python
-  has_callback: false
-  language: python
-  name: Training set
-  retry_config: null
-  status: executed
-  timeout: null
-  type: global_data_product
-  upstream_blocks: []
-  uuid: training_set
-- all_upstream_blocks_executed: true
-  color: null
-  configuration:
-    file_source:
-      path: null
-  downstream_blocks:
-  - xgboost
-  - dashboard_data_source
-  executor_config: null
-  executor_type: local_python
-  has_callback: false
-  language: python
-  name: xgboost Hyperparameter tuning
-  retry_config: null
-  status: executed
-  timeout: null
-  type: transformer
-  upstream_blocks:
-  - training_set
-  uuid: hyperparameter_tuning/xgboost
-- all_upstream_blocks_executed: true
-  color: null
-  configuration:
-    file_source:
-      path: null
-  downstream_blocks:
-  - dashboard_data_source
-  executor_config: null
-  executor_type: local_python
-  has_callback: false
-  language: python
-  name: XGBoost
-  retry_config: null
-  status: executed
-  timeout: null
-  type: data_exporter
-  upstream_blocks:
-  - hyperparameter_tuning/xgboost
-  uuid: xgboost
-- all_upstream_blocks_executed: true
-  color: pink
-  configuration:
-    file_source:
-      path: null
-  downstream_blocks: []
-  executor_config: null
-  executor_type: local_python
-  has_callback: false
-  language: python
-  name: Dashboard data source
-  retry_config: null
-  status: executed
-  timeout: null
-  type: custom
-  upstream_blocks:
-  - hyperparameter_tuning/xgboost
-  - xgboost
-  uuid: dashboard_data_source
+  - all_upstream_blocks_executed: true
+    color: null
+    configuration:
+      global_data_product:
+        uuid: training_set
+    downstream_blocks:
+      - xgboost
+    executor_config: null
+    executor_type: local_python
+    has_callback: false
+    language: python
+    name: Training set
+    retry_config: null
+    status: executed
+    timeout: null
+    type: global_data_product
+    upstream_blocks: []
+    uuid: training_set
+  - all_upstream_blocks_executed: true
+    color: null
+    configuration:
+      file_path: unit_2_training/transformers/hyperparameter_tuning/xgboost.py
+      file_source:
+        path: unit_2_training/transformers/hyperparameter_tuning/xgboost.py
+    downstream_blocks:
+      - xgboost
+      - dashboard_data_source
+    executor_config: null
+    executor_type: local_python
+    has_callback: false
+    language: python
+    name: xgboost Hyperparameter tuning
+    retry_config: null
+    status: executed
+    timeout: null
+    type: transformer
+    upstream_blocks:
+      - training_set
+    uuid: hyperparameter_tuning/xgboost
+  - all_upstream_blocks_executed: true
+    color: null
+    configuration:
+      file_path: unit_2_training/data_exporters/xgboost.py
+      file_source:
+        path: unit_2_training/data_exporters/xgboost.py
+    downstream_blocks:
+      - dashboard_data_source
+    executor_config: null
+    executor_type: local_python
+    has_callback: false
+    language: python
+    name: XGBoost
+    retry_config: null
+    status: executed
+    timeout: null
+    type: data_exporter
+    upstream_blocks:
+      - training_set
+      - hyperparameter_tuning/xgboost
+    uuid: xgboost
+  - all_upstream_blocks_executed: true
+    color: pink
+    configuration:
+      file_path: unit_2_training/custom/dashboard_data_source.py
+      file_source:
+        path: unit_2_training/custom/dashboard_data_source.py
+    downstream_blocks: []
+    executor_config: null
+    executor_type: local_python
+    has_callback: false
+    language: python
+    name: Dashboard data source
+    retry_config: null
+    status: executed
+    timeout: null
+    type: custom
+    upstream_blocks:
+      - xgboost
+      - hyperparameter_tuning/xgboost
+    uuid: dashboard_data_source
 cache_block_output_in_memory: false
 callbacks: []
 concurrency_config: {}
 conditionals: []
-created_at: '2024-05-07 18:27:34.902705+00:00'
+created_at: "2024-05-07 18:27:34.902705+00:00"
 data_integration: null
-description: 'XGBoost is a scalable and efficient implementation of gradient boosted
-  decision trees, a powerful ensemble machine learning technique. '
+description: "XGBoost is a scalable and efficient implementation of gradient boosted
+  decision trees, a powerful ensemble machine learning technique. "
 executor_config: {}
 executor_count: 1
 executor_type: null
@@ -101,5 +106,5 @@ variables:
   early_stopping_rounds: 1
   max_depth: 1
   max_evaluations: 1
-variables_dir: /home/src/mage_data/unit_2_training
+variables_dir: /home/src/mage_data/unit_4_triggering
 widgets: []

mlops/unit_3_observability/pipelines/xgboost_training/metadata.yaml

@@ -4,7 +4,8 @@ blocks:
     configuration:
       global_data_product:
         uuid: training_set
-    downstream_blocks: []
+    downstream_blocks:
+      - xgboost
     executor_config: null
     executor_type: local_python
     has_callback: false
@@ -19,8 +20,9 @@ blocks:
   - all_upstream_blocks_executed: true
     color: null
     configuration:
+      file_path: unit_2_training/transformers/hyperparameter_tuning/xgboost.py
       file_source:
-        path: null
+        path: unit_2_training/transformers/hyperparameter_tuning/xgboost.py
     downstream_blocks:
       - xgboost
       - dashboard_data_source
@@ -39,8 +41,9 @@ blocks:
   - all_upstream_blocks_executed: true
     color: null
     configuration:
+      file_path: unit_2_training/data_exporters/xgboost.py
       file_source:
-        path: null
+        path: unit_2_training/data_exporters/xgboost.py
     downstream_blocks:
       - dashboard_data_source
     executor_config: null
@@ -53,13 +56,15 @@ blocks:
     timeout: null
     type: data_exporter
     upstream_blocks:
+      - training_set
       - hyperparameter_tuning/xgboost
     uuid: xgboost
   - all_upstream_blocks_executed: true
     color: pink
     configuration:
+      file_path: unit_2_training/custom/dashboard_data_source.py
       file_source:
-        path: null
+        path: unit_2_training/custom/dashboard_data_source.py
     downstream_blocks: []
     executor_config: null
     executor_type: local_python
@@ -71,8 +76,8 @@ blocks:
     timeout: null
     type: custom
     upstream_blocks:
-      - hyperparameter_tuning/xgboost
       - xgboost
+      - hyperparameter_tuning/xgboost
     uuid: dashboard_data_source
 cache_block_output_in_memory: false
 callbacks: []
@@ -101,4 +106,5 @@ variables:
   early_stopping_rounds: 1
   max_depth: 1
   max_evaluations: 1
+variables_dir: /home/src/mage_data/unit_4_triggering
 widgets: []

mlops/unit_4_triggering/custom/dashboard_data_source.py

@@ -10,12 +10,12 @@
 
 @custom
 def source(
+    model: Booster, 
     settings: Tuple[
         Dict[str, Union[bool, float, int, str]],
         csr_matrix,
         Series,
     ],
-    model: Booster, 
     **kwargs,
 ) -> Tuple[Booster, csr_matrix, csr_matrix]:
     _, X_train, y_train = settings