Merge pull request #8 from elimu-ai/1-import-video-datasets-from-webapp

Import video datasets from webapp

.github/workflows/python-run-all-steps.yml

@@ -32,9 +32,9 @@ jobs:
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Test with pytest
-      run: |
-        pytest
+    # - name: Test with pytest
+    #   run: |
+    #     pytest
     - name: Run All Steps
       run: |
         python run_all_steps.py

run_all_steps.py

@@ -0,0 +1,18 @@
+import os
+
+print('\n*** Step 1. Prepare Data 🌏 ***')
+os.chdir('step1_prepare')
+print(os.path.basename(__file__), f'os.getcwd(): {os.getcwd()}')
+import step1_prepare.step1_download_data
+#import step1_prepare.step1_preprocess_data
+#import step1_prepare.step1_split_data
+
+print('\n*** Step 2. Train Model 🌏🚀 ***')
+#os.chdir('../step2_train')
+#print(os.path.basename(__file__), f'os.getcwd(): {os.getcwd()}')
+# TODO
+
+print('\n*** Step 3. Make Predictions 🌏 🚀✨ ***')
+#os.chdir('../step3_predict')
+#print(os.path.basename(__file__), f'os.getcwd(): {os.getcwd()}')
+# TODO

step1_prepare/step1_download_data.py

@@ -0,0 +1,16 @@
+import os
+import pandas
+
+# Read the storybooks CSV into a DataFrame, and write the DataFrame to a CSV file
+videos_csv_url = 'https://raw.githubusercontent.com/elimu-ai/webapp/main/src/main/resources/db/content_PROD/eng/videos.csv'
+print(os.path.basename(__file__), 'videos_csv_url: {}'.format(videos_csv_url))
+videos_dataframe = pandas.read_csv(videos_csv_url)
+print(os.path.basename(__file__), 'videos_dataframe: \n{}'.format(videos_dataframe))
+videos_dataframe.to_csv('step1_videos.csv', index=False)
+
+# Read the learning events CSV into a DataFrame, and write the DataFrame to a CSV file
+video_learning_events_csv_url = 'http://eng.elimu.ai/analytics/video-learning-event/list/video-learning-events.csv'
+print(os.path.basename(__file__), f'video_learning_events_csv_url: {video_learning_events_csv_url}')
+video_learning_events_dataframe = pandas.read_csv(video_learning_events_csv_url)
+print(os.path.basename(__file__), f'video_learning_events_dataframe: \n{video_learning_events_dataframe}')
+video_learning_events_dataframe.to_csv('step1_video_learning_events.csv', index=False)