[td] Add tags to pipeline cache (mage-ai#4162)

* [td] Add tags to pipeline cache

* [td] Fix sorting by blocks and triggers

* fix

* fix

Author: tommydangerous

mage_ai/api/resources/PipelineResource.py

@@ -207,25 +207,6 @@ async def get_pipeline(uuid: str) -> Pipeline:
             )
 
         pipelines = [p for p in pipelines if p is not None]
-        if sorts:
-            def _sort_key(p, sorts=sorts, reverse_sort=reverse_sort):
-                bools = []
-                vals = []
-                for k in sorts:
-                    if hasattr(p, k):
-                        val = getattr(p, k)
-                        vals.append(val)
-                        bools.append(val is None if not reverse_sort else val is not None)
-                    else:
-                        bools.append(False)
-
-                return tuple(bools + vals)
-
-            pipelines = sorted(
-                pipelines,
-                key=_sort_key,
-                reverse=reverse_sort,
-            )
 
         @safe_db_query
         def query_pipeline_schedules(pipeline_uuids):
@@ -287,6 +268,34 @@ def query_pipeline_schedules(pipeline_uuids):
                 if pipeline.uuid in history_by_pipeline_uuid:
                     pipeline.history = history_by_pipeline_uuid.get(pipeline.uuid)
 
+        if sorts:
+            def _sort_key(p, sorts=sorts, reverse_sort=reverse_sort):
+                bools = []
+                vals = []
+                for k in sorts:
+                    if 'blocks' == k.lower():
+                        val = len(p.blocks_by_uuid)
+                        vals.append(val)
+                        bools.append(val is None if not reverse_sort else val is not None)
+                    elif 'triggers' == k.lower():
+                        val = len(p.blocks_by_uuid)
+                        vals.append(val)
+                        bools.append(val is None if not reverse_sort else val is not None)
+                    elif hasattr(p, k):
+                        val = getattr(p, k)
+                        vals.append(val)
+                        bools.append(val is None if not reverse_sort else val is not None)
+                    else:
+                        bools.append(False)
+
+                return tuple(bools + vals)
+
+            pipelines = sorted(
+                pipelines,
+                key=_sort_key,
+                reverse=reverse_sort,
+            )
+
         if offset_limit_applied:
             results = pipelines
         else:

mage_ai/cache/utils.py

@@ -6,6 +6,7 @@
 PIPELINE_KEYS = [
     'description',
     'name',
+    'tags',
     'type',
     'updated_at',
     'uuid',
@@ -28,6 +29,7 @@ def build_pipeline_dict(
     pipeline_output_dict = dict(
         description=None,
         name=None,
+        tags=None,
         type=None,
         updated_at=None,
         uuid=None,

mage_ai/frontend/components/shared/Table/index.tsx

@@ -365,13 +365,15 @@ function Table({
 
       const column = columns?.[sortColIdx]?.uuid;
 
-      goToWithQuery(selectEntriesWithValues({
+      goToWithQuery({
+        ...selectEntriesWithValues({
+          [SortQueryEnum.SORT_COL_IDX]: sortColIdx,
+          [SortQueryEnum.SORT_DIRECTION]: sortDirection,
+        }),
         [MetaQueryEnum.ORDER_BY]: column
           ? `${sortedColumnDirection === SortDirectionEnum.DESC ? '-' : ''}${column}`
           : null,
-        [SortQueryEnum.SORT_COL_IDX]: sortColIdx,
-        [SortQueryEnum.SORT_DIRECTION]: sortDirection,
-      }), {
+      }, {
         pushHistory: true,
       });
     }

mage_ai/frontend/pages/pipelines/index.tsx

@@ -39,6 +39,7 @@ import ToggleSwitch from '@oracle/elements/Inputs/ToggleSwitch';
 import Toolbar from '@components/shared/Table/Toolbar';
 import api from '@api';
 import dark from '@oracle/styles/themes/dark';
+import useProject from '@utils/models/project/useProject';
 import { BORDER_RADIUS_SMALL } from '@oracle/styles/units/borders';
 import { BlockTypeEnum } from '@interfaces/BlockType';
 import {
@@ -138,6 +139,11 @@ function PipelineListPage() {
   const refPaginate = useRef(null);
   const timeout = useRef(null);
 
+  const {
+    fetchProjects,
+    project,
+  } = useProject();
+
   const [buttonTabsHeight, setButtonTabsHeight] = useState<number>(null);
 
   const [selectedPipeline, setSelectedPipeline] = useState<PipelineType>(null);
@@ -185,8 +191,6 @@ function PipelineListPage() {
     refButtonTabs,
   ]);
 
-  const { data: dataProjects, mutate: fetchProjects } = api.projects.list();
-  const project: ProjectType = useMemo(() => dataProjects?.projects?.[0], [dataProjects]);
   const displayLocalTimezone = useMemo(
     () => storeLocalTimezoneSetting(project?.features?.[FeatureUUIDEnum.LOCAL_TIMEZONE]),
     [project?.features],
@@ -202,8 +206,6 @@ function PipelineListPage() {
     include_schedules: 1,
   }, {
     revalidateOnFocus: false,
-  }, {
-    pauseFetch: operationHistoryEnabled && selectedTabUUID && TAB_RECENT.uuid === selectedTabUUID,
   });
 
   const fromHistoryDays = useMemo(() => q?.[PipelineQueryEnum.HISTORY_DAYS] || 7, [q]);
@@ -351,6 +353,16 @@ function PipelineListPage() {
     }
   );
 
+  useEffect(() => {
+    if (query?.[PipelineQueryEnum.SEARCH] && searchText === null) {
+      setSearchTextState(query?.[PipelineQueryEnum.SEARCH]);
+    }
+  }, [
+    query,
+    searchText,
+    setSearchTextState,
+  ]);
+
   useEffect(() => {
     let queryFinal = {};
 
@@ -462,12 +474,7 @@ function PipelineListPage() {
       setFilters(selectEntriesWithValues(f));
     }
 
-    if (queryFinal?.[PipelineQueryEnum.SEARCH]) {
-      setSearchText(queryFinal?.[PipelineQueryEnum.SEARCH]);
-    }
-
     if (!isEmptyObject(queryFinal)) {
-      console.log('wtf', queryFinal, selectEntriesWithValues(queryFinal))
       goToWithQuery(selectEntriesWithValues(queryFinal), {
         pushHistory: false,
       });

mage_ai/tests/cache/example_pipeline.json

@@ -1 +1 @@
-{"added_at":null,"pipeline":{"description":"Mage is powerful!","name":"example_pipeline","type":"python","updated_at":"2023-09-22 22:59:13","uuid":"example_pipeline","blocks":[{"downstream_blocks":["fill_in_missing_values","summer_dew","wandering_voice","blue_haze","solitary_moon","spring_feather","bar_chart_for_load_titanic_1695423547036"],"language":"python","name":"load_titanic","type":"data_loader","upstream_blocks":[],"uuid":"load_titanic"},{"downstream_blocks":[],"language":"markdown","name":"spring feather","type":"markdown","upstream_blocks":["load_titanic"],"uuid":"spring_feather"},{"downstream_blocks":[],"language":"markdown","name":"polished field","type":"markdown","upstream_blocks":[],"uuid":"polished_field"},{"downstream_blocks":[],"language":"python","name":"solitary moon","type":"transformer","upstream_blocks":["load_titanic"],"uuid":"solitary_moon"},{"downstream_blocks":["autumn_sound"],"language":"python","name":"blue haze","type":"transformer","upstream_blocks":["load_titanic"],"uuid":"blue_haze"},{"downstream_blocks":[],"language":"yaml","name":"autumn sound","type":"dbt","upstream_blocks":["blue_haze"],"uuid":"autumn_sound"},{"downstream_blocks":[],"language":"python","name":"wandering voice","type":"transformer","upstream_blocks":["load_titanic"],"uuid":"wandering_voice"},{"downstream_blocks":[],"language":"python","name":"summer dew","type":"data_exporter","upstream_blocks":["load_titanic"],"uuid":"summer_dew"},{"downstream_blocks":["export_titanic_clean"],"language":"python","name":"fill_in_missing_values","type":"transformer","upstream_blocks":["load_titanic"],"uuid":"fill_in_missing_values"},{"downstream_blocks":["hidden_wildflower"],"language":"python","name":"export_titanic_clean","type":"data_exporter","upstream_blocks":["fill_in_missing_values"],"uuid":"export_titanic_clean"},{"downstream_blocks":["empty_shadow"],"language":"python","name":"hidden wildflower","type":"data_loader","upstream_blocks":["export_titanic_clean"],"uuid":"hidden_wildflower"},{"downstream_blocks":["aged_cloud"],"language":"python","name":"empty shadow","type":"transformer","upstream_blocks":["hidden_wildflower"],"uuid":"empty_shadow"},{"downstream_blocks":[],"language":"python","name":"aged cloud","type":"data_exporter","upstream_blocks":["empty_shadow"],"uuid":"aged_cloud"},{"downstream_blocks":[],"language":"markdown","name":"winter waterfall","type":"markdown","upstream_blocks":[],"uuid":"winter_waterfall"},{"downstream_blocks":[],"language":"python","name":"growth/test","type":"transformer","upstream_blocks":[],"uuid":"growth/test"}]}}
+{"added_at":null,"pipeline":{"description":null,"name":"example_pipeline","tags":[],"type":"python","updated_at":"2023-09-22 22:59:13","uuid":"example_pipeline","blocks":[{"downstream_blocks":["fill_in_missing_values","summer_dew","wandering_voice","blue_haze","solitary_moon","spring_feather","bar_chart_for_load_titanic_1695423547036"],"language":"python","name":"load_titanic","type":"data_loader","upstream_blocks":[],"uuid":"load_titanic"},{"downstream_blocks":[],"language":"markdown","name":"spring feather","type":"markdown","upstream_blocks":["load_titanic"],"uuid":"spring_feather"},{"downstream_blocks":[],"language":"markdown","name":"polished field","type":"markdown","upstream_blocks":[],"uuid":"polished_field"},{"downstream_blocks":[],"language":"python","name":"solitary moon","type":"transformer","upstream_blocks":["load_titanic"],"uuid":"solitary_moon"},{"downstream_blocks":["autumn_sound"],"language":"python","name":"blue haze","type":"transformer","upstream_blocks":["load_titanic"],"uuid":"blue_haze"},{"downstream_blocks":[],"language":"yaml","name":"autumn sound","type":"dbt","upstream_blocks":["blue_haze"],"uuid":"autumn_sound"},{"downstream_blocks":[],"language":"python","name":"wandering voice","type":"transformer","upstream_blocks":["load_titanic"],"uuid":"wandering_voice"},{"downstream_blocks":[],"language":"python","name":"summer dew","type":"data_exporter","upstream_blocks":["load_titanic"],"uuid":"summer_dew"},{"downstream_blocks":["export_titanic_clean"],"language":"python","name":"fill_in_missing_values","type":"transformer","upstream_blocks":["load_titanic"],"uuid":"fill_in_missing_values"},{"downstream_blocks":["hidden_wildflower"],"language":"python","name":"export_titanic_clean","type":"data_exporter","upstream_blocks":["fill_in_missing_values"],"uuid":"export_titanic_clean"},{"downstream_blocks":["empty_shadow"],"language":"python","name":"hidden wildflower","type":"data_loader","upstream_blocks":["export_titanic_clean"],"uuid":"hidden_wildflower"},{"downstream_blocks":["aged_cloud"],"language":"python","name":"empty shadow","type":"transformer","upstream_blocks":["hidden_wildflower"],"uuid":"empty_shadow"},{"downstream_blocks":[],"language":"python","name":"aged cloud","type":"data_exporter","upstream_blocks":["empty_shadow"],"uuid":"aged_cloud"},{"downstream_blocks":[],"language":"markdown","name":"winter waterfall","type":"markdown","upstream_blocks":[],"uuid":"winter_waterfall"},{"downstream_blocks":[],"language":"python","name":"growth/test","type":"transformer","upstream_blocks":[],"uuid":"growth/test"}]},"updated_at":1702284950.403907}

mage_ai/tests/cache/pipeline_details_mapping_template

@@ -165,6 +165,7 @@ async def test_initialize_cache_for_models(self):
                 ),
             ],
             name=uuid.uuid4().hex,
+            tags=[uuid.uuid4().hex, uuid.uuid4().hex],
             type=uuid.uuid4().hex,
             updated_at=uuid.uuid4().hex,
             uuid=uuid.uuid4().hex,