Hitl dashbaord model viz improve

droidlet/tools/hitl/dashboard_app/README.MD

@@ -1,9 +1,18 @@
 # Dashboard App for HITL
-Updated July 22 by Chuxi.
+Updated Aug 4 by Chuxi.
 
 This is a Dashboard app prototype for the HITL system.
 
 ## Update Note
+- Aug 4:
+    - Update support for visualizing the model:
+        - Changed from showing loss and accuracy of validation and text_span to showing loss and accuracy separately, and show training loss/accuracy & validation loss/accuracy in the same graph. 
+        - Updated the graph visualization container to be responsive to browser window changes. 
+        - Demo:
+            - Best models loss & accuracy for a pipeline:
+                - ![demo_best_models_for_pipeline](https://user-images.githubusercontent.com/51009396/182976894-a1eb380b-06fd-491b-8e4f-29abd7c2024d.gif)
+            - Best model in a run (showing loss and accuracy seperately, graph responsive to browser window changes):
+                - ![demo_best_model_for_a_run](https://user-images.githubusercontent.com/51009396/182976905-60713821-31d8-4187-889a-11b3b1d1ce82.gif)
 - July 22:
     - Updated frontend component to show the model line graph of loss and accuracy vs epoch.
     - Demo:
@@ -128,15 +137,22 @@ APIs are based on socket event, the following APIs are supported currently:
         - the key for the model, could be any key from the model, or "COMPLETE", indicating getting the complete model dict
     - output: the key and the value specific to the key for the model if the model exists and key is valid, otherwise error code
 - get_best_model_loss_acc_by_id
-    - get loss and accuracy from a model log for a specific run's best model, 
-    the loss and accuracy infomation is reterived by crawling the best model's log file
+    - get loss and accuracy from a model log for a specific run's best model, the loss and accuracy infomation is reterived by crawling the best model's log file
     - input:
         - batch id of a specific run
     - output:
-        - a dictionary containing:
-            - epoch loss and accuracy
-            - text_span loss and accuracy
+        - a list contains the following when a best model log file can be found:
+            - a dictionary containing:
+                - key: loss, value: loss list for training and validation dataset
+                - key: acc, value: accuracy list for training and validation dataset
+            - batch id 
         - or an error code indicating the best model log cannot be find
+- get_best_model_bids_by_pipeline
+    - get best models' batch ids for a specific pipeline
+    - input:
+        - pipeline name (can be nlu, tao or vision)
+    - output:
+        - a list of the batch ids of the best models
 
 ## Demo
 ![backend_api_demo](https://user-images.githubusercontent.com/51009396/175696481-532cec55-5b2e-4bae-bceb-9e7d3f2aa7b7.gif)

droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py

@@ -199,7 +199,7 @@ def get_best_model_loss_acc_by_id(batch_id: int):
     """
     Get best model loss and accuracy from model log file,
     return:
-        - a dict including epoch & text span loss and accuracy, and no error code if can find the best model log
+        - a dict including loss and accuracy for training and valiation, and no error code if can find the best model log
         - an error message with error code 404 if cannot find the best model log
     """
 
@@ -228,5 +228,20 @@ def get_best_model_name(batch_id: int):
         return f"Cannot find best model log file with batch_id = {batch_id}", 404
 
     # read best model log file
-    epoch_ls, text_span_ls = read_model_log_to_list(best_model_log_fname)
-    return {"epoch": epoch_ls, "text_span": text_span_ls}, None
+    loss_ls, acc_ls = read_model_log_to_list(best_model_log_fname)
+    print(f"loss length: {len(loss_ls)}, accuracy length: {len(acc_ls)}")
+    return {"loss": loss_ls, "acc": acc_ls}, None
+
+
+def get_best_models_by_pipeline(pipeline: str):
+    """
+    Get best models batch_id list for a pipeline.
+    The pipeline can be nlu, tao or vision
+    """
+    local_fname = _download_file(f"{pipeline}_model_viz_batch_list.txt")
+    if local_fname is None:
+        return f"Cannot find best model list for {pipeline}", 404
+
+    model_batch_id_list = _read_file(local_fname).split("\n")
+
+    return model_batch_id_list, None

droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py

@@ -10,6 +10,7 @@
 import json
 from droidlet.tools.hitl.dashboard_app.backend.dashboard_aws_helper import (
     get_best_model_loss_acc_by_id,
+    get_best_models_by_pipeline,
     get_dataset_by_name,
     get_dataset_indices_by_id,
     get_dataset_version_list_by_pipeline,
@@ -50,6 +51,7 @@ class DASHBOARD_EVENT(Enum):
     GET_MODEL_KEYS = "get_model_keys_by_id"
     GET_MODEL_VALUE = "get_model_value_by_id_n_key"
     GET_BEST_MODEL_LOSS_ACC = "get_best_model_loss_acc_by_id"
+    GET_BEST_MODELS = "get_best_model_bids_by_pipeline"
 
 
 # constants for model related apis
@@ -235,9 +237,11 @@ def get_best_model_loss_acc(batch_id: int):
     - input:
         - batch id of a specific run
     - output:
-        - a dictionary containing:
-            - epoch loss and accuracy
-            - text_span loss and accuracy
+        - a list contains the following when a best model log file can be found:
+            - a dictionary containing:
+                - key: loss, value: loss list for training and validation dataset
+                - key: acc, value: accuracy list for training and validation dataset
+            - batch id
         - or an error code indicating the best model log cannot be find
     """
     print(
@@ -247,7 +251,24 @@ def get_best_model_loss_acc(batch_id: int):
     if error_code:
         emit(DASHBOARD_EVENT.GET_BEST_MODEL_LOSS_ACC.value, error_code)
     else:
-        emit(DASHBOARD_EVENT.GET_BEST_MODEL_LOSS_ACC.value, loss_acc_dict)
+        emit(DASHBOARD_EVENT.GET_BEST_MODEL_LOSS_ACC.value, [loss_acc_dict, int(batch_id)])
+
+
+@socketio.on(DASHBOARD_EVENT.GET_BEST_MODELS.value)
+def get_best_model_batches(pipeline: str):
+    """
+    get best models' batch ids for a specific pipeline
+    - input:
+        - pipeline name (can be nlu, tao or vision)
+    - output:
+        - a list of the batch ids of the best models
+    """
+    print(f"Request received: {DASHBOARD_EVENT.GET_BEST_MODELS.value}, pipeline = {pipeline}")
+    model_dict, error_code = get_best_models_by_pipeline(pipeline)
+    if error_code:
+        emit(DASHBOARD_EVENT.GET_BEST_MODELS.value, error_code)
+    else:
+        emit(DASHBOARD_EVENT.GET_BEST_MODELS.value, model_dict)
 
 
 if __name__ == "__main__":

droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js

@@ -10,18 +10,29 @@ import { Button, Card, Descriptions, Divider, Tooltip, Typography } from "antd";
 import React, { useCallback, useContext, useEffect, useState } from "react";
 import { SocketContext } from "../../../../context/socket";
 import ModelAtrributeModal from "./modelAttributeDetailModal";
-import { LineChart, Line, XAxis, YAxis, CartesianGrid, Legend, Tooltip as ChartTooltip } from 'recharts';
+import { LineChart, Line, XAxis, YAxis, CartesianGrid, Legend, Tooltip as ChartTooltip, ResponsiveContainer } from "recharts";
 
 const { Meta } = Card;
-const LOSS_ACC_TYPES = [{ "label": "Epoch", "value": "epoch" }, { "label": "Text Span", "value": "text_span" }]
 
-const ModelLossAccGraph = (props) => {
-    const data = props.data.map((o, idx) => ({ Loss: o.loss, Accuracy: o.acc, Epoch: idx }));
+export const ModelLossAccGraph = (props) => {
+    const width = props.width;
+    const height = props.height;
+    const yAxisName = props.yAxisName;
 
-    return <div style={{ width: "100%", height: "100%" }}>
+    const getScaled = (yName, value) => {
+        // if is loss, get log2 scaled
+        if ((yName) === "Loss") {
+            return value ? Math.log2(value): 0;
+        } else {
+            return value;
+        }
+    }
+    const data = props.data.map((o, idx) => ({ Training: getScaled(yAxisName, o.training), Validation:  getScaled(yAxisName, o.validation), Epoch: idx }));
+
+    return <ResponsiveContainer height={"70%"} aspect={width / height}>
         <LineChart
-            width={750}
-            height={400}
+            width={width}
+            height={height}
             data={data}
             margin={{
                 top: 5,
@@ -32,31 +43,35 @@ const ModelLossAccGraph = (props) => {
         >
             <CartesianGrid strokeDasharray="3 3" />
             <XAxis type="number" dataKey="Epoch" label={{offset: 0, value: "Epoch", position: "insideBottom" }}/>
-            <YAxis />
+            <YAxis label={{offset: 0, value: (yAxisName === "Loss" ? yAxisName + "(Log2 Scaled)": yAxisName), position: "insideLeft", angle: -90}}/>
             <Legend />
             <ChartTooltip />
-            <Line type="monotone" dataKey="Loss" stroke="#ad2102" />
-            <Line type="monotone" dataKey="Accuracy" stroke="#1890ff" activeDot={{ r: 8 }} dot={{strokeWidth: 2}} strokeWidth={2} />
+            <Line type="monotone" dataKey="Training" stroke="#ad2102" />
+            <Line type="monotone" dataKey="Validation" stroke="#1890ff" activeDot={{ r: 8 }} dot={{strokeWidth: 2}} strokeWidth={2} />
         </LineChart>
-    </div>
+    </ResponsiveContainer>
 }
 
-const ViewLossAccCard = (props) => {
+export const ViewLossAccCard = (props) => {
+    const width = props.width ? props.width : 750;
+    const height = props.height ? props.height : 400;
     const lossAccData = props.data;
+
+    const LOSS_ACC_TYPES = [{ "label": "Accuracy", "value": "acc" }, { "label": "Loss", "value": "loss" }]
+
     const [activeTabKey, setActiveTabKey] = useState(LOSS_ACC_TYPES[0]["value"]);
 
     return <Card
         tabList={LOSS_ACC_TYPES.map((o) => ({ tab: o["label"], key: o["value"] }))}
         activeTabKey={activeTabKey}
-        onTabChange={(key) => setActiveTabKey(key)}
+        onTabChange={(key) => {setActiveTabKey(key)}}
     >
-        <ModelLossAccGraph data={lossAccData[activeTabKey]} />
+        <ModelLossAccGraph data={lossAccData[activeTabKey]} height = {height} width = {width} yAxisName={LOSS_ACC_TYPES.find((o) => (o.value === activeTabKey))["label"]} />
     </Card>
 }
 
 const ModelCard = (props) => {
     const batchId = props.batchId;
-    const pipelineType = props.pipelineType;
     const [modelArgs, setModelArgs] = useState(null);
     const [modelKeys, setModelKeys] = useState(null);
     const [loadingArgs, setLoadingArgs] = useState(true);
@@ -86,7 +101,7 @@ const ModelCard = (props) => {
     const handleReceivedLossAcc = useCallback((data) => {
         setLoadingLossAcc(false);
         if (data !== 404) {
-            setLossAccData(data);
+            setLossAccData(data[0]);
         }
     });
 
@@ -130,12 +145,8 @@ const ModelCard = (props) => {
         setAttrModalOpen(true);
     }
 
-    const handleViewModelLossAndAcc = (lossAccType) => {
-        alert(lossAccData[lossAccType].map((o) => (`loss: ${o.loss},  acc: ${o.acc}`)));
-    }
-
     return (
-        <div style={{ width: '70%' }}>
+        <div style={{ width: "70%" }}>
             <Card title="Model" loading={loadingKeys || loadingArgs || loadingLossAcc}>
                 <Meta />
                 {
@@ -165,7 +176,7 @@ const ModelCard = (props) => {
                                     lossAccData &&
                                     <>
                                         <Divider />
-                                        <Typography.Title level={5} style={{ paddingBottom: '18px' }}>Model Loss And Accuracy</Typography.Title>
+                                        <Typography.Title level={5} style={{ paddingBottom: "18px" }}>Model Loss And Accuracy</Typography.Title>
                                         <ViewLossAccCard data={lossAccData} />
                                     </>
                                 }
@@ -174,7 +185,7 @@ const ModelCard = (props) => {
                             <div>NA</div>
                     )
                 }
-                {/* modal showing a specific model attribute's field (anything other than args) */}
+                {/* modal showing a specific model attribute"s field (anything other than args) */}
                 {currentModelKey
                     &&
                     <ModelAtrributeModal

droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js

@@ -0,0 +1,113 @@
+/*
+Copyright (c) Facebook, Inc. and its affiliates.
+
+The content for model visualization for a pipeline.
+
+The **pipeline** can be nlu/tao or vision; the pipeline information was reterived from path parameter so no props are required.
+
+Usage:
+<PipelineModelVizContent />
+*/
+
+import { SyncOutlined } from "@ant-design/icons";
+import { Progress, Tag, Typography } from "antd";
+import React, { useContext, useEffect, useState } from "react";
+import { useLocation } from "react-router-dom";
+import { SocketContext } from "../../../context/socket";
+import { ViewLossAccCard } from "../detail/asset/modelCard";
+
+const PipelineModelVizContent = (props) => {
+    const location = useLocation();
+    const socket = useContext(SocketContext);
+    const [modelBids, setModelBids] = useState(null);
+    const [currentIdx, setCurrentIdx] = useState(0);
+    const [modelDict, ] = useState({});
+    const [modelCombined, setModelCombined] = useState(null);
+
+    const handleReceivedModelBids = (data) => {
+        setModelBids(data.map((bid) => parseInt(bid)));
+    }
+
+    const handleReceivedModelLossAcc = (data) => {
+        if (data !== 404) {
+            modelDict[data[1]] = data[0];
+
+            if (modelBids && modelBids !== 404 && Object.keys(modelDict).length < modelBids.length) {
+                setCurrentIdx(Object.keys(modelDict).length);
+            } else if (modelBids && modelBids !== 404 && Object.keys(modelDict).length === modelBids.length) {
+                // finished loading, combine based on bid sequence
+                const modelArr = modelBids.map((bid) => (modelDict[bid]));
+                let comb = {}
+
+                for (let i = 0; i < modelArr.length; i++) {
+                    const o = modelArr[i];
+                    for (let j = 0; j < Object.keys(o).length; j++) {
+                        const key = Object.keys(o)[j];
+                        if (!(key in comb)) {
+                            comb[key] = o[key];
+                        } else {
+                            comb[key] = [...comb[key], ...o[key]]
+                        }
+                    }
+                }
+                setModelCombined(comb);
+            }
+        }
+    }
+
+    useEffect(() => {
+        socket.on("get_best_model_bids_by_pipeline", (data) => handleReceivedModelBids(data));
+        socket.on("get_best_model_loss_acc_by_id", (data) => handleReceivedModelLossAcc(data));
+    }, [socket, handleReceivedModelBids, handleReceivedModelLossAcc]);
+
+    useEffect(() => {
+        socket.emit("get_best_model_bids_by_pipeline", location.pathname.substring(1));
+    }, []);
+
+    useEffect(() => {
+        if (modelBids && modelBids !== 404 && currentIdx >= 0) {
+            socket.emit("get_best_model_loss_acc_by_id", modelBids[currentIdx]);
+        }
+    }, [modelBids, currentIdx]);
+
+    return <div>
+        <Typography.Title level={4}>{"View Model Accuracy & Loss"}</Typography.Title>
+        {
+            modelBids && modelBids !== 404 
+            && 
+            <div
+                style={{padding: "0 0 12px 0"}}
+            >
+                Showing {modelBids.map((bid) => <Tag>{bid}</Tag>)} 
+            </div>
+        }
+        <div style={{ padding: "0 24px 0 24px" }}>
+            {
+                // show progress bar after getting model batch ids
+                modelBids && modelBids !== 404 && (Object.keys(modelDict).length) !== modelBids.length &&
+                <div> 
+                    <span><SyncOutlined spin /> Loading Model Data... </span>
+                    <Progress percent={Math.trunc((Object.keys(modelDict).length) / modelBids.length * 100)} />
+                </div>
+            }
+
+            {
+                // show graph when loading is done
+                modelBids && modelBids !== 404 && (Object.keys(modelDict).length) === modelBids.length && modelCombined &&
+                <div>
+                    <ViewLossAccCard data={modelCombined} width={1500}/>
+                </div>
+            } 
+            {
+                // show no data when not having any batch ids
+                modelBids && modelBids === 404 && 
+                <div>
+                    Sorry, no model data is avaible yet. 
+                </div>
+            }
+        </div>
+
+    </div>
+}
+
+export default PipelineModelVizContent;

droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/panel.js

@@ -13,6 +13,7 @@ import React, { useEffect, useState } from 'react';
 import { Link, Outlet, useLocation, useParams } from 'react-router-dom';
 import { TAB_ITEMS } from '../../constants/pipelineConstants';
 import InfoBlock from './metaInfo/infoBlock';
+import PipelineModelVizContent from './metaInfo/modelViz';
 import RunList from './metaInfo/runList';
 
 const menuItems = Object.values(TAB_ITEMS);
@@ -107,6 +108,9 @@ const PipelinePanel = (props) => {
                                                 // render job list if key is jobs, otherwise render info block
                                                 item.key === TAB_ITEMS.RUNS.key ?
                                                     <RunList pipelineType={pipelineType} /> :
+                                                    item.key === TAB_ITEMS.MODEL.key ? 
+                                                    <PipelineModelVizContent />
+                                                    : 
                                                     <InfoBlock infoType={item.key} pipelineType={pipelineType} />
                                             }
                                         </TabPane>