Merge pull request #22 from mircealungu/graphs

Implementing more visualizations

Author: kloostert

dashboard/database/endpoint.py

@@ -1,7 +1,7 @@
 """
 Contains all functions that access a single endpoint
 """
-from sqlalchemy import func, text
+from sqlalchemy import func, text, asc
 from sqlalchemy.orm.exc import NoResultFound
 import datetime
 
@@ -24,12 +24,38 @@ def get_line_results(endpoint):
         return data
 
 
+def get_num_requests(endpoint):
+    """ Returns a list with all dates on which an endpoint is accessed.
+        :param endpoint: if None, the result is the sum of all endpoints
+    """
+    with session_scope() as db_session:
+        query = text("""select
+                datetime(CAST(strftime('%s', time)/3600 AS INT)*3600, 'unixepoch') AS newTime,
+                count(execution_time) as count
+                from functioncalls
+                where (endpoint=:val OR :val='None') group by newTime""")
+        result = db_session.execute(query, {'val': str(endpoint)})
+        data = result.fetchall()
+        return data
+
+
 def get_endpoint_column(endpoint, column):
+    """ Returns a list of entries from column in which the endpoint is involved. """
+    with session_scope() as db_session:
+        result = db_session.query(column,
+                                  func.min(FunctionCall.time).label('startedUsingOn')). \
+            filter(FunctionCall.endpoint == endpoint). \
+            group_by(column).order_by(asc('startedUsingOn')).all()
+        db_session.expunge_all()
+        return result
+
+
+def get_endpoint_column_user_sorted(endpoint, column):
     """ Returns a list of entries from column in which the endpoint is involved. """
     with session_scope() as db_session:
         result = db_session.query(column). \
             filter(FunctionCall.endpoint == endpoint). \
-            group_by(column).all()
+            group_by(column).order_by(asc(column)).all()
         db_session.expunge_all()
         return result
 

dashboard/database/function_calls.py

@@ -3,12 +3,25 @@
 """
 
 from flask import request
-from sqlalchemy import func, desc
+from sqlalchemy import func, desc, text, asc
 from dashboard import config
 import datetime
 from dashboard.database import session_scope, FunctionCall
 
 
+def get_reqs_endpoint_day():
+    """ Retrieves the number of requests per endpoint per day. """
+    with session_scope() as db_session:
+        query = text("""select strftime('%Y-%m-%d', time) AS newTime,
+                               count(endpoint) AS cnt,
+                               endpoint
+                        from functioncalls 
+                        group by newTime, endpoint""")
+        result = db_session.execute(query)
+        data = result.fetchall()
+        return data
+
+
 def add_function_call(time, endpoint):
     """ Add a measurement to the database. """
     with session_scope() as db_session:
@@ -45,3 +58,38 @@ def get_data():
                                   FunctionCall.ip).all()
         db_session.expunge_all()
         return result
+
+
+def get_data_per_version(version):
+    """ Returns all data in the FuctionCall table, grouped by their version. """
+    with session_scope() as db_session:
+        result = db_session.query(FunctionCall.execution_time, FunctionCall.version). \
+            filter(FunctionCall.version == version).all()
+        db_session.expunge_all()
+        return result
+
+
+def get_versions():
+    with session_scope() as db_session:
+        result = db_session.query(FunctionCall.version,
+                                  func.min(FunctionCall.time).label('startedUsingOn')). \
+            group_by(FunctionCall.version).order_by(asc('startedUsingOn')).all()
+        db_session.expunge_all()
+        return result
+
+
+def get_data_per_endpoint(end):
+    with session_scope() as db_session:
+        result = db_session.query(FunctionCall.execution_time, FunctionCall.endpoint). \
+            filter(FunctionCall.endpoint == end).all()
+        db_session.expunge_all()
+        return result
+
+
+def get_endpoints():
+    with session_scope() as db_session:
+        result = db_session.query(FunctionCall.endpoint,
+                                  func.count(FunctionCall.endpoint).label('cnt')). \
+            group_by(FunctionCall.endpoint).order_by(asc('cnt')).all()
+        db_session.expunge_all()
+        return result

dashboard/routings/result.py

@@ -6,8 +6,10 @@
 from dashboard import blueprint, config
 from dashboard.database import FunctionCall
 from dashboard.database.endpoint import get_endpoint_column, get_endpoint_results, get_monitor_rule, \
-    get_last_accessed_times, get_line_results, get_all_measurement_per_column
-from dashboard.database.function_calls import get_times
+    get_last_accessed_times, get_line_results, get_all_measurement_per_column, get_num_requests, \
+    get_endpoint_column_user_sorted
+from dashboard.database.function_calls import get_times, get_data_per_version, get_versions, get_reqs_endpoint_day, \
+    get_endpoints, get_data_per_endpoint
 from dashboard.security import secure
 
 import plotly
@@ -17,9 +19,62 @@
 @blueprint.route('/measurements')
 @secure
 def measurements():
-    t = get_times()
-    la = get_last_accessed_times()
-    return render_template('measurements.html', link=config.link, curr=2, times=t, access=la, session=session)
+    return render_template('measurements.html', link=config.link, curr=2, times=get_times(),
+                           access=get_last_accessed_times(), session=session,
+                           heatmap=get_heatmap(end=None), etpv=get_boxplot_per_version(), rpepd=get_stacked_bar(),
+                           etpe=get_boxplot_per_endpoint())
+
+
+def get_boxplot_per_version():
+    """
+    Creates a graph with the execution times per version
+    :return:
+    """
+    versions = [str(v.version) for v in get_versions()]
+
+    data = []
+    for v in versions:
+        values = [c.execution_time for c in get_data_per_version(v)]
+        data.append(go.Box(x=values, name=v))
+
+    layout = go.Layout(
+        autosize=False,
+        width=900,
+        height=350 + 40 * len(versions),
+        plot_bgcolor='rgba(249,249,249,1)',
+        showlegend=False,
+        title='Execution time for every version',
+        xaxis=dict(title='Execution time (ms)'),
+        yaxis=dict(title='Version')
+    )
+    return plotly.offline.plot(go.Figure(data=data, layout=layout), output_type='div', show_link=False)
+
+
+def get_boxplot_per_endpoint():
+    """
+    Creates a graph with the execution times per endpoint
+    :return:
+    """
+    endpoints = [str(e.endpoint) for e in get_endpoints()]
+
+    data = []
+    for e in endpoints:
+        values = [c.execution_time for c in get_data_per_endpoint(e)]
+        if len(e) > 16:
+            e = '...' + e[-14:]
+        data.append(go.Box(x=values, name=e))
+
+    layout = go.Layout(
+        autosize=False,
+        width=900,
+        height=350 + 40 * len(endpoints),
+        plot_bgcolor='rgba(249,249,249,1)',
+        showlegend=False,
+        title='Execution time for every endpoint',
+        xaxis=dict(title='Execution time (ms)'),
+        yaxis=dict(tickangle=-45)
+    )
+    return plotly.offline.plot(go.Figure(data=data, layout=layout), output_type='div', show_link=False)
 
 
 def formatter(ms):
@@ -28,7 +83,7 @@ def formatter(ms):
     :param ms: the number of ms
     :return: a string representing the same amount, but now represented in seconds and ms.
     """
-    sec = math.floor(ms/1000)
+    sec = math.floor(ms / 1000)
     ms = round(ms % 1000, 2)
     if sec == 0:
         return '{0}ms'.format(ms)
@@ -81,6 +136,33 @@ def get_graphs_per_hour(end):
     return graph1.render_data_uri(), graph2.render_data_uri()
 
 
+def get_stacked_bar():
+    data = get_reqs_endpoint_day()
+    graph = pygal.graph.horizontalstackedbar.HorizontalStackedBar(legend_at_bottom=True, height=100 + len(data) * 7)
+    graph.title = 'Number of requests per endpoint per day'
+    graph.x_labels = []
+    endpoints = []
+    for d in data:
+        if d.newTime not in graph.x_labels:
+            graph.x_labels.append(d.newTime)
+        if d.endpoint not in endpoints:
+            endpoints.append(d.endpoint)
+
+    for e in endpoints:
+        lst = []
+        for t in graph.x_labels:
+            found = False
+            for d in data:
+                if e == d.endpoint and t == d.newTime:
+                    found = True
+                    lst.append(d.cnt)
+            if not found:
+                lst.append(0)
+        graph.add(e, lst)
+
+    return graph.render_data_uri()
+
+
 def get_dot_charts(end, versions):
     """
     Function that builds two dot charts:
@@ -148,33 +230,85 @@ def get_boxplots(end, versions):
         width=900,
         height=350 + 40 * len(versions),
         plot_bgcolor='rgba(249,249,249,1)',
-        showlegend=False
+        showlegend=False,
+        title='Execution time for every version',
+        xaxis=dict(title='Execution time (ms)'),
+        yaxis=dict(title='Version')
     )
     graph1 = plotly.offline.plot(go.Figure(data=data, layout=layout), output_type='div', show_link=False)
 
-    # boxplot: execution time per versions
-    users = [str(c.group_by) for c in get_endpoint_column(endpoint=end, column=FunctionCall.group_by)]
-
+    users = [str(c.group_by) for c in get_endpoint_column_user_sorted(endpoint=end, column=FunctionCall.group_by)]
     data = []
     for u in users:
         values = [str(c.execution_time) for c in
                   get_all_measurement_per_column(endpoint=end, column=FunctionCall.group_by, value=u)]
-        data.append(go.Box(x=values, name=u))
+        data.append(go.Box(x=values, name='{0}  -'.format(u)))
 
     layout = go.Layout(
         autosize=False,
         width=900,
         height=350 + 40 * len(users),
         plot_bgcolor='rgba(249,249,249,1)',
-        showlegend=False
+        showlegend=False,
+        title='Execution time for every user',
+        xaxis=dict(title='Execution time (ms)'),
+        yaxis=dict(title='User')
     )
     graph2 = plotly.offline.plot(go.Figure(data=data, layout=layout), output_type='div', show_link=False)
     return graph1, graph2
 
 
-@blueprint.route('/show-graph/<end>')
+def get_heatmap(end):
+    # list of hours: 1:00 - 23:00
+    hours = ['0' + str(hour) + ':00' for hour in range(0, 10)] + \
+            [str(hour) + ':00' for hour in range(10, 24)]
+
+    data = get_num_requests(end)
+    # list of days (format: year-month-day)
+    days = [str(d.newTime[:10]) for d in data]
+    # remove duplicates and sort the result
+    days = sorted(list(set(days)))
+
+    # create empty 2D-dictionary with the keys: [hour][day]
+    requests = {}
+    for hour in hours:
+        requests_day = {}
+        for day in days:
+            requests_day[day] = 0
+        requests[hour] = requests_day
+
+    # add data to the dictionary
+    for d in data:
+        day = str(d.newTime[:10])
+        hour = str(d.newTime[11:16])
+        requests[hour][day] = d.count
+
+    # create a 2D-list out of the dictionary
+    requests_list = []
+    for hour in hours:
+        day_list = []
+        for day in days:
+            day_list.append(requests[hour][day])
+        requests_list.append(day_list)
+
+    layout = go.Layout(
+        autosize=False,
+        width=900,
+        height=800,
+        plot_bgcolor='rgba(249,249,249,1)',
+        showlegend=False,
+        title='Heatmap of number of requests',
+        xaxis=dict(title='Date'),
+        yaxis=dict(title='Time')
+    )
+
+    trace = go.Heatmap(z=requests_list, x=days, y=hours)
+    return plotly.offline.plot(go.Figure(data=[trace], layout=layout), output_type='div', show_link=False)
+
+
+@blueprint.route('/result/<end>')
 @secure
-def show_graph(end):
+def result(end):
     rule = get_monitor_rule(end)
     url = get_url(end)
     versions = [str(c.version) for c in get_endpoint_column(end, FunctionCall.version)]
@@ -188,6 +322,9 @@ def show_graph(end):
     # (5) Execution time per version and (6) Execution time per user
     graph5, graph6 = get_boxplots(end, versions)
 
-    return render_template('show-graph.html', link=config.link, session=session, rule=rule, url=url,
+    # (7) Number of requests per hour
+    graph7 = get_heatmap(end)
+
+    return render_template('endpoint.html', link=config.link, session=session, rule=rule, url=url,
                            times_data=graph1, hits_data=graph2, dot_chart_user=graph3,
-                           dot_chart_ip=graph4, div_versions=graph5, div_users=graph6)
+                           dot_chart_ip=graph4, div_versions=graph5, div_users=graph6, div_heatmap=graph7)