Merge branch 'development'

Author: mircealungu

README.md

@@ -49,14 +49,15 @@ The following things can be configured:
     GUEST_USERNAME=guest
     GUEST_PASSWORD=['dashboardguest!', 'second_pw!']
     DATABASE=sqlite:////<path to your project>/dashboard.db
-    GIT=/<path to your project>/dashboard/.git/
-    TEST_DIR=/<path to your project>/dashboard/tests/
+    GIT=/<path to your project>/.git/
+    TEST_DIR=/<path to your project>/tests/
+    LOG_DIR=/<path to your project>/
     N=5
     SUBMIT_RESULTS_URL=http://0.0.0.0:5000/dashboard/submit-test-results
     OUTLIER_DETECTION_CONSTANT=2.5
     COLORS={'main':[0,97,255], 'static':[255,153,0]}
 
-For more information: [see this file](dashboard/config.py)
+For more information, please refer to [this file](dashboard/config.py)
 
 When running your app, the dashboard van be viewed by default in the route:
 
@@ -68,13 +69,12 @@ To enable Travis to run your unit tests and send the results to the dashboard, f
 
 First off, the file 'collect_performance.py' (which comes with the dashboard) should be copied to the directory where your '.travis.yml' file resides.
 
-Secondly, your config file for the dashboard ('config.cfg') should be updated to include three additional values, TEST_DIR, SUBMIT_RESULTS_URL and N.
-The first specifies where your unit tests reside, the second where Travis should upload the test results to, and the third specifies the number of times Travis should run each unit test.
-See the sample config file in the section above for more details.
+Secondly, your config file for the dashboard ('config.cfg') should be updated to include four additional values, TEST_DIR, LOG_DIR, SUBMIT_RESULTS_URL and N.
+The first specifies where your unit tests reside, the second where the logs should be placed, the third where Travis should upload the test results to, and the last specifies the number of times Travis should run each unit test.
+See the sample config file in the section above for an example.
 
-Then, a dependency link to the dashboard has to be added to the 'setup.py' file of your app:
+Then, the installation requirement for the dashboard has to be added to the 'setup.py' file of your app:
 
-    dependency_links=["git+https://github.com/mircealungu/automatic-monitoring-dashboard.git#egg=flask_monitoring_dashboard"],
     install_requires=('flask_monitoring_dashboard')
 
 Lastly, in your '.travis.yml' file, two script commands should be added:

dashboard/collect_performance.py

@@ -4,6 +4,7 @@
 import datetime
 import os
 import sys
+import csv
 from unittest import TestLoader
 
 # Abort if config file is not specified.
@@ -14,6 +15,7 @@
     sys.exit(0)
 
 n = 1
+url = None
 parser = configparser.RawConfigParser()
 try:
     parser.read(config)
@@ -24,15 +26,19 @@
     else:
         print('No test directory specified in your config file. Please do so.')
         sys.exit(0)
+    if not parser.has_option('dashboard', 'LOG_DIR'):
+        print('No log directory specified in your config file. Please do so.')
+        sys.exit(0)
     if parser.has_option('dashboard', 'SUBMIT_RESULTS_URL'):
         url = parser.get('dashboard', 'SUBMIT_RESULTS_URL')
     else:
         print('No url specified in your config file for submitting test results. Please do so.')
-        sys.exit(0)
-except configparser.Error:
-    raise
+except configparser.Error as e:
+    print("Something went wrong while parsing the configuration file:\n{}".format(e))
 
-data = {'test_runs': []}
+data = {'test_runs': [], 'grouped_tests': []}
+log = open("test_runs.log", "w")
+log.write("\"start_time\",\"stop_time\",\"test_name\"\n")
 
 if test_dir:
     suites = TestLoader().discover(test_dir, pattern="*test*.py")
@@ -41,17 +47,47 @@
             for case in suite:
                 for test in case:
                     result = None
+                    t1 = str(datetime.datetime.now())
                     time1 = time.time()
                     result = test.run(result)
                     time2 = time.time()
+                    t2 = str(datetime.datetime.now())
+                    log.write("\"{}\",\"{}\",\"{}\"\n".format(t1, t2, str(test)))
                     t = (time2 - time1) * 1000
                     data['test_runs'].append({'name': str(test), 'exec_time': t, 'time': str(datetime.datetime.now()),
                                               'successful': result.wasSuccessful(), 'iter': i + 1})
 
-# Try to send test results to the dashboard
-try:
-    requests.post(url, json=data)
-    print('Sent unit test results to the dashboard.')
-except:
-    print('Sending unit test results to the dashboard failed.')
-    raise
+log.close()
+
+# Read and parse the log containing the test runs
+runs = []
+with open('test_runs.log') as log:
+    reader = csv.DictReader(log)
+    for row in reader:
+        runs.append([datetime.datetime.strptime(row["start_time"], "%Y-%m-%d %H:%M:%S.%f"),
+                     datetime.datetime.strptime(row["stop_time"], "%Y-%m-%d %H:%M:%S.%f"),
+                     row['test_name']])
+
+# Read and parse the log containing the endpoint hits
+hits = []
+with open('endpoint_hits.log') as log:
+    reader = csv.DictReader(log)
+    for row in reader:
+        hits.append([datetime.datetime.strptime(row["time"], "%Y-%m-%d %H:%M:%S.%f"),
+                     row['endpoint']])
+
+# Analyze logs to find out which endpoints are hit by which unit tests
+for h in hits:
+    for r in runs:
+        if r[0] <= h[0] <= r[1]:
+            if {'endpoint': h[1], 'test_name': r[2]} not in data['grouped_tests']:
+                data['grouped_tests'].append({'endpoint': h[1], 'test_name': r[2]})
+            break
+
+# Try to send test results and endpoint-grouped unit tests to the dashboard
+if url:
+    try:
+        requests.post(url, json=data)
+        print('Sent unit test results to the dashboard.')
+    except Exception as e:
+        print('Sending unit test results to the dashboard failed:\n{}'.format(e))

dashboard/colors.py

@@ -14,4 +14,3 @@ def get_color(hash):
     else:
         rgb = ColorHash(hash).rgb
     return 'rgb({0}, {1}, {2})'.format(rgb[0], rgb[1], rgb[2])
-

dashboard/config.py

@@ -23,6 +23,7 @@ def __init__(self):
         self.guest_password = ['guest_password']
         self.outlier_detection_constant = 2.5
         self.colors = {}
+        self.log_dir = None
 
         # define a custom function to retrieve the session_id or username
         self.get_group_by = None
@@ -56,6 +57,10 @@ def from_file(self, config_file):
             :param config_file: a string pointing to the location of the config-file
         """
 
+        config = os.getenv('DASHBOARD_CONFIG')
+        if config:
+            config_file = config
+
         parser = configparser.RawConfigParser()
         try:
             parser.read(config_file)
@@ -67,6 +72,11 @@ def from_file(self, config_file):
                 self.database_name = parser.get('dashboard', 'DATABASE')
             if parser.has_option('dashboard', 'TEST_DIR'):
                 self.test_dir = parser.get('dashboard', 'TEST_DIR')
+            if parser.has_option('dashboard', 'LOG_DIR'):
+                self.log_dir = parser.get('dashboard', 'LOG_DIR')
+                log = open(self.log_dir + "endpoint_hits.log", "w")
+                log.write("\"time\",\"endpoint\"\n")
+                log.close()
 
             # For manually defining colors of specific endpoints
             if parser.has_option('dashboard', 'COLORS'):
@@ -101,6 +111,7 @@ def from_file(self, config_file):
 
             # when an outlier detection constant has been set up:
             if parser.has_option('dashboard', 'OUTLIER_DETECTION_CONSTANT'):
-                self.outlier_detection_constant = parser.get('dashboard', 'OUTLIER_DETECTION_CONSTANT')
+                self.outlier_detection_constant = ast.literal_eval(
+                    parser.get('dashboard', 'OUTLIER_DETECTION_CONSTANT'))
         except configparser.Error:
             raise

dashboard/database/__init__.py

@@ -102,6 +102,15 @@ class Outlier(Base):
     time = Column(DateTime)
 
 
+class TestsGrouped(Base):
+    """ Table for storing grouped tests on endpoints. """
+    __tablename__ = 'testsGrouped'
+    # Name of the endpoint
+    endpoint = Column(String(250), primary_key=True)
+    # Name of the unit test
+    test_name = Column(String(250), primary_key=True)
+
+
 # define the database
 engine = create_engine(config.database_name)
 

dashboard/database/endpoint.py

@@ -86,7 +86,8 @@ def get_monitor_rule(endpoint):
             return result
     except NoResultFound:
         with session_scope() as db_session:
-            db_session.add(MonitorRule(endpoint=endpoint, version_added=config.version, time_added=datetime.datetime.now()))
+            db_session.add(
+                MonitorRule(endpoint=endpoint, version_added=config.version, time_added=datetime.datetime.now()))
 
         # return new added row
         return get_monitor_rule(endpoint)

dashboard/database/tests_grouped.py

@@ -0,0 +1,25 @@
+"""
+Contains all functions that operate on the testsGrouped table
+"""
+from dashboard.database import session_scope, TestsGrouped
+
+
+def reset_tests_grouped():
+    """ Resets the testsGrouped table of the database. """
+    with session_scope() as db_session:
+        db_session.query(TestsGrouped).delete()
+
+
+def add_tests_grouped(json):
+    """ Adds endpoint - unit tests combinations to the database. """
+    with session_scope() as db_session:
+        for combination in json:
+            db_session.add(TestsGrouped(endpoint=combination['endpoint'], test_name=combination['test_name']))
+
+
+def get_tests_grouped():
+    """ Return all existing endpoint - unit tests combinations. """
+    with session_scope() as db_session:
+        result = db_session.query(TestsGrouped).all()
+        db_session.expunge_all()
+        return result

dashboard/forms.py

@@ -25,4 +25,4 @@ class ChangeSetting(FlaskForm):
 
 class RunTests(FlaskForm):
     """ Used for serving a login form on /{{ link }}/testmonitor. """
-    submit = SubmitField('Run selected tests')
+    submit = SubmitField('Run selected tests')

dashboard/main.py

@@ -11,6 +11,7 @@ def get_session_id():
     # implement here your own custom function
     return '12345'
 
+
 dashboard.config.get_group_by = get_session_id
 dashboard.bind(app=user_app)
 

dashboard/measurement.py

@@ -44,6 +44,7 @@ def track_performance(func, endpoint):
     :param func: the function to be measured
     :param endpoint: the name of the endpoint
     """
+
     @wraps(func)
     def wrapper(*args, **kwargs):
         # compute average
@@ -54,12 +55,19 @@ def wrapper(*args, **kwargs):
             # start a thread to log the stacktrace after 'average' ms
             stack_info = StackInfo(average)
 
+        t1 = str(datetime.datetime.now())
         time1 = time.time()
         result = func(*args, **kwargs)
         time2 = time.time()
-        t = (time2-time1)*1000
+        t = (time2 - time1) * 1000
         add_function_call(time=t, endpoint=endpoint)
 
+        # Logging for grouping unit test results by endpoint
+        if config.log_dir:
+            log = open(config.log_dir + "endpoint_hits.log", "a")
+            log.write("\"{}\",\"{}\"\n".format(t1, endpoint))
+            log.close()
+
         # outlier detection
         endpoint_count[endpoint] += 1
         endpoint_sum[endpoint] += t
@@ -68,6 +76,7 @@ def wrapper(*args, **kwargs):
             add_outlier(endpoint, t, stack_info)
 
         return result
+
     wrapper.original = func
     return wrapper
 
@@ -78,10 +87,12 @@ def track_last_accessed(func, endpoint):
     :param func: the function to be measured
     :param endpoint: the name of the endpoint
     """
+
     @wraps(func)
     def wrapper(*args, **kwargs):
         update_last_accessed(endpoint=endpoint, value=datetime.datetime.now())
         return func(*args, **kwargs)
+
     return wrapper
 
 

dashboard/outlier.py

@@ -5,7 +5,6 @@
     Moreover, it logs cpu- and memory-info.
 """
 
-
 import time
 import traceback
 from threading import Thread, enumerate
@@ -14,15 +13,14 @@
 
 
 class StackInfo(object):
-
     def __init__(self, average):
         self.average = average
         self.stacktrace = ''
         self.cpu_percent = ''
         self.memory = ''
 
         try:
-            thread = Thread(target=log_stack_trace, args=(self, ))
+            thread = Thread(target=log_stack_trace, args=(self,))
             thread.start()
         except Exception:
             print('Can\'t log traceback information')
@@ -31,7 +29,7 @@ def __init__(self, average):
 
 def log_stack_trace(stack_info):
     # average is in ms, sleep requires seconds
-    time.sleep(stack_info.average/1000.0)
+    time.sleep(stack_info.average / 1000.0)
 
     # iterate through every active thread and get the stack-trace
     stack_list = []

dashboard/routings/export_data.py

@@ -2,6 +2,7 @@
 from dashboard.security import admin_secure
 from dashboard.database.function_calls import get_data
 from dashboard.database.tests import add_or_update_test, add_test_result, get_suite_nr
+from dashboard.database.tests_grouped import reset_tests_grouped, add_tests_grouped
 from dashboard import blueprint, config
 
 import datetime
@@ -42,4 +43,8 @@ def submit_test_results():
         time = datetime.datetime.strptime(result['time'], '%Y-%m-%d %H:%M:%S.%f')
         add_or_update_test(result['name'], time, result['successful'])
         add_test_result(result['name'], result['exec_time'], time, config.version, suite, result['iter'])
+
+    reset_tests_grouped()
+    add_tests_grouped(request.get_json()['grouped_tests'])
+
     return '', 204

dashboard/routings/measurements.py

@@ -39,7 +39,7 @@ def page_number_of_requests_per_endpoint():
     colors = {}
     for result in get_times():
         colors[result.endpoint] = get_color(result.endpoint)
-    return render_template('dashboard/measurement.html', link=config.link, curr=2,  session=session, index=2,
+    return render_template('dashboard/measurement.html', link=config.link, curr=2, session=session, index=2,
                            graph=get_stacked_bar())
 
 

dashboard/routings/result.py

@@ -18,7 +18,6 @@
 from dashboard.database.outlier import get_outliers
 from dashboard.colors import get_color
 
-
 # Constants
 BUBBLE_SIZE_RATIO = 1250
 

dashboard/routings/setup.py

@@ -5,6 +5,7 @@
 from dashboard.database.monitor_rules import reset_monitor_endpoints
 from dashboard.database.tests import get_tests, get_results, get_suites, get_test_measurements
 from dashboard.database.tests import get_res_current, get_measurements
+from dashboard.database.tests_grouped import get_tests_grouped
 from dashboard.forms import MonitorDashboard
 from dashboard.measurement import track_performance
 from dashboard.security import secure, admin_secure
@@ -83,8 +84,18 @@ def test_result(test):
 @blueprint.route('/testmonitor')
 @secure
 def testmonitor():
+    tests = get_tests_grouped()
+    grouped = {}
+    cols = {}
+    for t in tests:
+        if t.endpoint not in grouped:
+            grouped[t.endpoint] = []
+            cols[t.endpoint] = get_color(t.endpoint)
+        if t.test_name not in grouped[t.endpoint]:
+            grouped[t.endpoint].append(t.test_name)
+
     return render_template('dashboard/testmonitor.html', link=config.link, session=session, curr=3,
-                           tests=get_tests(), results=get_results(),
+                           tests=get_tests(), results=get_results(), groups=grouped, colors=cols,
                            res_current_version=get_res_current(config.version), boxplot=get_boxplot(None))