Merge pull request #18 from mircealungu/development

Moved setup.py to the root of the repo.

Author: mircealungu

MANIFEST.in

@@ -0,0 +1,2 @@
+recursive-include dashboard/static *
+recursive-include dashboard/templates *

README.md

@@ -20,18 +20,31 @@ Adding the extension to your flask app is simple:
     from flask import Flask
     import dashboard
 
-    app = Flask(__name__)
-    dashboard.bind(app)
+    user_app = Flask(__name__)
+    dashboard.config.from_file('/<path to your config file>/config.cfg')
+
+    def get_session_id():
+        # Implement your own function for obtaining the user variable here.
+        return "12345"
+
+    dashboard.config.get_group_by = get_session_id
+    dashboard.bind(app=user_app)
 
 Usage
 =====
 Once the setup is done, a config file ('config.cfg') should be set next to the python file that contains the entry point of the app.
-The following things should be set for best performance:
+The following things can be configured:
 
     [dashboard]
-    DATABASE=sqlite:////<path to your project>/flask-dashboard.db
-    GIT=/<path to your project>/flask-dashboard/.git/
-    TEST_DIR=/<path to your project>/flask-dashboard/tests/
+    APP_VERSION=1.0
+    CUSTOM_LINK=dashboard
+    USERNAME=admin
+    PASSWORD=admin
+    DATABASE=sqlite:////<path to your project>/dashboard.db
+    GIT=/<path to your project>/dashboard/.git/
+    TEST_DIR=/<path to your project>/dashboard/tests/
+
+For more information: [see this file](dashboard/config.py)
 
 When running your app, the dashboard van be viewed by default in the route:
 

flask-dashboard/dashboard/__init__.py renamed to dashboard/__init__.py

@@ -23,8 +23,8 @@
 
 # get current location of the project
 def loc():
-    s = os.path.abspath(os.path.dirname(__file__))
-    return str(s.rsplit('/', 1)[0]) + '/'
+    return os.path.abspath(os.path.dirname(__file__)) + '/'
+
 
 # define the blueprint
 blueprint = Blueprint('dashboard', __name__, template_folder=loc() + 'templates')
@@ -50,15 +50,16 @@ def bind(app):
     # register the blueprint to the app
     app.register_blueprint(blueprint, url_prefix='/' + config.link)
 
-    # search for tests
-    from dashboard.database.tests import add_test, get_tests
-    suites = TestLoader().discover(config.test_dir, pattern="*test*.py")
-    existing_tests = get_tests()
-    tests = []
-    for t in existing_tests:
-        tests.append(t.name)
-    for suite in suites:
-        for case in suite:
-            for test in case:
-                if str(test) not in tests:
-                    add_test(str(test))
+    # search for tests if test dir specified
+    if config.test_dir:
+        from dashboard.database.tests import add_test, get_tests
+        suites = TestLoader().discover(config.test_dir, pattern="*test*.py")
+        existing_tests = get_tests()
+        tests = []
+        for t in existing_tests:
+            tests.append(t.name)
+        for suite in suites:
+            for case in suite:
+                for test in case:
+                    if str(test) not in tests:
+                        add_test(str(test))

flask-dashboard/dashboard/config.py renamed to dashboard/config.py

@@ -15,7 +15,9 @@ def __init__(self):
         self.version = '1.0'
         self.link = 'dashboard'
         self.database_name = 'sqlite:///flask-dashboard.db'
-        self.test_dir = './'
+        self.test_dir = None
+        self.username = 'admin'
+        self.password = 'admin'
 
         # define a custom function to retrieve the session_id or username
         self.get_group_by = None
@@ -35,7 +37,10 @@ def from_file(self, config_file):
             version automatically retrieved by reading the commit-id (hashed value):
             GIT = If you're using git, then it is easier to set the location to the .git-folder, 
                 The location is relative to the config-file.
-            
+
+            USERNAME: for logging into the dashboard, a username and password is required. The
+                username can be set using this variable.
+            PASSWORD: same as for the username, but this is the password variable.
             
             :param config_file: a string pointing to the location of the config-file
         """
@@ -51,6 +56,7 @@ def from_file(self, config_file):
                 self.database_name = parser.get('dashboard', 'DATABASE')
             if parser.has_option('dashboard', 'TEST_DIR'):
                 self.test_dir = parser.get('dashboard', 'TEST_DIR')
+
             # When the option git is selected, it overrides the given version
             if parser.has_option('dashboard', 'GIT'):
                 git = parser.get('dashboard', 'GIT')
@@ -65,5 +71,11 @@ def from_file(self, config_file):
                 except IOError:
                     print("Error reading one of the files to retrieve the current git-version.")
                     raise
+
+            # provide username and/or password
+            if parser.has_option('dashboard', 'USERNAME'):
+                self.username = parser.get('dashboard', 'USERNAME')
+            if parser.has_option('dashboard', 'PASSWORD'):
+                self.password = parser.get('dashboard', 'PASSWORD')
         except configparser.Error:
             raise

flask-dashboard/dashboard/database/__init__.py renamed to dashboard/database/__init__.py

@@ -69,14 +69,6 @@ class FunctionCall(Base):
     ip = Column(String(25), nullable=False)
 
 
-class Setting(Base):
-    """ Table for storing the values of certain variables """
-    __tablename__ = 'settings'
-    # the name of the variable must be unique
-    variable = Column(String(100), primary_key=True)
-    # the corresponding value
-    value = Column(String(100))
-
 # define the database
 engine = create_engine(config.database_name)
 

flask-dashboard/dashboard/database/endpoint.py renamed to dashboard/database/endpoint.py

@@ -1,7 +1,7 @@
 """
 Contains all functions that access a single endpoint
 """
-from sqlalchemy import func, text
+from sqlalchemy import func, text, asc
 from sqlalchemy.orm.exc import NoResultFound
 import datetime
 
@@ -24,12 +24,38 @@ def get_line_results(endpoint):
         return data
 
 
+def get_num_requests(endpoint):
+    """ Returns a list with all dates on which an endpoint is accessed.
+        :param endpoint: if None, the result is the sum of all endpoints
+    """
+    with session_scope() as db_session:
+        query = text("""select
+                datetime(CAST(strftime('%s', time)/3600 AS INT)*3600, 'unixepoch') AS newTime,
+                count(execution_time) as count
+                from functioncalls
+                where (endpoint=:val OR :val='None') group by newTime""")
+        result = db_session.execute(query, {'val': str(endpoint)})
+        data = result.fetchall()
+        return data
+
+
 def get_endpoint_column(endpoint, column):
+    """ Returns a list of entries from column in which the endpoint is involved. """
+    with session_scope() as db_session:
+        result = db_session.query(column,
+                                  func.min(FunctionCall.time).label('startedUsingOn')). \
+            filter(FunctionCall.endpoint == endpoint). \
+            group_by(column).order_by(asc('startedUsingOn')).all()
+        db_session.expunge_all()
+        return result
+
+
+def get_endpoint_column_user_sorted(endpoint, column):
     """ Returns a list of entries from column in which the endpoint is involved. """
     with session_scope() as db_session:
         result = db_session.query(column). \
             filter(FunctionCall.endpoint == endpoint). \
-            group_by(column).all()
+            group_by(column).order_by(asc(column)).all()
         db_session.expunge_all()
         return result
 
@@ -74,13 +100,22 @@ def update_monitor_rule(endpoint, value):
 
 
 def get_all_measurement(endpoint):
-    """Return all entries with measurements from a given endpoint. Uses for creating a histogram. """
+    """Return all entries with measurements from a given endpoint. Used for creating a histogram. """
     with session_scope() as db_session:
         result = db_session.query(FunctionCall).filter(FunctionCall.endpoint == endpoint).all()
         db_session.expunge_all()
         return result
 
 
+def get_all_measurement_per_column(endpoint, column, value):
+    """Return all entries with measurements from a given endpoint for which the column has a specific value.
+    Used for creating a box plot. """
+    with session_scope() as db_session:
+        result = db_session.query(FunctionCall).filter(FunctionCall.endpoint == endpoint, column == value).all()
+        db_session.expunge_all()
+        return result
+
+
 def get_last_accessed_times():
     """ Returns a list of all endpoints and their last accessed time. """
     with session_scope() as db_session:

dashboard/database/function_calls.py

@@ -0,0 +1,96 @@
+"""
+Contains all functions that access any functionCall-object
+"""
+
+from flask import request
+from sqlalchemy import func, desc, text, asc
+from dashboard import config
+import datetime
+from dashboard.database import session_scope, FunctionCall
+
+
+def get_reqs_endpoint_day():
+    """ Retrieves the number of requests per endpoint per day. """
+    with session_scope() as db_session:
+        query = text("""select strftime('%Y-%m-%d', time) AS newTime,
+                               count(endpoint) AS cnt,
+                               endpoint
+                        from functioncalls 
+                        group by newTime, endpoint""")
+        result = db_session.execute(query)
+        data = result.fetchall()
+        return data
+
+
+def add_function_call(time, endpoint):
+    """ Add a measurement to the database. """
+    with session_scope() as db_session:
+        group_by = None
+        if config.get_group_by:
+            group_by = config.get_group_by()
+
+        ip = request.environ['REMOTE_ADDR']
+        call = FunctionCall(endpoint=endpoint, execution_time=time, version=config.version,
+                            time=datetime.datetime.now(), group_by=str(group_by), ip=ip)
+        db_session.add(call)
+
+
+def get_times():
+    """ Return all entries of measurements with the average and total number of execution times. The results are 
+    grouped by their endpoint. """
+    with session_scope() as db_session:
+        result = db_session.query(FunctionCall.endpoint,
+                                  func.count(FunctionCall.execution_time).label('count'),
+                                  func.avg(FunctionCall.execution_time).label('average')
+                                  ).group_by(FunctionCall.endpoint).order_by(desc('count')).all()
+        db_session.expunge_all()
+        return result
+
+
+def get_data():
+    """ Returns all data in the FunctionCall table, for the export data option. """
+    with session_scope() as db_session:
+        result = db_session.query(FunctionCall.endpoint,
+                                  FunctionCall.execution_time,
+                                  FunctionCall.time,
+                                  FunctionCall.version,
+                                  FunctionCall.group_by,
+                                  FunctionCall.ip).all()
+        db_session.expunge_all()
+        return result
+
+
+def get_data_per_version(version):
+    """ Returns all data in the FuctionCall table, grouped by their version. """
+    with session_scope() as db_session:
+        result = db_session.query(FunctionCall.execution_time, FunctionCall.version). \
+            filter(FunctionCall.version == version).all()
+        db_session.expunge_all()
+        return result
+
+
+def get_versions(end=None):
+    with session_scope() as db_session:
+        result = db_session.query(FunctionCall.version,
+                                  func.min(FunctionCall.time).label('startedUsingOn')). \
+            filter((FunctionCall.endpoint == end) | (end is None)).group_by(FunctionCall.version).order_by(
+            asc('startedUsingOn')).all()
+    db_session.expunge_all()
+    return result
+
+
+def get_data_per_endpoint(end):
+    with session_scope() as db_session:
+        result = db_session.query(FunctionCall.execution_time, FunctionCall.endpoint). \
+            filter(FunctionCall.endpoint == end).all()
+        db_session.expunge_all()
+        return result
+
+
+def get_endpoints():
+    with session_scope() as db_session:
+        result = db_session.query(FunctionCall.endpoint,
+                                  func.count(FunctionCall.endpoint).label('cnt')). \
+            group_by(FunctionCall.endpoint).order_by(asc('cnt')).all()
+        db_session.expunge_all()
+        return result

flask-dashboard/dashboard/database/monitor_rules.py renamed to dashboard/database/monitor_rules.py

@@ -1,6 +1,5 @@
 from flask_wtf import FlaskForm
-from wtforms import validators
-from wtforms import SubmitField, PasswordField, StringField
+from wtforms import validators, SubmitField, PasswordField, StringField, SelectMultipleField
 
 
 class MonitorDashboard(FlaskForm):
@@ -10,7 +9,7 @@ class MonitorDashboard(FlaskForm):
 
 
 class Login(FlaskForm):
-    """ Used for serving a login form on /{{ link }}/login. """
+    """ Used for serving a login form. """
     name = StringField('Username', [validators.required()])
     password = PasswordField('Password', [validators.required()])
     submit = SubmitField('Login')
@@ -26,4 +25,4 @@ class ChangeSetting(FlaskForm):
 
 class RunTests(FlaskForm):
     """ Used for serving a login form on /{{ link }}/testmonitor. """
-    submit = SubmitField('Run selected tests')
+    submit = SubmitField('Run selected tests')

flask-dashboard/dashboard/database/tests.py renamed to dashboard/database/tests.py

@@ -9,7 +9,7 @@
 
 def get_session_id():
     # implement here your own custom function
-    return "12345"
+    return "1234"
 
 dashboard.config.get_group_by = get_session_id
 dashboard.bind(app=user_app)

flask-dashboard/dashboard/forms.py renamed to dashboard/forms.py

@@ -10,6 +10,8 @@
 import dashboard.routings.login
 import dashboard.routings.setup
 import dashboard.routings.result
+import dashboard.routings.export_data
+import dashboard.routings.measurements
 
 # Provide a secret-key for using WTF-forms
 if user_app.secret_key is None:
@@ -25,4 +27,4 @@ def static(filename):
 # All rules below are for viewing the dashboard-pages
 @blueprint.route('/')
 def index():
-    return redirect(url_for('dashboard.measurements'))
+    return redirect(url_for('dashboard.measurements', index=0))

flask-dashboard/main.py renamed to dashboard/main.py

@@ -0,0 +1,34 @@
+from flask import make_response, render_template, session
+from dashboard.security import secure
+from dashboard.database.function_calls import get_data
+from dashboard import blueprint, config
+
+import datetime
+
+
+@blueprint.route('/download-csv')
+@secure
+def download_csv():
+    csv = "\"ENDPOINT\",\"EXECUTION_TIME\",\"TIME_OF_EXECUTION\",\"VERSION\",\"GROUP_BY\",\"IP_ADDRESS\"\n"
+    data = get_data()
+    for entry in data:
+        csv += "\"{0}\",{1},\"{2}\",\"{3}\",\"{4}\",\"{5}\"\n".format(entry.endpoint, entry.execution_time, entry.time,
+                                                                      entry.version, entry.group_by, entry.ip)
+
+    response = make_response(csv)
+    response.headers["Content-Disposition"] = "attachment; filename=measurements_{0}.csv".format(
+        str(datetime.datetime.now()).replace(" ", "_").replace(":", "-")[:19])
+    return response
+
+
+@blueprint.route('/export-data')
+@secure
+def export_data():
+    csv = ["\"ENDPOINT\",\"EXECUTION_TIME\",\"TIME_OF_EXECUTION\",\"VERSION\",\"GROUP_BY\",\"IP_ADDRESS\""]
+    data = get_data()
+    for entry in data:
+        csv.append("\"{0}\",{1},\"{2}\",\"{3}\",\"{4}\",\"{5}\"".format(entry.endpoint, entry.execution_time,
+                                                                        entry.time, entry.version, entry.group_by,
+                                                                        entry.ip))
+
+    return render_template('export-data.html', link=config.link, session=session, data=csv)