Merge branch 'release/0.27.0'

Author: Roey Darwish Dror

.gitignore

@@ -58,3 +58,4 @@ celerybeat.pid
 client_secret.json
 webapp/tmp
 conf.d
+tests/.cache

_lib/db.py

@@ -58,8 +58,6 @@ def ensure():
     elif match.group('db_type') == 'postgresql':
         _create_postgres(match)
 
-    with app.app_context():
-        db.create_all()
     logbook.info("DB successfully created")
 
 

_lib/deployment.py

@@ -1,10 +1,9 @@
 import os
-import sys
 from multiprocessing import cpu_count
 
 import click
 
-from .bootstrapping import from_env, from_env_bin, requires_env, from_project_root
+from .bootstrapping import from_env_bin, requires_env, from_project_root
 from .params import APP_NAME
 
 _UNIX_SOCKET_NAME = "/var/run/{}/wsgi.sock".format(APP_NAME)
@@ -13,6 +12,7 @@
 @click.command()
 @requires_env("app")
 def run_gunicorn():
+    num_workers = (2 * cpu_count()) + 1
     gunicorn_bin = from_env_bin('gunicorn')
-    cmd = [gunicorn_bin, '--log-syslog', '-b', 'unix://{}'.format(_UNIX_SOCKET_NAME), 'flask_app.wsgi:app', '--chdir', from_project_root('.')]
+    cmd = [gunicorn_bin, '--log-syslog', '-b', 'unix://{}'.format(_UNIX_SOCKET_NAME), 'flask_app.wsgi:app', '--chdir', from_project_root('.'), '-w', str(num_workers)]
     os.execv(gunicorn_bin, cmd)

ansible/roles/db/handlers/main.yml

@@ -0,0 +1,3 @@
+---
+- name: restart postgres
+  service: name=postgresql state=restarted

ansible/roles/db/tasks/main.yml

@@ -44,14 +44,26 @@
 - name: enable local authentication in postresql
   lineinfile: dest=/var/lib/pgsql/data/pg_hba.conf regexp="^host.*::1/128" line="host all all ::1/128 md5" state=present
   when: ansible_distribution == 'CentOS'
+- name: config postgres local hosts
+  lineinfile: dest=/etc/postgresql/{{postgres_version}}/main/pg_hba.conf regexp='^local.*all.*all.*[pm]' line="local  all  all  md5"
+  notify: restart postgres
+  when: groups['db'][0] != groups['webapp'][0] and ansible_distribution == 'Debian'
+- name: enable remote authentication in postresql
+  lineinfile: dest=/etc/postgresql/{{postgres_version}}/main/pg_hba.conf regexp='0\.0\.0\.0' line="host  all  all  0.0.0.0/0  md5"
+  notify: restart postgres
+  when: groups['db'][0] != groups['webapp'][0] and ansible_distribution == 'Debian'
+- name: config postgresql
+  lineinfile: dest=/etc/postgresql/{{postgres_version}}/main/postgresql.conf regexp=^listen_addresses line="listen_addresses='*'"
+  notify: restart postgres
+  when: groups['db'][0] != groups['webapp'][0] and ansible_distribution == 'Debian'
 - service: name=postgresql state=started enabled=True
 - name: setup db
   action: postgresql_db name={{app_name}} encoding=utf8
-  sudo_user: postgres
-  sudo: true
+  become: true
+  become_user: postgres
   tags: db
 - name: setup db user
   action: postgresql_user name={{app_name}} password={{app_name}} db={{app_name}} priv=ALL role_attr_flags=SUPERUSER
-  sudo_user: postgres
-  sudo: true
+  become: true
+  become_user: postgres
   tags: db

ansible/roles/db/vars/main.yml

@@ -0,0 +1 @@
+postgres_version: 9.4

ansible/roles/webapp/tasks/install_sources.yml

@@ -1,15 +1,15 @@
 - name: untar to temporary location
   shell: cd /tmp/ && rm -rf {{webapp_archive_location}}.untarred && mkdir {{webapp_archive_location}}.untarred && cd {{webapp_archive_location}}.untarred && tar xzf {{webapp_archive_location}}
-- name: rsync to src dir
-  shell: rsync -avP --delete {{webapp_archive_location}}.untarred/ {{src_root}}/
+- synchronize: src="{{webapp_archive_location}}.untarred/" dest="{{src_root}}/"
+  delegate_to: "{{ inventory_hostname }}"
 - name: bootstrap
   shell: python {{src_root}}/manage.py bootstrap --app
 - name: ensure private config
   shell: python {{src_root}}/manage.py ensure-secret {{deploy_root}}/conf.d/000-private.yml
 - name: migrate db
   shell: cd {{src_root}} && python manage.py db upgrade
 - name: fix permissions
-  shell: chown -R {{user_name}}:{{group_name}} {{deploy_root}}
+  file: path={{deploy_root}} owner={{user_name}} group={{group_name}} recurse=true
 - name: fix permissions of static folder
   file: path={{src_root}}/static owner={{nginx_user}} group={{nginx_group}} recurse=true state=directory
 - name: ensure that nginx sites directory exists

ansible/roles/webapp/tasks/main.yml

@@ -52,21 +52,13 @@
 - name: ensure /var/run/{{app_name}} directory on boot
   template: src=../templates/tmpfiles.d.conf.j2 dest=/etc/tmpfiles.d/{{app_name}}.conf
 - file: path=/var/run/{{app_name}} state=directory mode=0770 owner={{app_name}} group={{nginx_group}}
-- name: install systemd service
-  action: template src=../templates/gunicorn.service.j2 dest=/lib/systemd/system/{{app_name}}-wsgi.service
-  when: sources.changed or configuration.changed
-  register: systemd_change
-- name: install systemd service (Celery worker)
-  action: template src=../templates/celery-worker.service.j2 dest=/lib/systemd/system/{{app_name}}-celery-worker.service
-  when: sources.changed or configuration.changed
-  register: systemd_change
-- name: install systemd service (Celery beat)
-  action: template src=../templates/celery-beat.service.j2 dest=/lib/systemd/system/{{app_name}}-celery-beat.service
-  when: sources.changed or configuration.changed
-  register: systemd_change
+
+- name: Configure systemd services
+  include: systemd_services.yml
+  register: systemd_services
 - name: reload systemd daemon
   action: shell systemctl daemon-reload
-  when: systemd_change.changed
+  when: systemd_services.changed
 - name: enable the services
   action: service name={{item}} enabled=true state=started
   with_items:
@@ -79,6 +71,6 @@
     - "{{app_name}}-wsgi"
     - "{{app_name}}-celery-worker"
     - "{{app_name}}-celery-beat"
-  when: systemd_change.changed or sources.changed or configuration.changed
+  when: systemd_services.changed or sources.changed or configuration.changed
 - name: try to get local page
   shell: curl http://localhost/

ansible/roles/webapp/tasks/systemd_services.yml

@@ -0,0 +1,10 @@
+- name: install systemd service
+  action: template src=../templates/gunicorn.service.j2 dest=/lib/systemd/system/{{app_name}}-wsgi.service
+  when: sources.changed or configuration.changed
+- name: install systemd service (Celery worker)
+  action: template src=../templates/celery-worker.service.j2 dest=/lib/systemd/system/{{app_name}}-celery-worker.service
+  when: sources.changed or configuration.changed
+- name: install systemd service (Celery beat)
+  action: template src=../templates/celery-beat.service.j2 dest=/lib/systemd/system/{{app_name}}-celery-beat.service
+  when: sources.changed or configuration.changed
+  

ansible/roles/webapp/templates/deployment_conf.yml.j2

@@ -1,4 +1,5 @@
-SQLALCHEMY_DATABASE_URI: "postgresql://{{app_name}}:{{app_name}}@localhost/{{app_name}}"
+SQLALCHEMY_DATABASE_URI: "postgresql://{{app_name}}:{{app_name}}@{% if groups['db'][0] != groups['webapp'][0] %}{{groups['db'][0]}}{% else %}localhost{% endif %}/{{app_name}}"
+
 STORAGE_PATH: /var/scotty
 SENTRY_DSN: "{{sentry_dsn}}"
 

ansible/roles/webapp/templates/nginx_site_conf.conf.j2

@@ -26,6 +26,9 @@ server {
 
     location /file_contents {
         alias /var/scotty;
+        types {
+          text/plain    log txt;
+        }
     }
 
 

conf.d/.gitkeep

@@ -2,8 +2,8 @@ alembic>=0.7.3
 psycopg2
 Flask>=0.10.1
 Flask-Mail
-Flask-Login>=0.3.0
-Flask-Security
+Flask-Login>=0.3.0,<0.4.0
+Flask-Security>=1.7.5
 Flask-Migrate>=1.3.0
 Flask-SQLAlchemy
 URLObject

deps/app.txt

@@ -1,4 +1,4 @@
 app_name: scotty
-APP_VERSION: 0.26.2
+APP_VERSION: 0.27.0
 SECURITY_PASSWORD_HASH: sha512_crypt
 SQLALCHEMY_TRACK_MODIFICATIONS: false

flask_app/app.yml

@@ -64,7 +64,7 @@ def get_all():
     if "filter" in request.args and request.args["filter"]:
         query = query.filter(File.file_name.like("%{}%".format(request.args['filter'])))
 
-    query = query.order_by(File.storage_name)
+    query = query.order_by(File.file_name)
 
     total = query.count()
 

flask_app/blueprints/files.py

@@ -25,6 +25,9 @@
 def create():
     data = request.json['issue']
     id_in_tracker = data['id_in_tracker'].strip()
+    if not id_in_tracker:
+        return 'Invalid issue id', http.client.CONFLICT
+
     issue = Issue(tracker_id=data['tracker_id'], id_in_tracker=id_in_tracker, open=True)
     db.session.add(issue)
     try:

flask_app/blueprints/issues.py

@@ -75,7 +75,6 @@ def testserver(tmux, livereload, port):
     if tmux:
         return _run_tmux_frontend(port=port)
     from flask_app.app import create_app
-    app = create_app({'DEBUG': True, 'TESTING': True, 'SECRET_KEY': 'dummy', 'SECURITY_PASSWORD_SALT': 'dummy'})
 
     extra_files=[
         from_project_root("flask_app", "app.yml")

manage.py

@@ -63,8 +63,9 @@ def dump(self):
     def delete(self):
         os.unlink(self._path)
 
-    def create_issue(self):
-        id_in_tracker = str(uuid.uuid4())
+    def create_issue(self, id_in_tracker=None):
+        if id_in_tracker is None:
+            id_in_tracker = str(uuid.uuid4())
         self._issues[id_in_tracker] = True
         id_in_scotty = self._scotty.create_issue(self._id, id_in_tracker)
         issue = self.Issue(self, id_in_scotty, id_in_tracker)

tests/slashconf.py

@@ -1 +0,0 @@
-

tests/test_deployment/__init__.py

@@ -1,3 +1,5 @@
+import http
+import requests
 import slash
 import uuid
 from itertools import chain, combinations
@@ -8,6 +10,13 @@ def powerset(iterable):
     return (frozenset(p) for p in chain.from_iterable((combinations(s, r)) for r in range(len(s)+1)))
 
 
+def test_empty_issue(tracker):
+    with slash.assert_raises(requests.exceptions.HTTPError) as e:
+        tracker.create_issue(' ')
+
+    assert e.exception.response.status_code == http.client.CONFLICT
+
+
 def test_issue_creation(beam, issue):
     beam, _ = beam
     assert len(beam.associated_issues) == 0

tests/test_deployment/test__issue_tracker.py

@@ -18,4 +18,6 @@ def test_independent_beam(beam, local_beam_dir, download_dir):
     for file_ in beam.beam.iter_files():
         file_.download(download_dir)
 
+    assert beam.beam.completed
+
     subprocess.check_call(['diff', '-rq', local_beam_dir, download_dir])

tests/test_deployment/test__sanity.py

@@ -1,6 +1,6 @@
 import Ember from 'ember';
-import Resolver from 'ember/resolver';
-import loadInitializers from 'ember/load-initializers';
+import Resolver from './resolver';
+import loadInitializers from 'ember-load-initializers';
 import config from './config/environment';
 
 let App;
@@ -13,55 +13,6 @@ App = Ember.Application.extend({
   Resolver
 });
 
-App.Pollster = Ember.Object.extend({
-  interval: function() {
-    return 5000; // Time between polls (in ms)
-  }.property(),
-
-  // Schedules the function `f` to be executed every `interval` time.
-  schedule: function(f) {
-    return Ember.run.later(this, function() {
-      if (f.apply(this)) {
-        this.set('timer', this.schedule(f));
-      }
-    }, this.get('interval'));
-  },
-
-  // Stops the pollster
-  stop: function() {
-    Ember.run.cancel(this.get('timer'));
-  },
-
-  // Starts the pollster, i.e. executes the `onPoll` function every interval.
-  start: function() {
-    this.set('timer', this.schedule(this.get('onPoll')));
-  },
-
-  onPoll: function() {
-    // Issue JSON request and add data to the store
-  }
-});
-
-App.instanceInitializer({
-  name: "relative_time_update",
-
-  initialize: function(instance) {
-    let storage = instance.lookup("service:store");
-    App.update_reltime = App.Pollster.create({
-      onPoll: function() {
-        let beams = storage.peekAll("beam").content;
-        for (var i = 0; i < beams.length; i++) {
-          let beam = beams[i].getRecord();
-          beam.set("tick", beam.get("tick") + 1);
-        }
-        return true;
-      }
-    });
-    App.update_reltime.set("interval", 60000);
-    App.update_reltime.start();
-  }
-});
-
 loadInitializers(App, config.modulePrefix);
 config.torii.providers['google-oauth2'].redirectUri = window.location.origin;
 export default App;