Refactor docbot service

To make the project more flexible and extendable, tarantoolbot.py file
was split on the several files. Bottle framework was replaced with
Flask as it is easier to connect to Elastic APM server.

- `app.py` contains Flask config, WSGI application and endpoints.
- `github.py` has GitHub class to interact with its API.
- `handles.py` contains handlers which parse request data and
  transfers data to processors
- `processors.py` keeps GitHum commit and issue messages processors
- `settigns.py` - project config
- `utils.py` - project helpers

Added possibility to add any repository to the bot through
doc_repo_urls mapping in `settings.py`.

Added `deploy` workflow to push the project to Dokku server. The
workflow pushes code with `--force` flag, so it will override project
in Dokku even if Dokku repo has different branch or commits order.

`Dockerfile` was refactored for using lightweight python-3.10.4 image
based on Alpine.

To add new repository to the bot, do the following steps:
- add @TarantoolBot user to the source project and to the corresponding
  doc project as well.
- add source to docs repos mapping to `settings.py` in doc_repo_urls
  dictionary.
- push the changes to master, the project will be deployed.
- set up webhook to the source repository and point it to docbot
  service.

Closes #13
Closes #16
Part of tarantool/infra#77

Author: artembo

.github/workflows/deploy.yml

@@ -0,0 +1,21 @@
+name: Deploy to Dokku
+
+on:
+  push:
+    branches:
+      - master
+jobs:
+  deploy:
+    runs-on: ubuntu-20.04-self-hosted
+    steps:
+      - uses: actions/checkout@v2
+        with:
+            fetch-depth: 0
+      - id: deploy
+        name: Deploy to dokku
+        uses: idoberko2/dokku-deploy-github-action@v1
+        with:
+            ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+            dokku-host: ${{ secrets.DOCBOT_HOST }}
+            app-name: ${{ secrets.DOCBOT_APP }}
+            git-push-flags: '--force'

.gitignore

@@ -0,0 +1,5 @@
+venv
+.idea
+__pycache__
+*.pyc
+*.pyi

.travis.yml

@@ -1,17 +1,13 @@
-FROM python:2
+FROM python:3.10.4-alpine
 
-RUN mkdir -p /usr/src/bot
-WORKDIR /usr/src/bot
+COPY requirements.txt /tmp
+RUN pip install --upgrade pip --no-cache-dir && \
+    pip install -r tmp/requirements.txt --no-cache-dir
 
-RUN pip install --upgrade pip
-
-COPY requirements.txt /usr/src/bot
-COPY tarantoolbot.py /usr/src/bot
-COPY write_credentials.sh /usr/src/bot
-
-RUN pip install -r requirements.txt
+COPY docbot /app/docbot
 
 ENV PORT=5000
 EXPOSE 5000
 
-CMD gunicorn --bind 0.0.0.0:$PORT tarantoolbot:app
+WORKDIR /app
+CMD gunicorn --bind 0.0.0.0:$PORT docbot.app:app

Dockerfile

@@ -1 +1 @@
-web: gunicorn --bind 0.0.0.0:$PORT tarantoolbot:app
+web: gunicorn --bind 0.0.0.0:$PORT docbot.app:app

Procfile

@@ -0,0 +1,23 @@
+from elasticapm.contrib.flask import ElasticAPM
+from flask import Flask, request
+
+from .handlers import webhook_handler, list_events_handler
+
+app = Flask(__name__)
+
+app.config['ELASTIC_APM'] = {
+    'SERVICE_NAME': 'docbot',
+}
+apm = ElasticAPM(app)
+
+
+@app.route("/", methods=['GET'])
+def index() -> str:
+    return list_events_handler()
+
+
+@app.route("/", methods=['POST'])
+def webhook() -> str:
+    data: dict = request.json
+    event: str = request.headers.get('X-GitHub-Event')
+    return webhook_handler(data, event)

docbot/__init__.py

@@ -0,0 +1,39 @@
+import requests
+
+from .utils import create_event
+
+
+class GitHub:
+    def __init__(self, token: str) -> None:
+        self.token: str = token
+
+    def send_comment(self, body, issue_api, to):
+        create_event(to, 'send_comment', body)
+        body = {'body': '@{}: {}'.format(to, body)}
+        url = '{}/comments'.format(issue_api)
+        status_code = self._send_request(url, body)
+        print('Sent comment: {}'.format(status_code))
+
+    def get_comments(self, issue_api) -> dict:
+        body = {'since': '1970-01-01T00:00:00Z'}
+        url = '{}/comments'.format(issue_api)
+        r = self._send_request(url, body, method='get')
+        return r.json()
+
+    def create_issue(self, title, description, src_url, author, doc_repo_url):
+        create_event(author, 'create_issue', title)
+        description = '{}\nRequested by @{} in {}.'.format(description, author,
+                                                           src_url)
+        body = {'title': title, 'body': description}
+        url = '{}/issues'.format(doc_repo_url)
+        status_code = self._send_request(url, body)
+        print('Created issue: {}'.format(status_code))
+
+    def _send_request(self, url, body, method='post'):
+        headers = {
+            'Authorization': 'token {}'.format(self.token),
+        }
+        r = requests.request(method, url, json=body, headers=headers)
+        print(r.status_code)
+        r.raise_for_status()
+        return r

docbot/app.py

@@ -0,0 +1,44 @@
+from .utils import last_events
+from . import settings
+from .processors import process_issue_comment, process_issue_state_change, \
+    process_commit
+
+
+def webhook_handler(data: dict, event: str) -> str:
+    if event is None or data is None:
+        return 'Event or data was not found'
+
+    if 'issue' in data:
+        issue = data['issue']
+        if 'state' not in issue:
+            return 'Event is not needed.'
+        issue_state = issue['state']
+        issue_api = issue['url']
+        issue_url = issue['html_url']
+        issue_repo = data['repository']['full_name']
+
+        if event == 'issue_comment':
+            return process_issue_comment(data, issue_state,
+                                         issue_api)
+        elif event == 'issues':
+            doc_repo_url = settings.doc_repo_urls.get(issue_repo)
+            return process_issue_state_change(data, issue_api,
+                                              issue_url, doc_repo_url)
+        else:
+            return 'Event "{}" is not needed.'.format(event)
+    elif event == 'push':
+        repo = data['repository']
+        branch = '/'.join(data['ref'].split('/')[2:])
+        is_master_push = repo['master_branch'] == branch
+        issue_repo = settings.doc_repo_urls.get(repo['full_name'])
+        for c in data['commits']:
+            process_commit(c, is_master_push, issue_repo)
+        return 'Webhook was processed'
+    else:
+        return 'Event is not needed.'
+
+
+def list_events_handler() -> str:
+    return ('<h1>{}</h1><table border="1" cellspacing="2" ' +
+            'cellpadding="2">{}</table>').format('TarantoolBot Journal',
+                                                 ' '.join(last_events))

docbot/github.py

@@ -0,0 +1,79 @@
+from . import settings
+from .github import GitHub
+from .utils import create_event
+
+github = GitHub(settings.token)
+
+
+def parse_comment(body):
+    if not body.startswith(settings.bot_name):
+        return None, None
+    offset = len(settings.bot_name)
+    for dr in settings.doc_requests:
+        if body.startswith(dr, offset):
+            offset += len(dr)
+            break
+    else:
+        return None, 'Invalid request type.'
+    if not body.startswith(settings.title_header, offset):
+        return None, 'Title not found.'
+    offset += len(settings.title_header)
+    pos = body.find('\n', offset)
+    if pos == -1:
+        pos = len(body)
+    return {'title': body[offset:pos],
+            'description': body[pos:]}, None
+
+
+def process_issue_comment(body, issue_state, issue_api) -> str:
+    action = body['action']
+    if (action != 'created' and action != 'edited') or \
+       issue_state != 'open':
+        return 'Not needed.'
+    comment = body['comment']
+    author = comment['user']['login']
+    comment, error = parse_comment(comment['body'])
+    if error:
+        print('Error during request processing: {}'.format(error))
+        github.send_comment(error, issue_api, author)
+    elif comment:
+        print('Request is processed ok')
+        if action == 'edited':
+            github.send_comment('Accept edited.', issue_api, author)
+        else:
+            github.send_comment('Accept.', issue_api, author)
+    else:
+        print('Ignore non-request comments')
+    return 'Doc request is processed.'
+
+
+def process_issue_state_change(body, issue_api, issue_url, doc_repo_url):
+    action = body['action']
+    if action != 'closed':
+        return 'Not needed.'
+    comments = github.get_comments(issue_api)
+    for c in comments:
+        comment, error = parse_comment(c['body'])
+        if comment:
+            github.create_issue(comment["title"], comment["description"],
+                                issue_url, c['user']['login'], doc_repo_url)
+    return 'Issue is processed.'
+
+
+def process_commit(c, is_master_push, doc_repo_url):
+    body = c['message']
+    request_pos = body.find(settings.bot_name)
+    if request_pos == -1:
+        return
+    request = body[request_pos:]
+    author = c['author']['username']
+    comment, error = parse_comment(request)
+    if error:
+        print('Error during request processing: {}'.format(error))
+        create_event(author, 'process_commit', error)
+    else:
+        create_event(author, 'process_commit', 'Accept')
+        if is_master_push:
+            github.create_issue(comment['title'],
+                                comment['description'], c['url'],
+                                author, doc_repo_url)

docbot/handlers.py

@@ -0,0 +1,16 @@
+import os
+
+token = os.environ.get('GITHUB_TOKEN')
+assert token is not None
+doc_requests = [' document\r\n', ' document\n']
+bot_name = '@TarantoolBot'
+title_header = 'Title:'
+api = 'https://api.github.com/repos/tarantool/'
+doc_repo_urls = {
+    f'tarantool/tarantool': f'{api}doc',
+    f'tarantool/tarantool-ee': f'{api}enterprise_doc',
+    f'tarantool/sdk': f'{api}enterprise_doc',
+    f'tarantool/tdg': f'{api}tdg-doc',
+    f'tarantool/tdg2': f'{api}tdg-doc',
+}
+LAST_EVENTS_SIZE = 30

docbot/processors.py

@@ -0,0 +1,18 @@
+import datetime
+
+from . import settings
+
+last_events = []
+
+
+def create_event(author, action, body):
+    time = datetime.datetime.now().strftime('%b %d %H:%M:%S')
+    result = '<tr>'
+    result += '<td>{}</td>'.format(time)
+    result += '<td>{}</td>'.format(author)
+    result += '<td>{}</td>'.format(action)
+    result += '<td>{}</td>'.format(body)
+    result += '</tr>'
+    last_events.append(result)
+    if len(last_events) > settings.LAST_EVENTS_SIZE:
+        last_events.pop(0)

docbot/settings.py

@@ -1,3 +1,14 @@
-bottle==0.12.13
-requests>=2.20.0
-gunicorn==19.9.0
+blinker==1.4
+certifi==2022.6.15
+charset-normalizer==2.0.12
+click==8.1.3
+elastic-apm==6.9.1
+Flask==2.1.2
+gunicorn==20.1.0
+idna==3.3
+itsdangerous==2.1.2
+Jinja2==3.1.2
+MarkupSafe==2.1.1
+requests==2.28.0
+urllib3==1.26.9
+Werkzeug==2.1.2

docbot/utils.py

@@ -1 +1 @@
-python-2.7.15
+python-3.10.4