Refactor docbot service

.github/workflows/deploy.yml

@@ -0,0 +1,21 @@
+name: Deploy to Dokku
+
+on:
+  push:
+    branches:
+      - master
+jobs:
+  deploy:
+    runs-on: ubuntu-20.04-self-hosted
+    steps:
+      - uses: actions/checkout@v2
+        with:
+            fetch-depth: 0
+      - id: deploy
+        name: Deploy to dokku
+        uses: idoberko2/dokku-deploy-github-action@v1
+        with:
+            ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+            dokku-host: ${{ secrets.DOCBOT_HOST }}
+            app-name: ${{ secrets.DOCBOT_APP }}
+            git-push-flags: '--force'

.gitignore

@@ -0,0 +1,5 @@
+venv
+.idea
+__pycache__
+*.pyc
+*.pyi

Dockerfile

@@ -1,17 +1,13 @@
-FROM python:2
+FROM python:3.10.4-alpine
 
-RUN mkdir -p /usr/src/bot
-WORKDIR /usr/src/bot
+COPY requirements.txt /tmp
+RUN pip install --upgrade pip --no-cache-dir && \
+    pip install -r tmp/requirements.txt --no-cache-dir
 
-RUN pip install --upgrade pip
-
-COPY requirements.txt /usr/src/bot
-COPY tarantoolbot.py /usr/src/bot
-COPY write_credentials.sh /usr/src/bot
-
-RUN pip install -r requirements.txt
+COPY docbot /app/docbot
 
 ENV PORT=5000
 EXPOSE 5000
 
-CMD gunicorn --bind 0.0.0.0:$PORT tarantoolbot:app
+WORKDIR /app
+CMD gunicorn --bind 0.0.0.0:$PORT docbot.app:app

Procfile

@@ -1 +1 @@
-web: gunicorn --bind 0.0.0.0:$PORT tarantoolbot:app
+web: gunicorn --bind 0.0.0.0:$PORT docbot.app:app

docbot/app.py

@@ -0,0 +1,23 @@
+from elasticapm.contrib.flask import ElasticAPM
+from flask import Flask, request
+
+from .handlers import webhook_handler, list_events_handler
+
+app = Flask(__name__)
+
+app.config['ELASTIC_APM'] = {
+    'SERVICE_NAME': 'docbot',
+}
+apm = ElasticAPM(app)
+
+
+@app.route("/", methods=['GET'])
+def index() -> str:
+    return list_events_handler()
+
+
+@app.route("/", methods=['POST'])
+def webhook() -> str:
+    data: dict = request.json
+    event: str = request.headers.get('X-GitHub-Event')
+    return webhook_handler(data, event)

docbot/github.py

@@ -0,0 +1,39 @@
+import requests
+
+from .utils import create_event
+
+
+class GitHub:
+    def __init__(self, token: str) -> None:
+        self.token: str = token
+
+    def send_comment(self, body, issue_api, to):
+        create_event(to, 'send_comment', body)
+        body = {'body': '@{}: {}'.format(to, body)}
+        url = '{}/comments'.format(issue_api)
+        status_code = self._send_request(url, body)
+        print('Sent comment: {}'.format(status_code))
+
+    def get_comments(self, issue_api) -> dict:
+        body = {'since': '1970-01-01T00:00:00Z'}
+        url = '{}/comments'.format(issue_api)
+        r = self._send_request(url, body, method='get')
+        return r.json()
+
+    def create_issue(self, title, description, src_url, author, doc_repo_url):
+        create_event(author, 'create_issue', title)
+        description = '{}\nRequested by @{} in {}.'.format(description, author,
+                                                           src_url)
+        body = {'title': title, 'body': description}
+        url = '{}/issues'.format(doc_repo_url)
+        status_code = self._send_request(url, body)
+        print('Created issue: {}'.format(status_code))
+
+    def _send_request(self, url, body, method='post'):
+        headers = {
+            'Authorization': 'token {}'.format(self.token),
+        }
+        r = requests.request(method, url, json=body, headers=headers)
+        print(r.status_code)
+        r.raise_for_status()
+        return r

docbot/handlers.py

@@ -0,0 +1,44 @@
+from .utils import last_events
+from . import settings
+from .processors import process_issue_comment, process_issue_state_change, \
+    process_commit
+
+
+def webhook_handler(data: dict, event: str) -> str:
+    if event is None or data is None:
+        return 'Event or data was not found'
+
+    if 'issue' in data:
+        issue = data['issue']
+        if 'state' not in issue:
+            return 'Event is not needed.'
+        issue_state = issue['state']
+        issue_api = issue['url']
+        issue_url = issue['html_url']
+        issue_repo = data['repository']['full_name']
+
+        if event == 'issue_comment':
+            return process_issue_comment(data, issue_state,
+                                         issue_api)
+        elif event == 'issues':
+            doc_repo_url = settings.doc_repo_urls.get(issue_repo)
+            return process_issue_state_change(data, issue_api,
+                                              issue_url, doc_repo_url)
+        else:
+            return 'Event "{}" is not needed.'.format(event)
+    elif event == 'push':
+        repo = data['repository']
+        branch = '/'.join(data['ref'].split('/')[2:])
+        is_master_push = repo['master_branch'] == branch
+        issue_repo = settings.doc_repo_urls.get(repo['full_name'])
+        for c in data['commits']:
+            process_commit(c, is_master_push, issue_repo)
+        return 'Webhook was processed'
+    else:
+        return 'Event is not needed.'
+
+
+def list_events_handler() -> str:
+    return ('<h1>{}</h1><table border="1" cellspacing="2" ' +
+            'cellpadding="2">{}</table>').format('TarantoolBot Journal',
+                                                 ' '.join(last_events))

docbot/processors.py

@@ -0,0 +1,79 @@
+from . import settings
+from .github import GitHub
+from .utils import create_event
+
+github = GitHub(settings.token)
+
+
+def parse_comment(body):
+    if not body.startswith(settings.bot_name):
+        return None, None
+    offset = len(settings.bot_name)
+    for dr in settings.doc_requests:
+        if body.startswith(dr, offset):
+            offset += len(dr)
+            break
+    else:
+        return None, 'Invalid request type.'
+    if not body.startswith(settings.title_header, offset):
+        return None, 'Title not found.'
+    offset += len(settings.title_header)
+    pos = body.find('\n', offset)
+    if pos == -1:
+        pos = len(body)
+    return {'title': body[offset:pos],
+            'description': body[pos:]}, None
+
+
+def process_issue_comment(body, issue_state, issue_api) -> str:
+    action = body['action']
+    if (action != 'created' and action != 'edited') or \
+       issue_state != 'open':
+        return 'Not needed.'
+    comment = body['comment']
+    author = comment['user']['login']
+    comment, error = parse_comment(comment['body'])
+    if error:
+        print('Error during request processing: {}'.format(error))
+        github.send_comment(error, issue_api, author)
+    elif comment:
+        print('Request is processed ok')
+        if action == 'edited':
+            github.send_comment('Accept edited.', issue_api, author)
+        else:
+            github.send_comment('Accept.', issue_api, author)
+    else:
+        print('Ignore non-request comments')
+    return 'Doc request is processed.'
+
+
+def process_issue_state_change(body, issue_api, issue_url, doc_repo_url):
+    action = body['action']
+    if action != 'closed':
+        return 'Not needed.'
+    comments = github.get_comments(issue_api)
+    for c in comments:
+        comment, error = parse_comment(c['body'])
+        if comment:
+            github.create_issue(comment["title"], comment["description"],
+                                issue_url, c['user']['login'], doc_repo_url)
+    return 'Issue is processed.'
+
+
+def process_commit(c, is_master_push, doc_repo_url):
+    body = c['message']
+    request_pos = body.find(settings.bot_name)
+    if request_pos == -1:
+        return
+    request = body[request_pos:]
+    author = c['author']['username']
+    comment, error = parse_comment(request)
+    if error:
+        print('Error during request processing: {}'.format(error))
+        create_event(author, 'process_commit', error)
+    else:
+        create_event(author, 'process_commit', 'Accept')
+        if is_master_push:
+            github.create_issue(comment['title'],
+                                comment['description'], c['url'],
+                                author, doc_repo_url)

docbot/settings.py

@@ -0,0 +1,16 @@
+import os
+
+token = os.environ.get('GITHUB_TOKEN')
+assert token is not None
+doc_requests = [' document\r\n', ' document\n']
+bot_name = '@TarantoolBot'
+title_header = 'Title:'
+api = 'https://api.github.com/repos/tarantool/'
+doc_repo_urls = {
+    f'tarantool/tarantool': f'{api}doc',
+    f'tarantool/tarantool-ee': f'{api}enterprise_doc',
+    f'tarantool/sdk': f'{api}enterprise_doc',
+    f'tarantool/tdg': f'{api}tdg-doc',
+    f'tarantool/tdg2': f'{api}tdg-doc',
+}
+LAST_EVENTS_SIZE = 30

docbot/utils.py

@@ -0,0 +1,18 @@
+import datetime
+
+from . import settings
+
+last_events = []
+
+
+def create_event(author, action, body):
+    time = datetime.datetime.now().strftime('%b %d %H:%M:%S')
+    result = '<tr>'
+    result += '<td>{}</td>'.format(time)
+    result += '<td>{}</td>'.format(author)
+    result += '<td>{}</td>'.format(action)
+    result += '<td>{}</td>'.format(body)
+    result += '</tr>'
+    last_events.append(result)
+    if len(last_events) > settings.LAST_EVENTS_SIZE:
+        last_events.pop(0)

requirements.txt

@@ -1,3 +1,14 @@
-bottle==0.12.20
-requests>=2.20.0
-gunicorn==19.9.0
+blinker==1.4
+certifi==2022.6.15
+charset-normalizer==2.0.12
+click==8.1.3
+elastic-apm==6.9.1
+Flask==2.1.2
+gunicorn==20.1.0
+idna==3.3
+itsdangerous==2.1.2
+Jinja2==3.1.2
+MarkupSafe==2.1.1
+requests==2.28.0
+urllib3==1.26.9
+Werkzeug==2.1.2

runtime.txt

@@ -1 +1 @@
-python-2.7.15
+python-3.10.4