diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..3bd475a --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,21 @@ +name: Deploy to Dokku + +on: + push: + branches: + - master +jobs: + deploy: + runs-on: ubuntu-20.04-self-hosted + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - id: deploy + name: Deploy to dokku + uses: idoberko2/dokku-deploy-github-action@v1 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + dokku-host: ${{ secrets.DOCBOT_HOST }} + app-name: ${{ secrets.DOCBOT_APP }} + git-push-flags: '--force' diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..82f30bf --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +venv +.idea +__pycache__ +*.pyc +*.pyi diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index fbc6dc4..0000000 --- a/.travis.yml +++ /dev/null @@ -1,28 +0,0 @@ -sudo: required -services: - - docker - -branches: - only: - - master - -env: CONTAINER_NAME=tarantool/docbot:latest - -script: - - docker build -t ${CONTAINER_NAME} . - -after_success: - - docker login -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD} - - docker push ${CONTAINER_NAME} - - - "( apt-get update -y && apt-get install openssh-client -y ) || ( which ssh-agent - || yum install openssh -y )" - - - openssl aes-256-cbc -K $encrypted_2272aa30aa52_key -iv $encrypted_2272aa30aa52_iv - -in travis_key.enc -out travis_key -d - - - eval $(ssh-agent -s) - - chmod 600 travis_key - - ssh-add travis_key - - - ssh -o "StrictHostKeyChecking no" docbot-deployer@try.tarantool.org diff --git a/Dockerfile b/Dockerfile index 5a0546d..eb4bc95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,13 @@ -FROM python:2 +FROM python:3.10.4-alpine -RUN mkdir -p /usr/src/bot -WORKDIR /usr/src/bot +COPY requirements.txt /tmp +RUN pip install --upgrade pip --no-cache-dir && \ + pip install -r tmp/requirements.txt --no-cache-dir -RUN pip install --upgrade pip - -COPY requirements.txt /usr/src/bot -COPY tarantoolbot.py /usr/src/bot -COPY write_credentials.sh /usr/src/bot - -RUN pip install -r requirements.txt +COPY docbot /app/docbot ENV PORT=5000 EXPOSE 5000 -CMD gunicorn --bind 0.0.0.0:$PORT tarantoolbot:app +WORKDIR /app +CMD gunicorn --bind 0.0.0.0:$PORT docbot.app:app diff --git a/Procfile b/Procfile index 9321b17..9180f6d 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: gunicorn --bind 0.0.0.0:$PORT tarantoolbot:app \ No newline at end of file +web: gunicorn --bind 0.0.0.0:$PORT docbot.app:app diff --git a/docbot/__init__.py b/docbot/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/docbot/app.py b/docbot/app.py new file mode 100644 index 0000000..5f56060 --- /dev/null +++ b/docbot/app.py @@ -0,0 +1,23 @@ +from elasticapm.contrib.flask import ElasticAPM +from flask import Flask, request + +from .handlers import webhook_handler, list_events_handler + +app = Flask(__name__) + +app.config['ELASTIC_APM'] = { + 'SERVICE_NAME': 'docbot', +} +apm = ElasticAPM(app) + + +@app.route("/", methods=['GET']) +def index() -> str: + return list_events_handler() + + +@app.route("/", methods=['POST']) +def webhook() -> str: + data: dict = request.json + event: str = request.headers.get('X-GitHub-Event') + return webhook_handler(data, event) diff --git a/docbot/github.py b/docbot/github.py new file mode 100644 index 0000000..90121df --- /dev/null +++ b/docbot/github.py @@ -0,0 +1,39 @@ +import requests + +from .utils import create_event + + +class GitHub: + def __init__(self, token: str) -> None: + self.token: str = token + + def send_comment(self, body, issue_api, to): + create_event(to, 'send_comment', body) + body = {'body': '@{}: {}'.format(to, body)} + url = '{}/comments'.format(issue_api) + status_code = self._send_request(url, body) + print('Sent comment: {}'.format(status_code)) + + def get_comments(self, issue_api) -> dict: + body = {'since': '1970-01-01T00:00:00Z'} + url = '{}/comments'.format(issue_api) + r = self._send_request(url, body, method='get') + return r.json() + + def create_issue(self, title, description, src_url, author, doc_repo_url): + create_event(author, 'create_issue', title) + description = '{}\nRequested by @{} in {}.'.format(description, author, + src_url) + body = {'title': title, 'body': description} + url = '{}/issues'.format(doc_repo_url) + status_code = self._send_request(url, body) + print('Created issue: {}'.format(status_code)) + + def _send_request(self, url, body, method='post'): + headers = { + 'Authorization': 'token {}'.format(self.token), + } + r = requests.request(method, url, json=body, headers=headers) + print(r.status_code) + r.raise_for_status() + return r diff --git a/docbot/handlers.py b/docbot/handlers.py new file mode 100644 index 0000000..f32bebf --- /dev/null +++ b/docbot/handlers.py @@ -0,0 +1,44 @@ +from .utils import last_events +from . import settings +from .processors import process_issue_comment, process_issue_state_change, \ + process_commit + + +def webhook_handler(data: dict, event: str) -> str: + if event is None or data is None: + return 'Event or data was not found' + + if 'issue' in data: + issue = data['issue'] + if 'state' not in issue: + return 'Event is not needed.' + issue_state = issue['state'] + issue_api = issue['url'] + issue_url = issue['html_url'] + issue_repo = data['repository']['full_name'] + + if event == 'issue_comment': + return process_issue_comment(data, issue_state, + issue_api) + elif event == 'issues': + doc_repo_url = settings.doc_repo_urls.get(issue_repo) + return process_issue_state_change(data, issue_api, + issue_url, doc_repo_url) + else: + return 'Event "{}" is not needed.'.format(event) + elif event == 'push': + repo = data['repository'] + branch = '/'.join(data['ref'].split('/')[2:]) + is_master_push = repo['master_branch'] == branch + issue_repo = settings.doc_repo_urls.get(repo['full_name']) + for c in data['commits']: + process_commit(c, is_master_push, issue_repo) + return 'Webhook was processed' + else: + return 'Event is not needed.' + + +def list_events_handler() -> str: + return ('

{}

{}
').format('TarantoolBot Journal', + ' '.join(last_events)) diff --git a/docbot/processors.py b/docbot/processors.py new file mode 100644 index 0000000..dcd07dc --- /dev/null +++ b/docbot/processors.py @@ -0,0 +1,79 @@ +from . import settings +from .github import GitHub +from .utils import create_event + +github = GitHub(settings.token) + + +def parse_comment(body): + if not body.startswith(settings.bot_name): + return None, None + offset = len(settings.bot_name) + for dr in settings.doc_requests: + if body.startswith(dr, offset): + offset += len(dr) + break + else: + return None, 'Invalid request type.' + if not body.startswith(settings.title_header, offset): + return None, 'Title not found.' + offset += len(settings.title_header) + pos = body.find('\n', offset) + if pos == -1: + pos = len(body) + return {'title': body[offset:pos], + 'description': body[pos:]}, None + + +def process_issue_comment(body, issue_state, issue_api) -> str: + action = body['action'] + if (action != 'created' and action != 'edited') or \ + issue_state != 'open': + return 'Not needed.' + comment = body['comment'] + author = comment['user']['login'] + comment, error = parse_comment(comment['body']) + if error: + print('Error during request processing: {}'.format(error)) + github.send_comment(error, issue_api, author) + elif comment: + print('Request is processed ok') + if action == 'edited': + github.send_comment('Accept edited.', issue_api, author) + else: + github.send_comment('Accept.', issue_api, author) + else: + print('Ignore non-request comments') + return 'Doc request is processed.' + + +def process_issue_state_change(body, issue_api, issue_url, doc_repo_url): + action = body['action'] + if action != 'closed': + return 'Not needed.' + comments = github.get_comments(issue_api) + for c in comments: + comment, error = parse_comment(c['body']) + if comment: + github.create_issue(comment["title"], comment["description"], + issue_url, c['user']['login'], doc_repo_url) + return 'Issue is processed.' + + +def process_commit(c, is_master_push, doc_repo_url): + body = c['message'] + request_pos = body.find(settings.bot_name) + if request_pos == -1: + return + request = body[request_pos:] + author = c['author']['username'] + comment, error = parse_comment(request) + if error: + print('Error during request processing: {}'.format(error)) + create_event(author, 'process_commit', error) + else: + create_event(author, 'process_commit', 'Accept') + if is_master_push: + github.create_issue(comment['title'], + comment['description'], c['url'], + author, doc_repo_url) diff --git a/docbot/settings.py b/docbot/settings.py new file mode 100644 index 0000000..bb4e979 --- /dev/null +++ b/docbot/settings.py @@ -0,0 +1,16 @@ +import os + +token = os.environ.get('GITHUB_TOKEN') +assert token is not None +doc_requests = [' document\r\n', ' document\n'] +bot_name = '@TarantoolBot' +title_header = 'Title:' +api = 'https://api.github.com/repos/tarantool/' +doc_repo_urls = { + f'tarantool/tarantool': f'{api}doc', + f'tarantool/tarantool-ee': f'{api}enterprise_doc', + f'tarantool/sdk': f'{api}enterprise_doc', + f'tarantool/tdg': f'{api}tdg-doc', + f'tarantool/tdg2': f'{api}tdg-doc', +} +LAST_EVENTS_SIZE = 30 diff --git a/docbot/utils.py b/docbot/utils.py new file mode 100644 index 0000000..c77367d --- /dev/null +++ b/docbot/utils.py @@ -0,0 +1,18 @@ +import datetime + +from . import settings + +last_events = [] + + +def create_event(author, action, body): + time = datetime.datetime.now().strftime('%b %d %H:%M:%S') + result = '' + result += '{}'.format(time) + result += '{}'.format(author) + result += '{}'.format(action) + result += '{}'.format(body) + result += '' + last_events.append(result) + if len(last_events) > settings.LAST_EVENTS_SIZE: + last_events.pop(0) diff --git a/requirements.txt b/requirements.txt index cc0039a..f30bd69 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,14 @@ -bottle==0.12.20 -requests>=2.20.0 -gunicorn==19.9.0 +blinker==1.4 +certifi==2022.6.15 +charset-normalizer==2.0.12 +click==8.1.3 +elastic-apm==6.9.1 +Flask==2.1.2 +gunicorn==20.1.0 +idna==3.3 +itsdangerous==2.1.2 +Jinja2==3.1.2 +MarkupSafe==2.1.1 +requests==2.28.0 +urllib3==1.26.9 +Werkzeug==2.1.2 diff --git a/runtime.txt b/runtime.txt index f27f1cc..73e24dc 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -python-2.7.15 +python-3.10.4 diff --git a/tarantoolbot.py b/tarantoolbot.py deleted file mode 100644 index d9cad94..0000000 --- a/tarantoolbot.py +++ /dev/null @@ -1,262 +0,0 @@ -from bottle import run, post, get, request, default_app -import requests -import json -import datetime -import os - -doc_requests = [' document\r\n', ' document\n'] -bot_name = '@TarantoolBot' -title_header = 'Title:' -doc_repo_url = 'https://api.github.com/repos/tarantool/doc' - -if os.getenv('GITHUB_TOKEN'): - token = os.getenv('GITHUB_TOKEN') -else: - with open('credentials.json') as f: - credentials = json.loads(f.read()) - token = credentials['GitHub']['token'] - -LAST_EVENTS_SIZE = 30 -last_events = [] - -## -# Log an event at the end of the events queue. Once the queue -# reaches size LAST_EVENTS_SIZE, the oldest event is deleted. -# @param author GitHub login of the author. -# @param action What to log. -# @param body Message body. - - -def create_event(author, action, body): - time = datetime.datetime.now().strftime('%b %d %H:%M:%S') - result = '' - result += '{}'.format(time) - result += '{}'.format(author) - result += '{}'.format(action) - result += '{}'.format(body) - result += '' - last_events.append(result) - if len(last_events) > LAST_EVENTS_SIZE: - last_events.pop(0) - -## -# Bot to track documentation update requests -# and open issues in doc repository with a -# specified body and title, when an issue is -# closed. -# - -## -# Parse a comment content. Try to -# extract title and body. -# @param body String comment. -# -# @retval None, None - @A body is not a request. -# @retval None, not None - @A body is a request, -# but an error occured. The second retval -# is the error string. -# @retval not None - @A body is a well-formatted -# request. Returned value is a dictionary -# with 'title' and 'description'. - - -def parse_comment(body): - if not body.startswith(bot_name): - return None, None - offset = len(bot_name) - for dr in doc_requests: - if body.startswith(dr, offset): - offset += len(dr) - break - else: - return None, 'Invalid request type.' - if not body.startswith(title_header, offset): - return None, 'Title not found.' - offset += len(title_header) - pos = body.find('\n', offset) - if pos == -1: - pos = len(body) - return {'title': body[offset:pos], - 'description': body[pos:]}, None -## -# Send a comment to an issue. Sent comment is either -# parser error message, or notification about accepting -# a request. - - -def send_comment(body, issue_api, to): - create_event(to, 'send_comment', body) - body = {'body': '@{}: {}'.format(to, body)} - url = '{}/comments'.format(issue_api) - headers = { - 'Authorization': 'token {}'.format(token), - } - r = requests.post(url, json=body, headers=headers) - r.raise_for_status() - print('Sent comment: {}'.format(r.status_code)) - -## -# Get all the comments of an issue. - - -def get_comments(issue_api): - body = {'since': '1970-01-01T00:00:00Z'} - url = '{}/comments'.format(issue_api) - headers = { - 'Authorization': 'token {}'.format(token), - } - r = requests.get(url, json=body, headers=headers) - r.raise_for_status() - return r.json() - -## -# Open a new issue in a documentation repository. -# @param title Issue title. -# @param description Issue description. -# @param src_url Link to a webpage with the original issue or -# commit. -# @param author Github login of the request author. - - -def create_issue(title, description, src_url, author): - create_event(author, 'create_issue', title) - description = '{}\nRequested by @{} in {}.'.format(description, author, - src_url) - body = {'title': title, 'body': description} - url = '{}/issues'.format(doc_repo_url) - headers = { - 'Authorization': 'token {}'.format(token), - } - r = requests.post(url, json=body, headers=headers) - r.raise_for_status() - print('Created issue: {}'.format(r.status_code)) - -## -# Check that a new or edited comment for an issue is the request. -# If it is, then try to parse it and send to a caller -# notification about a result. -# @param body GitHub hook body. -# @param issue_state Issue status: closed, open, created. -# @param issue_api URL of the commented issue. Here a -# response is wrote. -# -# @retval Response to a GitHub hook. - - -def process_issue_comment(body, issue_state, issue_api): - action = body['action'] - if (action != 'created' and action != 'edited') or \ - issue_state != 'open': - return 'Not needed.' - comment = body['comment'] - author = comment['user']['login'] - comment, error = parse_comment(comment['body']) - if error: - print('Error during request processing: {}'.format(error)) - send_comment(error, issue_api, author) - elif comment: - print('Request is processed ok') - if action == 'edited': - send_comment('Accept edited.', issue_api, author) - else: - send_comment('Accept.', issue_api, author) - else: - print('Ignore non-request comments') - return 'Doc request is processed.' - -## -# Check that a just closed issue contains contains -# a well-formatted doc update request. If it does, then -# open a new issue in doc repository. For multiple doc requests -# multiple issues will be created. -# @param body GitHub hook body. -# @param issue_state Issue status: closed, open, created. -# @param issue_api API URL of the original issue. -# @param issue_url Public URL of the original issue. -# -# @retval Response to a GitHub hook. - - -def process_issue_state_change(body, issue_state, issue_api, issue_url): - action = body['action'] - if action != 'closed': - return 'Not needed.' - comments = get_comments(issue_api) - comment = None - for c in comments: - comment, error = parse_comment(c['body']) - if comment: - create_issue(comment["title"], comment["description"], - issue_url, c['user']['login']) - return 'Issue is processed.' - -## -# Process a commit event, triggered on any push. If in the commit -# message a request is found, then parse it and create an issue on -# documentation, if the push is done into master. -# @param c Commit object. -# @param is_master_push True if the commit is pushed into the -# master branch. - - -def process_commit(c, is_master_push): - body = c['message'] - request_pos = body.find(bot_name) - if request_pos == -1: - return - request = body[request_pos:] - author = c['author']['username'] - comment, error = parse_comment(request) - if error: - print('Error during request processing: {}'.format(error)) - create_event(author, 'process_commit', error) - else: - create_event(author, 'process_commit', 'Accept') - if is_master_push: - create_issue(comment['title'], comment['description'], - c['url'], author) - - -@post('/') -def index_post(): - r = request.json - t = request.get_header('X-GitHub-Event') - if 'issue' in r: - issue = r['issue'] - if not 'state' in issue: - return 'Event is not needed.' - issue_state = issue['state'] - issue_api = issue['url'] - issue_url = issue['html_url'] - - if t == 'issue_comment': - return process_issue_comment(r, issue_state, issue_api) - elif t == 'issues': - return process_issue_state_change(r, issue_state, - issue_api, issue_url) - else: - return 'Event "{}" is not needed.'.format(t) - elif t == 'push': - repo = r['repository'] - # Strip 'refs/heads/' from beginning. - branch = '/'.join(r['ref'].split('/')[2:]) - is_master_push = repo['master_branch'] == branch - for c in r['commits']: - process_commit(c, is_master_push) - else: - return 'Event is not needed.' - - -@get('/') -def index_get(): - return ('

{}

{}
').format('TarantoolBot Journal', - ' '.join(last_events)) - - -print('Starting bot') -if __name__ == "__main__": - run(host='0.0.0.0', port=5000) - -# allow to run under gunicorn -app = default_app() diff --git a/travis_key.enc b/travis_key.enc deleted file mode 100644 index a147052..0000000 Binary files a/travis_key.enc and /dev/null differ diff --git a/write_credentials.sh b/write_credentials.sh deleted file mode 100755 index ea615bc..0000000 --- a/write_credentials.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -touch credentials.json - -echo -e "{" > credentials.json -echo -e "\t\"GitHub\": {" >> credentials.json -echo -e "\t\t\"token\": \"${GITHUB_TOKEN}\"" >> credentials.json -echo -e "\t}\n}" >> credentials.json