diff --git a/doajtest/helpers.py b/doajtest/helpers.py
index a8f33d52d5..84441a4fc6 100644
--- a/doajtest/helpers.py
+++ b/doajtest/helpers.py
@@ -145,7 +145,14 @@ def create_app_patch(cls):
'ENABLE_EMAIL': False,
"FAKER_SEED": 1,
"EVENT_SEND_FUNCTION": "portality.events.shortcircuit.send_event",
- 'CMS_BUILD_ASSETS_ON_STARTUP': False
+ 'CMS_BUILD_ASSETS_ON_STARTUP': False,
+
+ # disable send plausible request
+ 'PLAUSIBLE_URL': '',
+ 'PLAUSIBLE_API_URL': '',
+
+ 'RATE_LIMITS_PER_MIN_DEFAULT': 1000000,
+ 'RATE_LIMITS_PER_MIN_T2 ': 1000000,
}
@classmethod
diff --git a/doajtest/unit/api_tests/test_api_rate_limited.py b/doajtest/unit/api_tests/test_api_rate_limited.py
new file mode 100644
index 0000000000..ce9d5b25d8
--- /dev/null
+++ b/doajtest/unit/api_tests/test_api_rate_limited.py
@@ -0,0 +1,80 @@
+import json
+
+from doajtest import fixtures
+from doajtest.helpers import DoajTestCase, patch_config
+from portality.app import setup_dev_log
+from portality.bll import DOAJ
+from portality.core import app
+from portality.models import Account
+
+api_rate_serv = DOAJ.apiRateService()
+
+
+def send_test_req(client, api_key):
+ url = '/api/search/applications/issn%3A0000-0000?api_key=' + api_key
+ return client.get(url)
+
+
+def assert_is_too_many_requests(resp):
+ data = json.loads(resp.data)
+ assert data['status'] == 'too_many_requests'
+ assert resp.status_code == 429
+
+
+def assert_not_too_many_requests(resp):
+ data = json.loads(resp.data)
+ assert data['query']
+ assert resp.status_code != 429
+
+
+class TestApiRateLimited(DoajTestCase):
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ super().setUpClass()
+ cls.originals = patch_config(
+ app,
+ {
+ 'RATE_LIMITS_PER_MIN_DEFAULT': 10,
+ 'RATE_LIMITS_PER_MIN_T2 ': 1000,
+ })
+ setup_dev_log()
+
+ def setUp(self):
+ super().setUp()
+ self.t2_user = Account(**fixtures.accounts.PUBLISHER_SOURCE)
+ self.t2_user.generate_api_key()
+ self.t2_user.add_role('api_rate_t2')
+
+ self.normal_user = Account(**fixtures.accounts.PUBLISHER_B_SOURCE)
+ self.normal_user.generate_api_key()
+
+ Account.save_all([self.normal_user, self.t2_user])
+ Account.refresh()
+
+ def send_multi_req_more_than_default_limit(self, api_key=None):
+ with self.app_test.test_client() as t_client:
+ for _ in range(app.config['RATE_LIMITS_PER_MIN_DEFAULT'] + 5):
+ resp = send_test_req(t_client, api_key=api_key)
+ return resp
+
+ def send_one_req(self, api_key=None):
+ with self.app_test.test_client() as t_client:
+ resp = send_test_req(t_client, api_key=api_key)
+ return resp
+
+ def test_normal__normal_user(self):
+ resp = self.send_one_req(api_key=self.normal_user.api_key)
+ assert_not_too_many_requests(resp)
+
+ def test_normal__t2_user(self):
+ resp = self.send_one_req(api_key=self.t2_user.api_key)
+ assert_not_too_many_requests(resp)
+
+ def test_multi_req__too_many_requests__normal_user(self):
+ resp = self.send_multi_req_more_than_default_limit(api_key=self.normal_user.api_key)
+ assert_is_too_many_requests(resp)
+
+ def test_multi_req__normal__t2_user(self):
+ resp = self.send_multi_req_more_than_default_limit(api_key=self.t2_user.api_key)
+ assert_not_too_many_requests(resp)
diff --git a/docs/dictionary.md b/docs/dictionary.md
index 49ba798b8f..02ee217810 100644
--- a/docs/dictionary.md
+++ b/docs/dictionary.md
@@ -1,10 +1,11 @@
-| Short | Description |
-|---------|------------------------------|
-| bgjob | background job |
-| noti | notification |
-| noqa | NO-QA (NO Quality Assurance) |
-| inst | instance |
-| fmt | format |
-| exparam | extra parameter |
-| maned | Managing Editor |
-| gsheet | Google Sheet |
\ No newline at end of file
+| Short | Description |
+|----------|------------------------------|
+| bgjob | background job |
+| noti | notification |
+| noqa | NO-QA (NO Quality Assurance) |
+| inst | instance |
+| fmt | format |
+| exparam | extra parameter |
+| maned | Managing Editor |
+| gsheet | Google Sheet |
+| serv | service |
\ No newline at end of file
diff --git a/portality/api/common.py b/portality/api/common.py
index 5a5bbf9932..ce84c4ef02 100644
--- a/portality/api/common.py
+++ b/portality/api/common.py
@@ -1,10 +1,13 @@
-#~~API:Feature~~
-import json, uuid
-from portality.core import app
-from flask import request
+# ~~API:Feature~~
+import json
+import uuid
from copy import deepcopy
+
+from flask import request
from link_header import LinkHeader, Link
+from portality.core import app
+
LINK_HEADERS = ['next', 'prev', 'last']
TOTAL_RESULTS_COUNT = ['total']
@@ -16,17 +19,17 @@ class Api(object):
# ~~->Swagger:Feature~~
# ~~->API:Documentation~~
SWAG_TEMPLATE = {
- "description" : "",
+ "description": "",
"responses": {},
"parameters": [],
"tags": []
}
R200 = {"schema": {}, "description": "A successful request/response"}
R201 = {"schema": {"properties": CREATED_TEMPLATE}, "description": "Resource created successfully, response "
- "contains the new resource ID and location."}
- R201_BULK = {"schema": {"items": {"properties" : CREATED_TEMPLATE, "type" : "object"}, "type" : "array"},
- "description": "Resources created successfully, response contains the new resource IDs "
- "and locations."}
+ "contains the new resource ID and location."}
+ R201_BULK = {"schema": {"items": {"properties": CREATED_TEMPLATE, "type": "object"}, "type": "array"},
+ "description": "Resources created successfully, response contains the new resource IDs "
+ "and locations."}
R204 = {"description": "OK (Request succeeded), No Content"}
R400 = {"schema": {"properties": ERROR_TEMPLATE}, "description": "Bad Request. Your request body was missing a "
"required field, or the data in one of the "
@@ -125,6 +128,13 @@ class Api409Error(Exception):
pass
+class Api429Error(Exception):
+ """
+ Too many requests
+ """
+ pass
+
+
class Api500Error(Exception):
pass
@@ -201,7 +211,7 @@ def generate_link_headers(metadata):
links.append(Link(v, rel=k)) # e.g. Link("https://example.com/foo", rel="next")
return str(LinkHeader(links)) # RFC compliant headers e.g.
- # ; rel=next, ; rel=last
+ # ; rel=next, ; rel=last
def respond(data, status, metadata=None):
@@ -226,6 +236,16 @@ def respond(data, status, metadata=None):
return app.response_class(data, status, headers, mimetype='application/json')
+def resp_err(error, log_msg, status_code, status_msg):
+ err_ref_id = uuid.uuid1()
+ err_msg = str(error) + " (ref: {y})".format(y=err_ref_id)
+ app.logger.info(log_msg + f' -- {err_msg}')
+ t = deepcopy(ERROR_TEMPLATE)
+ t['status'] = status_msg
+ t['error'] = err_msg
+ return respond(json.dumps(t), status_code)
+
+
@app.errorhandler(Api400Error)
def bad_request(error):
magic = uuid.uuid1()
@@ -266,6 +286,13 @@ def forbidden(error):
return respond(json.dumps(t), 403)
+@app.errorhandler(Api429Error)
+def too_many_requests(error):
+ return resp_err(error,
+ f"Sending 429 Too Many Requests from client",
+ 429, 'too_many_requests')
+
+
@app.errorhandler(Api500Error)
def bad_request(error):
magic = uuid.uuid1()
diff --git a/portality/app.py b/portality/app.py
index efd9e5b170..309a3f4c14 100644
--- a/portality/app.py
+++ b/portality/app.py
@@ -9,34 +9,34 @@
~~DOAJ:WebApp~~
"""
+import logging
+import os
+import sys
+from datetime import datetime
-import os, sys
-import tzlocal
import pytz
-
+import tzlocal
from flask import request, abort, render_template, redirect, send_file, url_for, jsonify, send_from_directory
from flask_login import login_user, current_user
-from datetime import datetime
-
import portality.models as models
-from portality.core import app, es_connection, initialise_index
from portality import settings
+from portality.core import app, es_connection, initialise_index
from portality.lib import edges, dates
from portality.lib.dates import FMT_DATETIME_STD, FMT_YEAR
-
from portality.view.account import blueprint as account
from portality.view.admin import blueprint as admin
-from portality.view.publisher import blueprint as publisher
-from portality.view.query import blueprint as query
-from portality.view.doaj import blueprint as doaj
-from portality.view.oaipmh import blueprint as oaipmh
-from portality.view.openurl import blueprint as openurl
+from portality.view.apply import blueprint as apply
from portality.view.atom import blueprint as atom
-from portality.view.editor import blueprint as editor
+from portality.view.doaj import blueprint as doaj
from portality.view.doajservices import blueprint as services
+from portality.view.editor import blueprint as editor
from portality.view.jct import blueprint as jct
-from portality.view.apply import blueprint as apply
+from portality.view.oaipmh import blueprint as oaipmh
+from portality.view.openurl import blueprint as openurl
+from portality.view.publisher import blueprint as publisher
+from portality.view.query import blueprint as query
+
if 'api1' in app.config['FEATURES']:
from portality.view.api_v1 import blueprint as api_v1
if 'api2' in app.config['FEATURES']:
@@ -51,35 +51,35 @@
if app.config.get("DEBUG", False) and app.config.get("TESTDRIVE_ENABLED", False):
from portality.view.testdrive import blueprint as testdrive
-app.register_blueprint(account, url_prefix='/account') #~~->Account:Blueprint~~
-app.register_blueprint(admin, url_prefix='/admin') #~~-> Admin:Blueprint~~
-app.register_blueprint(publisher, url_prefix='/publisher') #~~-> Publisher:Blueprint~~
-app.register_blueprint(query, name='query', url_prefix='/query') # ~~-> Query:Blueprint~~
+app.register_blueprint(account, url_prefix='/account') # ~~->Account:Blueprint~~
+app.register_blueprint(admin, url_prefix='/admin') # ~~-> Admin:Blueprint~~
+app.register_blueprint(publisher, url_prefix='/publisher') # ~~-> Publisher:Blueprint~~
+app.register_blueprint(query, name='query', url_prefix='/query') # ~~-> Query:Blueprint~~
app.register_blueprint(query, name='admin_query', url_prefix='/admin_query')
app.register_blueprint(query, name='publisher_query', url_prefix='/publisher_query')
app.register_blueprint(query, name='editor_query', url_prefix='/editor_query')
app.register_blueprint(query, name='associate_query', url_prefix='/associate_query')
app.register_blueprint(query, name='dashboard_query', url_prefix="/dashboard_query")
-app.register_blueprint(editor, url_prefix='/editor') # ~~-> Editor:Blueprint~~
-app.register_blueprint(services, url_prefix='/service') # ~~-> Services:Blueprint~~
+app.register_blueprint(editor, url_prefix='/editor') # ~~-> Editor:Blueprint~~
+app.register_blueprint(services, url_prefix='/service') # ~~-> Services:Blueprint~~
if 'api1' in app.config['FEATURES']:
- app.register_blueprint(api_v1, url_prefix='/api/v1') # ~~-> APIv1:Blueprint~~
+ app.register_blueprint(api_v1, url_prefix='/api/v1') # ~~-> APIv1:Blueprint~~
if 'api2' in app.config['FEATURES']:
- app.register_blueprint(api_v2, url_prefix='/api/v2') # ~~-> APIv2:Blueprint~~
+ app.register_blueprint(api_v2, url_prefix='/api/v2') # ~~-> APIv2:Blueprint~~
if 'api3' in app.config['FEATURES']:
- app.register_blueprint(api_v3, name='api', url_prefix='/api') # ~~-> APIv3:Blueprint~~
- app.register_blueprint(api_v3, name='api_v3', url_prefix='/api/v3') # ~~-> APIv3:Blueprint~~
-app.register_blueprint(status, name='status', url_prefix='/status') # ~~-> Status:Blueprint~~
+ app.register_blueprint(api_v3, name='api', url_prefix='/api') # ~~-> APIv3:Blueprint~~
+ app.register_blueprint(api_v3, name='api_v3', url_prefix='/api/v3') # ~~-> APIv3:Blueprint~~
+app.register_blueprint(status, name='status', url_prefix='/status') # ~~-> Status:Blueprint~~
app.register_blueprint(status, name='_status', url_prefix='/_status')
-app.register_blueprint(apply, url_prefix='/apply') # ~~-> Apply:Blueprint~~
-app.register_blueprint(jct, url_prefix="/jct") # ~~-> JCT:Blueprint~~
-app.register_blueprint(dashboard, url_prefix="/dashboard") #~~-> Dashboard:Blueprint~~
+app.register_blueprint(apply, url_prefix='/apply') # ~~-> Apply:Blueprint~~
+app.register_blueprint(jct, url_prefix="/jct") # ~~-> JCT:Blueprint~~
+app.register_blueprint(dashboard, url_prefix="/dashboard") # ~~-> Dashboard:Blueprint~~
app.register_blueprint(tours, url_prefix="/tours") # ~~-> Tours:Blueprint~~
-app.register_blueprint(oaipmh) # ~~-> OAIPMH:Blueprint~~
-app.register_blueprint(openurl) # ~~-> OpenURL:Blueprint~~
-app.register_blueprint(atom) # ~~-> Atom:Blueprint~~
-app.register_blueprint(doaj) # ~~-> DOAJ:Blueprint~~
+app.register_blueprint(oaipmh) # ~~-> OAIPMH:Blueprint~~
+app.register_blueprint(openurl) # ~~-> OpenURL:Blueprint~~
+app.register_blueprint(atom) # ~~-> Atom:Blueprint~~
+app.register_blueprint(doaj) # ~~-> DOAJ:Blueprint~~
if app.config.get("DEBUG", False) and app.config.get("TESTDRIVE_ENABLED", False):
app.logger.warning('Enabling TESTDRIVE at /testdrive')
@@ -91,6 +91,7 @@
# putting it here ensures it will run under any web server
initialise_index(app, es_connection)
+
# serve static files from multiple potential locations
# this allows us to override the standard static file handling with our own dynamic version
# ~~-> Assets:WebRoute~~
@@ -108,11 +109,14 @@ def custom_static(path):
return send_from_directory(os.path.dirname(target), os.path.basename(target))
abort(404)
+
# Configure Analytics
# ~~-> PlausibleAnalytics:ExternalService~~
from portality.lib import plausible
+
plausible.create_logfile(app.config.get('PLAUSIBLE_LOG_DIR', None))
+
# Redirects from previous DOAJ app.
# RJ: I have decided to put these here so that they can be managed
# alongside the DOAJ codebase. I know they could also go into the
@@ -139,6 +143,7 @@ def legacy():
def another_legacy_csv_route():
return redirect("/csv"), 301
+
###################################################
# ~~-> DOAJArticleXML:Schema~~
@@ -146,9 +151,9 @@ def another_legacy_csv_route():
def legacy_doaj_XML_schema():
schema_fn = 'doajArticles.xsd'
return send_file(
- os.path.join(app.config.get("STATIC_DIR"), "doaj", schema_fn),
- mimetype="application/xml", as_attachment=True, attachment_filename=schema_fn
- )
+ os.path.join(app.config.get("STATIC_DIR"), "doaj", schema_fn),
+ mimetype="application/xml", as_attachment=True, attachment_filename=schema_fn
+ )
# ~~-> CrossrefArticleXML:WebRoute~~
@@ -182,7 +187,7 @@ def set_current_context():
"app": app,
"current_year": dates.now_str(FMT_YEAR),
"base_url": app.config.get('BASE_URL'),
- }
+ }
# Jinja2 Template Filters
@@ -193,7 +198,7 @@ def bytes_to_filesize(size):
units = ["bytes", "Kb", "Mb", "Gb"]
scale = 0
while size > 1000 and scale < len(units):
- size = float(size) / 1000.0 # note that it is no longer 1024
+ size = float(size) / 1000.0 # note that it is no longer 1024
scale += 1
return "{size:.1f}{unit}".format(size=size, unit=units[scale])
@@ -286,6 +291,7 @@ def form_diff_table_subject_expand(val):
return ", ".join(results)
+
@app.template_filter("is_in_the_past")
def is_in_the_past(dttm):
return dates.is_before(dttm, dates.today())
@@ -297,6 +303,7 @@ def is_in_the_past(dttm):
def search_query_source_wrapper():
def search_query_source(**params):
return edges.make_url_query(**params)
+
return dict(search_query_source=search_query_source)
@@ -311,6 +318,7 @@ def maned_of():
if len(egs) > 0:
assignments = models.Application.assignment_to_editor_groups(egs)
return egs, assignments
+
return dict(maned_of=maned_of)
@@ -325,8 +333,10 @@ def editor_of():
if len(egs) > 0:
assignments = models.Application.assignment_to_editor_groups(egs)
return egs, assignments
+
return dict(editor_of=editor_of)
+
@app.context_processor
def associate_of_wrapper():
def associate_of():
@@ -338,8 +348,10 @@ def associate_of():
if len(egs) > 0:
assignments = models.Application.assignment_to_editor_groups(egs)
return egs, assignments
+
return dict(associate_of=associate_of)
+
# ~~-> Account:Model~~
# ~~-> AuthNZ:Feature~~
@app.before_request
@@ -434,6 +446,23 @@ def page_not_found(e):
return render_template('500.html'), 500
+is_dev_log_setup_completed = False
+
+
+def setup_dev_log():
+ global is_dev_log_setup_completed
+ if not is_dev_log_setup_completed:
+ is_dev_log_setup_completed = True
+ app.logger.handlers = []
+ log = logging.getLogger()
+ log.setLevel(logging.DEBUG)
+ ch = logging.StreamHandler()
+ ch.setLevel(logging.DEBUG)
+ ch.setFormatter(logging.Formatter('%(asctime)s %(levelname).4s %(processName)s%(threadName)s - '
+ '%(message)s --- [%(name)s][%(funcName)s:%(lineno)d]'))
+ log.addHandler(ch)
+
+
def run_server(host=None, port=None, fake_https=False):
"""
:param host:
@@ -443,6 +472,10 @@ def run_server(host=None, port=None, fake_https=False):
that can help for debugging Plausible
:return:
"""
+
+ if app.config.get('DEBUG_DEV_LOG', False):
+ setup_dev_log()
+
pycharm_debug = app.config.get('DEBUG_PYCHARM', False)
if len(sys.argv) > 1:
if sys.argv[1] == '-d':
@@ -467,4 +500,3 @@ def run_server(host=None, port=None, fake_https=False):
if __name__ == "__main__":
run_server()
-
diff --git a/portality/bll/doaj.py b/portality/bll/doaj.py
index bd756e8b59..063f52568b 100644
--- a/portality/bll/doaj.py
+++ b/portality/bll/doaj.py
@@ -130,4 +130,12 @@ def tourService(cls):
@classmethod
def autochecksService(cls, autocheck_plugins=None):
from portality.bll.services import autochecks
- return autochecks.AutocheckService(autocheck_plugins=autocheck_plugins)
\ No newline at end of file
+ return autochecks.AutocheckService(autocheck_plugins=autocheck_plugins)
+ @classmethod
+ def apiRateService(cls):
+ """
+ Obtain an instance of the api_rate service ~~->ApiRate:Service~~
+ :return: ApiRateService
+ """
+ from portality.bll.services import api_rate
+ return api_rate.ApiRateService()
diff --git a/portality/bll/services/api_rate.py b/portality/bll/services/api_rate.py
new file mode 100644
index 0000000000..49d3138404
--- /dev/null
+++ b/portality/bll/services/api_rate.py
@@ -0,0 +1,79 @@
+import datetime
+import functools
+
+from portality.api import Api429Error
+from portality.core import app
+from portality.lib import flask_utils, dates
+from portality.models import Account
+from portality.models.api_log import ApiLog, ApiRateQuery
+
+ROLE_API_RATE_T2 = 'api_rate_t2'
+
+
+def count_api_rate(src: str, target: str) -> float:
+ minutes = 1
+ since = dates.now() - datetime.timedelta(minutes=minutes)
+ ApiLog.refresh()
+ count = ApiLog.count(body=ApiRateQuery(src=src, target=target, since=since).query())
+ rate = count / minutes
+ return rate
+
+
+class ApiRateService:
+
+ @staticmethod
+ def track_api_rate(endpoint_fn):
+ """
+ Decorator for endpoint function to track API rate
+ it will add api_log and throw 429 error if rate limit exceeded
+ """
+
+ @functools.wraps(endpoint_fn)
+ def fn(*args, **kwargs):
+ from flask import request
+
+ # define src
+ src = None
+ api_user = None
+ if 'api_key' in request.values:
+ api_key = request.values['api_key']
+ api_user = Account.pull_by_api_key(api_key)
+ if api_user is None:
+ app.logger.debug(f'api_key not found [{api_key}]')
+ else:
+ src = api_key
+
+ if src is None:
+ raise ValueError('api_key not found')
+
+ target = request.url_rule.endpoint
+
+ # rate checking
+ cur_api_rate = count_api_rate(src, target)
+ limited_api_rate = ApiRateService.get_allowed_rate(api_user)
+ app.logger.debug(f'track_api_rate src[{src}] target[{target}] '
+ f'cur_rate[{cur_api_rate}] limited[{limited_api_rate}]')
+ if cur_api_rate >= limited_api_rate:
+ app.logger.info(f'reject src[{src}] target[{target}] rate[{cur_api_rate} >= {limited_api_rate}]')
+ raise Api429Error('Rate limit exceeded')
+ else:
+ ApiLog.create(src, target)
+
+ return endpoint_fn(*args, **kwargs)
+
+ return fn
+
+ @staticmethod
+ def get_allowed_rate(api_user: Account = None) -> float:
+ if api_user is not None and ApiRateService.is_t2_user(api_user):
+ return app.config.get('RATE_LIMITS_PER_MIN_T2', 1000)
+
+ return ApiRateService.get_default_api_rate()
+
+ @staticmethod
+ def get_default_api_rate():
+ return app.config.get('RATE_LIMITS_PER_MIN_DEFAULT', 10)
+
+ @staticmethod
+ def is_t2_user(api_user: Account) -> bool:
+ return isinstance(api_user, Account) and api_user.has_role(ROLE_API_RATE_T2)
diff --git a/portality/dao.py b/portality/dao.py
index 1c2b32da5f..bd06f12c06 100644
--- a/portality/dao.py
+++ b/portality/dao.py
@@ -851,10 +851,9 @@ def all(cls, size=10000, **kwargs):
return cls.q2obj(size=size, **kwargs)
@classmethod
- def count(cls):
- res = ES.count(index=cls.index_name(), doc_type=cls.doc_type())
+ def count(cls, **kwargs):
+ res = ES.count(index=cls.index_name(), doc_type=cls.doc_type(), **kwargs)
return res.get("count")
- # return requests.get(cls.target() + '_count').json()['count']
@classmethod
def hit_count(cls, query, **kwargs):
diff --git a/portality/lib/flask_utils.py b/portality/lib/flask_utils.py
new file mode 100644
index 0000000000..401c197a95
--- /dev/null
+++ b/portality/lib/flask_utils.py
@@ -0,0 +1,7 @@
+from flask import request
+
+
+def get_remote_addr():
+ if request:
+ return request.headers.get("cf-connecting-ip", request.remote_addr)
+ return None
diff --git a/portality/lib/plausible.py b/portality/lib/plausible.py
index 72dca5cf48..c3743ef658 100644
--- a/portality/lib/plausible.py
+++ b/portality/lib/plausible.py
@@ -1,15 +1,15 @@
# ~~ PlausibleAnalytics:ExternalService~~
-import json
import logging
import os
-import requests
-
from functools import wraps
from threading import Thread
-from portality.core import app
+import requests
from flask import request
+from portality.core import app
+from portality.lib import flask_utils
+
logger = logging.getLogger(__name__)
# Keep track of when this is misconfigured so we don't spam the logs with skip messages
@@ -50,7 +50,7 @@ def send_event(goal: str, on_completed=None, **props_kwargs):
headers = {'Content-Type': 'application/json'}
if request:
# Add IP from CloudFlare header or remote_addr - this works because we have ProxyFix on the app
- headers["X-Forwarded-For"] = request.headers.get("cf-connecting-ip", request.remote_addr)
+ headers["X-Forwarded-For"] = flask_utils.get_remote_addr()
user_agent_key = 'User-Agent'
user_agent_val = request.headers.get(user_agent_key)
if user_agent_val:
diff --git a/portality/models/api_log.py b/portality/models/api_log.py
new file mode 100644
index 0000000000..b2a55bcf6c
--- /dev/null
+++ b/portality/models/api_log.py
@@ -0,0 +1,77 @@
+import datetime
+
+from portality.dao import DomainObject
+from portality.lib import dates
+
+
+class ApiLog(DomainObject):
+ """~~ApiLog:Model->DomainObject:Model~~"""
+ __type__ = "api_log"
+
+ def __init__(self, **kwargs):
+ super(ApiLog, self).__init__(**kwargs)
+
+ @property
+ def src(self):
+ return self.data.get("src")
+
+ @src.setter
+ def src(self, val: str):
+ """
+ Parameters
+ ----------
+ val
+ can be IP address or API key
+ """
+ self.data["src"] = val
+
+ @property
+ def target(self):
+ return self.data.get("target")
+
+ @target.setter
+ def target(self, target: str):
+ """
+
+ Parameters
+ ----------
+ target
+ value format should be "METHOD /api/endpoint"
+
+ Returns
+ -------
+
+ """
+ self.data["target"] = target
+
+ @classmethod
+ def create(cls, src: str, target: str):
+ api_log = ApiLog()
+ api_log.src = src
+ api_log.target = target
+ api_log.set_created()
+ api_log.save()
+ return api_log
+
+
+class ApiRateQuery:
+
+ def __init__(self, src: str, target: str, since):
+ if isinstance(since, datetime.datetime):
+ since = dates.format(since)
+ self._src = src
+ self._target = target
+ self._since = since
+
+ def query(self):
+ return {
+ 'query': {
+ 'bool': {
+ 'must': [
+ {'range': {'created_date': {'gte': self._since}}},
+ {'term': {'src': self._src}},
+ {'term': {'target': self._target}},
+ ]
+ }
+ }
+ }
diff --git a/portality/settings.py b/portality/settings.py
index 744e4b1108..880d18e866 100644
--- a/portality/settings.py
+++ b/portality/settings.py
@@ -709,6 +709,7 @@
MAPPINGS['provenance'] = MAPPINGS["account"] #~~->Provenance:Model~~
MAPPINGS['preserve'] = MAPPINGS["account"] #~~->Preservation:Model~~
MAPPINGS['notification'] = MAPPINGS["account"] #~~->Notification:Model~~
+MAPPINGS['api_log'] = MAPPINGS["account"] #~~->ApiLog:Model~~
#########################################
# Query Routes
@@ -1385,6 +1386,7 @@
TASK_DATA_RETENTION_DAYS = {
"notification": 180, # ~~-> Notifications:Feature ~~
"background_job": 180, # ~~-> BackgroundJobs:Feature ~~
+ "api_log": 180, # ~~-> ApiLog:Feature ~~
}
########################################
@@ -1551,3 +1553,18 @@
AUTOCHECK_RESOURCE_ISSN_ORG_TIMEOUT = 10
AUTOCHECK_RESOURCE_ISSN_ORG_THROTTLE = 1 # seconds between requests
+
+
+
+##################################################
+# ApiRate
+# ~~->ApiRate:Feature~~
+
+# api rate limits per minute for user have no api key or normal api key
+RATE_LIMITS_PER_MIN_DEFAULT = 10
+
+# api rate limits per minute for user have two-tiered api key
+RATE_LIMITS_PER_MIN_T2 = 1000
+
+
+
diff --git a/portality/static/vendor/edges b/portality/static/vendor/edges
index 990f422016..fa15f74f85 160000
--- a/portality/static/vendor/edges
+++ b/portality/static/vendor/edges
@@ -1 +1 @@
-Subproject commit 990f4220163a3e18880f0bdc3ad5c80d234d22dd
+Subproject commit fa15f74f858c558d4ba62d4fbb10c2c71eb902c6
diff --git a/portality/tasks/old_data_cleanup.py b/portality/tasks/old_data_cleanup.py
index 4c764694ee..a4a92f85f7 100644
--- a/portality/tasks/old_data_cleanup.py
+++ b/portality/tasks/old_data_cleanup.py
@@ -8,10 +8,16 @@
from portality.lib import dates
from portality.lib.es_queries import ES_DATETIME_FMT
from portality.models import Notification, BackgroundJob
+from portality.models.api_log import ApiLog
from portality.tasks.helpers import background_helper
from portality.tasks.redis_huey import schedule, long_running
target_queue = long_running
+MODELS_TOBE_CLEANUP = [
+ Notification,
+ BackgroundJob,
+ ApiLog,
+]
class RetentionQuery:
@@ -59,7 +65,7 @@ def clean_all_old_data(logger_fn=None):
if logger_fn is None:
logger_fn = print
- for klazz in [Notification, BackgroundJob]:
+ for klazz in MODELS_TOBE_CLEANUP:
_clean_old_data(klazz, logger_fn=logger_fn)
logger_fn("old data cleanup completed")
diff --git a/portality/templates/api/current/swagger_description.html b/portality/templates/api/current/swagger_description.html
index 3db704c2c2..a67b278db3 100644
--- a/portality/templates/api/current/swagger_description.html
+++ b/portality/templates/api/current/swagger_description.html
@@ -21,6 +21,18 @@ Authenticated routes
If you already have an account, please log in, click 'My Account' and 'Settings' to see your API key. If you do not see an API key then please contact us
{% endif %}
+
+Rate limiting
+
+ {% if is_default_api_rate %}
+ Default rate limit is {{ api_rate_limit }} requests per minute. API rate can be increase to {{ config.get('RATE_LIMITS_PER_MIN_T2') }}.
+ {% else %}
+ Your API key is rate limited to {{ api_rate_limit }} requests per minute.
+ {% endif %}
+ If you exceed this limit, you will receive a 429 status code.
+ If you need a higher rate limit, please contact us.
+
+
Help and support
We have 3 API groups that you can join for API announcements and discussion:
diff --git a/portality/view/api_v3.py b/portality/view/api_v3.py
index adfaa4fb15..6c98e58e57 100644
--- a/portality/view/api_v3.py
+++ b/portality/view/api_v3.py
@@ -9,6 +9,7 @@
from portality.api.current import DiscoveryApi, DiscoveryException
from portality.api.current import jsonify_models, jsonify_data_object, Api400Error, Api404Error, created, \
no_content, bulk_created
+from portality.bll import DOAJ
from portality.core import app
from portality.decorators import api_key_required, api_key_optional, swag, write_required
from portality.lib import plausible
@@ -20,6 +21,7 @@
# Google Analytics category for API events
ANALYTICS_CATEGORY = app.config.get('ANALYTICS_CATEGORY_API', 'API Hit')
ANALYTICS_ACTIONS = app.config.get('ANALYTICS_ACTIONS_API', {})
+api_rate_serv = DOAJ.apiRateService()
@blueprint.route('/')
@@ -33,11 +35,18 @@ def docs():
if current_user.is_authenticated:
account_url = url_for('account.username', username=current_user.id, _external=True,
_scheme=app.config.get('PREFERRED_URL_SCHEME', 'https'))
+
+ api_rate_limit = api_rate_serv.get_allowed_rate(current_user)
+ is_default_api_rate = api_rate_limit == api_rate_serv.get_default_api_rate()
+
return render_template('api/current/api_docs.html',
api_version=API_VERSION_NUMBER,
base_url=app.config.get("BASE_API_URL", url_for('.api_v3_root')),
contact_us_url=url_for('doaj.contact'),
- account_url=account_url)
+ account_url=account_url,
+ api_rate_limit=api_rate_limit,
+ is_default_api_rate=is_default_api_rate,
+ )
@blueprint.route('/swagger.json')
@@ -52,11 +61,13 @@ def api_spec():
# Handle wayward paths by raising an API404Error
-@blueprint.route("/", methods=["POST", "GET", "PUT", "DELETE", "PATCH", "HEAD"]) # leaving out methods should mean all, but tests haven't shown that behaviour.
+# leaving out methods should mean all, but tests haven't shown that behaviour.
+@blueprint.route("/", methods=["POST", "GET", "PUT", "DELETE", "PATCH", "HEAD"])
def missing_resource(invalid_path):
docs_url = app.config.get("BASE_URL", "") + url_for('.docs')
spec_url = app.config.get("BASE_URL", "") + url_for('.api_spec')
- raise Api404Error("No endpoint at {0}. See {1} for valid paths or read the documentation at {2}.".format(invalid_path, spec_url, docs_url))
+ raise Api404Error("No endpoint at {0}. See {1} for valid paths or read the documentation at {2}.".format(
+ invalid_path, spec_url, docs_url))
@swag(swag_summary='Search your applications [Authenticated, not public]',
@@ -65,6 +76,7 @@ def missing_resource(invalid_path):
@api_key_required
@plausible.pa_event(ANALYTICS_CATEGORY, action=ANALYTICS_ACTIONS.get('search_applications', 'Search applications'),
record_value_of_which_arg='search_query')
+@api_rate_serv.track_api_rate
def search_applications(search_query):
# get the values for the 2 other bits of search info: the page number and the page size
page = request.values.get("page", 1)
@@ -163,6 +175,7 @@ def search_articles(search_query):
@swag(swag_summary='Create an application [Authenticated, not public]',
swag_spec=ApplicationsCrudApi.create_swag()) # must be applied after @api_key_(optional|required) decorators. They don't preserve func attributes.
@plausible.pa_event(ANALYTICS_CATEGORY, action=ANALYTICS_ACTIONS.get('create_application', 'Create application'))
+@api_rate_serv.track_api_rate
def create_application():
# get the data from the request
try:
@@ -183,6 +196,7 @@ def create_application():
swag_spec=ApplicationsCrudApi.retrieve_swag()) # must be applied after @api_key_(optional|required) decorators. They don't preserve func attributes.
@plausible.pa_event(ANALYTICS_CATEGORY, action=ANALYTICS_ACTIONS.get('retrieve_application', 'Retrieve application'),
record_value_of_which_arg='application_id')
+@api_rate_serv.track_api_rate
def retrieve_application(application_id):
a = ApplicationsCrudApi.retrieve(application_id, current_user)
return jsonify_models(a)
@@ -195,6 +209,7 @@ def retrieve_application(application_id):
swag_spec=ApplicationsCrudApi.update_swag()) # must be applied after @api_key_(optional|required) decorators. They don't preserve func attributes.
@plausible.pa_event(ANALYTICS_CATEGORY, action=ANALYTICS_ACTIONS.get('update_application', 'Update application'),
record_value_of_which_arg='application_id')
+@api_rate_serv.track_api_rate
def update_application(application_id):
# get the data from the request
try:
@@ -216,6 +231,7 @@ def update_application(application_id):
swag_spec=ApplicationsCrudApi.delete_swag()) # must be applied after @api_key_(optional|required) decorators. They don't preserve func attributes.
@plausible.pa_event(ANALYTICS_CATEGORY, action=ANALYTICS_ACTIONS.get('delete_application', 'Delete application'),
record_value_of_which_arg='application_id')
+@api_rate_serv.track_api_rate
def delete_application(application_id):
ApplicationsCrudApi.delete(application_id, current_user._get_current_object())
return no_content()
@@ -230,6 +246,7 @@ def delete_application(application_id):
@swag(swag_summary='Create an article [Authenticated, not public]',
swag_spec=ArticlesCrudApi.create_swag()) # must be applied after @api_key_(optional|required) decorators. They don't preserve func attributes.
@plausible.pa_event(ANALYTICS_CATEGORY, action=ANALYTICS_ACTIONS.get('create_article', 'Create article'))
+@api_rate_serv.track_api_rate
def create_article():
# get the data from the request
try:
@@ -262,6 +279,7 @@ def retrieve_article(article_id):
swag_spec=ArticlesCrudApi.update_swag()) # must be applied after @api_key_(optional|required) decorators. They don't preserve func attributes.
@plausible.pa_event(ANALYTICS_CATEGORY, action=ANALYTICS_ACTIONS.get('update_article', 'Update article'),
record_value_of_which_arg='article_id')
+@api_rate_serv.track_api_rate
def update_article(article_id):
# get the data from the request
try:
@@ -283,6 +301,7 @@ def update_article(article_id):
swag_spec=ArticlesCrudApi.delete_swag()) # must be applied after @api_key_(optional|required) decorators. They don't preserve func attributes.
@plausible.pa_event(ANALYTICS_CATEGORY, action=ANALYTICS_ACTIONS.get('delete_article', 'Delete article'),
record_value_of_which_arg='article_id')
+@api_rate_serv.track_api_rate
def delete_article(article_id):
ArticlesCrudApi.delete(article_id, current_user)
return no_content()
@@ -309,7 +328,9 @@ def retrieve_journal(journal_id):
@write_required(api=True)
@swag(swag_summary='Create applications in bulk [Authenticated, not public]',
swag_spec=ApplicationsBulkApi.create_swag()) # must be applied after @api_key_(optional|required) decorators. They don't preserve func attributes.
-@plausible.pa_event(ANALYTICS_CATEGORY, action=ANALYTICS_ACTIONS.get('bulk_application_create', 'Bulk application create'))
+@plausible.pa_event(ANALYTICS_CATEGORY,
+ action=ANALYTICS_ACTIONS.get('bulk_application_create', 'Bulk application create'))
+@api_rate_serv.track_api_rate
def bulk_application_create():
# get the data from the request
try:
@@ -334,7 +355,9 @@ def bulk_application_create():
@write_required(api=True)
@swag(swag_summary='Delete applications in bulk [Authenticated, not public]',
swag_spec=ApplicationsBulkApi.delete_swag()) # must be applied after @api_key_(optional|required) decorators. They don't preserve func attributes.
-@plausible.pa_event(ANALYTICS_CATEGORY, action=ANALYTICS_ACTIONS.get('bulk_application_delete', 'Bulk application delete'))
+@plausible.pa_event(ANALYTICS_CATEGORY,
+ action=ANALYTICS_ACTIONS.get('bulk_application_delete', 'Bulk application delete'))
+@api_rate_serv.track_api_rate
def bulk_application_delete():
# get the data from the request
try:
@@ -356,6 +379,7 @@ def bulk_application_delete():
@swag(swag_summary='Bulk article creation [Authenticated, not public]',
swag_spec=ArticlesBulkApi.create_swag()) # must be applied after @api_key_(optional|required) decorators. They don't preserve func attributes.
@plausible.pa_event(ANALYTICS_CATEGORY, action=ANALYTICS_ACTIONS.get('bulk_article_create', 'Bulk article create'))
+@api_rate_serv.track_api_rate
def bulk_article_create():
# get the data from the request
try:
@@ -381,6 +405,7 @@ def bulk_article_create():
@swag(swag_summary='Bulk article delete [Authenticated, not public]',
swag_spec=ArticlesBulkApi.delete_swag()) # must be applied after @api_key_(optional|required) decorators. They don't preserve func attributes.
@plausible.pa_event(ANALYTICS_CATEGORY, action=ANALYTICS_ACTIONS.get('bulk_article_delete', 'Bulk article delete'))
+@api_rate_serv.track_api_rate
def bulk_article_delete():
# get the data from the request
try: