From 15b3545b064b65fc2130f539a6760dad4bbbe9a3 Mon Sep 17 00:00:00 2001 From: Stefan Tatschner Date: Tue, 30 Jul 2024 13:35:17 +0200 Subject: [PATCH 1/6] Drop ujson and switch to orjson Implemented as requested in this comment: https://github.com/python-lsp/python-lsp-server/pull/579#pullrequestreview-2206408333 --- examples/langserver.py | 4 ++-- examples/langserver_ext.py | 4 ++-- pylsp_jsonrpc/streams.py | 4 ++-- pyproject.toml | 2 +- test/test_streams.py | 27 +++++++++++---------------- 5 files changed, 18 insertions(+), 23 deletions(-) diff --git a/examples/langserver.py b/examples/langserver.py index a1ddc8f..5a40f1c 100644 --- a/examples/langserver.py +++ b/examples/langserver.py @@ -5,8 +5,8 @@ from pylsp_jsonrpc import dispatchers, endpoint try: - import ujson as json -except Exception: # pylint: disable=broad-except + import orjson as json +except ImportError: import json log = logging.getLogger(__name__) diff --git a/examples/langserver_ext.py b/examples/langserver_ext.py index 3371a20..19d227c 100644 --- a/examples/langserver_ext.py +++ b/examples/langserver_ext.py @@ -7,8 +7,8 @@ from pylsp_jsonrpc import streams try: - import ujson as json -except Exception: # pylint: disable=broad-except + import orjson as json +except ImportError: import json log = logging.getLogger(__name__) diff --git a/pylsp_jsonrpc/streams.py b/pylsp_jsonrpc/streams.py index 40048a9..0a85a19 100644 --- a/pylsp_jsonrpc/streams.py +++ b/pylsp_jsonrpc/streams.py @@ -5,8 +5,8 @@ import threading try: - import ujson as json -except Exception: # pylint: disable=broad-except + import orjson as json +except ImportError: import json log = logging.getLogger(__name__) diff --git a/pyproject.toml b/pyproject.toml index 376785c..2c4d196 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ authors = [{name = "Python Language Server Contributors"}] description = "JSON RPC 2.0 server library" license = {text = "MIT"} requires-python = ">=3.8" -dependencies = ["ujson>=3.0.0"] +dependencies = ["orjson>=3.10.0"] dynamic = ["version"] classifiers = [ "License :: OSI Approved :: MIT License", diff --git a/test/test_streams.py b/test/test_streams.py index 14fe1bb..4dde134 100644 --- a/test/test_streams.py +++ b/test/test_streams.py @@ -77,25 +77,20 @@ def test_reader_bad_json(rfile, reader): def test_writer(wfile, writer): - writer.write({ + data = { 'id': 'hello', 'method': 'method', 'params': {} - }) - if 'ujson' in sys.modules: - assert wfile.getvalue() == ( - b'Content-Length: 44\r\n' - b'Content-Type: application/vscode-jsonrpc; charset=utf8\r\n' - b'\r\n' - b'{"id":"hello","method":"method","params":{}}' - ) - else: - assert wfile.getvalue() == ( - b'Content-Length: 49\r\n' - b'Content-Type: application/vscode-jsonrpc; charset=utf8\r\n' - b'\r\n' - b'{"id": "hello", "method": "method", "params": {}}' - ) + } + writer.write(data) + + raw_result = wfile.getvalue().decode() + raw_result_lines = raw_result.split() + + assert raw_result_lines[0].split(":") == "Content-Length" + assert raw_result_lines[1] == 'Content-Type: application/vscode-jsonrpc; charset=utf8' + assert raw_result_lines[2] == '' + assert json.loads(raw_result_lines[3]) == data class JsonDatetime(datetime.datetime): From 383d92c457dd4deac9653cc4fd9ac80b6f076a05 Mon Sep 17 00:00:00 2001 From: valrus Date: Sun, 11 Aug 2024 14:44:12 -0700 Subject: [PATCH 2/6] Modify opts and encodings for using orjson --- pylsp_jsonrpc/streams.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/pylsp_jsonrpc/streams.py b/pylsp_jsonrpc/streams.py index 0a85a19..1080deb 100644 --- a/pylsp_jsonrpc/streams.py +++ b/pylsp_jsonrpc/streams.py @@ -3,6 +3,7 @@ import logging import threading +import sys try: import orjson as json @@ -83,7 +84,15 @@ class JsonRpcStreamWriter: def __init__(self, wfile, **json_dumps_args): self._wfile = wfile self._wfile_lock = threading.Lock() - self._json_dumps_args = json_dumps_args + + if 'orjson' in sys.modules and json_dumps_args.pop('sort_keys'): + # orjson needs different option handling + self._json_dumps_args = {'option': json.OPT_SORT_KEYS} + self._json_dumps_args.update(**json_dumps_args) + else: + self._json_dumps_args = json_dumps_args + # omit unnecessary whitespace for consistency with orjson + self._json_dumps_args.setdefault('separators', (',', ':')) def close(self): with self._wfile_lock: @@ -96,16 +105,16 @@ def write(self, message): try: body = json.dumps(message, **self._json_dumps_args) - # Ensure we get the byte length, not the character length - content_length = len(body) if isinstance(body, bytes) else len(body.encode('utf-8')) + # orjson gives bytes, builtin json gives str. ensure we have bytes + body_bytes = body if isinstance(body, bytes) else body.encode('utf-8') response = ( - f"Content-Length: {content_length}\r\n" - f"Content-Type: application/vscode-jsonrpc; charset=utf8\r\n\r\n" - f"{body}" - ) + b"Content-Length: %(length)i\r\n" + b"Content-Type: application/vscode-jsonrpc; charset=utf8\r\n\r\n" + b"%(body)s" + ) % {b'length': len(body_bytes), b'body': body_bytes} - self._wfile.write(response.encode('utf-8')) + self._wfile.write(response) self._wfile.flush() except Exception: # pylint: disable=broad-except log.exception("Failed to write message to output file %s", message) From c6dfb305c3841e97cba649e4dabbba5185b5a0bd Mon Sep 17 00:00:00 2001 From: valrus Date: Sun, 11 Aug 2024 14:44:28 -0700 Subject: [PATCH 3/6] Add tests, including one for using stdlib json --- test/test_streams.py | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/test/test_streams.py b/test/test_streams.py index 4dde134..2fed744 100644 --- a/test/test_streams.py +++ b/test/test_streams.py @@ -8,6 +8,7 @@ import sys from unittest import mock import pytest +import json from pylsp_jsonrpc.streams import JsonRpcStreamReader, JsonRpcStreamWriter @@ -31,7 +32,6 @@ def reader(rfile): def writer(wfile): return JsonRpcStreamWriter(wfile, sort_keys=True) - def test_reader(rfile, reader): rfile.write( b'Content-Length: 49\r\n' @@ -84,13 +84,41 @@ def test_writer(wfile, writer): } writer.write(data) - raw_result = wfile.getvalue().decode() - raw_result_lines = raw_result.split() + assert wfile.getvalue() == ( + b'Content-Length: 44\r\n' + b'Content-Type: application/vscode-jsonrpc; charset=utf8\r\n' + b'\r\n' + b'{"id":"hello","method":"method","params":{}}' + ) - assert raw_result_lines[0].split(":") == "Content-Length" - assert raw_result_lines[1] == 'Content-Type: application/vscode-jsonrpc; charset=utf8' - assert raw_result_lines[2] == '' - assert json.loads(raw_result_lines[3]) == data + +def test_writer_builtin_json(wfile): + """Test the stream writer using the standard json lib.""" + data = { + 'id': 'hello', + 'method': 'method', + 'params': {} + } + orig_modules = sys.modules + try: + # Pretend orjson wasn't imported when initializing the writer. + sys.modules = {'json': json} + std_json_writer = JsonRpcStreamWriter(wfile, sort_keys=True) + finally: + sys.modules = orig_modules + + with mock.patch('pylsp_jsonrpc.streams.json') as streams_json: + # Mock the imported json's dumps function to use the stdlib's dumps, + # whether orjson is available or not. + streams_json.dumps = json.dumps + std_json_writer.write(data) + + assert wfile.getvalue() == ( + b'Content-Length: 44\r\n' + b'Content-Type: application/vscode-jsonrpc; charset=utf8\r\n' + b'\r\n' + b'{"id":"hello","method":"method","params":{}}' + ) class JsonDatetime(datetime.datetime): From 1c699f77fa01ef4c27fd5d618f979ca822429395 Mon Sep 17 00:00:00 2001 From: valrus Date: Sun, 11 Aug 2024 14:58:05 -0700 Subject: [PATCH 4/6] Rename test; json is stdlib, not builtin --- test/test_streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_streams.py b/test/test_streams.py index 2fed744..c71c912 100644 --- a/test/test_streams.py +++ b/test/test_streams.py @@ -92,7 +92,7 @@ def test_writer(wfile, writer): ) -def test_writer_builtin_json(wfile): +def test_writer_stdlib_json(wfile): """Test the stream writer using the standard json lib.""" data = { 'id': 'hello', From eec1388159ecbb5374fd8cad67ff7b065b1051d9 Mon Sep 17 00:00:00 2001 From: valrus Date: Mon, 12 Aug 2024 08:33:44 -0700 Subject: [PATCH 5/6] fix linter issues --- pylsp_jsonrpc/streams.py | 5 +++-- test/test_streams.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pylsp_jsonrpc/streams.py b/pylsp_jsonrpc/streams.py index 1080deb..aa74a1d 100644 --- a/pylsp_jsonrpc/streams.py +++ b/pylsp_jsonrpc/streams.py @@ -86,8 +86,9 @@ def __init__(self, wfile, **json_dumps_args): self._wfile_lock = threading.Lock() if 'orjson' in sys.modules and json_dumps_args.pop('sort_keys'): - # orjson needs different option handling - self._json_dumps_args = {'option': json.OPT_SORT_KEYS} + # orjson needs different option handling; + # pylint has an erroneous error here https://github.com/pylint-dev/pylint/issues/9762 + self._json_dumps_args = {'option': json.OPT_SORT_KEYS} # pylint: disable=maybe-no-member self._json_dumps_args.update(**json_dumps_args) else: self._json_dumps_args = json_dumps_args diff --git a/test/test_streams.py b/test/test_streams.py index c71c912..d97b3a2 100644 --- a/test/test_streams.py +++ b/test/test_streams.py @@ -7,8 +7,8 @@ import datetime import sys from unittest import mock -import pytest import json +import pytest from pylsp_jsonrpc.streams import JsonRpcStreamReader, JsonRpcStreamWriter From da259528557cfa8e8d66165bf430e7003aa6c58b Mon Sep 17 00:00:00 2001 From: valrus Date: Fri, 16 Aug 2024 17:10:36 -0700 Subject: [PATCH 6/6] add'l lint fixes --- pylsp_jsonrpc/streams.py | 2 +- test/test_streams.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pylsp_jsonrpc/streams.py b/pylsp_jsonrpc/streams.py index aa74a1d..7dacbea 100644 --- a/pylsp_jsonrpc/streams.py +++ b/pylsp_jsonrpc/streams.py @@ -88,7 +88,7 @@ def __init__(self, wfile, **json_dumps_args): if 'orjson' in sys.modules and json_dumps_args.pop('sort_keys'): # orjson needs different option handling; # pylint has an erroneous error here https://github.com/pylint-dev/pylint/issues/9762 - self._json_dumps_args = {'option': json.OPT_SORT_KEYS} # pylint: disable=maybe-no-member + self._json_dumps_args = {'option': json.OPT_SORT_KEYS} # pylint: disable=maybe-no-member self._json_dumps_args.update(**json_dumps_args) else: self._json_dumps_args = json_dumps_args diff --git a/test/test_streams.py b/test/test_streams.py index d97b3a2..aff10da 100644 --- a/test/test_streams.py +++ b/test/test_streams.py @@ -32,6 +32,7 @@ def reader(rfile): def writer(wfile): return JsonRpcStreamWriter(wfile, sort_keys=True) + def test_reader(rfile, reader): rfile.write( b'Content-Length: 49\r\n'