Merge pull request #63 from ydb-platform/generate-protobuf

Generate protobuf from ydb-api-protos

Author: gridnevvvit

.gitignore

@@ -1,2 +1,6 @@
 __pycache__
 ydb.egg-info/
+/.idea
+/tox
+/venv
+/ydb_certs

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "ydb-api-protos"]
+	path = ydb-api-protos
+	url = https://github.com/ydb-platform/ydb-api-protos.git

BUILD.md

@@ -45,3 +45,11 @@ Use the command below to run unit tests.
 ```sh
 tox -epy38
 ```
+
+### Regenerate protobuf
+
+Use the command below for regenerate protobuf code.
+
+```sh
+make protobuf
+```

CHANGELOG.md

@@ -1,3 +1,6 @@
+* Regenerate protobuf code from public api protos (some private protobufs was removed)
+* Remove internal S3 client code
+
 ## 2.11.0 ##
 
 * removed unused experimental client from ydb python sdk.

Makefile

@@ -0,0 +1,3 @@
+protobuf:
+	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env
+	docker run --rm -it -v $${PWD}:$${PWD} -w $${PWD} ydb-python-sdk-proto-generator-env python generate_protoc.py

generate-protobuf.Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.9.15
+RUN \
+    python -m pip install --upgrade pip && \
+    python -m pip install grpcio==1.39.0 && \
+    python -m pip install grpclib && \
+    python -m pip install protobuf==3.20.3 && \
+    python -m pip install grpcio-tools==1.38.0
+
+ENV PROTOC_VER=21.8
+RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VER}/protoc-${PROTOC_VER}-linux-x86_64.zip && \
+    unzip protoc-*.zip && \
+    rm -f protoc-*.zip && \
+    mv bin/protoc /usr/local/bin/protoc && \
+    mv include well-known-protos

generate_protoc.py

@@ -1,22 +1,70 @@
 import os
+import pathlib
+import shutil
+from typing import List
+
 from grpc_tools import command
 
-command.build_package_protos('.')
 
-init_files_to_create = []
+def files_filter(dir, items: List[str]) -> List[str]:
+    ignored_names = ['.git']
+
+    ignore = []
+    for item in items:
+        fullpath = os.path.join(dir, item)
+        if os.path.isdir(fullpath) and item not in ignored_names:
+            continue
+        if item.endswith(".proto"):
+            continue
+        ignore.append(item)
+
+    return ignore
+
+
+def create_init_files(rootdirpath: str):
+    for root, _dirs, _files in os.walk(rootdirpath):
+        init_path = pathlib.Path(os.path.join(root, '__init__.py'))
+        if not init_path.exists():
+            init_path.touch()
+
+
+def remove_protos(rootdirpath: str):
+    for root, _dirs, files in os.walk(rootdirpath):
+        for file in files:
+            if file.endswith(".proto"):
+                os.remove(os.path.join(root, file))
+
+
+def fix_file_contents(rootdir: str):
+    flake_ignore_line = "# flake8: " + "noqa"  # prevent ignore the file
+
+    for dirpath, _, fnames in os.walk(rootdir):
+        for fname in fnames:
+            if not fname.endswith(".py"):
+                continue
+
+            with open(os.path.join(dirpath, fname), 'r+t') as f:
+                content = f.read()
+
+                # Fix imports
+                content = content.replace("from protos", "from ydb._grpc.protos")
+
+                # Add ignore style check
+                content = content.replace("# -*- coding: utf-8 -*-", "# -*- coding: utf-8 -*-\n" + flake_ignore_line)
+                f.seek(0)
+                f.write(content)
+
+
+def generate_protobuf(src_proto_dir: str, dst_dir: str):
+    shutil.rmtree(dst_dir, ignore_errors=True)
 
-for root, dirs, files in os.walk('kikimr'):
-    if '__init__.py' in files:
-        continue
+    shutil.copytree(src_proto_dir, dst_dir, ignore=files_filter)
+    create_init_files(dst_dir)
 
-    init_files_to_create.append(
-        os.path.join(
-            root,
-            '__init__.py'
-        )
-    )
+    command.build_package_protos(dst_dir)
+    remove_protos(dst_dir)
+    fix_file_contents(dst_dir)
 
 
-for filename in init_files_to_create:
-    with open(filename, 'w') as f:
-        pass
+if __name__ == '__main__':
+    generate_protobuf("ydb-api-protos", "ydb/_grpc")

requirements.txt

@@ -1,4 +1,4 @@
-grpcio==1.38.0
-protobuf>3.17.3
+grpcio==1.39.0
+protobuf>3.17.3,<4.0.0
 pytest==6.2.4
 aiohttp==3.7.4

test-requirements.txt

@@ -10,15 +10,16 @@ docker==5.0.0
 docker-compose==1.29.2
 dockerpty==0.4.1
 docopt==0.6.2
-grpcio>=1.38.0
+grpcio==1.39.0
+grpcio-tools==1.39.0
 idna==3.2
 importlib-metadata==4.6.1
 iniconfig==1.1.1
 jsonschema==3.2.0
 packaging==21.0
 paramiko==2.10.1
 pluggy==0.13.1
-protobuf>3.17.3
+protobuf>3.17.3,<4.0.0
 py==1.10.0
 pycparser==2.20
 PyNaCl==1.4.0

tox.ini

@@ -46,13 +46,13 @@ commands =
 skip_install = true
 deps = black
 commands =
-    black ydb examples tests --extend-exclude ydb/public/api
+    black ydb examples tests --extend-exclude ydb/_grpc
 
 [testenv:black]
 skip_install = true
 deps = black
 commands =
-    black --check ydb examples tests --extend-exclude ydb/public/api
+    black --check ydb examples tests --extend-exclude ydb/_grpc
 
 [testenv:pylint]
 deps = pylint

ydb-api-protos

@@ -7,7 +7,6 @@
 from .settings import *  # noqa
 from .resolver import *  # noqa
 from .export import *  # noqa
-from .s3list import *  # noqa
 from .auth_helpers import *  # noqa
 from .operation import *  # noqa
 from .scripting import *  # noqa

ydb/__init__.py

@@ -1,20 +1,20 @@
 # -*- coding: utf-8 -*-
-from ydb.public.api.grpc import (
+from ydb._grpc import (
     ydb_cms_v1_pb2_grpc,
     ydb_discovery_v1_pb2_grpc,
     ydb_scheme_v1_pb2_grpc,
     ydb_table_v1_pb2_grpc,
 )
-from ydb.public.api.protos import (
+from ydb._grpc.protos import (
     ydb_status_codes_pb2,
     ydb_discovery_pb2,
     ydb_scheme_pb2,
     ydb_table_pb2,
     ydb_value_pb2,
 )
-from ydb.public.api.protos import ydb_operation_pb2
-from ydb.public.api.protos import ydb_common_pb2
-from ydb.public.api.grpc import ydb_operation_v1_pb2_grpc
+from ydb._grpc.protos import ydb_operation_pb2
+from ydb._grpc.protos import ydb_common_pb2
+from ydb._grpc import ydb_operation_v1_pb2_grpc
 
 
 StatusIds = ydb_status_codes_pb2.StatusIds