From 878a63e449b33e5e2c424fca5e04c11685b2890c Mon Sep 17 00:00:00 2001 From: Clemens Rudert Date: Sun, 1 Sep 2024 14:40:04 +0200 Subject: [PATCH] fix precommit --- .dockerignore | 3 + .env.example | 4 ++ .github/workflows/image.yml | 23 +++++++ .gitignore | 3 + .pre-commit-config.yaml | 2 +- Dockerfile | 44 +++++++++++-- MANIFEST.in | 1 + Makefile | 3 +- README.md | 6 +- docker-compose.yml | 14 ++-- setup.py | 95 +++++++++++++++------------ src/qgis_server_light/worker/redis.py | 50 +++++++++----- 12 files changed, 172 insertions(+), 76 deletions(-) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .github/workflows/image.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f10d6b8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.venv +build +dist diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..49434fc --- /dev/null +++ b/.env.example @@ -0,0 +1,4 @@ +QSL_DATA_MOUNT= +QSL_LOG_LEVEL=debug +QSL_REPLICAS=1 +QSL_REDIS_URL=redis://redis diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml new file mode 100644 index 0000000..c99e3b9 --- /dev/null +++ b/.github/workflows/image.yml @@ -0,0 +1,23 @@ +name: ci + +on: + push: + branches: + - master + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push worker + uses: docker/build-push-action@v3 + with: + context: . + file: Dockerfile + target: release diff --git a/.gitignore b/.gitignore index 0255f6a..fac98d2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ *.egg-info/ __pycache__/ src/*.egg-info +.env +build +dist diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3500614..a8e69b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: # Remove unused imports/variables - repo: https://github.com/myint/autoflake - rev: v1.4 + rev: v2.3.1 hooks: - id: autoflake args: diff --git a/Dockerfile b/Dockerfile index b708f49..ff30fc2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,25 @@ -FROM ghcr.io/opengisch/qgis-slim:3.34.10 AS dev +FROM ghcr.io/opengisch/qgis-slim:3.34.10 AS base + +# switch to root user for install +USER 0 + +RUN apt-get update && \ + apt-get install -y \ + python3-pip \ + python3-setuptools + +######################### +# DEV +######################### +FROM base AS dev LABEL org.opengisch.author="Clemens Rudert " LABEL org.opengisch.image.title="QGIS-Server-Light" USER 0 -RUN apt-get update -y && \ - apt-get install -y \ - python3-full \ - python3-dev \ - make \ - build-essential +RUN apt-get install -y \ + python3-venv \ + make WORKDIR /opt/qgis-server-light/ ADD requirements.worker.txt . @@ -29,3 +39,23 @@ RUN VENV_PATH=${VENV_PATH} make dev ENTRYPOINT ["/tini", "--", "/opt/qgis-server-light/venv/bin/python3", "-m"] CMD ["qgis_server_light.worker.redis"] + +######################### +# BUILDER (FOR RELEASE) +######################### +FROM base AS builder + +WORKDIR /app +COPY ./ . +RUN WITH_WORKER=true python3 setup.py bdist_wheel + +######################### +# RELEASE +######################### +FROM base AS release +COPY --from=builder /app/dist/*.whl /tmp +RUN pip3 install /tmp/*.whl + +USER 1001 + +CMD ["redis_worker"] diff --git a/MANIFEST.in b/MANIFEST.in index a7c77e6..3179d29 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,3 +4,4 @@ include CHANGES.md include README.md include LICENSE.md include pytest.ini +include requirements.* diff --git a/Makefile b/Makefile index 4a74236..a9e9936 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ $(QGIS_VENV_PATH): echo "/usr/share/qgis/python" > $@ $(PIP_REQUIREMENTS): $(VENV_REQUIREMENTS) requirements.interface.txt requirements.worker.txt requirements.exporter.txt $(QGIS_VENV_PATH) - $(VENV_BIN)/$(PIP_COMMAND) install --upgrade pip + $(VENV_BIN)/$(PIP_COMMAND) install --upgrade pip wheel $(VENV_BIN)/$(PIP_COMMAND) install -r requirements.interface.txt -r requirements.worker.txt -r requirements.exporter.txt touch $@ @@ -55,6 +55,7 @@ install: $(PIP_REQUIREMENTS) .PHONY: build build: $(BUILD_DEPS) + $(VENV_BIN)/python setup.py bdist_wheel .PHONY: clean clean: diff --git a/README.md b/README.md index 2b7a221..260ec0e 100644 --- a/README.md +++ b/README.md @@ -81,12 +81,12 @@ the recommended way of running this stack is Docker (docker need to be installed First you need to build the image: ```shell -docker build -t qgis-server-light:3.34.8-dev --target dev . +docker build -t opengisch/qgis-server-light:dev --target dev . ``` Then run: ```shell -docker run --rm -e REDIS_URL=redis://localhost:1234 -e LOG_LEVEL=debug -v $(pwd):/app --net host qgis-server-light:3.34.8-dev +docker run --rm -e REDIS_URL=redis://localhost:1234 -e LOG_LEVEL=debug -v $(pwd):/app --net host opengisch/qgis-server-light:dev ``` Congratulations... Your QGIS-Server-Light is running and waits for jobs in the redis queue. @@ -102,7 +102,7 @@ If you have issues with matching QGIS version on your system and the one used by you should prefer the docker way with the source code mounted to the running container. ```shell -docker run --rm -e REDIS_URL=redis://localhost:1234 -e LOG_LEVEL=debug -v $(pwd):/app --net host qgis-server-light:3.34.8-dev +docker run --rm -e REDIS_URL=redis://localhost:1234 -e LOG_LEVEL=debug -v $(pwd):/app --net host opengisch/qgis-server-light:dev ``` That way the current changes you make are transparently available in the container. diff --git a/docker-compose.yml b/docker-compose.yml index b364d9d..b9c1f1b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,17 +1,21 @@ services: qgis-server-light: - image: qgis-server-light:3.34.8-dev + image: opengisch/qgis-server-light:dev + env_file: + - .env build: context: . dockerfile: ./Dockerfile target: dev restart: unless-stopped environment: - - "REDIS_URL=redis://redis" - - "LOG_LEVEL=debug" + - "QSL_DATA_ROOT=/io/data" volumes: - - ../data/project_store/:/io/themes:ro - - ./:/app + - "../data/project_store/:/io/themes:ro" + - "./:/app" + - "${QSL_DATA_MOUNT}:/io/data" + deploy: + replicas: ${QSL_REPLICAS} redis: image: redis:7.4.0 ports: diff --git a/setup.py b/setup.py index 9b9b641..a9906dc 100644 --- a/setup.py +++ b/setup.py @@ -1,62 +1,73 @@ import os -import re -from setuptools import setup, find_packages +from setuptools import setup here = os.path.abspath(os.path.dirname(__name__)) -with open(os.path.join(here, 'README.md')) as f: +with open(os.path.join(here, "README.md")) as f: README = f.read() -with open(os.path.join(here, 'CHANGES.md')) as f: +with open(os.path.join(here, "CHANGES.md")) as f: CHANGES = f.read() -with open(os.path.join(here, 'requirements.interface.txt')) as f: - re_ = re.compile(r'(.+)==') - recommend = f.read().splitlines() -requires = [re_.match(r).group(1) for r in recommend] +with open(os.path.join(here, "requirements.interface.txt")) as f: + install_requires = f.read().splitlines() -tests_require = [ - 'pytest', # includes virtualenv - 'pytest-cov' -] +tests_require = ["pytest", "pytest-cov"] # includes virtualenv +worker_files = {} +worker_modules = [] +worker_packages = [] +worker_scripts = [] +if os.environ.get("WITH_WORKER", False): + with open(os.path.join(here, "requirements.worker.txt")) as f: + install_requires = install_requires + f.read().splitlines() + + worker_files = {"qgis_server_light.worker": ["*.py"]} + worker_modules = [ + "qgis_server_light/worker/engine", + "qgis_server_light/worker/image_utils", + "qgis_server_light/worker/qgis", + "qgis_server_light/worker/redis", + "qgis_server_light/worker/runner", + ] + worker_packages = ["qgis_server_light.worker"] + worker_scripts = ["redis_worker=qgis_server_light.worker.redis:main"] + +package_data = {"qgis_server_light.interface": ["*.py"]} +package_data.update(worker_files) setup( - name='qgis_server_light', - version='v0.0.1', - description='qgis renderer as a python process', - long_description=README + '\n\n' + CHANGES, + name="qgis_server_light", + version="v0.0.1", + description="qgis renderer as a python process", + long_description=README + "\n\n" + CHANGES, classifiers=[ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", "Programming Language :: Python", - "Framework :: Pyramid", + "Programming Language :: Python :: 3.10", "Topic :: Internet :: WWW/HTTP", - "Topic :: Internet :: WWW/HTTP :: WSGI :: Application", + "Topic :: Internet :: WWW/HTTP :: ASGI :: Application", + "Typing :: Typed", ], - author='Clemens Rudert (OPENGIS.ch)', - author_email='clemens@opengis.ch', - url='https://github.com/opengisch/qgis-server-light', - keywords=['web worker qgis qgis-server processing'], - packages=['qgis_server_light.interface'], - package_dir={'': 'src'}, - package_data={ - "qgis_server_light.interface": ["*.py"] - }, + author="Clemens Rudert (OPENGIS.ch)", + author_email="clemens@opengis.ch", + url="https://github.com/opengisch/qgis-server-light", + keywords=["web worker qgis qgis-server processing"], + packages=["qgis_server_light.interface"] + worker_packages, + package_dir={"": "src"}, + package_data=package_data, py_modules=[ - 'qgis_server_light/interface/job', - 'qgis_server_light/interface/dispatcher', - 'qgis_server_light/interface/qgis' - - ], + "qgis_server_light/interface/job", + "qgis_server_light/interface/dispatcher", + "qgis_server_light/interface/qgis", + ] + + worker_modules, include_package_data=True, zip_safe=False, project_urls={ - 'Documentation': 'https://github.com/opengisch/qgis-server-light', - 'Changelog': 'https://github.com/opengisch/qgis-server-light/blob/master/CHANGES.md', - 'Issue Tracker': 'https://github.com/opengisch/qgis-server-light/issues', - }, - extras_require={ - 'recommend': recommend, - 'no-version': requires, - 'testing': tests_require, + "Documentation": "https://github.com/opengisch/qgis-server-light", + "Changelog": "https://github.com/opengisch/qgis-server-light/blob/master/CHANGES.md", + "Issue Tracker": "https://github.com/opengisch/qgis-server-light/issues", }, - entry_points={ - } + install_requires=install_requires, + entry_points={"console_scripts": [] + worker_scripts}, ) diff --git a/src/qgis_server_light/worker/redis.py b/src/qgis_server_light/worker/redis.py index 591d1b6..45c7e6e 100644 --- a/src/qgis_server_light/worker/redis.py +++ b/src/qgis_server_light/worker/redis.py @@ -7,25 +7,33 @@ import pickle import signal import time -from typing import List, Optional +from typing import List +from typing import Optional + import redis -from xsdata.formats.dataclass.parsers import DictDecoder, JsonParser -from qgis_server_light.interface.job import QslGetMapJob, QslGetFeatureInfoJob, QslLegendJob, \ - JobRunnerInfoQslGetMapJob, JobRunnerInfoQslGetFeatureInfoJob, JobRunnerInfoQslLegendJob -from qgis_server_light.worker.engine import Engine, EngineContext +from xsdata.formats.dataclass.parsers import JsonParser + +from qgis_server_light.interface.job import JobRunnerInfoQslGetFeatureInfoJob +from qgis_server_light.interface.job import JobRunnerInfoQslGetMapJob +from qgis_server_light.interface.job import JobRunnerInfoQslLegendJob +from qgis_server_light.worker.engine import Engine +from qgis_server_light.worker.engine import EngineContext class RedisEngine(Engine): - def __init__(self, context: EngineContext, svg_paths: Optional[List] = None) -> None: + def __init__( + self, context: EngineContext, svg_paths: Optional[List] = None + ) -> None: super().__init__(context, svg_paths) self.shutdown = False def exit_gracefully(self, signum, frame): print("Received:", signum) self.shutdown = True + # actually exit the programm (for some reason it is not working with the shutdown switch) + exit(0) def run(self, redis_url): - log = logging.getLogger(__name__) signal.signal(signal.SIGINT, self.exit_gracefully) signal.signal(signal.SIGTERM, self.exit_gracefully) @@ -50,11 +58,19 @@ def run(self, redis_url): _, job_info_json = r.blpop("jobs") job_info_dict = json.loads(job_info_json) if JobRunnerInfoQslGetMapJob.__name__ == job_info_dict["type"]: - job_info = JsonParser().from_bytes(job_info_json, JobRunnerInfoQslGetMapJob) - elif JobRunnerInfoQslGetFeatureInfoJob.__name__ == job_info_dict["type"]: - job_info = JsonParser().from_bytes(job_info_json, JobRunnerInfoQslGetFeatureInfoJob) + job_info = JsonParser().from_bytes( + job_info_json, JobRunnerInfoQslGetMapJob + ) + elif ( + JobRunnerInfoQslGetFeatureInfoJob.__name__ == job_info_dict["type"] + ): + job_info = JsonParser().from_bytes( + job_info_json, JobRunnerInfoQslGetFeatureInfoJob + ) elif JobRunnerInfoQslLegendJob.__name__ == job_info_dict["type"]: - job_info = JsonParser().from_bytes(job_info_json, JobRunnerInfoQslLegendJob) + job_info = JsonParser().from_bytes( + job_info_json, JobRunnerInfoQslLegendJob + ) else: raise NotImplementedError( @@ -100,33 +116,33 @@ def main() -> None: "--redis-url", type=str, help="redis url", - default=os.environ.get("REDIS_URL"), + default=os.environ.get("QSL_REDIS_URL"), ) parser.add_argument( "--log-level", type=str, help="log level (debug, info, warning or error)", - default=os.environ.get("LOG_LEVEL") or "info", + default=os.environ.get("QSL_LOG_LEVEL") or "info", ) parser.add_argument( "--data-root", type=str, help="Absolute path to the data dir.", - default=os.environ.get("DATA_ROOT") or "/io/data", + default=os.environ.get("QSL_DATA_ROOT") or "/io/data", ) parser.add_argument( "--svg-path", type=str, help="Absolute path to additional svg files. Multiple paths can be separated by `:`. Defaults to /io/svg", - default=os.environ.get("SVG_PATH") or "/io/svg", + default=os.environ.get("QSL_SVG_PATH") or "/io/svg", ) args = parser.parse_args() - LOG_LEVEL = os.environ.get("LOG_LEVEL", "WARNING").upper() + LOG_LEVEL = os.environ.get("QSL_LOG_LEVEL", "WARNING").upper() logging.basicConfig( level=LOG_LEVEL, format="%(asctime)s [%(levelname)s] %(message)s" @@ -136,7 +152,7 @@ def main() -> None: log.info(json.dumps(dict(os.environ), indent=2)) if not args.redis_url: - raise AssertionError("no redis host specified (--redis-url, REDIS_URL)") + raise AssertionError("no redis host specified (--redis-url, QSL_REDIS_URL)") svg_paths = args.svg_path.split(":") engine = RedisEngine(EngineContext(args.data_root), svg_paths)