From 28a5133412132b187812a512d65d8f5de42fc2c1 Mon Sep 17 00:00:00 2001 From: "alexander.dityatin" Date: Wed, 8 Jan 2025 16:55:10 +0300 Subject: [PATCH 1/8] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B2=D0=BE=D0=B5=20?= =?UTF-8?q?=D0=B7=D0=B0=D0=B4=D0=B0=D0=BD=D0=B8=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 387719232d74bcb8d110eb1079fe9b494ce3da67) --- task1.drawio | 496 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 496 insertions(+) create mode 100644 task1.drawio diff --git a/task1.drawio b/task1.drawio new file mode 100644 index 00000000..5d6a89f4 --- /dev/null +++ b/task1.drawio @@ -0,0 +1,496 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From cbd94e6dcca2a2707f1d48c67ec4dc1f1651194d Mon Sep 17 00:00:00 2001 From: "alexander.dityatin" Date: Wed, 8 Jan 2025 20:45:11 +0300 Subject: [PATCH 2/8] =?UTF-8?q?=D0=92=D1=82=D0=BE=D1=80=D0=BE=D0=B5=20?= =?UTF-8?q?=D0=B7=D0=B0=D0=B4=D0=B0=D0=BD=D0=B8=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 5d6b7c191780b90db0d1c71a90eaa147241cdebf) (cherry picked from commit e1357133f8deedcd015fab7ebe32b44892b4b726) --- mongo-sharding/README.md | 25 +++ mongo-sharding/api_app/Dockerfile | 10 ++ mongo-sharding/api_app/app.py | 193 ++++++++++++++++++++++++ mongo-sharding/api_app/requirements.txt | 6 + mongo-sharding/compose.yaml | 129 ++++++++++++++++ mongo-sharding/scripts/mongo-init.sh | 57 +++++++ 6 files changed, 420 insertions(+) create mode 100644 mongo-sharding/README.md create mode 100644 mongo-sharding/api_app/Dockerfile create mode 100644 mongo-sharding/api_app/app.py create mode 100644 mongo-sharding/api_app/requirements.txt create mode 100644 mongo-sharding/compose.yaml create mode 100755 mongo-sharding/scripts/mongo-init.sh diff --git a/mongo-sharding/README.md b/mongo-sharding/README.md new file mode 100644 index 00000000..38658e3d --- /dev/null +++ b/mongo-sharding/README.md @@ -0,0 +1,25 @@ +## Как запустить + +Запускаем mongodb и приложение + +```shell +docker compose up -d +``` + +Конфигурируем кластер и заполняем данными +> **_NOTE:_** если скрипт не отработал с первого раза, перезапустите его, +> возможно кластер не развернулся полностью + +```shell +./scripts/mongo-init.sh +``` + +## Как проверить + +### Если вы запускаете проект на локальной машине + +Откройте в браузере http://localhost:8080 + +## Доступные эндпоинты + +Список доступных эндпоинтов, swagger http://:8080/docs \ No newline at end of file diff --git a/mongo-sharding/api_app/Dockerfile b/mongo-sharding/api_app/Dockerfile new file mode 100644 index 00000000..46f6c9d0 --- /dev/null +++ b/mongo-sharding/api_app/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12.1-slim +WORKDIR /app +EXPOSE 8080 +COPY requirements.txt ./ +# Устанавливаем зависимости python не пересобирая их +RUN pip install --no-cache --no-cache-dir -r requirements.txt +# Копирование кода приложения +COPY app.py /app/ +ENTRYPOINT ["uvicorn"] +CMD ["app:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/mongo-sharding/api_app/app.py b/mongo-sharding/api_app/app.py new file mode 100644 index 00000000..339675f7 --- /dev/null +++ b/mongo-sharding/api_app/app.py @@ -0,0 +1,193 @@ +import json +import logging +import os +import time +from typing import List, Optional + +import motor.motor_asyncio +from bson import ObjectId +from fastapi import Body, FastAPI, HTTPException, status +from fastapi_cache import FastAPICache +from fastapi_cache.backends.redis import RedisBackend +from fastapi_cache.decorator import cache +from logmiddleware import RouterLoggingMiddleware, logging_config +from pydantic import BaseModel, ConfigDict, EmailStr, Field +from pydantic.functional_validators import BeforeValidator +from pymongo import errors +from redis import asyncio as aioredis +from typing_extensions import Annotated + +# Configure JSON logging +logging.config.dictConfig(logging_config) +logger = logging.getLogger(__name__) + +app = FastAPI() +app.add_middleware( + RouterLoggingMiddleware, + logger=logger, +) + +DATABASE_URL = os.environ["MONGODB_URL"] +DATABASE_NAME = os.environ["MONGODB_DATABASE_NAME"] +REDIS_URL = os.getenv("REDIS_URL", None) + + +def nocache(*args, **kwargs): + def decorator(func): + return func + + return decorator + + +if REDIS_URL: + cache = cache +else: + cache = nocache + + +client = motor.motor_asyncio.AsyncIOMotorClient(DATABASE_URL) + +db = client[DATABASE_NAME] + +# Represents an ObjectId field in the database. +# It will be represented as a `str` on the model so that it can be serialized to JSON. +PyObjectId = Annotated[str, BeforeValidator(str)] + + +@app.on_event("startup") +async def startup(): + if REDIS_URL: + redis = aioredis.from_url(REDIS_URL, encoding="utf8", decode_responses=True) + FastAPICache.init(RedisBackend(redis), prefix="api:cache") + + +class UserModel(BaseModel): + """ + Container for a single user record. + """ + + id: Optional[PyObjectId] = Field(alias="_id", default=None) + age: int = Field(...) + name: str = Field(...) + + +class UserCollection(BaseModel): + """ + A container holding a list of `UserModel` instances. + """ + + users: List[UserModel] + + +@app.get("/") +async def root(): + collection_names = await db.list_collection_names() + collections = {} + for collection_name in collection_names: + collection = db.get_collection(collection_name) + collections[collection_name] = { + "documents_count": await collection.count_documents({}) + } + try: + replica_status = await client.admin.command("replSetGetStatus") + replica_status = json.dumps(replica_status, indent=2, default=str) + except errors.OperationFailure: + replica_status = "No Replicas" + + topology_description = client.topology_description + read_preference = client.client_options.read_preference + topology_type = topology_description.topology_type_name + replicaset_name = topology_description.replica_set_name + + shards = None + if topology_type == "Sharded": + shards_list = await client.admin.command("listShards") + shards = {} + for shard in shards_list.get("shards", {}): + shards[shard["_id"]] = shard["host"] + + cache_enabled = False + if REDIS_URL: + cache_enabled = FastAPICache.get_enable() + + return { + "mongo_topology_type": topology_type, + "mongo_replicaset_name": replicaset_name, + "mongo_db": DATABASE_NAME, + "read_preference": str(read_preference), + "mongo_nodes": client.nodes, + "mongo_primary_host": client.primary, + "mongo_secondary_hosts": client.secondaries, + "mongo_address": client.address, + "mongo_is_primary": client.is_primary, + "mongo_is_mongos": client.is_mongos, + "collections": collections, + "shards": shards, + "cache_enabled": cache_enabled, + "status": "OK", + } + + +@app.get("/{collection_name}/count") +async def collection_count(collection_name: str): + collection = db.get_collection(collection_name) + items_count = await collection.count_documents({}) + # status = await client.admin.command('replSetGetStatus') + # import ipdb; ipdb.set_trace() + return {"status": "OK", "mongo_db": DATABASE_NAME, "items_count": items_count} + + +@app.get( + "/{collection_name}/users", + response_description="List all users", + response_model=UserCollection, + response_model_by_alias=False, +) +@cache(expire=60 * 1) +async def list_users(collection_name: str): + """ + List all of the user data in the database. + The response is unpaginated and limited to 1000 results. + """ + time.sleep(1) + collection = db.get_collection(collection_name) + return UserCollection(users=await collection.find().to_list(1000)) + + +@app.get( + "/{collection_name}/users/{name}", + response_description="Get a single user", + response_model=UserModel, + response_model_by_alias=False, +) +async def show_user(collection_name: str, name: str): + """ + Get the record for a specific user, looked up by `name`. + """ + + collection = db.get_collection(collection_name) + if (user := await collection.find_one({"name": name})) is not None: + return user + + raise HTTPException(status_code=404, detail=f"User {name} not found") + + +@app.post( + "/{collection_name}/users", + response_description="Add new user", + response_model=UserModel, + status_code=status.HTTP_201_CREATED, + response_model_by_alias=False, +) +async def create_user(collection_name: str, user: UserModel = Body(...)): + """ + Insert a new user record. + + A unique `id` will be created and provided in the response. + """ + collection = db.get_collection(collection_name) + new_user = await collection.insert_one( + user.model_dump(by_alias=True, exclude=["id"]) + ) + created_user = await collection.find_one({"_id": new_user.inserted_id}) + return created_user diff --git a/mongo-sharding/api_app/requirements.txt b/mongo-sharding/api_app/requirements.txt new file mode 100644 index 00000000..61b29b87 --- /dev/null +++ b/mongo-sharding/api_app/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.110.2 +uvicorn[standard]==0.29.0 +motor==3.5.3 +redis==4.4.2 +fastapi-cache2==0.2.0 +logmiddleware==0.0.4 \ No newline at end of file diff --git a/mongo-sharding/compose.yaml b/mongo-sharding/compose.yaml new file mode 100644 index 00000000..90ed55b0 --- /dev/null +++ b/mongo-sharding/compose.yaml @@ -0,0 +1,129 @@ +services: + pymongo_api: + container_name: pymongo_api + build: + context: mongo-sharding + dockerfile: Dockerfile + image: kazhem/pymongo_api:1.0.0 + depends_on: + - shard1 + - shard2 + ports: + - 8080:8080 + networks: + app-network: + ipv4_address: 173.17.0.15 + environment: + MONGODB_URL: "mongodb://mongos_router:27020" + MONGODB_DATABASE_NAME: "somedb" + + configSrv: + image: mongo:8.0.4 + container_name: configSrv + restart: always + ports: + - "27017:27017" + networks: + app-network: + ipv4_address: 173.17.0.10 + volumes: + - config-data:/data/db + command: + [ + "--configsvr", + "--replSet", + "config_server", + "--bind_ip_all", + "--port", + "27017" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + mongos_router: + image: mongo:8.0.4 + container_name: mongos_router + restart: always + ports: + - "27020:27020" + networks: + app-network: + ipv4_address: 173.17.0.7 + command: + [ + "mongos", + "--configdb", + "config_server/configSrv:27017", + "--bind_ip_all", + "--port", + "27020" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard1: + image: mongo:8.0.4 + container_name: shard1 + restart: always + ports: + - "27018:27018" + networks: + app-network: + ipv4_address: 173.17.0.9 + volumes: + - shard1-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27018" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard2: + image: mongo:8.0.4 + container_name: shard2 + restart: always + ports: + - "27019:27019" + networks: + app-network: + ipv4_address: 173.17.0.8 + volumes: + - shard2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27019" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + +networks: + app-network: + driver: bridge + ipam: + driver: default + config: + - subnet: 173.17.0.0/16 + +volumes: + config-data: + shard1-data: + shard2-data: \ No newline at end of file diff --git a/mongo-sharding/scripts/mongo-init.sh b/mongo-sharding/scripts/mongo-init.sh new file mode 100755 index 00000000..a863c210 --- /dev/null +++ b/mongo-sharding/scripts/mongo-init.sh @@ -0,0 +1,57 @@ +#!/bin/bash + + +### Инициализация конфигурационного сервиса + +docker compose exec -T configSrv mongosh --port 27017 --quiet < Date: Sun, 12 Jan 2025 00:31:04 +0300 Subject: [PATCH 3/8] =?UTF-8?q?=D0=A2=D1=80=D0=B5=D1=82=D1=8C=D0=B5=20?= =?UTF-8?q?=D0=B7=D0=B0=D0=B4=D0=B0=D0=BD=D0=B8=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 59f5101854d5a945d49953eb7d90bf5ccb8cb7d0) --- mongo-sharding-repl/README.md | 25 ++ mongo-sharding-repl/api_app/Dockerfile | 10 + mongo-sharding-repl/api_app/app.py | 193 +++++++++++ mongo-sharding-repl/api_app/requirements.txt | 6 + mongo-sharding-repl/compose.yaml | 336 +++++++++++++++++++ mongo-sharding-repl/scripts/mongo-init.sh | 63 ++++ 6 files changed, 633 insertions(+) create mode 100644 mongo-sharding-repl/README.md create mode 100644 mongo-sharding-repl/api_app/Dockerfile create mode 100644 mongo-sharding-repl/api_app/app.py create mode 100644 mongo-sharding-repl/api_app/requirements.txt create mode 100644 mongo-sharding-repl/compose.yaml create mode 100755 mongo-sharding-repl/scripts/mongo-init.sh diff --git a/mongo-sharding-repl/README.md b/mongo-sharding-repl/README.md new file mode 100644 index 00000000..38658e3d --- /dev/null +++ b/mongo-sharding-repl/README.md @@ -0,0 +1,25 @@ +## Как запустить + +Запускаем mongodb и приложение + +```shell +docker compose up -d +``` + +Конфигурируем кластер и заполняем данными +> **_NOTE:_** если скрипт не отработал с первого раза, перезапустите его, +> возможно кластер не развернулся полностью + +```shell +./scripts/mongo-init.sh +``` + +## Как проверить + +### Если вы запускаете проект на локальной машине + +Откройте в браузере http://localhost:8080 + +## Доступные эндпоинты + +Список доступных эндпоинтов, swagger http://:8080/docs \ No newline at end of file diff --git a/mongo-sharding-repl/api_app/Dockerfile b/mongo-sharding-repl/api_app/Dockerfile new file mode 100644 index 00000000..46f6c9d0 --- /dev/null +++ b/mongo-sharding-repl/api_app/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12.1-slim +WORKDIR /app +EXPOSE 8080 +COPY requirements.txt ./ +# Устанавливаем зависимости python не пересобирая их +RUN pip install --no-cache --no-cache-dir -r requirements.txt +# Копирование кода приложения +COPY app.py /app/ +ENTRYPOINT ["uvicorn"] +CMD ["app:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/mongo-sharding-repl/api_app/app.py b/mongo-sharding-repl/api_app/app.py new file mode 100644 index 00000000..339675f7 --- /dev/null +++ b/mongo-sharding-repl/api_app/app.py @@ -0,0 +1,193 @@ +import json +import logging +import os +import time +from typing import List, Optional + +import motor.motor_asyncio +from bson import ObjectId +from fastapi import Body, FastAPI, HTTPException, status +from fastapi_cache import FastAPICache +from fastapi_cache.backends.redis import RedisBackend +from fastapi_cache.decorator import cache +from logmiddleware import RouterLoggingMiddleware, logging_config +from pydantic import BaseModel, ConfigDict, EmailStr, Field +from pydantic.functional_validators import BeforeValidator +from pymongo import errors +from redis import asyncio as aioredis +from typing_extensions import Annotated + +# Configure JSON logging +logging.config.dictConfig(logging_config) +logger = logging.getLogger(__name__) + +app = FastAPI() +app.add_middleware( + RouterLoggingMiddleware, + logger=logger, +) + +DATABASE_URL = os.environ["MONGODB_URL"] +DATABASE_NAME = os.environ["MONGODB_DATABASE_NAME"] +REDIS_URL = os.getenv("REDIS_URL", None) + + +def nocache(*args, **kwargs): + def decorator(func): + return func + + return decorator + + +if REDIS_URL: + cache = cache +else: + cache = nocache + + +client = motor.motor_asyncio.AsyncIOMotorClient(DATABASE_URL) + +db = client[DATABASE_NAME] + +# Represents an ObjectId field in the database. +# It will be represented as a `str` on the model so that it can be serialized to JSON. +PyObjectId = Annotated[str, BeforeValidator(str)] + + +@app.on_event("startup") +async def startup(): + if REDIS_URL: + redis = aioredis.from_url(REDIS_URL, encoding="utf8", decode_responses=True) + FastAPICache.init(RedisBackend(redis), prefix="api:cache") + + +class UserModel(BaseModel): + """ + Container for a single user record. + """ + + id: Optional[PyObjectId] = Field(alias="_id", default=None) + age: int = Field(...) + name: str = Field(...) + + +class UserCollection(BaseModel): + """ + A container holding a list of `UserModel` instances. + """ + + users: List[UserModel] + + +@app.get("/") +async def root(): + collection_names = await db.list_collection_names() + collections = {} + for collection_name in collection_names: + collection = db.get_collection(collection_name) + collections[collection_name] = { + "documents_count": await collection.count_documents({}) + } + try: + replica_status = await client.admin.command("replSetGetStatus") + replica_status = json.dumps(replica_status, indent=2, default=str) + except errors.OperationFailure: + replica_status = "No Replicas" + + topology_description = client.topology_description + read_preference = client.client_options.read_preference + topology_type = topology_description.topology_type_name + replicaset_name = topology_description.replica_set_name + + shards = None + if topology_type == "Sharded": + shards_list = await client.admin.command("listShards") + shards = {} + for shard in shards_list.get("shards", {}): + shards[shard["_id"]] = shard["host"] + + cache_enabled = False + if REDIS_URL: + cache_enabled = FastAPICache.get_enable() + + return { + "mongo_topology_type": topology_type, + "mongo_replicaset_name": replicaset_name, + "mongo_db": DATABASE_NAME, + "read_preference": str(read_preference), + "mongo_nodes": client.nodes, + "mongo_primary_host": client.primary, + "mongo_secondary_hosts": client.secondaries, + "mongo_address": client.address, + "mongo_is_primary": client.is_primary, + "mongo_is_mongos": client.is_mongos, + "collections": collections, + "shards": shards, + "cache_enabled": cache_enabled, + "status": "OK", + } + + +@app.get("/{collection_name}/count") +async def collection_count(collection_name: str): + collection = db.get_collection(collection_name) + items_count = await collection.count_documents({}) + # status = await client.admin.command('replSetGetStatus') + # import ipdb; ipdb.set_trace() + return {"status": "OK", "mongo_db": DATABASE_NAME, "items_count": items_count} + + +@app.get( + "/{collection_name}/users", + response_description="List all users", + response_model=UserCollection, + response_model_by_alias=False, +) +@cache(expire=60 * 1) +async def list_users(collection_name: str): + """ + List all of the user data in the database. + The response is unpaginated and limited to 1000 results. + """ + time.sleep(1) + collection = db.get_collection(collection_name) + return UserCollection(users=await collection.find().to_list(1000)) + + +@app.get( + "/{collection_name}/users/{name}", + response_description="Get a single user", + response_model=UserModel, + response_model_by_alias=False, +) +async def show_user(collection_name: str, name: str): + """ + Get the record for a specific user, looked up by `name`. + """ + + collection = db.get_collection(collection_name) + if (user := await collection.find_one({"name": name})) is not None: + return user + + raise HTTPException(status_code=404, detail=f"User {name} not found") + + +@app.post( + "/{collection_name}/users", + response_description="Add new user", + response_model=UserModel, + status_code=status.HTTP_201_CREATED, + response_model_by_alias=False, +) +async def create_user(collection_name: str, user: UserModel = Body(...)): + """ + Insert a new user record. + + A unique `id` will be created and provided in the response. + """ + collection = db.get_collection(collection_name) + new_user = await collection.insert_one( + user.model_dump(by_alias=True, exclude=["id"]) + ) + created_user = await collection.find_one({"_id": new_user.inserted_id}) + return created_user diff --git a/mongo-sharding-repl/api_app/requirements.txt b/mongo-sharding-repl/api_app/requirements.txt new file mode 100644 index 00000000..61b29b87 --- /dev/null +++ b/mongo-sharding-repl/api_app/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.110.2 +uvicorn[standard]==0.29.0 +motor==3.5.3 +redis==4.4.2 +fastapi-cache2==0.2.0 +logmiddleware==0.0.4 \ No newline at end of file diff --git a/mongo-sharding-repl/compose.yaml b/mongo-sharding-repl/compose.yaml new file mode 100644 index 00000000..393b5d58 --- /dev/null +++ b/mongo-sharding-repl/compose.yaml @@ -0,0 +1,336 @@ +services: + pymongo_api: + container_name: pymongo_api + build: + context: mongo-sharding-repl + dockerfile: Dockerfile + image: kazhem/pymongo_api:1.0.0 + depends_on: + - shard1_1 + - shard1_2 + - shard1_3 + - shard2_1 + - shard2_2 + - shard2_3 + ports: + - 8080:8080 + networks: + app-network: + ipv4_address: 173.17.0.10 + environment: + MONGODB_URL: "mongodb://mongos_router0:27010,mongos_router1:27011,mongos_router2:27012" + MONGODB_DATABASE_NAME: "somedb" + + configSrv0: + image: mongo:8.0.4 + container_name: configSrv0 + restart: always + ports: + - "27000:27000" + networks: + app-network: + ipv4_address: 173.17.0.2 + volumes: + - config-data0:/data/db + command: + [ + "--configsvr", + "--replSet", + "config_server", + "--bind_ip_all", + "--port", + "27000" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + configSrv1: + image: mongo:8.0.4 + container_name: configSrv1 + restart: always + ports: + - "27001:27001" + networks: + app-network: + ipv4_address: 173.17.0.3 + volumes: + - config-data1:/data/db + command: + [ + "--configsvr", + "--replSet", + "config_server", + "--bind_ip_all", + "--port", + "27001" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + configSrv2: + image: mongo:8.0.4 + container_name: configSrv2 + restart: always + ports: + - "27002:27002" + networks: + app-network: + ipv4_address: 173.17.0.4 + volumes: + - config-data2:/data/db + command: + [ + "--configsvr", + "--replSet", + "config_server", + "--bind_ip_all", + "--port", + "27002" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + mongos_router0: + image: mongo:8.0.4 + container_name: mongos_router0 + restart: always + ports: + - "27010:27010" + networks: + app-network: + ipv4_address: 173.17.0.5 + command: + [ + "mongos", + "--configdb", + "config_server/configSrv0:27000,configSrv1:27001,configSrv2:27002", + "--bind_ip_all", + "--port", + "27010" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + mongos_router1: + image: mongo:8.0.4 + container_name: mongos_router1 + restart: always + ports: + - "27011:27011" + networks: + app-network: + ipv4_address: 173.17.0.6 + command: + [ + "mongos", + "--configdb", + "config_server/configSrv0:27000,configSrv1:27001,configSrv2:27002", + "--bind_ip_all", + "--port", + "27011" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + mongos_router2: + image: mongo:8.0.4 + container_name: mongos_router2 + restart: always + ports: + - "27012:27012" + networks: + app-network: + ipv4_address: 173.17.0.7 + command: + [ + "mongos", + "--configdb", + "config_server/configSrv0:27000,configSrv1:27001,configSrv2:27002", + "--bind_ip_all", + "--port", + "27012" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard1_1: + image: mongo:8.0.4 + container_name: shard1_1 + restart: always + ports: + - "27018:27018" + networks: + app-network: + ipv4_address: 173.17.0.8 + volumes: + - shard1_1-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27018" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard1_2: + image: mongo:8.0.4 + container_name: shard1_2 + restart: always + ports: + - "27028:27028" + networks: + app-network: + ipv4_address: 173.17.0.9 + volumes: + - shard1_2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27028" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard1_3: + image: mongo:8.0.4 + container_name: shard1_3 + restart: always + ports: + - "27038:27038" + networks: + app-network: + ipv4_address: 173.17.0.11 + volumes: + - shard1_3-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27038" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard2_1: + image: mongo:8.0.4 + container_name: shard2_1 + restart: always + ports: + - "27019:27019" + networks: + app-network: + ipv4_address: 173.17.0.12 + volumes: + - shard2_1-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27019" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + + shard2_2: + image: mongo:8.0.4 + container_name: shard2_2 + restart: always + ports: + - "27029:27029" + networks: + app-network: + ipv4_address: 173.17.0.13 + volumes: + - shard2_2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27029" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard2_3: + image: mongo:8.0.4 + container_name: shard2_3 + restart: always + ports: + - "27039:27039" + networks: + app-network: + ipv4_address: 173.17.0.14 + volumes: + - shard2_3-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27039" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + +networks: + app-network: + driver: bridge + ipam: + driver: default + config: + - subnet: 173.17.0.0/16 + +volumes: + config-data0: + config-data1: + config-data2: + shard1_1-data: + shard1_2-data: + shard1_3-data: + shard2_1-data: + shard2_2-data: + shard2_3-data: \ No newline at end of file diff --git a/mongo-sharding-repl/scripts/mongo-init.sh b/mongo-sharding-repl/scripts/mongo-init.sh new file mode 100755 index 00000000..33e81a23 --- /dev/null +++ b/mongo-sharding-repl/scripts/mongo-init.sh @@ -0,0 +1,63 @@ +#!/bin/bash + + +### Инициализация конфигурационного сервиса + +docker compose exec -T configSrv0 mongosh --port 27000 --quiet < Date: Mon, 13 Jan 2025 00:13:57 +0300 Subject: [PATCH 4/8] =?UTF-8?q?=D0=A7=D0=B5=D1=82=D0=B2=D1=91=D1=80=D1=82?= =?UTF-8?q?=D0=BE=D0=B5=20=D0=B7=D0=B0=D0=B4=D0=B0=D0=BD=D0=B8=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit e7d21b93c5f5fd98e7a4ef0892aa187c95aade6b) --- sharding-repl-cache/README.md | 25 ++ sharding-repl-cache/api_app/Dockerfile | 10 + sharding-repl-cache/api_app/app.py | 193 ++++++++++ sharding-repl-cache/api_app/requirements.txt | 6 + sharding-repl-cache/compose.yaml | 348 +++++++++++++++++++ sharding-repl-cache/scripts/mongo-init.sh | 63 ++++ 6 files changed, 645 insertions(+) create mode 100644 sharding-repl-cache/README.md create mode 100644 sharding-repl-cache/api_app/Dockerfile create mode 100644 sharding-repl-cache/api_app/app.py create mode 100644 sharding-repl-cache/api_app/requirements.txt create mode 100644 sharding-repl-cache/compose.yaml create mode 100755 sharding-repl-cache/scripts/mongo-init.sh diff --git a/sharding-repl-cache/README.md b/sharding-repl-cache/README.md new file mode 100644 index 00000000..38658e3d --- /dev/null +++ b/sharding-repl-cache/README.md @@ -0,0 +1,25 @@ +## Как запустить + +Запускаем mongodb и приложение + +```shell +docker compose up -d +``` + +Конфигурируем кластер и заполняем данными +> **_NOTE:_** если скрипт не отработал с первого раза, перезапустите его, +> возможно кластер не развернулся полностью + +```shell +./scripts/mongo-init.sh +``` + +## Как проверить + +### Если вы запускаете проект на локальной машине + +Откройте в браузере http://localhost:8080 + +## Доступные эндпоинты + +Список доступных эндпоинтов, swagger http://:8080/docs \ No newline at end of file diff --git a/sharding-repl-cache/api_app/Dockerfile b/sharding-repl-cache/api_app/Dockerfile new file mode 100644 index 00000000..46f6c9d0 --- /dev/null +++ b/sharding-repl-cache/api_app/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12.1-slim +WORKDIR /app +EXPOSE 8080 +COPY requirements.txt ./ +# Устанавливаем зависимости python не пересобирая их +RUN pip install --no-cache --no-cache-dir -r requirements.txt +# Копирование кода приложения +COPY app.py /app/ +ENTRYPOINT ["uvicorn"] +CMD ["app:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/sharding-repl-cache/api_app/app.py b/sharding-repl-cache/api_app/app.py new file mode 100644 index 00000000..339675f7 --- /dev/null +++ b/sharding-repl-cache/api_app/app.py @@ -0,0 +1,193 @@ +import json +import logging +import os +import time +from typing import List, Optional + +import motor.motor_asyncio +from bson import ObjectId +from fastapi import Body, FastAPI, HTTPException, status +from fastapi_cache import FastAPICache +from fastapi_cache.backends.redis import RedisBackend +from fastapi_cache.decorator import cache +from logmiddleware import RouterLoggingMiddleware, logging_config +from pydantic import BaseModel, ConfigDict, EmailStr, Field +from pydantic.functional_validators import BeforeValidator +from pymongo import errors +from redis import asyncio as aioredis +from typing_extensions import Annotated + +# Configure JSON logging +logging.config.dictConfig(logging_config) +logger = logging.getLogger(__name__) + +app = FastAPI() +app.add_middleware( + RouterLoggingMiddleware, + logger=logger, +) + +DATABASE_URL = os.environ["MONGODB_URL"] +DATABASE_NAME = os.environ["MONGODB_DATABASE_NAME"] +REDIS_URL = os.getenv("REDIS_URL", None) + + +def nocache(*args, **kwargs): + def decorator(func): + return func + + return decorator + + +if REDIS_URL: + cache = cache +else: + cache = nocache + + +client = motor.motor_asyncio.AsyncIOMotorClient(DATABASE_URL) + +db = client[DATABASE_NAME] + +# Represents an ObjectId field in the database. +# It will be represented as a `str` on the model so that it can be serialized to JSON. +PyObjectId = Annotated[str, BeforeValidator(str)] + + +@app.on_event("startup") +async def startup(): + if REDIS_URL: + redis = aioredis.from_url(REDIS_URL, encoding="utf8", decode_responses=True) + FastAPICache.init(RedisBackend(redis), prefix="api:cache") + + +class UserModel(BaseModel): + """ + Container for a single user record. + """ + + id: Optional[PyObjectId] = Field(alias="_id", default=None) + age: int = Field(...) + name: str = Field(...) + + +class UserCollection(BaseModel): + """ + A container holding a list of `UserModel` instances. + """ + + users: List[UserModel] + + +@app.get("/") +async def root(): + collection_names = await db.list_collection_names() + collections = {} + for collection_name in collection_names: + collection = db.get_collection(collection_name) + collections[collection_name] = { + "documents_count": await collection.count_documents({}) + } + try: + replica_status = await client.admin.command("replSetGetStatus") + replica_status = json.dumps(replica_status, indent=2, default=str) + except errors.OperationFailure: + replica_status = "No Replicas" + + topology_description = client.topology_description + read_preference = client.client_options.read_preference + topology_type = topology_description.topology_type_name + replicaset_name = topology_description.replica_set_name + + shards = None + if topology_type == "Sharded": + shards_list = await client.admin.command("listShards") + shards = {} + for shard in shards_list.get("shards", {}): + shards[shard["_id"]] = shard["host"] + + cache_enabled = False + if REDIS_URL: + cache_enabled = FastAPICache.get_enable() + + return { + "mongo_topology_type": topology_type, + "mongo_replicaset_name": replicaset_name, + "mongo_db": DATABASE_NAME, + "read_preference": str(read_preference), + "mongo_nodes": client.nodes, + "mongo_primary_host": client.primary, + "mongo_secondary_hosts": client.secondaries, + "mongo_address": client.address, + "mongo_is_primary": client.is_primary, + "mongo_is_mongos": client.is_mongos, + "collections": collections, + "shards": shards, + "cache_enabled": cache_enabled, + "status": "OK", + } + + +@app.get("/{collection_name}/count") +async def collection_count(collection_name: str): + collection = db.get_collection(collection_name) + items_count = await collection.count_documents({}) + # status = await client.admin.command('replSetGetStatus') + # import ipdb; ipdb.set_trace() + return {"status": "OK", "mongo_db": DATABASE_NAME, "items_count": items_count} + + +@app.get( + "/{collection_name}/users", + response_description="List all users", + response_model=UserCollection, + response_model_by_alias=False, +) +@cache(expire=60 * 1) +async def list_users(collection_name: str): + """ + List all of the user data in the database. + The response is unpaginated and limited to 1000 results. + """ + time.sleep(1) + collection = db.get_collection(collection_name) + return UserCollection(users=await collection.find().to_list(1000)) + + +@app.get( + "/{collection_name}/users/{name}", + response_description="Get a single user", + response_model=UserModel, + response_model_by_alias=False, +) +async def show_user(collection_name: str, name: str): + """ + Get the record for a specific user, looked up by `name`. + """ + + collection = db.get_collection(collection_name) + if (user := await collection.find_one({"name": name})) is not None: + return user + + raise HTTPException(status_code=404, detail=f"User {name} not found") + + +@app.post( + "/{collection_name}/users", + response_description="Add new user", + response_model=UserModel, + status_code=status.HTTP_201_CREATED, + response_model_by_alias=False, +) +async def create_user(collection_name: str, user: UserModel = Body(...)): + """ + Insert a new user record. + + A unique `id` will be created and provided in the response. + """ + collection = db.get_collection(collection_name) + new_user = await collection.insert_one( + user.model_dump(by_alias=True, exclude=["id"]) + ) + created_user = await collection.find_one({"_id": new_user.inserted_id}) + return created_user diff --git a/sharding-repl-cache/api_app/requirements.txt b/sharding-repl-cache/api_app/requirements.txt new file mode 100644 index 00000000..61b29b87 --- /dev/null +++ b/sharding-repl-cache/api_app/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.110.2 +uvicorn[standard]==0.29.0 +motor==3.5.3 +redis==4.4.2 +fastapi-cache2==0.2.0 +logmiddleware==0.0.4 \ No newline at end of file diff --git a/sharding-repl-cache/compose.yaml b/sharding-repl-cache/compose.yaml new file mode 100644 index 00000000..b533bbcc --- /dev/null +++ b/sharding-repl-cache/compose.yaml @@ -0,0 +1,348 @@ +services: + pymongo_api: + container_name: pymongo_api + build: + context: mongo-sharding-repl-cache + dockerfile: Dockerfile + image: kazhem/pymongo_api:1.0.0 + depends_on: + - shard1_1 + - shard1_2 + - shard1_3 + - shard2_1 + - shard2_2 + - shard2_3 + - redis + ports: + - 8080:8080 + networks: + app-network: + ipv4_address: 173.17.0.10 + environment: + MONGODB_URL: "mongodb://mongos_router0:27010,mongos_router1:27011,mongos_router2:27012" + MONGODB_DATABASE_NAME: "somedb" + REDIS_URL: "redis://redis:6379" + + configSrv0: + image: mongo:8.0.4 + container_name: configSrv0 + restart: always + ports: + - "27000:27000" + networks: + app-network: + ipv4_address: 173.17.0.2 + volumes: + - config-data0:/data/db + command: + [ + "--configsvr", + "--replSet", + "config_server", + "--bind_ip_all", + "--port", + "27000" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + configSrv1: + image: mongo:8.0.4 + container_name: configSrv1 + restart: always + ports: + - "27001:27001" + networks: + app-network: + ipv4_address: 173.17.0.3 + volumes: + - config-data1:/data/db + command: + [ + "--configsvr", + "--replSet", + "config_server", + "--bind_ip_all", + "--port", + "27001" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + configSrv2: + image: mongo:8.0.4 + container_name: configSrv2 + restart: always + ports: + - "27002:27002" + networks: + app-network: + ipv4_address: 173.17.0.4 + volumes: + - config-data2:/data/db + command: + [ + "--configsvr", + "--replSet", + "config_server", + "--bind_ip_all", + "--port", + "27002" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + mongos_router0: + image: mongo:8.0.4 + container_name: mongos_router0 + restart: always + ports: + - "27010:27010" + networks: + app-network: + ipv4_address: 173.17.0.5 + command: + [ + "mongos", + "--configdb", + "config_server/configSrv0:27000,configSrv1:27001,configSrv2:27002", + "--bind_ip_all", + "--port", + "27010" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + mongos_router1: + image: mongo:8.0.4 + container_name: mongos_router1 + restart: always + ports: + - "27011:27011" + networks: + app-network: + ipv4_address: 173.17.0.6 + command: + [ + "mongos", + "--configdb", + "config_server/configSrv0:27000,configSrv1:27001,configSrv2:27002", + "--bind_ip_all", + "--port", + "27011" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + mongos_router2: + image: mongo:8.0.4 + container_name: mongos_router2 + restart: always + ports: + - "27012:27012" + networks: + app-network: + ipv4_address: 173.17.0.7 + command: + [ + "mongos", + "--configdb", + "config_server/configSrv0:27000,configSrv1:27001,configSrv2:27002", + "--bind_ip_all", + "--port", + "27012" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard1_1: + image: mongo:8.0.4 + container_name: shard1_1 + restart: always + ports: + - "27018:27018" + networks: + app-network: + ipv4_address: 173.17.0.8 + volumes: + - shard1_1-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27018" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard1_2: + image: mongo:8.0.4 + container_name: shard1_2 + restart: always + ports: + - "27028:27028" + networks: + app-network: + ipv4_address: 173.17.0.9 + volumes: + - shard1_2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27028" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard1_3: + image: mongo:8.0.4 + container_name: shard1_3 + restart: always + ports: + - "27038:27038" + networks: + app-network: + ipv4_address: 173.17.0.11 + volumes: + - shard1_3-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27038" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard2_1: + image: mongo:8.0.4 + container_name: shard2_1 + restart: always + ports: + - "27019:27019" + networks: + app-network: + ipv4_address: 173.17.0.12 + volumes: + - shard2_1-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27019" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + + shard2_2: + image: mongo:8.0.4 + container_name: shard2_2 + restart: always + ports: + - "27029:27029" + networks: + app-network: + ipv4_address: 173.17.0.13 + volumes: + - shard2_2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27029" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard2_3: + image: mongo:8.0.4 + container_name: shard2_3 + restart: always + ports: + - "27039:27039" + networks: + app-network: + ipv4_address: 173.17.0.14 + volumes: + - shard2_3-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27039" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + redis: + image: "redis:7.4.2" + container_name: redis + ports: + - "6379" + command: [ "redis-server" ] + networks: + app-network: + ipv4_address: 173.17.0.15 + +networks: + app-network: + driver: bridge + ipam: + driver: default + config: + - subnet: 173.17.0.0/16 + +volumes: + config-data0: + config-data1: + config-data2: + shard1_1-data: + shard1_2-data: + shard1_3-data: + shard2_1-data: + shard2_2-data: + shard2_3-data: \ No newline at end of file diff --git a/sharding-repl-cache/scripts/mongo-init.sh b/sharding-repl-cache/scripts/mongo-init.sh new file mode 100755 index 00000000..33e81a23 --- /dev/null +++ b/sharding-repl-cache/scripts/mongo-init.sh @@ -0,0 +1,63 @@ +#!/bin/bash + + +### Инициализация конфигурационного сервиса + +docker compose exec -T configSrv0 mongosh --port 27000 --quiet < Date: Mon, 13 Jan 2025 00:55:54 +0300 Subject: [PATCH 5/8] =?UTF-8?q?=D0=9F=D1=8F=D1=82=D0=BE=D0=B5=20=D0=B8=20?= =?UTF-8?q?=D1=88=D0=B5=D1=81=D1=82=D0=BE=D0=B5=20=D0=B7=D0=B0=D0=B4=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 32aadf8e3e9ca32b1dc237d6b71968e3d368f663) --- diagram.drawio | 1229 ++++++++++++++++++++++++++++++++++++++++++++++++ task1.drawio | 496 ------------------- 2 files changed, 1229 insertions(+), 496 deletions(-) create mode 100644 diagram.drawio delete mode 100644 task1.drawio diff --git a/diagram.drawio b/diagram.drawio new file mode 100644 index 00000000..0ad47741 --- /dev/null +++ b/diagram.drawio @@ -0,0 +1,1229 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/task1.drawio b/task1.drawio deleted file mode 100644 index 5d6a89f4..00000000 --- a/task1.drawio +++ /dev/null @@ -1,496 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From ad673ec0055622223eb7c8e212ff143df3bcb1e5 Mon Sep 17 00:00:00 2001 From: "alexander.dityatin" Date: Mon, 13 Jan 2025 01:28:22 +0300 Subject: [PATCH 6/8] update root/README.md rename dir (cherry picked from commit 466e08ebb97619727cb47704881a64892a1ed9f6) --- README.md | 32 ++----------------- .../README.md | 4 +-- .../api_app/Dockerfile | 0 .../api_app/app.py | 0 .../api_app/requirements.txt | 0 .../compose.yaml | 0 .../scripts/mongo-init.sh | 0 7 files changed, 4 insertions(+), 32 deletions(-) rename {sharding-repl-cache => mongo-sharding-repl-cache}/README.md (83%) rename {sharding-repl-cache => mongo-sharding-repl-cache}/api_app/Dockerfile (100%) rename {sharding-repl-cache => mongo-sharding-repl-cache}/api_app/app.py (100%) rename {sharding-repl-cache => mongo-sharding-repl-cache}/api_app/requirements.txt (100%) rename {sharding-repl-cache => mongo-sharding-repl-cache}/compose.yaml (100%) rename {sharding-repl-cache => mongo-sharding-repl-cache}/scripts/mongo-init.sh (100%) diff --git a/README.md b/README.md index b6ddb826..0ec97051 100644 --- a/README.md +++ b/README.md @@ -2,34 +2,6 @@ ## Как запустить -Запускаем mongodb и приложение +Запускаем mongodb и приложения -```shell -docker compose up -d -``` - -Заполняем mongodb данными - -```shell -./scripts/mongo-init.sh -``` - -## Как проверить - -### Если вы запускаете проект на локальной машине - -Откройте в браузере http://localhost:8080 - -### Если вы запускаете проект на предоставленной виртуальной машине - -Узнать белый ip виртуальной машины - -```shell -curl --silent http://ifconfig.me -``` - -Откройте в браузере http://:8080 - -## Доступные эндпоинты - -Список доступных эндпоинтов, swagger http://:8080/docs \ No newline at end of file +Для запуска перейдите в mongo-sharding-repl-cache и следуйте инструкции в README.md \ No newline at end of file diff --git a/sharding-repl-cache/README.md b/mongo-sharding-repl-cache/README.md similarity index 83% rename from sharding-repl-cache/README.md rename to mongo-sharding-repl-cache/README.md index 38658e3d..e7bcc29a 100644 --- a/sharding-repl-cache/README.md +++ b/mongo-sharding-repl-cache/README.md @@ -1,12 +1,12 @@ ## Как запустить -Запускаем mongodb и приложение +Запускаем стенд ```shell docker compose up -d ``` -Конфигурируем кластер и заполняем данными +Конфигурируем кластер mongo и заполняем данными > **_NOTE:_** если скрипт не отработал с первого раза, перезапустите его, > возможно кластер не развернулся полностью diff --git a/sharding-repl-cache/api_app/Dockerfile b/mongo-sharding-repl-cache/api_app/Dockerfile similarity index 100% rename from sharding-repl-cache/api_app/Dockerfile rename to mongo-sharding-repl-cache/api_app/Dockerfile diff --git a/sharding-repl-cache/api_app/app.py b/mongo-sharding-repl-cache/api_app/app.py similarity index 100% rename from sharding-repl-cache/api_app/app.py rename to mongo-sharding-repl-cache/api_app/app.py diff --git a/sharding-repl-cache/api_app/requirements.txt b/mongo-sharding-repl-cache/api_app/requirements.txt similarity index 100% rename from sharding-repl-cache/api_app/requirements.txt rename to mongo-sharding-repl-cache/api_app/requirements.txt diff --git a/sharding-repl-cache/compose.yaml b/mongo-sharding-repl-cache/compose.yaml similarity index 100% rename from sharding-repl-cache/compose.yaml rename to mongo-sharding-repl-cache/compose.yaml diff --git a/sharding-repl-cache/scripts/mongo-init.sh b/mongo-sharding-repl-cache/scripts/mongo-init.sh similarity index 100% rename from sharding-repl-cache/scripts/mongo-init.sh rename to mongo-sharding-repl-cache/scripts/mongo-init.sh From 699c89aa2521cee3a36fc706246fd483129aebf8 Mon Sep 17 00:00:00 2001 From: "alexander.dityatin" Date: Wed, 15 Jan 2025 23:39:30 +0300 Subject: [PATCH 7/8] fix issue --- diagram.drawio | 959 ++++-------------- mongo-sharding-repl-cache/compose.yaml | 100 +- .../scripts/mongo-init.sh | 2 - mongo-sharding-repl/compose.yaml | 100 +- mongo-sharding-repl/scripts/mongo-init.sh | 2 - 5 files changed, 226 insertions(+), 937 deletions(-) diff --git a/diagram.drawio b/diagram.drawio index 0ad47741..b82347c3 100644 --- a/diagram.drawio +++ b/diagram.drawio @@ -72,7 +72,7 @@ - + @@ -95,8 +95,10 @@ - - + + + + @@ -104,9 +106,6 @@ - - - @@ -136,187 +135,84 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - + - - - - + - + - - - - + - - + + - + - + - + - - - - + - + - - - - + - + - + - + - + - + - + - + - + @@ -325,191 +221,76 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - + - - + + - + - - + + - - - - - - - + - + - - - - + - + - + - + - - - - - - - - - - + + - - + + + + + - + - - - - - - @@ -529,18 +310,17 @@ - - + + + + - + - - - @@ -579,81 +359,8 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + @@ -661,30 +368,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - @@ -694,22 +377,13 @@ - - - - - - - - - - + @@ -719,35 +393,11 @@ - - - - - - - - - - - - - - - - - - - - - - - - - + - + @@ -755,30 +405,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - @@ -789,7 +415,7 @@ - + @@ -821,407 +447,270 @@ + + + + + + + + + + + - + - - - - - + + + + - - + + - - + + - - - - + + - - + + - - - + + + + + + + + - - + + + + + + + + + - - + + + + + + + + - - + + + + + - - + + + + + - - + + + + + - - + + - - + + - - + + - - + + - + - - + - + + + + - + - + - - - - - + + - + - - - - - + - - - - - + + - - + + - - - - - - - + + - + - - - - - - - - - - - - - - + + - + - - - - - - + + - + - - - - - - + - + - - - - - - - - + + - - + + - + - - - - - - - - - - - - - - + + - - - - + - - - - - - - - - - - - - - + + - + - - + + - - - - - - - - - - - + + - - - - + - - - - - - + + - - - - - - - - - - + + + + + - + - - + + - - + + - - - - - + + - - + + - + - - + + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - + - - + + - + - + - - + + - + - - + + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + diff --git a/mongo-sharding-repl-cache/compose.yaml b/mongo-sharding-repl-cache/compose.yaml index b533bbcc..b4ec7423 100644 --- a/mongo-sharding-repl-cache/compose.yaml +++ b/mongo-sharding-repl-cache/compose.yaml @@ -19,7 +19,7 @@ services: app-network: ipv4_address: 173.17.0.10 environment: - MONGODB_URL: "mongodb://mongos_router0:27010,mongos_router1:27011,mongos_router2:27012" + MONGODB_URL: "mongodb://mongos_router0:27010" MONGODB_DATABASE_NAME: "somedb" REDIS_URL: "redis://redis:6379" @@ -48,56 +48,6 @@ services: interval: 5s start_period: 10s - configSrv1: - image: mongo:8.0.4 - container_name: configSrv1 - restart: always - ports: - - "27001:27001" - networks: - app-network: - ipv4_address: 173.17.0.3 - volumes: - - config-data1:/data/db - command: - [ - "--configsvr", - "--replSet", - "config_server", - "--bind_ip_all", - "--port", - "27001" - ] - healthcheck: - test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] - interval: 5s - start_period: 10s - - configSrv2: - image: mongo:8.0.4 - container_name: configSrv2 - restart: always - ports: - - "27002:27002" - networks: - app-network: - ipv4_address: 173.17.0.4 - volumes: - - config-data2:/data/db - command: - [ - "--configsvr", - "--replSet", - "config_server", - "--bind_ip_all", - "--port", - "27002" - ] - healthcheck: - test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] - interval: 5s - start_period: 10s - mongos_router0: image: mongo:8.0.4 container_name: mongos_router0 @@ -121,52 +71,6 @@ services: interval: 5s start_period: 10s - mongos_router1: - image: mongo:8.0.4 - container_name: mongos_router1 - restart: always - ports: - - "27011:27011" - networks: - app-network: - ipv4_address: 173.17.0.6 - command: - [ - "mongos", - "--configdb", - "config_server/configSrv0:27000,configSrv1:27001,configSrv2:27002", - "--bind_ip_all", - "--port", - "27011" - ] - healthcheck: - test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] - interval: 5s - start_period: 10s - - mongos_router2: - image: mongo:8.0.4 - container_name: mongos_router2 - restart: always - ports: - - "27012:27012" - networks: - app-network: - ipv4_address: 173.17.0.7 - command: - [ - "mongos", - "--configdb", - "config_server/configSrv0:27000,configSrv1:27001,configSrv2:27002", - "--bind_ip_all", - "--port", - "27012" - ] - healthcheck: - test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] - interval: 5s - start_period: 10s - shard1_1: image: mongo:8.0.4 container_name: shard1_1 @@ -338,8 +242,6 @@ networks: volumes: config-data0: - config-data1: - config-data2: shard1_1-data: shard1_2-data: shard1_3-data: diff --git a/mongo-sharding-repl-cache/scripts/mongo-init.sh b/mongo-sharding-repl-cache/scripts/mongo-init.sh index 33e81a23..e4c2024d 100755 --- a/mongo-sharding-repl-cache/scripts/mongo-init.sh +++ b/mongo-sharding-repl-cache/scripts/mongo-init.sh @@ -10,8 +10,6 @@ rs.initiate( configsvr: true, members: [ { _id : 0, host : "configSrv0:27000" }, - { _id : 1, host : "configSrv1:27001" }, - { _id : 2, host : "configSrv2:27002" }, ] } ); diff --git a/mongo-sharding-repl/compose.yaml b/mongo-sharding-repl/compose.yaml index 393b5d58..709cebb3 100644 --- a/mongo-sharding-repl/compose.yaml +++ b/mongo-sharding-repl/compose.yaml @@ -18,7 +18,7 @@ services: app-network: ipv4_address: 173.17.0.10 environment: - MONGODB_URL: "mongodb://mongos_router0:27010,mongos_router1:27011,mongos_router2:27012" + MONGODB_URL: "mongodb://mongos_router0:27010" MONGODB_DATABASE_NAME: "somedb" configSrv0: @@ -46,56 +46,6 @@ services: interval: 5s start_period: 10s - configSrv1: - image: mongo:8.0.4 - container_name: configSrv1 - restart: always - ports: - - "27001:27001" - networks: - app-network: - ipv4_address: 173.17.0.3 - volumes: - - config-data1:/data/db - command: - [ - "--configsvr", - "--replSet", - "config_server", - "--bind_ip_all", - "--port", - "27001" - ] - healthcheck: - test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] - interval: 5s - start_period: 10s - - configSrv2: - image: mongo:8.0.4 - container_name: configSrv2 - restart: always - ports: - - "27002:27002" - networks: - app-network: - ipv4_address: 173.17.0.4 - volumes: - - config-data2:/data/db - command: - [ - "--configsvr", - "--replSet", - "config_server", - "--bind_ip_all", - "--port", - "27002" - ] - healthcheck: - test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] - interval: 5s - start_period: 10s - mongos_router0: image: mongo:8.0.4 container_name: mongos_router0 @@ -119,52 +69,6 @@ services: interval: 5s start_period: 10s - mongos_router1: - image: mongo:8.0.4 - container_name: mongos_router1 - restart: always - ports: - - "27011:27011" - networks: - app-network: - ipv4_address: 173.17.0.6 - command: - [ - "mongos", - "--configdb", - "config_server/configSrv0:27000,configSrv1:27001,configSrv2:27002", - "--bind_ip_all", - "--port", - "27011" - ] - healthcheck: - test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] - interval: 5s - start_period: 10s - - mongos_router2: - image: mongo:8.0.4 - container_name: mongos_router2 - restart: always - ports: - - "27012:27012" - networks: - app-network: - ipv4_address: 173.17.0.7 - command: - [ - "mongos", - "--configdb", - "config_server/configSrv0:27000,configSrv1:27001,configSrv2:27002", - "--bind_ip_all", - "--port", - "27012" - ] - healthcheck: - test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] - interval: 5s - start_period: 10s - shard1_1: image: mongo:8.0.4 container_name: shard1_1 @@ -326,8 +230,6 @@ networks: volumes: config-data0: - config-data1: - config-data2: shard1_1-data: shard1_2-data: shard1_3-data: diff --git a/mongo-sharding-repl/scripts/mongo-init.sh b/mongo-sharding-repl/scripts/mongo-init.sh index 33e81a23..e4c2024d 100755 --- a/mongo-sharding-repl/scripts/mongo-init.sh +++ b/mongo-sharding-repl/scripts/mongo-init.sh @@ -10,8 +10,6 @@ rs.initiate( configsvr: true, members: [ { _id : 0, host : "configSrv0:27000" }, - { _id : 1, host : "configSrv1:27001" }, - { _id : 2, host : "configSrv2:27002" }, ] } ); From d98d40d6d34ed6bc34cdb05dd80fbf551dfeed69 Mon Sep 17 00:00:00 2001 From: "alexander.dityatin" Date: Tue, 4 Feb 2025 22:35:15 +0300 Subject: [PATCH 8/8] fix issue --- diagram.drawio | 132 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 90 insertions(+), 42 deletions(-) diff --git a/diagram.drawio b/diagram.drawio index b82347c3..9aaa2275 100644 --- a/diagram.drawio +++ b/diagram.drawio @@ -1,4 +1,4 @@ - + @@ -462,7 +462,7 @@ - + @@ -471,7 +471,7 @@ - + @@ -536,72 +536,72 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -610,31 +610,31 @@ - + - + - + - + - + - + - + @@ -644,34 +644,34 @@ - + - + - + - + - + - + - + - + @@ -680,25 +680,25 @@ - + - + - + - + @@ -709,9 +709,57 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +