diff --git a/.gitignore b/.gitignore index f5e96dbf..b9cd02b6 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ -venv \ No newline at end of file +venv + +.idea +.$schem.drawio.bkp \ No newline at end of file diff --git a/README.md b/README.md index b6ddb826..88bbd032 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,11 @@ # pymongo-api -## Как запустить - -Запускаем mongodb и приложение - -```shell -docker compose up -d -``` - -Заполняем mongodb данными - -```shell -./scripts/mongo-init.sh -``` - -## Как проверить - -### Если вы запускаете проект на локальной машине - -Откройте в браузере http://localhost:8080 - -### Если вы запускаете проект на предоставленной виртуальной машине - -Узнать белый ip виртуальной машины - -```shell -curl --silent http://ifconfig.me -``` - -Откройте в браузере http://:8080 - -## Доступные эндпоинты - -Список доступных эндпоинтов, swagger http://:8080/docs \ No newline at end of file +### Схема +Файл с схемой лежит в корне проекта ```./schem.drawio``` + +### Проекты +В проекте есть 4 папки +1. `./mongo-origin` - исходный проект +2. `./mongo-origin-sharding` - проект с шардированием +3. `./mongo-origin-sharding-repl` - проект с шардированием и репликой +4. `./mongo-origin-sharding-repl-cache` - проект с шардированием, репликой и redis \ No newline at end of file diff --git a/mongo-origin/.gitignore b/mongo-origin/.gitignore new file mode 100644 index 00000000..f5e96dbf --- /dev/null +++ b/mongo-origin/.gitignore @@ -0,0 +1 @@ +venv \ No newline at end of file diff --git a/mongo-origin/README.md b/mongo-origin/README.md new file mode 100644 index 00000000..b6ddb826 --- /dev/null +++ b/mongo-origin/README.md @@ -0,0 +1,35 @@ +# pymongo-api + +## Как запустить + +Запускаем mongodb и приложение + +```shell +docker compose up -d +``` + +Заполняем mongodb данными + +```shell +./scripts/mongo-init.sh +``` + +## Как проверить + +### Если вы запускаете проект на локальной машине + +Откройте в браузере http://localhost:8080 + +### Если вы запускаете проект на предоставленной виртуальной машине + +Узнать белый ip виртуальной машины + +```shell +curl --silent http://ifconfig.me +``` + +Откройте в браузере http://:8080 + +## Доступные эндпоинты + +Список доступных эндпоинтов, swagger http://:8080/docs \ No newline at end of file diff --git a/api_app/Dockerfile b/mongo-origin/api_app/Dockerfile similarity index 100% rename from api_app/Dockerfile rename to mongo-origin/api_app/Dockerfile diff --git a/api_app/app.py b/mongo-origin/api_app/app.py similarity index 100% rename from api_app/app.py rename to mongo-origin/api_app/app.py diff --git a/api_app/requirements.txt b/mongo-origin/api_app/requirements.txt similarity index 100% rename from api_app/requirements.txt rename to mongo-origin/api_app/requirements.txt diff --git a/compose.yaml b/mongo-origin/compose.yaml similarity index 100% rename from compose.yaml rename to mongo-origin/compose.yaml diff --git a/scripts/mongo-init.sh b/mongo-origin/scripts/mongo-init.sh similarity index 100% rename from scripts/mongo-init.sh rename to mongo-origin/scripts/mongo-init.sh diff --git a/mongo-sharding-repl-cache/.gitignore b/mongo-sharding-repl-cache/.gitignore new file mode 100644 index 00000000..f5e96dbf --- /dev/null +++ b/mongo-sharding-repl-cache/.gitignore @@ -0,0 +1 @@ +venv \ No newline at end of file diff --git a/mongo-sharding-repl-cache/README.md b/mongo-sharding-repl-cache/README.md new file mode 100644 index 00000000..d3595114 --- /dev/null +++ b/mongo-sharding-repl-cache/README.md @@ -0,0 +1,43 @@ +# pymongo-api + +## Как запустить + +Запускаем mongodb и приложение + +```shell +docker compose up -d +``` + +Заполняем mongodb данными + +```shell +./scripts/mongo-init.sh +``` +Удаляем БД + +```shell +./scripts/db-drop.sh +``` + +## Как проверить + +### Если вы запускаете проект на локальной машине + +1. Откройте в браузере http://localhost:8080/docs +2. Запросите данные о пользователе в БД `GET /helloDoc/users/` +3. Посмотреть время ответа ```docker compose logs pymongo_api``` +4. Запросить еще несколько раз `GET /helloDoc/users/` +5. Посмотреть время ответа ```docker compose logs pymongo_api``` + +Или можно создать нового пользователя +1. Создайте пользователя `POST /helloDoc/users` +2. Запросите созданного пользователя `GET /helloDoc/users/` +3. Посмотрите логи контейнера с приложением ``` docker compose logs pymongo_api``` + +### Результат: +В моем случае были такие тайминги: +```0.456s``` ускорилось до ```0.0085s``` + +## Доступные эндпоинты + +Список доступных эндпоинтов, swagger http://:8080/docs \ No newline at end of file diff --git a/mongo-sharding-repl-cache/api_app/Dockerfile b/mongo-sharding-repl-cache/api_app/Dockerfile new file mode 100644 index 00000000..46f6c9d0 --- /dev/null +++ b/mongo-sharding-repl-cache/api_app/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12.1-slim +WORKDIR /app +EXPOSE 8080 +COPY requirements.txt ./ +# Устанавливаем зависимости python не пересобирая их +RUN pip install --no-cache --no-cache-dir -r requirements.txt +# Копирование кода приложения +COPY app.py /app/ +ENTRYPOINT ["uvicorn"] +CMD ["app:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/mongo-sharding-repl-cache/api_app/app.py b/mongo-sharding-repl-cache/api_app/app.py new file mode 100644 index 00000000..cadc93f2 --- /dev/null +++ b/mongo-sharding-repl-cache/api_app/app.py @@ -0,0 +1,193 @@ +import json +import logging +import os +import time +from typing import List, Optional + +import motor.motor_asyncio +from bson import ObjectId +from fastapi import Body, FastAPI, HTTPException, status +from fastapi_cache import FastAPICache +from fastapi_cache.backends.redis import RedisBackend +from fastapi_cache.decorator import cache +from logmiddleware import RouterLoggingMiddleware, logging_config +from pydantic import BaseModel, ConfigDict, EmailStr, Field +from pydantic.functional_validators import BeforeValidator +from pymongo import errors +from redis import asyncio as aioredis +from typing_extensions import Annotated + +# Configure JSON logging +logging.config.dictConfig(logging_config) +logger = logging.getLogger(__name__) + +app = FastAPI() +app.add_middleware( + RouterLoggingMiddleware, + logger=logger, +) + +DATABASE_URL = os.environ["MONGODB_URL"] +DATABASE_NAME = os.environ["MONGODB_DATABASE_NAME"] +REDIS_URL = os.getenv("REDIS_URL", None) + + +def nocache(*args, **kwargs): + def decorator(func): + return func + + return decorator + + +if REDIS_URL: + cache = cache +else: + cache = nocache + + +client = motor.motor_asyncio.AsyncIOMotorClient(DATABASE_URL) +db = client[DATABASE_NAME] + +# Represents an ObjectId field in the database. +# It will be represented as a `str` on the model so that it can be serialized to JSON. +PyObjectId = Annotated[str, BeforeValidator(str)] + + +@app.on_event("startup") +async def startup(): + if REDIS_URL: + redis = aioredis.from_url(REDIS_URL, encoding="utf8", decode_responses=True) + FastAPICache.init(RedisBackend(redis), prefix="api:cache") + + +class UserModel(BaseModel): + """ + Container for a single user record. + """ + + id: Optional[PyObjectId] = Field(alias="_id", default=None) + age: int = Field(...) + name: str = Field(...) + + +class UserCollection(BaseModel): + """ + A container holding a list of `UserModel` instances. + """ + + users: List[UserModel] + + +@app.get("/") +async def root(): + collection_names = await db.list_collection_names() + collections = {} + for collection_name in collection_names: + collection = db.get_collection(collection_name) + collections[collection_name] = { + "documents_count": await collection.count_documents({}) + } + try: + replica_status = await client.admin.command("replSetGetStatus") + replica_status = json.dumps(replica_status, indent=2, default=str) + except errors.OperationFailure: + replica_status = "No Replicas" + + topology_description = client.topology_description + read_preference = client.client_options.read_preference + topology_type = topology_description.topology_type_name + replicaset_name = topology_description.replica_set_name + + shards = None + if topology_type == "Sharded": + shards_list = await client.admin.command("listShards") + shards = {} + for shard in shards_list.get("shards", {}): + shards[shard["_id"]] = shard["host"] + + cache_enabled = False + if REDIS_URL: + cache_enabled = FastAPICache.get_enable() + + return { + "mongo_topology_type": topology_type, + "mongo_replicaset_name": replicaset_name, + "mongo_db": DATABASE_NAME, + "read_preference": str(read_preference), + "mongo_nodes": client.nodes, + "mongo_primary_host": client.primary, + "mongo_secondary_hosts": client.secondaries, + "mongo_address": client.address, + "mongo_is_primary": client.is_primary, + "mongo_is_mongos": client.is_mongos, + "collections": collections, + "shards": shards, + "cache_enabled": cache_enabled, + "status": "OK", + } + + +@app.get("/{collection_name}/count") +async def collection_count(collection_name: str): + collection = db.get_collection(collection_name) + items_count = await collection.count_documents({}) + # status = await client.admin.command('replSetGetStatus') + # import ipdb; ipdb.set_trace() + return {"status": "OK", "mongo_db": DATABASE_NAME, "items_count": items_count} + + +@app.get( + "/{collection_name}/users", + response_description="List all users", + response_model=UserCollection, + response_model_by_alias=False, +) +@cache(expire=60 * 1) +async def list_users(collection_name: str): + """ + List all of the user data in the database. + The response is unpaginated and limited to 1000 results. + """ + time.sleep(1) + collection = db.get_collection(collection_name) + return UserCollection(users=await collection.find().to_list(1000)) + + + +@app.get( + "/{collection_name}/users/{name}", + response_description="Get a single user", + response_model=UserModel, + response_model_by_alias=False, +) +async def show_user(collection_name: str, name: str): + """ + Get the record for a specific user, looked up by `name`. + """ + + collection = db.get_collection(collection_name) + if (user := await collection.find_one({"name": name})) is not None: + return user + + raise HTTPException(status_code=404, detail=f"User {name} not found") + + +@app.post( + "/{collection_name}/users", + response_description="Add new user", + response_model=UserModel, + status_code=status.HTTP_201_CREATED, + response_model_by_alias=False, +) +async def create_user(collection_name: str, user: UserModel = Body(...)): + """ + Insert a new user record. + + A unique `id` will be created and provided in the response. + """ + collection = db.get_collection(collection_name) + new_user = await collection.insert_one( + user.model_dump(by_alias=True, exclude=["id"]) + ) + created_user = await collection.find_one({"_id": new_user.inserted_id}) + return created_user diff --git a/mongo-sharding-repl-cache/api_app/requirements.txt b/mongo-sharding-repl-cache/api_app/requirements.txt new file mode 100644 index 00000000..61b29b87 --- /dev/null +++ b/mongo-sharding-repl-cache/api_app/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.110.2 +uvicorn[standard]==0.29.0 +motor==3.5.3 +redis==4.4.2 +fastapi-cache2==0.2.0 +logmiddleware==0.0.4 \ No newline at end of file diff --git a/mongo-sharding-repl-cache/compose.yaml b/mongo-sharding-repl-cache/compose.yaml new file mode 100644 index 00000000..90cefd18 --- /dev/null +++ b/mongo-sharding-repl-cache/compose.yaml @@ -0,0 +1,267 @@ +services: +# сервер конфигурации + configSrv: + image: mongo:latest # docker образ + container_name: configSrv + restart: always + ports: + - "27017:27017" + networks: + app-network: + ipv4_address: 173.17.0.10 + volumes: + - config-data:/data/db + command: + [ + "--configsvr", # запуск в режиме конфигурации + "--replSet", + "config_server", + "--bind_ip_all", + "--port", + "27017" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + +# 1-й шард + shard1: + image: mongo:latest + container_name: shard1 + restart: always + ports: + - "27018:27018" + networks: + app-network: + ipv4_address: 173.17.0.20 + volumes: + - shard1-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27018" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + s1mongodb1: + image: mongo:latest # docker образ + container_name: s1mongodb1 + restart: always + ports: + - "27027:27027" + networks: + app-network: + ipv4_address: 173.17.0.21 + volumes: + - s1mongodb1-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27027" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + s1mongodb2: + image: mongo:latest # docker образ + container_name: s1mongodb2 + restart: always + ports: + - "27028:27028" + networks: + app-network: + ipv4_address: 173.17.0.22 + volumes: + - s1mongodb2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27028" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + # 2-й шард + shard2: + image: mongo:latest + container_name: shard2 + restart: always + ports: + - "27019:27019" + networks: + app-network: + ipv4_address: 173.17.0.8 + volumes: + - shard2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27019" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + s2mongodb1: + image: mongo:latest # docker образ + container_name: s2mongodb1 + restart: always + ports: + - "27037:27037" + networks: + app-network: + ipv4_address: 173.17.0.31 + volumes: + - s2mongodb1-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27037" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + s2mongodb2: + image: mongo:latest # docker образ + container_name: s2mongodb2 + restart: always + ports: + - "27038:27038" + networks: + app-network: + ipv4_address: 173.17.0.32 + volumes: + - s2mongodb2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27038" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + + # роутер + mongos_router: + image: mongo:latest + container_name: mongos_router + restart: always + ports: + - "27020:27020" + networks: + app-network: + ipv4_address: 173.17.0.7 + command: + [ + "mongos", # обычная mongo в режиме роутера + "--configdb", + "config_server/configSrv:27017", # передача данных сервера конфигурации + "--bind_ip_all", + "--port", + "27020" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + redis_1: + image: "redis:latest" + container_name: redis_1 + ports: + - "6379" + volumes: + - redis_1_data:/data + - ./redis/redis.conf:/usr/local/etc/redis/redis.conf + command: [ "redis-server", "/usr/local/etc/redis/redis.conf" ] + networks: + app-network: + ipv4_address: 173.17.0.40 + + redis_2: + image: "redis:latest" + container_name: redis_2 + ports: + - "6379" + volumes: + - redis_2_data:/data + - ./redis/redis.conf:/usr/local/etc/redis/redis.conf + command: [ "redis-server", "/usr/local/etc/redis/redis.conf" ] + networks: + app-network: + ipv4_address: 173.17.0.41 + + pymongo_api: + container_name: pymongo_api + build: + context: api_app + dockerfile: Dockerfile + image: kazhem/pymongo_api:1.0.0 + depends_on: + - mongos_router + ports: + - 8080:8080 + networks: + app-network: + ipv4_address: 173.17.0.11 + environment: + MONGODB_URL: "mongodb://mongos_router:27020" + MONGODB_DATABASE_NAME: "somedb" + REDIS_URL: "redis://redis_1:6379" + +networks: + app-network: + driver: bridge + ipam: + driver: default + config: + - subnet: 173.17.0.0/16 + +volumes: + redis_1_data: { } + redis_2_data: { } + config-data: + shard1-data: + s1mongodb1-data: + s1mongodb2-data: + shard2-data: + s2mongodb1-data: + s2mongodb2-data: + diff --git a/mongo-sharding-repl-cache/redis/redis.conf b/mongo-sharding-repl-cache/redis/redis.conf new file mode 100644 index 00000000..c6591658 --- /dev/null +++ b/mongo-sharding-repl-cache/redis/redis.conf @@ -0,0 +1,5 @@ +port 6379 +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 5000 +appendonly yes \ No newline at end of file diff --git a/mongo-sharding-repl-cache/scripts/db-drop.sh b/mongo-sharding-repl-cache/scripts/db-drop.sh new file mode 100755 index 00000000..3068e4e7 --- /dev/null +++ b/mongo-sharding-repl-cache/scripts/db-drop.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +### +# Очистка бд +### + +docker compose exec -T mongos_router mongosh --port 27020 --quiet <:8080/docs \ No newline at end of file diff --git a/mongo-sharding-repl/api_app/Dockerfile b/mongo-sharding-repl/api_app/Dockerfile new file mode 100644 index 00000000..46f6c9d0 --- /dev/null +++ b/mongo-sharding-repl/api_app/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12.1-slim +WORKDIR /app +EXPOSE 8080 +COPY requirements.txt ./ +# Устанавливаем зависимости python не пересобирая их +RUN pip install --no-cache --no-cache-dir -r requirements.txt +# Копирование кода приложения +COPY app.py /app/ +ENTRYPOINT ["uvicorn"] +CMD ["app:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/mongo-sharding-repl/api_app/app.py b/mongo-sharding-repl/api_app/app.py new file mode 100644 index 00000000..9b19c017 --- /dev/null +++ b/mongo-sharding-repl/api_app/app.py @@ -0,0 +1,192 @@ +import json +import logging +import os +import time +from typing import List, Optional + +import motor.motor_asyncio +from bson import ObjectId +from fastapi import Body, FastAPI, HTTPException, status +from fastapi_cache import FastAPICache +from fastapi_cache.backends.redis import RedisBackend +from fastapi_cache.decorator import cache +from logmiddleware import RouterLoggingMiddleware, logging_config +from pydantic import BaseModel, ConfigDict, EmailStr, Field +from pydantic.functional_validators import BeforeValidator +from pymongo import errors +from redis import asyncio as aioredis +from typing_extensions import Annotated + +# Configure JSON logging +logging.config.dictConfig(logging_config) +logger = logging.getLogger(__name__) + +app = FastAPI() +app.add_middleware( + RouterLoggingMiddleware, + logger=logger, +) + +DATABASE_URL = os.environ["MONGODB_URL"] +DATABASE_NAME = os.environ["MONGODB_DATABASE_NAME"] +REDIS_URL = os.getenv("REDIS_URL", None) + + +def nocache(*args, **kwargs): + def decorator(func): + return func + + return decorator + + +if REDIS_URL: + cache = cache +else: + cache = nocache + + +client = motor.motor_asyncio.AsyncIOMotorClient(DATABASE_URL) +db = client[DATABASE_NAME] + +# Represents an ObjectId field in the database. +# It will be represented as a `str` on the model so that it can be serialized to JSON. +PyObjectId = Annotated[str, BeforeValidator(str)] + + +@app.on_event("startup") +async def startup(): + if REDIS_URL: + redis = aioredis.from_url(REDIS_URL, encoding="utf8", decode_responses=True) + FastAPICache.init(RedisBackend(redis), prefix="api:cache") + + +class UserModel(BaseModel): + """ + Container for a single user record. + """ + + id: Optional[PyObjectId] = Field(alias="_id", default=None) + age: int = Field(...) + name: str = Field(...) + + +class UserCollection(BaseModel): + """ + A container holding a list of `UserModel` instances. + """ + + users: List[UserModel] + + +@app.get("/") +async def root(): + collection_names = await db.list_collection_names() + collections = {} + for collection_name in collection_names: + collection = db.get_collection(collection_name) + collections[collection_name] = { + "documents_count": await collection.count_documents({}) + } + try: + replica_status = await client.admin.command("replSetGetStatus") + replica_status = json.dumps(replica_status, indent=2, default=str) + except errors.OperationFailure: + replica_status = "No Replicas" + + topology_description = client.topology_description + read_preference = client.client_options.read_preference + topology_type = topology_description.topology_type_name + replicaset_name = topology_description.replica_set_name + + shards = None + if topology_type == "Sharded": + shards_list = await client.admin.command("listShards") + shards = {} + for shard in shards_list.get("shards", {}): + shards[shard["_id"]] = shard["host"] + + cache_enabled = False + if REDIS_URL: + cache_enabled = FastAPICache.get_enable() + + return { + "mongo_topology_type": topology_type, + "mongo_replicaset_name": replicaset_name, + "mongo_db": DATABASE_NAME, + "read_preference": str(read_preference), + "mongo_nodes": client.nodes, + "mongo_primary_host": client.primary, + "mongo_secondary_hosts": client.secondaries, + "mongo_address": client.address, + "mongo_is_primary": client.is_primary, + "mongo_is_mongos": client.is_mongos, + "collections": collections, + "shards": shards, + "cache_enabled": cache_enabled, + "status": "OK", + } + + +@app.get("/{collection_name}/count") +async def collection_count(collection_name: str): + collection = db.get_collection(collection_name) + items_count = await collection.count_documents({}) + # status = await client.admin.command('replSetGetStatus') + # import ipdb; ipdb.set_trace() + return {"status": "OK", "mongo_db": DATABASE_NAME, "items_count": items_count} + + +@app.get( + "/{collection_name}/users", + response_description="List all users", + response_model=UserCollection, + response_model_by_alias=False, +) +@cache(expire=60 * 1) +async def list_users(collection_name: str): + """ + List all of the user data in the database. + The response is unpaginated and limited to 1000 results. + """ + time.sleep(1) + collection = db.get_collection(collection_name) + return UserCollection(users=await collection.find().to_list(1000)) + + +@app.get( + "/{collection_name}/users/{name}", + response_description="Get a single user", + response_model=UserModel, + response_model_by_alias=False, +) +async def show_user(collection_name: str, name: str): + """ + Get the record for a specific user, looked up by `name`. + """ + + collection = db.get_collection(collection_name) + if (user := await collection.find_one({"name": name})) is not None: + return user + + raise HTTPException(status_code=404, detail=f"User {name} not found") + + +@app.post( + "/{collection_name}/users", + response_description="Add new user", + response_model=UserModel, + status_code=status.HTTP_201_CREATED, + response_model_by_alias=False, +) +async def create_user(collection_name: str, user: UserModel = Body(...)): + """ + Insert a new user record. + + A unique `id` will be created and provided in the response. + """ + collection = db.get_collection(collection_name) + new_user = await collection.insert_one( + user.model_dump(by_alias=True, exclude=["id"]) + ) + created_user = await collection.find_one({"_id": new_user.inserted_id}) + return created_user diff --git a/mongo-sharding-repl/api_app/requirements.txt b/mongo-sharding-repl/api_app/requirements.txt new file mode 100644 index 00000000..61b29b87 --- /dev/null +++ b/mongo-sharding-repl/api_app/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.110.2 +uvicorn[standard]==0.29.0 +motor==3.5.3 +redis==4.4.2 +fastapi-cache2==0.2.0 +logmiddleware==0.0.4 \ No newline at end of file diff --git a/mongo-sharding-repl/compose.yaml b/mongo-sharding-repl/compose.yaml new file mode 100644 index 00000000..5bf970e4 --- /dev/null +++ b/mongo-sharding-repl/compose.yaml @@ -0,0 +1,238 @@ +services: +# сервер конфигурации + configSrv: + image: mongo:latest # docker образ + container_name: configSrv + restart: always + ports: + - "27017:27017" + networks: + app-network: + ipv4_address: 173.17.0.10 + volumes: + - config-data:/data/db + command: + [ + "--configsvr", # запуск в режиме конфигурации + "--replSet", + "config_server", + "--bind_ip_all", + "--port", + "27017" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + +# 1-й шард + shard1: + image: mongo:latest + container_name: shard1 + restart: always + ports: + - "27018:27018" + networks: + app-network: + ipv4_address: 173.17.0.20 + volumes: + - shard1-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27018" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + s1mongodb1: + image: mongo:latest # docker образ + container_name: s1mongodb1 + restart: always + ports: + - "27027:27027" + networks: + app-network: + ipv4_address: 173.17.0.21 + volumes: + - s1mongodb1-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27027" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + s1mongodb2: + image: mongo:latest # docker образ + container_name: s1mongodb2 + restart: always + ports: + - "27028:27028" + networks: + app-network: + ipv4_address: 173.17.0.22 + volumes: + - s1mongodb2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27028" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + # 2-й шард + shard2: + image: mongo:latest + container_name: shard2 + restart: always + ports: + - "27019:27019" + networks: + app-network: + ipv4_address: 173.17.0.8 + volumes: + - shard2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27019" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + s2mongodb1: + image: mongo:latest # docker образ + container_name: s2mongodb1 + restart: always + ports: + - "27037:27037" + networks: + app-network: + ipv4_address: 173.17.0.31 + volumes: + - s2mongodb1-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27037" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + s2mongodb2: + image: mongo:latest # docker образ + container_name: s2mongodb2 + restart: always + ports: + - "27038:27038" + networks: + app-network: + ipv4_address: 173.17.0.32 + volumes: + - s2mongodb2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27038" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + + # роутер + mongos_router: + image: mongo:latest + container_name: mongos_router + restart: always + ports: + - "27020:27020" + networks: + app-network: + ipv4_address: 173.17.0.7 + command: + [ + "mongos", # обычная mongo в режиме роутера + "--configdb", + "config_server/configSrv:27017", # передача данных сервера конфигурации + "--bind_ip_all", + "--port", + "27020" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + pymongo_api: + container_name: pymongo_api + build: + context: api_app + dockerfile: Dockerfile + image: kazhem/pymongo_api:1.0.0 + depends_on: + - mongos_router + ports: + - 8080:8080 + networks: + app-network: + ipv4_address: 173.17.0.11 + environment: + MONGODB_URL: "mongodb://mongos_router:27020" + MONGODB_DATABASE_NAME: "somedb" + +networks: + app-network: + driver: bridge + ipam: + driver: default + config: + - subnet: 173.17.0.0/16 + +volumes: + config-data: + shard1-data: + s1mongodb1-data: + s1mongodb2-data: + shard2-data: + s2mongodb1-data: + s2mongodb2-data: + diff --git a/mongo-sharding-repl/scripts/db-drop.sh b/mongo-sharding-repl/scripts/db-drop.sh new file mode 100755 index 00000000..3068e4e7 --- /dev/null +++ b/mongo-sharding-repl/scripts/db-drop.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +### +# Очистка бд +### + +docker compose exec -T mongos_router mongosh --port 27020 --quiet <:8080/docs \ No newline at end of file diff --git a/mongo-sharding/api_app/Dockerfile b/mongo-sharding/api_app/Dockerfile new file mode 100644 index 00000000..46f6c9d0 --- /dev/null +++ b/mongo-sharding/api_app/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12.1-slim +WORKDIR /app +EXPOSE 8080 +COPY requirements.txt ./ +# Устанавливаем зависимости python не пересобирая их +RUN pip install --no-cache --no-cache-dir -r requirements.txt +# Копирование кода приложения +COPY app.py /app/ +ENTRYPOINT ["uvicorn"] +CMD ["app:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/mongo-sharding/api_app/app.py b/mongo-sharding/api_app/app.py new file mode 100644 index 00000000..9b19c017 --- /dev/null +++ b/mongo-sharding/api_app/app.py @@ -0,0 +1,192 @@ +import json +import logging +import os +import time +from typing import List, Optional + +import motor.motor_asyncio +from bson import ObjectId +from fastapi import Body, FastAPI, HTTPException, status +from fastapi_cache import FastAPICache +from fastapi_cache.backends.redis import RedisBackend +from fastapi_cache.decorator import cache +from logmiddleware import RouterLoggingMiddleware, logging_config +from pydantic import BaseModel, ConfigDict, EmailStr, Field +from pydantic.functional_validators import BeforeValidator +from pymongo import errors +from redis import asyncio as aioredis +from typing_extensions import Annotated + +# Configure JSON logging +logging.config.dictConfig(logging_config) +logger = logging.getLogger(__name__) + +app = FastAPI() +app.add_middleware( + RouterLoggingMiddleware, + logger=logger, +) + +DATABASE_URL = os.environ["MONGODB_URL"] +DATABASE_NAME = os.environ["MONGODB_DATABASE_NAME"] +REDIS_URL = os.getenv("REDIS_URL", None) + + +def nocache(*args, **kwargs): + def decorator(func): + return func + + return decorator + + +if REDIS_URL: + cache = cache +else: + cache = nocache + + +client = motor.motor_asyncio.AsyncIOMotorClient(DATABASE_URL) +db = client[DATABASE_NAME] + +# Represents an ObjectId field in the database. +# It will be represented as a `str` on the model so that it can be serialized to JSON. +PyObjectId = Annotated[str, BeforeValidator(str)] + + +@app.on_event("startup") +async def startup(): + if REDIS_URL: + redis = aioredis.from_url(REDIS_URL, encoding="utf8", decode_responses=True) + FastAPICache.init(RedisBackend(redis), prefix="api:cache") + + +class UserModel(BaseModel): + """ + Container for a single user record. + """ + + id: Optional[PyObjectId] = Field(alias="_id", default=None) + age: int = Field(...) + name: str = Field(...) + + +class UserCollection(BaseModel): + """ + A container holding a list of `UserModel` instances. + """ + + users: List[UserModel] + + +@app.get("/") +async def root(): + collection_names = await db.list_collection_names() + collections = {} + for collection_name in collection_names: + collection = db.get_collection(collection_name) + collections[collection_name] = { + "documents_count": await collection.count_documents({}) + } + try: + replica_status = await client.admin.command("replSetGetStatus") + replica_status = json.dumps(replica_status, indent=2, default=str) + except errors.OperationFailure: + replica_status = "No Replicas" + + topology_description = client.topology_description + read_preference = client.client_options.read_preference + topology_type = topology_description.topology_type_name + replicaset_name = topology_description.replica_set_name + + shards = None + if topology_type == "Sharded": + shards_list = await client.admin.command("listShards") + shards = {} + for shard in shards_list.get("shards", {}): + shards[shard["_id"]] = shard["host"] + + cache_enabled = False + if REDIS_URL: + cache_enabled = FastAPICache.get_enable() + + return { + "mongo_topology_type": topology_type, + "mongo_replicaset_name": replicaset_name, + "mongo_db": DATABASE_NAME, + "read_preference": str(read_preference), + "mongo_nodes": client.nodes, + "mongo_primary_host": client.primary, + "mongo_secondary_hosts": client.secondaries, + "mongo_address": client.address, + "mongo_is_primary": client.is_primary, + "mongo_is_mongos": client.is_mongos, + "collections": collections, + "shards": shards, + "cache_enabled": cache_enabled, + "status": "OK", + } + + +@app.get("/{collection_name}/count") +async def collection_count(collection_name: str): + collection = db.get_collection(collection_name) + items_count = await collection.count_documents({}) + # status = await client.admin.command('replSetGetStatus') + # import ipdb; ipdb.set_trace() + return {"status": "OK", "mongo_db": DATABASE_NAME, "items_count": items_count} + + +@app.get( + "/{collection_name}/users", + response_description="List all users", + response_model=UserCollection, + response_model_by_alias=False, +) +@cache(expire=60 * 1) +async def list_users(collection_name: str): + """ + List all of the user data in the database. + The response is unpaginated and limited to 1000 results. + """ + time.sleep(1) + collection = db.get_collection(collection_name) + return UserCollection(users=await collection.find().to_list(1000)) + + +@app.get( + "/{collection_name}/users/{name}", + response_description="Get a single user", + response_model=UserModel, + response_model_by_alias=False, +) +async def show_user(collection_name: str, name: str): + """ + Get the record for a specific user, looked up by `name`. + """ + + collection = db.get_collection(collection_name) + if (user := await collection.find_one({"name": name})) is not None: + return user + + raise HTTPException(status_code=404, detail=f"User {name} not found") + + +@app.post( + "/{collection_name}/users", + response_description="Add new user", + response_model=UserModel, + status_code=status.HTTP_201_CREATED, + response_model_by_alias=False, +) +async def create_user(collection_name: str, user: UserModel = Body(...)): + """ + Insert a new user record. + + A unique `id` will be created and provided in the response. + """ + collection = db.get_collection(collection_name) + new_user = await collection.insert_one( + user.model_dump(by_alias=True, exclude=["id"]) + ) + created_user = await collection.find_one({"_id": new_user.inserted_id}) + return created_user diff --git a/mongo-sharding/api_app/requirements.txt b/mongo-sharding/api_app/requirements.txt new file mode 100644 index 00000000..61b29b87 --- /dev/null +++ b/mongo-sharding/api_app/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.110.2 +uvicorn[standard]==0.29.0 +motor==3.5.3 +redis==4.4.2 +fastapi-cache2==0.2.0 +logmiddleware==0.0.4 \ No newline at end of file diff --git a/mongo-sharding/compose.yaml b/mongo-sharding/compose.yaml new file mode 100644 index 00000000..b8b305fa --- /dev/null +++ b/mongo-sharding/compose.yaml @@ -0,0 +1,132 @@ +services: +# сервер конфигурации + configSrv: + image: mongo:latest # docker образ + container_name: configSrv + restart: always + ports: + - "27017:27017" + networks: + app-network: + ipv4_address: 173.17.0.10 + volumes: + - config-data:/data/db + command: + [ + "--configsvr", # запуск в режиме конфигурации + "--replSet", + "config_server", + "--bind_ip_all", + "--port", + "27017" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + +# 1-й шард + shard1: + image: mongo:latest + container_name: shard1 + restart: always + ports: + - "27018:27018" + networks: + app-network: + ipv4_address: 173.17.0.9 + volumes: + - shard1-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27018" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + # 2-й шард + shard2: + image: mongo:latest + container_name: shard2 + restart: always + ports: + - "27019:27019" + networks: + app-network: + ipv4_address: 173.17.0.8 + volumes: + - shard2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27019" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + # роутер + mongos_router: + image: mongo:latest + container_name: mongos_router + restart: always + ports: + - "27020:27020" + networks: + app-network: + ipv4_address: 173.17.0.7 + command: + [ + "mongos", # обычная mongo в режиме роутера + "--configdb", + "config_server/configSrv:27017", # передача данных сервера конфигурации + "--bind_ip_all", + "--port", + "27020" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + pymongo_api: + container_name: pymongo_api + build: + context: api_app + dockerfile: Dockerfile + image: kazhem/pymongo_api:1.0.0 + depends_on: + - mongos_router + ports: + - 8080:8080 + networks: + app-network: + ipv4_address: 173.17.0.11 + environment: + MONGODB_URL: "mongodb://mongos_router:27020" + MONGODB_DATABASE_NAME: "somedb" + +networks: + app-network: + driver: bridge + ipam: + driver: default + config: + - subnet: 173.17.0.0/16 + +volumes: + config-data: + shard1-data: + shard2-data: diff --git a/mongo-sharding/scripts/db-drop.sh b/mongo-sharding/scripts/db-drop.sh new file mode 100755 index 00000000..3068e4e7 --- /dev/null +++ b/mongo-sharding/scripts/db-drop.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +### +# Очистка бд +### + +docker compose exec -T mongos_router mongosh --port 27020 --quiet < + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +