create an all service make target

brevdev · Dec 2, 2024 · 1473217 · 1473217
1 parent ec35b56
commit 1473217
Show file tree

Hide file tree

Showing 7 changed files with 117 additions and 118 deletions.
diff --git a/Makefile b/Makefile
@@ -25,6 +25,13 @@ NC := \033[0m  # No Color
 # Explicitly use bash
 SHELL := /bin/bash
 
+# List of all services used - pdf model services
+NON_PDF_MODEL_SERVICES := redis minio api-service agent-service pdf-service tts-service jaeger
+
+PDF_MODEL_SERVICES := redis pdf-api celery-worker
+
+NVINGEST_URL := https://nv-ingest-rest-endpoint.brevlab.com/v1
+
 # Check if environment variables are set
 check_env:
 	@for var in $(REQUIRED_ENV_VARS); do \
@@ -43,51 +50,49 @@ uv:
 	@echo "$(GREEN)Setting up UV environment...$(NC)"
 	@bash setup.sh
 
-# Development target
-dev: check_env
+create-minio-data-dir:
 	@if [ ! -d "data/minio" ]; then \
 		echo "$(GREEN)Creating data/minio directory...$(NC)"; \
 		mkdir -p data/minio; \
 	fi
-	docker compose down
-	@if [ -z "$(MODEL_API_URL)" ]; then \
-		echo "$(GREEN)USING NV-INGEST$(NC)"; \
-	else \
-		echo "$(GREEN)USING DOCLING$(NC)"; \
-	fi
+
+# Development target that runs with external nvingest
+dev: check_env create-minio-data-dir
+	docker compose down $(NON_PDF_MODEL_SERVICES)
 	@echo "$(GREEN)Starting development environment...$(NC)"
 	@if [ "$(DETACH)" = "1" ]; then \
-		docker compose -f docker-compose.yaml --env-file .env up --build -d; \
+		MODEL_API_URL=$(NVINGEST_URL) docker compose -f docker-compose.yaml --env-file .env up $(NON_PDF_MODEL_SERVICES) --build -d; \
 	else \
-		docker compose -f docker-compose.yaml --env-file .env up --build; \
+		MODEL_API_URL=$(NVINGEST_URL) docker compose -f docker-compose.yaml --env-file .env up $(NON_PDF_MODEL_SERVICES) --build; \
 	fi
 
 # Development target for pdf model service
 model-dev:
-	docker compose -f services/PDFService/PDFModelService/docker-compose.yml down
+	docker compose down $(PDF_MODEL_SERVICES)
 	@echo "$(GREEN)Starting development environment...$(NC)"
-	docker compose -f services/PDFService/PDFModelService/docker-compose.yml up --build
+	docker compose up $(PDF_MODEL_SERVICES) --build
 
-# Production target
-prod: check_env
-	@if [ ! -d "data/minio" ]; then \
-		echo "$(GREEN)Creating data/minio directory...$(NC)"; \
-		mkdir -p data/minio; \
-	fi
+# Development target that will run all services including pdf model locally
+all-services: check_env create-minio-data-dir
 	docker compose down
-	@if [ -z "$(MODEL_API_URL)" ]; then \
-		echo "$(GREEN)USING NV-INGEST$(NC)"; \
+	@echo "$(GREEN)Starting development environment all-services...$(NC)"
+	@if [ "$(DETACH)" = "1" ]; then \
+		docker compose -f docker-compose.yaml --env-file .env up --build -d; \
 	else \
-		echo "$(GREEN)USING DOCLING$(NC)"; \
+		docker compose -f docker-compose.yaml --env-file .env up --build; \
 	fi
+
+# Production target
+prod: check_env create-minio-data-dir
+	docker compose down $(NON_PDF_MODEL_SERVICES)
 	@echo "$(GREEN)Starting production environment with version $(VERSION)...$(NC)"
-	VERSION=$(VERSION) docker compose -f docker-compose-remote.yaml --env-file .env up
+	VERSION=$(VERSION) MODEL_API_URL=$(NVINGEST_URL) docker compose -f docker-compose-remote.yaml --env-file .env up $(NON_PDF_MODEL_SERVICES)
 
 # Production target for pdf model service
 model-prod:
-	docker compose -f services/PDFService/PDFModelService/docker-compose-remote.yml down
+	docker compose -f docker-compose-remote.yaml down $(PDF_MODEL_SERVICES)
 	@echo "$(GREEN)Starting production environment with version $(VERSION)...$(NC)"
-	VERSION=$(VERSION) docker compose -f services/PDFService/PDFModelService/docker-compose-remote.yml up
+	VERSION=$(VERSION) docker compose -f docker-compose-remote.yaml up $(PDF_MODEL_SERVICES)
 
 # Version bump (minor) and release target
 version-bump:
@@ -127,4 +132,4 @@ format:
 
 ruff: lint format
 
-.PHONY: check_env dev clean ruff prod version-bump version-bump-major uv model-prod model-dev
+.PHONY: check_env dev clean ruff prod version-bump version-bump-major uv model-prod model-dev all-services create-minio-data-dir
diff --git a/docker-compose-remote.yaml b/docker-compose-remote.yaml
@@ -6,6 +6,7 @@ services:
     command: redis-server --appendonly no
     networks:
       - app-network
+
   minio:
     image: minio/minio:latest
     ports:
@@ -41,6 +42,7 @@ services:
       - jaeger
     networks:
       - app-network
+
   agent-service:
     image: nvcr.io/pfteb4cqjzrs/playground/agent-service:${VERSION:-1.1}
     ports:
@@ -55,19 +57,21 @@ services:
       - redis
     networks:
       - app-network
+
   pdf-service:
     image: nvcr.io/pfteb4cqjzrs/playground/pdf-service:${VERSION:-1.1}
     ports:
       - "8003:8003"
     environment:
       - REDIS_URL=redis://redis:6379
-      - MODEL_API_URL=${MODEL_API_URL:-https://nv-ingest-8t8cjywa2.brevlab.com/v1}
+      - MODEL_API_URL=${MODEL_API_URL:-http://pdf-api:8004}
     extra_hosts:
       - "host.docker.internal:host-gateway"
     depends_on:
       - redis
     networks:
       - app-network
+
   tts-service:
     image: nvcr.io/pfteb4cqjzrs/playground/tts-service:${VERSION:-1.1}
     ports:
@@ -96,6 +100,43 @@ services:
     networks:
       - app-network
 
+  pdf-api:
+    image: nvcr.io/pfteb4cqjzrs/playground/pdf-model-api:${VERSION}
+    ports:
+      - "8003:8003"
+    environment:
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - TEMP_FILE_DIR=/tmp/pdf_conversions
+    volumes:
+      - pdf_temp:/tmp/pdf_conversions
+    depends_on:
+      - redis
+      - celery-worker
+    restart: unless-stopped
+    networks:
+      - app-network
+
+  celery-worker:
+    image: nvcr.io/pfteb4cqjzrs/playground/pdf-model-worker:${VERSION}
+    environment:
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      - TEMP_FILE_DIR=/tmp/pdf_conversions
+    volumes:
+      - pdf_temp:/tmp/pdf_conversions
+    depends_on:
+      - redis
+    restart: unless-stopped
+    networks:
+      - app-network
+
+volumes:
+  redis_data:
+  pdf_temp:
+
 networks:
   app-network:
     driver: bridge
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -71,7 +71,7 @@ services:
       - "8003:8003"
     environment:
       - REDIS_URL=redis://redis:6379
-      - MODEL_API_URL=${MODEL_API_URL:-https://nv-ingest-8t8cjywa2.brevlab.com/v1}
+      - MODEL_API_URL=${MODEL_API_URL:-http://pdf-api:8004}
     extra_hosts:
       - "host.docker.internal:host-gateway"
     depends_on:
@@ -109,6 +109,47 @@ services:
       retries: 3
     networks:
       - app-network
+
+  pdf-api:
+    build:
+      context: services/PDFService/PDFModelService
+      dockerfile: Dockerfile.api
+    ports:
+      - "8004:8004"
+    environment:
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - TEMP_FILE_DIR=/tmp/pdf_conversions
+    volumes:
+      - pdf_temp:/tmp/pdf_conversions
+    depends_on:
+      - redis
+      - celery-worker
+    restart: unless-stopped
+    networks:
+      - app-network
+
+  celery-worker:
+    build:
+      context: services/PDFService/PDFModelService
+      dockerfile: Dockerfile.worker
+    environment:
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      - TEMP_FILE_DIR=/tmp/pdf_conversions
+    volumes:
+      - pdf_temp:/tmp/pdf_conversions
+    depends_on:
+      - redis
+    restart: unless-stopped
+    networks:
+      - app-network
+
+volumes:
+  redis_data:
+  pdf_temp:
 
 networks:
   app-network:

diff --git a/services/PDFService/PDFModelService/Dockerfile.api b/services/PDFService/PDFModelService/Dockerfile.api
@@ -22,6 +22,6 @@ COPY tasks.py /app/
 # Create directory for temporary files
 RUN mkdir -p /tmp/pdf_conversions
 
-EXPOSE 8003
+EXPOSE 8004
 
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8003"]
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8004"]
diff --git a/services/PDFService/PDFModelService/docker-compose-remote.yml b/services/PDFService/PDFModelService/docker-compose-remote.yml
diff --git a/services/PDFService/PDFModelService/docker-compose.yml b/services/PDFService/PDFModelService/docker-compose.yml
diff --git a/services/PDFService/PDFModelService/main.py b/services/PDFService/PDFModelService/main.py
@@ -127,4 +127,4 @@ async def health():
 if __name__ == "__main__":
     import uvicorn
 
-    uvicorn.run(app, host="0.0.0.0", port=8003)
+    uvicorn.run(app, host="0.0.0.0", port=8004)