Enhance CLI functionality and logging configuration

- Added command-line interface (CLI) commands for starting the system, analyzing DAGs, and generating reports.
- Introduced logging configuration with support for console and file logging.
- Updated configuration structure to include monitoring interval and log path.
- Created models for application configuration using Pydantic.
- Updated README with new CLI usage instructions.

.gitignore

@@ -29,3 +29,4 @@ ENV/
 *.swp
 .DS_Store
 config/config.yaml
+logs/

.pre-commit-config.yaml

@@ -33,6 +33,8 @@ repos:
     rev: v1.6.1
     hooks:
       - id: mypy
+        additional_dependencies:
+          - types-PyYAML
 
   # bandit: Security linting
   - repo: https://github.com/PyCQA/bandit

README.md

@@ -80,17 +80,27 @@ The application is configured through `config/config.yaml`:
 
 ## 📊 Usage
 
-### CLI Interface
+### Command-Line Interface (CLI)
 
+DAGnostics provides a CLI for managing the monitoring and reporting system. Use the following commands:
+
+#### Start the System
+```bash
+poetry run dagnostics start
+```
+
+#### Analyze a Specific DAG
 ```bash
-# Generate daily report
-dagnostics report daily
+poetry run dagnostics analyze <dag-name>
+```
 
-# Analyze specific DAG
-dagnostics analyze my-dag-name
+#### Generate a Report
+```bash
+# Generate a standard report
+poetry run dagnostics report
 
-# Start monitoring server
-dagnostics server start
+# Generate a daily report
+poetry run dagnostics report --daily
 ```
 
 ### Python API

config/config.yaml.example

@@ -4,6 +4,9 @@ llm:
   max_tokens: 500
 
 monitoring:
+  interval: 60
+  log_path: /var/log/dagnostics
+
   categories:
     - system_error
     - resource_constraint

config/logging.yaml

@@ -0,0 +1,48 @@
+version: 1
+disable_existing_loggers: False
+
+formatters:
+  standard:
+    format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+  json:
+    format: '{"time": "%(asctime)s", "name": "%(name)s", "level": "%(levelname)s", "message": "%(message)s"}'
+
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: INFO
+    formatter: standard
+    stream: ext://sys.stdout
+
+  file:
+    class: logging.handlers.RotatingFileHandler
+    level: DEBUG
+    formatter: standard
+    filename: logs/app.log
+    maxBytes: 10485760  # 10 MB
+    backupCount: 5
+    encoding: utf8
+
+  json_file:
+    class: logging.handlers.RotatingFileHandler
+    level: DEBUG
+    formatter: json
+    filename: logs/app.json.log
+    maxBytes: 10485760  # 10 MB
+    backupCount: 5
+    encoding: utf8
+
+loggers:
+  dagnostics:
+    level: DEBUG
+    handlers: [console, file]
+    propagate: False
+
+  dagnostics.api:
+    level: INFO
+    handlers: [json_file]
+    propagate: False
+
+root:
+  level: WARNING
+  handlers: [console]

mypy.ini

@@ -1,2 +1,3 @@
 [mypy]
 exclude = .venv
+ignore_missing_imports = false

pyproject.toml

@@ -2,9 +2,7 @@
 name = "dagnostics"
 version = "0.1.0"
 description = ""
-authors = [
-    {name = "rhasanm",email = "[email protected]"}
-]
+authors = [{ name = "rhasanm", email = "[email protected]" }]
 readme = "README.md"
 requires-python = ">=3.12,<4.0"
 dependencies = [
@@ -17,12 +15,11 @@ dependencies = [
     "uvicorn (>=0.34.0,<0.35.0)",
     "pydantic (>=2.10.5,<3.0.0)",
     "typer (>=0.15.1,<0.16.0)",
-    "rich (>=13.9.4,<14.0.0)"
+    "rich (>=13.9.4,<14.0.0)",
 ]
 
 [tool.poetry]
-packages = [{include = "dagnostics", from = "src"}]
-
+packages = [{ include = "dagnostics", from = "src" }]
 
 [tool.poetry.group.dev.dependencies]
 invoke = "^2.2.0"
@@ -33,6 +30,11 @@ isort = "^5.13.2"
 flake8 = "^7.1.1"
 mypy = "^1.14.1"
 pre-commit = "^4.0.1"
+types-pyyaml = "^6.0.12.20241230"
+
+[tool.poetry.scripts]
+start = "dagnostics.main:main"
+dagnostics = "dagnostics.cli.main:cli"
 
 [build-system]
 requires = ["poetry-core>=2.0.0,<3.0.0"]

src/dagnostics/cli/main.py

@@ -0,0 +1,38 @@
+import typer
+
+from dagnostics.core.config import load_config
+from dagnostics.monitoring.collector import start_monitoring
+from dagnostics.reporting.generator import setup_reporting
+
+app = typer.Typer()
+
+
+@app.command()
+def start():
+    """Start the DAGnostics monitoring and reporting system."""
+    config = load_config()
+    typer.echo("Starting DAGnostics...")
+    start_monitoring(config)
+    setup_reporting(config)
+
+
+@app.command()
+def analyze(dag_name: str):
+    """Analyze a specific DAG."""
+    # config = load_config()
+    typer.echo(f"Analyzing DAG: {dag_name}")
+
+
+@app.command()
+def report(daily: bool = False):
+    """Generate a report."""
+    # config = load_config()
+    if daily:
+        typer.echo("Generating daily report...")
+    else:
+        typer.echo("Generating report...")
+
+
+def cli():
+    """Entry point for the CLI."""
+    app()

src/dagnostics/core/config.py

@@ -0,0 +1,16 @@
+from pathlib import Path
+
+import yaml
+
+from dagnostics.core.models import AppConfig
+
+
+def load_config() -> AppConfig:
+    """Load and parse the application configuration."""
+    config_path = Path("config/config.yaml")
+    if config_path.exists():
+        with open(config_path, "r") as f:
+            raw_config = yaml.safe_load(f)
+            return AppConfig(**raw_config)
+    else:
+        raise FileNotFoundError("Configuration file not found: config/config.yaml")

src/dagnostics/core/models.py

@@ -0,0 +1,18 @@
+from pydantic import BaseModel, Field
+
+
+class MonitoringConfig(BaseModel):
+    interval: int = Field(gt=0, description="Monitoring interval in seconds")
+    log_path: str = Field(..., description="Path to store logs")
+
+
+class ReportingConfig(BaseModel):
+    format: str = Field(default="markdown", description="Report format")
+    output_dir: str = Field(
+        default="reports", description="Output directory for reports"
+    )
+
+
+class AppConfig(BaseModel):
+    monitoring: MonitoringConfig
+    reporting: ReportingConfig

src/dagnostics/main.py

@@ -0,0 +1,49 @@
+import logging.config
+from pathlib import Path
+
+import yaml
+
+from dagnostics.cli.main import cli
+from dagnostics.core.config import load_config
+from dagnostics.monitoring.collector import start_monitoring
+from dagnostics.reporting.generator import setup_reporting
+
+
+def setup_logging():
+    """Load logging configuration from logging.yaml and ensure logs directory exists."""
+    logging_config_path = Path("config/logging.yaml")
+    if logging_config_path.exists():
+        with open(logging_config_path, "r") as f:
+            config = yaml.safe_load(f)
+
+            logs_dir = Path("logs")
+            logs_dir.mkdir(exist_ok=True)
+
+            logging.config.dictConfig(config)
+    else:
+        logging.basicConfig(level=logging.INFO)
+        logging.warning(
+            "Logging configuration file not found. Using default logging settings."
+        )
+
+
+def main():
+    """Main entry point for the DAGnostics application."""
+    setup_logging()
+    logger = logging.getLogger(__name__)
+    logger.info("Starting DAGnostics...")
+
+    config = load_config()
+    logger.info("Configuration loaded successfully.")
+
+    start_monitoring(config)
+    logger.info("Monitoring started.")
+
+    setup_reporting(config)
+    logger.info("Reporting setup complete.")
+
+    cli()
+
+
+if __name__ == "__main__":
+    main()

src/dagnostics/monitoring/collector.py

@@ -0,0 +1,14 @@
+import logging
+
+from dagnostics.core.config import AppConfig
+
+logger = logging.getLogger(__name__)
+
+
+def start_monitoring(config: AppConfig):
+    """Start monitoring DAG failures."""
+    logger.info("Starting monitoring system...")
+    logger.info(f"Monitoring interval: {config.monitoring.interval}")
+    logger.info(f"Log path: {config.monitoring.log_path}")
+
+    logger.info("Monitoring system started.")

src/dagnostics/reporting/generator.py

@@ -0,0 +1,14 @@
+import logging
+
+from dagnostics.core.config import AppConfig
+
+logger = logging.getLogger(__name__)
+
+
+def setup_reporting(config: AppConfig):
+    """Set up the reporting system."""
+    logger.info("Setting up reporting system...")
+    logger.info(f"Report format: {config.reporting.format}")
+    logger.info(f"Output directory: {config.reporting.output_dir}")
+
+    logger.info("Reporting system setup complete.")