docker-compose.yml

version: "3"

networks:
  app-tier:
    driver: bridge

services:
  memgraph-mage:
    build: ./memgraph
    ports:
      - '7687:7687'
      - '7444:7444'
    volumes:
      - ./memgraph/import-data:/usr/lib/memgraph/import-data
    entrypoint: [
      "/usr/lib/memgraph/memgraph",
      "--telemetry-enabled=false",
      "--query-modules-directory=/transformations,/usr/lib/memgraph/query_modules",
      "--log-level=TRACE"]
    networks:
      - app-tier

  zookeeper:
    image: confluentinc/cp-zookeeper:7.1.1
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVERS: zookeeper:2888:3888
    networks:
      - app-tier

  kafka:
    build: ./kafka
    container_name: kafka
    ports:
      - "9092:9092"
      - "9093:9093"
      - "9999:9999"
    environment:
      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka:9092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:SASL_PLAINTEXT,LISTENER_DOCKER_EXTERNAL:SASL_PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
      KAFKA_BROKER_ID: 1
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_JMX_PORT: 9999
      KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1}
      KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
      KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf"
      KAFKA_SASL_ENABLED_MECHANISMS: PLAIN
      KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
      ZOOKEEPER_SASL_ENABLED: "false"
    volumes:
      - ./kafka/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
      - ./kafka/connect.properties:/etc/kafka/connect.properties
    command: sh -c "/etc/confluent/docker/run"
    depends_on:
      - zookeeper
    networks:
      - app-tier

  redpanda:
    command:
      - redpanda
      - start
      - --smp
      - "1"
      - --reserve-memory
      - 0M
      - --overprovisioned
      - --node-id
      - "0"
      - --kafka-addr
      - PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9094
      - --advertise-kafka-addr
      - PLAINTEXT://redpanda:29092,OUTSIDE://0.0.0.0:9094
    image: docker.vectorized.io/vectorized/redpanda:v21.9.5
    ports:
      - 9094:9094
      - 29092:29092
    networks:
      - app-tier

  rabbitmq:
    image: rabbitmq:3-management-alpine
    ports:
      - 5672:5672
      - 15672:15672
    networks:
      - app-tier

  pulsar:
    image: apachepulsar/pulsar:2.6.0
    ports:
      - 8080:8080
      - 6650:6650
    environment:
      PULSAR_MEM: " -Xms512m -Xmx512m -XX:MaxDirectMemorySize=1g"
    command: bin/pulsar standalone
    networks:
      - app-tier

  core:
    image: tianon/true
    restart: "no"
    depends_on:
      - kafka
      - redpanda
      - pulsar
      - rabbitmq

  art-blocks:
    build:
      context: ./
      dockerfile: ./datasets/art-blocks/Dockerfile
    entrypoint: [ "python3", "produce.py", "--stream-delay", "1.0", "--consumer" ]
    env_file: platform_variables.env
    environment:
      KAFKA_TOPIC: "sales"
      REDPANDA_TOPIC: "sales"
      RABBITMQ_QUEUE: "sales"
      PULSAR_TOPIC: "sales"
    networks:
      - app-tier

  github:
    build:
      context: ./
      dockerfile: ./datasets/github/Dockerfile
    entrypoint: [ "python3", "produce.py", "--stream-delay", "1.0", "--consumer" ]
    env_file: platform_variables.env
    environment:
      KAFKA_TOPIC: "github"
      REDPANDA_TOPIC: "github"
      RABBITMQ_QUEUE: "github"
      PULSAR_TOPIC: "github"
    networks:
      - app-tier

  movielens:
    build:
      context: ./
      dockerfile: ./datasets/movielens/Dockerfile
    entrypoint: [ "python3", "produce.py", "--stream-delay", "1.0", "--consumer" ]
    env_file:
      platform_variables.env
    environment:
      KAFKA_TOPIC: "ratings"
      REDPANDA_TOPIC: "ratings"
      RABBITMQ_QUEUE: "ratings"
      PULSAR_TOPIC: "ratings"
    networks:
      - app-tier

  amazon-books:
    build:
      context: ./
      dockerfile: ./datasets/amazon-books/Dockerfile
    entrypoint: [ "python3", "produce.py", "--stream-delay", "1.0", "--consumer" ]
    env_file:
      platform_variables.env
    environment:
      KAFKA_TOPIC: "book-ratings"
      REDPANDA_TOPIC: "book-ratings"
      RABBITMQ_QUEUE: "book-ratings"
      PULSAR_TOPIC: "book-ratings"
    networks:
      - app-tier
      

  art-blocks-analysis:
    build:
      context: ./
      dockerfile: ./data-analysis/Dockerfile
    entrypoint: [ "python3", "art-blocks-analysis.py" ]
    env_file: platform_variables.env
    environment:
      KAFKA_TOPIC: "sales"
      REDPANDA_TOPIC: "sales"
      RABBITMQ_QUEUE: "sales"
      PULSAR_TOPIC: "sales"
    networks:
      - app-tier

  art-blocks-memgraph:
    build:
      context: ./
      dockerfile: ./data-analysis/Dockerfile
    entrypoint: [ "python3", "art-blocks-memgraph.py" ]
    environment:
      MEMGRAPH_IP: memgraph-mage
      MEMGRAPH_PORT: "7687"
    depends_on:
      - memgraph-mage
    networks:
      - app-tier

  github-analysis:
    build:
      context: ./
      dockerfile: ./data-analysis/Dockerfile
    entrypoint: [ "python3", "github-analysis.py" ]
    env_file: platform_variables.env
    environment:
      KAFKA_TOPIC: "github"
      REDPANDA_TOPIC: "github"
      RABBITMQ_QUEUE: "github"
      PULSAR_TOPIC: "github"
    networks:
      - app-tier

  movielens-memgraph:
    build:
      context: ./
      dockerfile: ./data-analysis/Dockerfile
    entrypoint: [ "python3", "movielens-memgraph.py" ]
    environment:
      MEMGRAPH_IP: memgraph-mage
      MEMGRAPH_PORT: "7687"
    depends_on:
      - memgraph-mage
    networks:
      - app-tier

  amazon-books-memgraph:
    build:
      context: ./
      dockerfile: ./data-analysis/Dockerfile
    entrypoint: [ "python3", "amazon-books-memgraph.py" ]
    environment:
      MEMGRAPH_IP: memgraph-mage
      MEMGRAPH_PORT: "7687"
    depends_on:
      - memgraph-mage
    networks:
      - app-tier

  github-commits-memgraph:
    build:
      context: ./
      dockerfile: ./data-analysis/Dockerfile
    entrypoint: [ "python3", "github-commits-memgraph.py" ]
    environment:
      MEMGRAPH_IP: memgraph-mage
      MEMGRAPH_PORT: "7687"
    depends_on:
      - memgraph-mage
    networks:
      - app-tier