docker-compose.yml

# Local Kafka development environment setup
# ref: https://github.com/simplesteph/kafka-stack-docker-compose/blob/master/full-stack.yml
version: "3"

services:
  airflow:
    image: puckel/docker-airflow
    ports:
      - "8080:8080"
    volumes:
      - ./app:/usr/local/airflow/dags
      -  /var/run/docker.sock:/var/run/docker.sock
      -  ./requirements.txt:/requirements.txt
    command: webserver
    depends_on:
      - zoo1
      - kafka1
      - pg-docker
      - connect
  pg-docker:
    image: debezium/postgres
    hostname: postgres
    ports:
      - "5432:5432"
    environment:
      - POSTGRES_PASSWORD=postgres
    volumes:
      - $HOME/docker/volumes/postgres:/var/lib/psql/data
  connect:
    image: debezium/connect
    ports:
      - 8083:8083
    links:
      - zoo1
      - kafka1
      - pg-docker
    environment:
      - BOOTSTRAP_SERVERS=kafka1:19092
      - GROUP_ID=1
      - CONFIG_STORAGE_TOPIC=my_connect_configs
      - OFFSET_STORAGE_TOPIC=my_connect_offsets
      - STATUS_STORAGE_TOPIC=my_source_connect_statuses
    depends_on:
      - zoo1
      - kafka1
      - pg-docker

  zoo1:
    image: zookeeper:3.4.9
    restart: unless-stopped
    hostname: zoo1
    ports:
      - "2181:2181"
    environment:
      ZOO_MY_ID: 1
      ZOO_PORT: 2181
      ZOO_SERVERS: server.1=zoo1:2888:3888
    volumes:
      - ./full-stack/zoo1/data:/data
      - ./full-stack/zoo1/datalog:/datalog

  kafka1:
    image: confluentinc/cp-kafka:5.2.1
    hostname: kafka1
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092, LISTENER_DOCKER_EXTERNAL://localhost:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
      KAFKA_BROKER_ID: 1
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CREATE_TOPICS: "trackingtj:1:1"
    volumes:
      - ./full-stack/kafka1/data:/var/lib/kafka/data
    depends_on:
      - zoo1
  # kafka2:
  #   image: confluentinc/cp-kafka:5.2.1
  #   hostname: kafka2
  #   ports:
  #     - "9093:9093"
  #   environment:
  #     KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1},:9093
  #     KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
  #     KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
  #     KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
  #     KAFKA_BROKER_ID: 2
  #     KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
  #     KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR:
  #     KAFKA_CREATE_TOPICS: "test:1:1"
  #   volumes:
  #     - ./full-stack/kafka2/data:/var/lib/kafka/data
  #   depends_on:
  #     - zoo1