docker-compose.yaml

version: "3.5"

services:

  aggregator:
    networks:
    - back-tier
    restart: on-failure
    container_name: aggregator
    image: ${DOCKER_REGISTRY}/${AGGREGATOR_IMAGE}:${image_tag}
    build:
      context: ./
      dockerfile: aggregator/Dockerfile
      args:
        http_proxy: ${http_proxy}
        https_proxy: ${https_proxy}
    working_dir: /aggregator
    volumes:
    - './aggregator/src:/aggregator'
    environment:
      LOGGING_LEVEL: ${LOGGING_LEVEL}
      PYTHONUNBUFFERED: "true"
      CASSANDRA_KEYSPACE: ${CASSANDRA_KEYSPACE}
      CASSANDRA_HOST: ${CASSANDRA_HOST}
      CASSANDRA_PORT: ${CASSANDRA_PORT}
      CASSANDRA_USER: ${CASSANDRA_USER}
      CASSANDRA_PASSWORD: ${CASSANDRA_PASSWORD}
      CASSANDRA_USE_CLUSTER: ${CASSANDRA_USE_CLUSTER}
      CASSANDRA_NODES: ${CASSANDRA_NODES}
      AGGREGATOR_SCHEDULING_MINUTES: ${AGGREGATOR_SCHEDULING_MINUTES}
      MIN_RELEVANT_FLOOD_PROBABILITY: ${MIN_RELEVANT_FLOOD_PROBABILITY}
      FLOOD_PROBABILITY_RANGES: ${FLOOD_PROBABILITY_RANGES}
      NUM_RELEVANT_TWEETS_AGGREGATED: ${NUM_RELEVANT_TWEETS_AGGREGATED}
      MYSQL_HOST: ${MYSQL_HOST}
      MYSQL_DBNAME: ${MYSQL_DBNAME}
      MYSQL_USER: ${MYSQL_USER}
      MYSQL_PASSWORD: ${MYSQL_PASSWORD}
      http_proxy: ${http_proxy}
      https_proxy: ${https_proxy}
    command: -- python3.6 start.py ${AGGREGATOR_RUN_CONF}

    depends_on:
    - mysql
    - cassandrasmfr

  annotator:
    restart: always
    container_name: annotator
    image: ${DOCKER_REGISTRY}/${ANNOTATOR_IMAGE}:${image_tag}
    build:
      context: ./
      dockerfile: annotator/Dockerfile
      args:
        http_proxy: ${http_proxy}
        https_proxy: ${https_proxy}
        models_repo: ${GIT_REPO_MODELS}
    working_dir: /annotator
    ports:
      - '5556:5556'
      - '6006:6006'
    networks:
      - back-tier
    environment:
      LOGGING_LEVEL: ${LOGGING_LEVEL}
      TF_CPP_MIN_LOG_LEVEL: '2'
      PYTHONUNBUFFERED: 'true'
      KERAS_BACKEND: tensorflow
      PERSISTER_KAFKA_TOPIC: ${PERSISTER_KAFKA_TOPIC}
      ANNOTATOR_KAFKA_TOPIC: ${ANNOTATOR_KAFKA_TOPIC}
      DEVELOPMENT: ${DEVELOPMENT}
      KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS}
      CASSANDRA_HOST: ${CASSANDRA_HOST}
      CASSANDRA_PORT: ${CASSANDRA_PORT}
      CASSANDRA_USER: ${CASSANDRA_USER}
      CASSANDRA_PASSWORD: ${CASSANDRA_PASSWORD}
      CASSANDRA_KEYSPACE: ${CASSANDRA_KEYSPACE}
      CASSANDRA_USE_CLUSTER: ${CASSANDRA_USE_CLUSTER}
      CASSANDRA_NODES: ${CASSANDRA_NODES}
      MYSQL_HOST: ${MYSQL_HOST}
      MYSQL_DBNAME: ${MYSQL_DBNAME}
      MYSQL_USER: ${MYSQL_USER}
      MYSQL_PASSWORD: ${MYSQL_PASSWORD}
      MODELS_PATH: /models
      GIT_REPO_MODELS: ${GIT_REPO_MODELS}
      http_proxy: ${http_proxy}
      https_proxy: ${https_proxy}
    volumes:
      - './annotator/src:/annotator'
    depends_on:
      - kafka
      - cassandrasmfr
      - persister

  geocoder:
    restart: always
    container_name: geocoder
    image: ${DOCKER_REGISTRY}/${GEOCODER_IMAGE}:${image_tag}
    build:
      context: ./
      dockerfile: geocoder/Dockerfile
      args:
        http_proxy: ${http_proxy}
        https_proxy: ${https_proxy}
    working_dir: /geocoder
    volumes:
    - './geocoder/src:/geocoder'
    ports:
    - '5557:5557'
    - '6007:6006'
    environment:
      LOGGING_LEVEL: ${LOGGING_LEVEL}
      TF_CPP_MIN_LOG_LEVEL: '2'
      PYTHONUNBUFFERED: 'true'
      PERSISTER_KAFKA_TOPIC: ${PERSISTER_KAFKA_TOPIC}
      GEOCODER_KAFKA_TOPIC: ${GEOCODER_KAFKA_TOPIC}
      KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS}
      CASSANDRA_HOST: ${CASSANDRA_HOST}
      CASSANDRA_PORT: ${CASSANDRA_PORT}
      CASSANDRA_USER: ${CASSANDRA_USER}
      CASSANDRA_PASSWORD: ${CASSANDRA_PASSWORD}
      CASSANDRA_KEYSPACE: ${CASSANDRA_KEYSPACE}
      CASSANDRA_USE_CLUSTER: ${CASSANDRA_USE_CLUSTER}
      CASSANDRA_NODES: ${CASSANDRA_NODES}
      MYSQL_HOST: ${MYSQL_HOST}
      MYSQL_DBNAME: ${MYSQL_DBNAME}
      MYSQL_USER: ${MYSQL_USER}
      MYSQL_PASSWORD: ${MYSQL_PASSWORD}
      http_proxy: ${http_proxy}
      https_proxy: ${https_proxy}
    networks:
    - back-tier

    depends_on:
    - kafka
    - cassandrasmfr
    - geonames
    - persister

  persister:
    restart: always
    container_name: persister
    image: ${DOCKER_REGISTRY}/${PERSISTER_IMAGE}:${image_tag}
    build:
      context: ./
      args:
        http_proxy: ${http_proxy}
        https_proxy: ${https_proxy}
      dockerfile: persister/Dockerfile
    working_dir: /persister
    environment:
      LOGGING_LEVEL: ${LOGGING_LEVEL}
      PYTHONUNBUFFERED: 'true'
      BACKGROUND_COLLECTION_ID: ${BACKGROUND_COLLECTION_ID}
      PERSISTER_KAFKA_TOPIC: ${PERSISTER_KAFKA_TOPIC}
      ANNOTATOR_KAFKA_TOPIC: ${ANNOTATOR_KAFKA_TOPIC}
      GEOCODER_KAFKA_TOPIC: ${GEOCODER_KAFKA_TOPIC}
      KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS}
      MYSQL_HOST: ${MYSQL_HOST}
      MYSQL_DBNAME: ${MYSQL_DBNAME}
      MYSQL_USER: ${MYSQL_USER}
      MYSQL_PASSWORD: ${MYSQL_PASSWORD}
      CASSANDRA_HOST: ${CASSANDRA_HOST}
      CASSANDRA_PORT: ${CASSANDRA_PORT}
      CASSANDRA_USER: ${CASSANDRA_USER}
      CASSANDRA_PASSWORD: ${CASSANDRA_PASSWORD}
      CASSANDRA_KEYSPACE: ${CASSANDRA_KEYSPACE}
      CASSANDRA_USE_CLUSTER: ${CASSANDRA_USE_CLUSTER}
      CASSANDRA_NODES: ${CASSANDRA_NODES}
      ANNOTATOR_PORT: ${ANNOTATOR_PORT}
      GEOCODER_PORT: ${GEOCODER_PORT}
      ANNOTATOR_HOST: ${ANNOTATOR_HOST}
      GEOCODER_HOST: ${GEOCODER_HOST}
      http_proxy: ${http_proxy}
      https_proxy: ${https_proxy}
    ports:
      - '5558:5558'
    volumes:
      - './persister/src:/persister'
      - './persister/logs:/logs'
    networks:
      - back-tier
    depends_on:
      - kafka
      - cassandrasmfr

  products:
    restart: on-failure
    container_name: products
    image: ${DOCKER_REGISTRY}/${PRODUCTS_IMAGE}:${image_tag}
    build:
      context: ./
      dockerfile: products/Dockerfile
      args:
        http_proxy: ${http_proxy}
        https_proxy: ${https_proxy}
        sftp_host: ${KAJO_FTP_SERVER}
        ftp_host: ${FTP_HOST}
        ramnode_host: ${RAMNODE_FTP_SERVER}
        download_folder: ${DOWNLOAD_FOLDER}
    working_dir: /products
    volumes:
      - ./products/src:/products
      - ${PRODUCTS_OUTPUT}:/output:rw
    environment:
      LOGGING_LEVEL: ${LOGGING_LEVEL}
      DEVELOPMENT: ${DEVELOPMENT}
      INCLUDE_PAST_COLLECTIONS: ${INCLUDE_PAST_COLLECTIONS}
      PYTHONUNBUFFERED: "true"
      PERSISTER_KAFKA_TOPIC: ${PERSISTER_KAFKA_TOPIC}
      RESTSERVER_BASEPATH: '/1.0'
      RESTSERVER_PORT: ${RESTSERVER_PORT}
      RRA_FETCH_SCHEDULING: ${RRA_FETCH_SCHEDULING}
      FTP_USER: ${FTP_USER}
      FTP_PASS: ${FTP_PASS}
      FTP_HOST: ${FTP_HOST}
      FTP_PATH: ${FTP_PATH}
      DOWNLOAD_FOLDER: ${DOWNLOAD_FOLDER}
      RRA_ONDEMAND_FILENAME: ${RRA_ONDEMAND_FILENAME}
      FLOOD_PROBABILITY_RANGES: ${FLOOD_PROBABILITY_RANGES}
      THRESHOLDS: ${THRESHOLDS}
      NUM_RELEVANT_TWEETS_PRODUCTS: ${NUM_RELEVANT_TWEETS_PRODUCTS}
      HERE_APP_ID: ${HERE_APP_ID}
      HERE_APP_CODE: ${HERE_APP_CODE}
      MYSQL_HOST: ${MYSQL_HOST}
      MYSQL_DBNAME: ${MYSQL_DBNAME}
      MYSQL_USER: ${MYSQL_USER}
      MYSQL_PASSWORD: ${MYSQL_PASSWORD}
      CASSANDRA_KEYSPACE: ${CASSANDRA_KEYSPACE}
      CASSANDRA_HOST: ${CASSANDRA_HOST}
      CASSANDRA_PORT: ${CASSANDRA_PORT}
      CASSANDRA_USER: ${CASSANDRA_USER}
      CASSANDRA_PASSWORD: ${CASSANDRA_PASSWORD}
      CASSANDRA_USE_CLUSTER: ${CASSANDRA_USE_CLUSTER}
      CASSANDRA_NODES: ${CASSANDRA_NODES}
      KAJO_FTP_SERVER: ${KAJO_FTP_SERVER}
      KAJO_FTP_USER: ${KAJO_FTP_USER}
      KAJO_FTP_PASSWORD: ${KAJO_FTP_PASSWORD}
      KAJO_FTP_FOLDER: ${KAJO_FTP_FOLDER}
      RAMNODE_FTP_SERVER: ${RAMNODE_FTP_SERVER}
      RAMNODE_FTP_USER: ${RAMNODE_FTP_USER}
      RAMNODE_FTP_PASSWORD: ${RAMNODE_FTP_PASSWORD}
      RAMNODE_FTP_FOLDER: ${RAMNODE_FTP_FOLDER}
      http_proxy: ${http_proxy}
      https_proxy: ${https_proxy}
    networks:
      - back-tier
    depends_on:
      - mysql
      - cassandrasmfr

  collectors:
    restart: always
    container_name: collectors
    image: ${DOCKER_REGISTRY}/${COLLECTORS_IMAGE}:${image_tag}
    build:
      context: ./
      dockerfile: collectors/Dockerfile
      args:
        http_proxy: ${http_proxy}
        https_proxy: ${https_proxy}
        ftp_host: ${FTP_HOST}
        download_folder: ${DOWNLOAD_FOLDER}
    working_dir: /collectors
    environment:
      LOGGING_LEVEL: ${LOGGING_LEVEL}
      DEVELOPMENT: ${DEVELOPMENT}
      START_COLLECTORS: ${START_COLLECTORS}
      MYSQL_HOST: ${MYSQL_HOST}
      MYSQL_USER: ${MYSQL_USER}
      MYSQL_PASSWORD: ${MYSQL_PASSWORD}
      MYSQL_DBNAME: ${MYSQL_DBNAME}
      CASSANDRA_HOST: ${CASSANDRA_HOST}
      CASSANDRA_PORT: ${CASSANDRA_PORT}
      CASSANDRA_USER: ${CASSANDRA_USER}
      CASSANDRA_PASSWORD: ${CASSANDRA_PASSWORD}
      CASSANDRA_KEYSPACE: ${CASSANDRA_KEYSPACE}
      CASSANDRA_USE_CLUSTER: ${CASSANDRA_USE_CLUSTER}
      CASSANDRA_NODES: ${CASSANDRA_NODES}
      TWITTERBG_CONSUMER_KEY: ${TWITTERBG_CONSUMER_KEY}
      TWITTERBG_CONSUMER_SECRET: ${TWITTERBG_CONSUMER_SECRET}
      TWITTERBG_ACCESS_TOKEN: ${TWITTERBG_ACCESS_TOKEN}
      TWITTERBG_ACCESS_TOKEN_SECRET: ${TWITTERBG_ACCESS_TOKEN_SECRET}
      TWITTEROD_CONSUMER_KEY: ${TWITTEROD_CONSUMER_KEY}
      TWITTEROD_CONSUMER_SECRET: ${TWITTEROD_CONSUMER_SECRET}
      TWITTEROD_ACCESS_TOKEN: ${TWITTEROD_ACCESS_TOKEN}
      TWITTEROD_ACCESS_TOKEN_SECRET: ${TWITTEROD_ACCESS_TOKEN_SECRET}
      TWITTERMA_CONSUMER_KEY: ${TWITTERMA_CONSUMER_KEY}
      TWITTERMA_CONSUMER_SECRET: ${TWITTERMA_CONSUMER_SECRET}
      TWITTERMA_ACCESS_TOKEN: ${TWITTERMA_ACCESS_TOKEN}
      TWITTERMA_ACCESS_TOKEN_SECRET: ${TWITTERMA_ACCESS_TOKEN_SECRET}
      KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS}
      PERSISTER_KAFKA_TOPIC: ${PERSISTER_KAFKA_TOPIC}
      CQLENG_ALLOW_SCHEMA_MANAGEMENT: 1
      CHECK_ONDEMAND_RUNTIME_SCHEDULING: ${CHECK_ONDEMAND_RUNTIME_SCHEDULING}
      CHECK_JOBS_INTERVAL_SECONDS: ${CHECK_JOBS_INTERVAL_SECONDS}
      PYTHONUNBUFFERED: 'true'
      http_proxy: ${http_proxy}
      https_proxy: ${http_proxy}
    networks:
      - back-tier
    volumes:
      - './collectors/src:/collectors'
      - '${FLOOD_KEYWORDS_PATH}:/configuration/flood_keywords.yaml'
    depends_on:
      - mysql
      - kafka
      - cassandrasmfr
      - persister
      - annotator
      - geocoder
      - restserver
      - products
      - aggregator
      - web

  restserver:
    restart: always
    container_name: restserver
    image: ${DOCKER_REGISTRY}/${RESTSERVER_IMAGE}:${image_tag}
    build:
      context: ./
      dockerfile: restserver/Dockerfile
      args:
        http_proxy: ${http_proxy}
        https_proxy: ${https_proxy}
        ftp_host: ${FTP_HOST}
    working_dir: /restserver
    environment:
      LOGGING_LEVEL: ${LOGGING_LEVEL}
      DEVELOPMENT: ${DEVELOPMENT}
      INCLUDE_PAST_COLLECTIONS: ${INCLUDE_PAST_COLLECTIONS}
      MYSQL_HOST: ${MYSQL_HOST}
      MYSQL_USER: ${MYSQL_USER}
      MYSQL_PASSWORD: ${MYSQL_PASSWORD}
      MYSQL_DBNAME: ${MYSQL_DBNAME}
      CASSANDRA_HOST: ${CASSANDRA_HOST}
      CASSANDRA_PORT: ${CASSANDRA_PORT}
      CASSANDRA_USER: ${CASSANDRA_USER}
      CASSANDRA_PASSWORD: ${CASSANDRA_PASSWORD}
      CASSANDRA_KEYSPACE: ${CASSANDRA_KEYSPACE}
      CASSANDRA_USE_CLUSTER: ${CASSANDRA_USE_CLUSTER}
      CASSANDRA_NODES: ${CASSANDRA_NODES}
      RESTSERVER_BASEPATH: '/1.0'
      RESTSERVER_PORT: ${RESTSERVER_PORT}
      ANNOTATOR_PORT: ${ANNOTATOR_PORT}
      GEOCODER_PORT: ${GEOCODER_PORT}
      PERSISTER_PORT: ${PERSISTER_PORT}
      ANNOTATOR_HOST: ${ANNOTATOR_HOST}
      GEOCODER_HOST: ${GEOCODER_HOST}
      PERSISTER_HOST: ${PERSISTER_HOST}
      KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS}
      PERSISTER_KAFKA_TOPIC: ${PERSISTER_KAFKA_TOPIC}
      FLASK_APP: smfr.py
      SERVER_PATH_UPLOADS: /smfr/uploads/
      CQLENG_ALLOW_SCHEMA_MANAGEMENT: 1
      CHECK_ONDEMAND_RUNTIME_SCHEDULING: ${CHECK_ONDEMAND_RUNTIME_SCHEDULING}
      CHECK_JOBS_INTERVAL_SECONDS: ${CHECK_JOBS_INTERVAL_SECONDS}
      PYTHONUNBUFFERED: 'true'
      http_proxy: ${http_proxy}
      https_proxy: ${http_proxy}
    ports:
      - '5555:5555'
    networks:
      - front-tier
      - back-tier
    volumes:
      - './restserver/src:/restserver'
    depends_on:
      - mysql
      - kafka
      - cassandrasmfr
      - annotator
      - geocoder
      - persister
      - aggregator

  web:
    restart: always
    container_name: web
    image: ${DOCKER_REGISTRY}/${WEB_IMAGE}:${image_tag}
    build:
      context: ./
      dockerfile: web/Dockerfile
      args:
        http_proxy: ${http_proxy}
        https_proxy: ${http_proxy}
    working_dir: /app
    volumes:
      - './web/src:/app'
      - '${PRODUCTS_OUTPUT}:/products'
    environment:
      PYTHONUNBUFFERED: 'true'
      RESTSERVER_PORT: ${RESTSERVER_PORT}
      RESTSERVER_BASEPATH: '/1.0'
      PRODUCTS_OUTPUT_FOLDER: '/products'
      http_proxy: ${http_proxy}
      https_proxy: ${https_proxy}
      DEVELOPMENT: ${DEVELOPMENT}
    ports:
      - '8888:8888'
    depends_on:
      - restserver
      - products
    networks:
      - front-tier

networks:
  front-tier: