diff --git a/docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh b/docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh index 0084e50f..f346243e 100755 --- a/docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh +++ b/docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh @@ -8,9 +8,16 @@ # *********** Setting ES_JAVA_OPTS *************** if [[ -z "$ES_JAVA_OPTS" ]]; then - ES_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024/2}' /proc/meminfo) - if [ $ES_MEMORY -gt 31 ]; then - ES_MEMORY=31 + AVAILABLE_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024}' /proc/meminfo) + if [ $AVAILABLE_MEMORY -ge 8 -a $AVAILABLE_MEMORY -le 12 ]; then + ES_MEMORY=2 + elif [$AVAILABLE_MEMORY -ge 13 -a $AVAILABLE_MEMORY -le 16]; then + ES_MEMORY=4 + else + ES_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024/2}' /proc/meminfo) + if [ $ES_MEMORY -gt 31 ]; then + ES_MEMORY=31 + fi fi export ES_JAVA_OPTS="-Xms${ES_MEMORY}g -Xmx${ES_MEMORY}g" fi diff --git a/docker/helk-jupyter/Dockerfile b/docker/helk-jupyter/Dockerfile index 68307401..df072d03 100644 --- a/docker/helk-jupyter/Dockerfile +++ b/docker/helk-jupyter/Dockerfile @@ -29,6 +29,11 @@ RUN apt-get update -qq \ jupyterlab==0.35.4 \ jupyterhub==0.9.4 \ ipywidgets==7.4.2 \ + matplotlib==3.0.2 \ + scipy==1.2.0 \ + scikit-learn==0.20.2 \ + Keras==2.2.4 \ + s3fs==0.2.0 \ # *********** Setting Jupyter Hub & Jupyter ********************** && curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash - \ && apt-get install -y --no-install-recommends nodejs \ diff --git a/docker/helk-jupyter/kernels/pyspark_kernel.json b/docker/helk-jupyter/kernels/pyspark_kernel.json index 0ae67c6c..ad24c8df 100644 --- a/docker/helk-jupyter/kernels/pyspark_kernel.json +++ b/docker/helk-jupyter/kernels/pyspark_kernel.json @@ -11,7 +11,6 @@ "env": { "SPARK_HOME": "/opt/helk/spark/", "PYTHONPATH": "/opt/helk/spark/python/:/opt/helk/spark/python/lib/py4j-0.10.7-src.zip", - "PYTHONSTARTUP": "/opt/helk/spark/python/pyspark/shell.py", "PYSPARK_PYTHON": "/usr/bin/python3" } } \ No newline at end of file diff --git a/docker/helk-jupyter/scripts/jupyter-entrypoint.sh b/docker/helk-jupyter/scripts/jupyter-entrypoint.sh index 053c0aab..0b265571 100755 --- a/docker/helk-jupyter/scripts/jupyter-entrypoint.sh +++ b/docker/helk-jupyter/scripts/jupyter-entrypoint.sh @@ -37,7 +37,7 @@ if [[ $HELK_USER_EXISTS == "1" ]]; then echo "[HELK-JUPYTER-DOCKER-INSTALLATION-INFO] Creating JupyterHub Group..." groupadd -g ${JUPYTERHUB_GID} jupyterhub - # ************* Create notebooks folder if it is not provided in comose file ****************** + # ************* Create notebooks folder if it is not provided in compose file ****************** mkdir -p ${JUPYTER_NOTEBOOKS} # ************* Creating JupyterHub Admin *************** diff --git a/docker/helk-jupyter/hive-site.xml b/docker/helk-jupyter/spark/hive-site.xml similarity index 100% rename from docker/helk-jupyter/hive-site.xml rename to docker/helk-jupyter/spark/hive-site.xml diff --git a/docker/helk-jupyter/spark/spark-defaults.conf b/docker/helk-jupyter/spark/spark-defaults.conf index 228ff1c8..09e663c3 100644 --- a/docker/helk-jupyter/spark/spark-defaults.conf +++ b/docker/helk-jupyter/spark/spark-defaults.conf @@ -12,7 +12,7 @@ # Logs the effective SparkConf as INFO when a SparkContext is started. Default: false spark.logConf true # The cluster manager to connect to. -spark.master spark://helk-spark-master:7077 +# spark.master spark://helk-spark-master:7077 # Restarts the driver automatically if it fails with a non-zero exit status spark.driver.supervise true @@ -25,7 +25,7 @@ spark.executor.logs.rolling.strategy spark.executor.logs.rolling.time.interval spark.jars /opt/helk/es-hadoop/elasticsearch-hadoop-6.5.4.jar # Comma-separated list of Maven coordinates of jars to include on the driver and executor classpaths. # The coordinates should be groupId:artifactId:version. -spark.jars.packages graphframes:graphframes:0.6.0-spark2.3-s_2.11,org.apache.spark:spark-sql-kafka-0-10_2.11:2.4.0 +spark.jars.packages graphframes:graphframes:0.7.0-spark2.4-s_2.11,org.apache.spark:spark-sql-kafka-0-10_2.11:2.4.0 #spark.jars.packages org.apache.spark:spark-sql-kafka-0-10_2.11:2.3.1,databricks:spark-sklearn:0.2.3 # ************ Spark UI **************** diff --git a/docker/helk-kibana-notebook-analysis-basic.yml b/docker/helk-kibana-notebook-analysis-basic.yml index 9e51932d..42c69f80 100644 --- a/docker/helk-kibana-notebook-analysis-basic.yml +++ b/docker/helk-kibana-notebook-analysis-basic.yml @@ -81,13 +81,13 @@ services: networks: helk: helk-jupyter: - image: cyb3rward0g/helk-jupyter:0.0.9 + image: cyb3rward0g/helk-jupyter:0.1.0 container_name: helk-jupyter volumes: - ./helk-jupyter/notebooks:/opt/helk/jupyter/notebooks environment: JUPYTER_HELK_PWD: hunting - JUPYTER_USERS: hunter1, hunter2 + JUPYTER_USERS: hunter1 restart: always depends_on: - helk-logstash @@ -161,7 +161,7 @@ services: KSQL_KSQL_COMMIT_INTERVAL_MS: 2000 KSQL_KSQL_CACHE_MAX_BYTES_BUFFERING: 10000000 KSQL_KSQL_STREAMS_AUTO_OFFSET_RESET: earliest - KSQL_HEAP_OPTS: -Xmx1g + KSQL_HEAP_OPTS: -Xmx500m ports: - 8088:8088 networks: @@ -172,7 +172,7 @@ services: depends_on: - helk-ksql-server environment: - KSQL_HEAP_OPTS: -Xmx550m + KSQL_HEAP_OPTS: -Xmx500m entrypoint: /bin/sh tty: true networks: diff --git a/docker/helk-kibana-notebook-analysis-trial.yml b/docker/helk-kibana-notebook-analysis-trial.yml index d047f884..6928123a 100644 --- a/docker/helk-kibana-notebook-analysis-trial.yml +++ b/docker/helk-kibana-notebook-analysis-trial.yml @@ -83,13 +83,13 @@ services: networks: helk: helk-jupyter: - image: cyb3rward0g/helk-jupyter:0.0.9 + image: cyb3rward0g/helk-jupyter:0.1.0 container_name: helk-jupyter volumes: - ./helk-jupyter/notebooks:/opt/helk/jupyter/notebooks environment: JUPYTER_HELK_PWD: hunting - JUPYTER_USERS: hunter1, hunter2 + JUPYTER_USERS: hunter1 restart: always depends_on: - helk-logstash @@ -143,7 +143,7 @@ services: ADVERTISED_LISTENER: ${ADVERTISED_LISTENER} ZOOKEEPER_NAME: helk-zookeeper KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, winsysmon, winsecurity - KAFKA_HEAP_OPTS: -Xmx1G -Xms1G + KAFKA_HEAP_OPTS: -Xmx1g -Xms1g LOG_RETENTION_HOURS: 4 ports: - "9092:9092" @@ -163,7 +163,7 @@ services: KSQL_KSQL_COMMIT_INTERVAL_MS: 2000 KSQL_KSQL_CACHE_MAX_BYTES_BUFFERING: 10000000 KSQL_KSQL_STREAMS_AUTO_OFFSET_RESET: earliest - KSQL_HEAP_OPTS: -Xmx1g + KSQL_HEAP_OPTS: -Xmx500m ports: - 8088:8088 networks: @@ -174,7 +174,7 @@ services: depends_on: - helk-ksql-server environment: - KSQL_HEAP_OPTS: -Xmx550m + KSQL_HEAP_OPTS: -Xmx500m entrypoint: /bin/sh tty: true networks: diff --git a/docker/helk-logstash/scripts/logstash-entrypoint.sh b/docker/helk-logstash/scripts/logstash-entrypoint.sh index 5da3c65e..0c766dfc 100755 --- a/docker/helk-logstash/scripts/logstash-entrypoint.sh +++ b/docker/helk-logstash/scripts/logstash-entrypoint.sh @@ -79,13 +79,6 @@ for file in ${DIR}/*.json; do done done -# ********* Setting LS_JAVA_OPTS *************** -if [[ -z "$LS_JAVA_OPTS" ]]; then - LS_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/4}' /proc/meminfo) - export LS_JAVA_OPTS="-Xms${LS_MEMORY}m -Xmx${LS_MEMORY}m" -fi -echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Setting LS_JAVA_OPTS to $LS_JAVA_OPTS" - # ********** Install Plugin ***************** echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Installing Logstash plugins.." if logstash-plugin list 'prune'; then @@ -94,6 +87,21 @@ else logstash-plugin install logstash-filter-prune fi +# ********* Setting LS_JAVA_OPTS *************** +if [[ -z "$LS_JAVA_OPTS" ]]; then + while true; do + LS_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/4}' /proc/meminfo) + if [ $LS_MEMORY -gt 980 ]; then + export LS_JAVA_OPTS="-Xms${LS_MEMORY}m -Xmx${LS_MEMORY}m" + break + else + echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] $LS_MEMORY MB is not enough memory for Logstash yet.." + sleep 1 + fi + done +fi +echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Setting LS_JAVA_OPTS to $LS_JAVA_OPTS" + # ********** Starting Logstash ***************** echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Running docker-entrypoint script.." /usr/local/bin/docker-entrypoint \ No newline at end of file diff --git a/docker/helk_docker_install.sh b/docker/helk_docker_install.sh index 97c1a090..e2c85341 100755 --- a/docker/helk_docker_install.sh +++ b/docker/helk_docker_install.sh @@ -19,12 +19,72 @@ echoerror() { } # ********* Globals ********************** -systemKernel="$(uname -s)" +SYSTEM_KERNEL="$(uname -s)" + +echo "[HELK-DOCKER-INSTALLATION-INFO] Checking distribution list and product version" +if [ "$SYSTEM_KERNEL" == "Linux" ]; then + # *********** Check distribution list *************** + LSB_DIST="$(. /etc/os-release && echo "$ID")" + LSB_DIST="$(echo "$LSB_DIST" | tr '[:upper:]' '[:lower:]')" + # *********** Check distribution version *************** + case "$LSB_DIST" in + ubuntu) + if [ -x "$(command -v lsb_release)" ]; then + DIST_VERSION="$(lsb_release --codename | cut -f2)" + fi + if [ -z "$DIST_VERSION" ] && [ -r /etc/lsb-release ]; then + DIST_VERSION="$(. /etc/lsb-release && echo "$DISTRIB_CODENAME")" + fi + # ********* Commenting Out CDROM ********************** + sed -i "s/\(^deb cdrom.*$\)/\#/g" /etc/apt/sources.list + ;; + debian|raspbian) + DIST_VERSION="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')" + case "$DIST_VERSION" in + 9) DIST_VERSION="stretch";; + 8) DIST_VERSION="jessie";; + 7) DIST_VERSION="wheezy";; + esac + # ********* Commenting Out CDROM ********************** + sed -i "s/\(^deb cdrom.*$\)/\#/g" /etc/apt/sources.list + ;; + centos) + if [ -z "$DIST_VERSION" ] && [ -r /etc/os-release ]; then + DIST_VERSION="$(. /etc/os-release && echo "$VERSION_ID")" + fi + ;; + rhel|ol|sles) + ee_notice "$LSB_DIST" + exit 1 + ;; + *) + if [ -x "$(command -v lsb_release)" ]; then + DIST_VERSION="$(lsb_release --release | cut -f2)" + fi + if [ -z "$DIST_VERSION" ] && [ -r /etc/os-release ]; then + DIST_VERSION="$(. /etc/os-release && echo "$VERSION_ID")" + fi + ;; + esac + ERROR=$? + if [ $ERROR -ne 0 ]; then + echoerror "Could not verify distribution or version of the OS (Error Code: $ERROR)." + fi + echo "[HELK-DOCKER-INSTALLATION-INFO] You're using $LSB_DIST version $DIST_VERSION" +elif [ "$SYSTEM_KERNEL" == "Darwin" ]; then + PRODUCT_NAME="$(sw_vers -productName)" + PRODUCT_VERSION="$(sw_vers -productVersion)" + BUILD_VERSION="$(sw_vers -buildVersion)" + echo "[HELK-DOCKER-INSTALLATION-INFO] You're using $PRODUCT_NAME version $PRODUCT_VERSION" +else + echo "[HELK-DOCKER-INSTALLATION-INFO] We cannot figure out the SYSTEM_KERNEL, distribution or version of the OS" +fi + # ********** Install Curl ******************** install_curl(){ echo "[HELK-DOCKER-INSTALLATION-INFO] Installing curl before installing docker.." - case "$lsb_dist" in + case "$LSB_DIST" in ubuntu|debian|raspbian) apt-get install -y curl >> $LOGFILE 2>&1 ;; @@ -32,7 +92,7 @@ install_curl(){ yum install curl >> $LOGFILE 2>&1 ;; *) - echo "[HELK-DOCKER-INSTALLATION-INFO] Please install curl for $lsb_dist $dist_version.." + echo "[HELK-DOCKER-INSTALLATION-INFO] Please install curl for $LSB_DIST $DIST_VERSION .." exit 1 ;; esac @@ -81,8 +141,8 @@ install_docker_compose(){ fi } -# *********** Main steps -if [ "$systemKernel" == "Linux" ]; then +# *********** Main steps ********************* +if [ "$SYSTEM_KERNEL" == "Linux" ]; then # *********** Check if curl is installed *************** if [ -x "$(command -v curl)" ]; then echo "[HELK-DOCKER-INSTALLATION-INFO] curl is already installed" @@ -110,7 +170,7 @@ else if [ -x "$(command -v docker)" ] && [ -x "$(command -v docker-compose)" ]; then echo "[HELK-DOCKER-INSTALLATION-INFO] Docker & Docker-compose already installed" else - echo "[HELK-DOCKER-INSTALLATION-INFO] Please innstall Docker & Docker-compose for $systemKernel" + echo "[HELK-DOCKER-INSTALLATION-INFO] Please innstall Docker & Docker-compose for $SYSTEM_KERNEL" exit 1 fi fi \ No newline at end of file diff --git a/docker/helk_install.sh b/docker/helk_install.sh index cbdca81a..79a8e012 100755 --- a/docker/helk_install.sh +++ b/docker/helk_install.sh @@ -36,7 +36,7 @@ check_min_requirements(){ echo "[HELK-INSTALLATION-ERROR] Installation Wiki: https://github.com/Cyb3rWard0g/HELK/wiki/Installation" exit 1 fi - if [ "${AVAILABLE_MEMORY}" -ge "11" ] && [ "${AVAILABLE_DISK}" -ge "25" ]; then + if [ "${AVAILABLE_MEMORY}" -ge "12" ] && [ "${AVAILABLE_DISK}" -ge "25" ]; then echo "[HELK-INSTALLATION-INFO] Available Memory: $AVAILABLE_MEMORY" echo "[HELK-INSTALLATION-INFO] Available Disk: $AVAILABLE_DISK" else @@ -442,7 +442,7 @@ show_banner(){ echo "** HELK - THE HUNTING ELK **" echo "** **" echo "** Author: Roberto Rodriguez (@Cyb3rWard0g) **" - echo "** HELK build version: v0.1.6-alpha01312019 **" + echo "** HELK build version: v0.1.6-alpha02022019 **" echo "** HELK ELK version: 6.5.4 **" echo "** License: GPL-3.0 **" echo "**********************************************" @@ -483,9 +483,9 @@ install_helk(){ check_system_info set_helk_build set_helk_subscription + set_network set_kibana_ui_password set_elasticsearch_password - set_network prepare_helk build_helk sleep 180