From 485f1cd1c532b69146f76ef491c18866fe393425 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Thu, 13 May 2021 11:43:50 -0700 Subject: [PATCH 1/2] Upgrade Suricata to 6.0.2 (latest) --- Dockerfile | 69 ++++++++++++++---------------------------- suricata_/suricata_.py | 4 ++- 2 files changed, 25 insertions(+), 48 deletions(-) diff --git a/Dockerfile b/Dockerfile index cfedbde..29d0be4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM cccs/assemblyline-v4-service-base:latest AS base ENV SERVICE_PATH suricata_.suricata_.Suricata -ENV SURICATA_VERSION 4.1.2 +ENV SURICATA_VERSION 6.0.2 USER root @@ -9,51 +9,16 @@ RUN echo 'deb http://deb.debian.org/debian stretch-backports main' >> /etc/apt/s # Install APT dependancies RUN apt-get update && apt-get install -y \ - git \ - libpcre3 \ - libpcap0.8 \ - libnet1 \ - libyaml-0-2 \ - zlib1g \ - libcap-ng0 \ - libhtp2 \ - libjansson4 \ - liblua5.1-0 \ - libnss3 \ - liblz4-1 \ - && rm -rf /var/lib/apt/lists/* + git wget curl \ + libpcre3 libpcre3-dbg libpcre3-dev build-essential libpcap-dev \ + libnet1-dev libyaml-0-2 libyaml-dev pkg-config zlib1g zlib1g-dev \ + libcap-ng-dev libcap-ng0 make libmagic-dev libjansson-dev\ + libnss3-dev libgeoip-dev liblua5.1-dev libhiredis-dev libevent-dev \ + python-yaml rustc cargo autoconf \ + && rm -rf /var/lib/apt/lists/* FROM base AS build -# Install APT dependancies -RUN apt-get update && apt-get install -y \ - git \ - libpcre3 \ - libpcre3-dbg \ - libpcre3-dev \ - build-essential \ - autoconf \ - automake \ - libtool \ - libpcap-dev \ - libnet1-dev \ - libyaml-0-2 \ - libyaml-dev \ - zlib1g \ - zlib1g-dev \ - libcap-ng-dev \ - libcap-ng0 \ - make \ - libmagic-dev \ - libjansson-dev \ - libjansson4 \ - pkg-config \ - cargo \ - liblua5.1-0-dev \ - libnss3-dev \ - liblz4-dev \ - wget && rm -rf /var/lib/apt/lists/* - # Install PIP dependancies USER assemblyline RUN touch /tmp/before-pip @@ -67,18 +32,26 @@ RUN pip install --no-cache-dir --user \ USER root RUN ln -s /var/lib/assemblyline/.local /root/.local +# Install rustup (purge rustc) +RUN apt remove --purge -y rustc +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y +SHELL ["bash", "-lc"] +RUN source $HOME/.cargo/env + # Build suricata RUN wget -O /tmp/suricata-${SURICATA_VERSION}.tar.gz https://www.openinfosecfoundation.org/download/suricata-${SURICATA_VERSION}.tar.gz RUN tar -xvzf /tmp/suricata-${SURICATA_VERSION}.tar.gz -C /tmp WORKDIR /tmp/suricata-${SURICATA_VERSION} RUN ./configure --disable-gccmarch-native --prefix=/build/ --sysconfdir=/etc/ --localstatedir=/var/ \ - --enable-python --enable-rust --enable-lua + --enable-python --enable-rust --enable-lua RUN make -C /tmp/suricata-${SURICATA_VERSION} RUN make -C /tmp/suricata-${SURICATA_VERSION} install -RUN ldconfig RUN make -C /tmp/suricata-${SURICATA_VERSION} install-full +RUN ldconfig /usr/local/lib # Install suricata pip package +ENV PATH="/build/bin:$PATH" +ENV TMPDIR=/tmp/suricata-${SURICATA_VERSION} RUN pip install --no-cache-dir --user /tmp/suricata-${SURICATA_VERSION}/python # Install stripe @@ -99,7 +72,9 @@ COPY --chown=assemblyline:assemblyline --from=build /var/lib/assemblyline/.local COPY --from=build /build/ /usr/local/ COPY --from=build /etc/suricata/ /etc/suricata/ COPY --from=build /var/log/suricata/ /var/log/suricata/ +COPY --from=build /usr/lib /usr/lib +ENV LD_LIBRARY_PATH=/usr/local/lib # Create all suricata directories and set permissions RUN mkdir -p /mount/updates && chown -R assemblyline /mount/updates RUN mkdir -p /etc/suricata && chown -R assemblyline /etc/suricata @@ -111,7 +86,7 @@ RUN mkdir -p /var/run/suricata && chown -R assemblyline /var/run/suricata COPY suricata_/conf/suricata.yaml /etc/suricata/ RUN chown assemblyline /etc/suricata/suricata.yaml RUN sed -i -e 's/__HOME_NET__/any/g' /etc/suricata/suricata.yaml -RUN sed -i -e 's/__RULE_FILES__/rule_files: []/g' /etc/suricata/suricata.yaml +RUN sed -i -e 's/__RULE_FILES__/rule-files: []/g' /etc/suricata/suricata.yaml # Update local rules using suricata-update script here RUN touch /etc/suricata/suricata-rules-update @@ -131,4 +106,4 @@ USER root RUN sed -i -e "s/\$SERVICE_TAG/$version/g" service_manifest.yml # Switch to assemblyline user -USER assemblyline \ No newline at end of file +USER assemblyline diff --git a/suricata_/suricata_.py b/suricata_/suricata_.py index 7af6370..dd9fa7a 100644 --- a/suricata_/suricata_.py +++ b/suricata_/suricata_.py @@ -26,7 +26,7 @@ def __init__(self, config=None): super(Suricata, self).__init__(config) self.home_net = self.config.get("home_net", "any") - self.rules_config = yaml.safe_dump({"rule_files": []}) + self.rules_config = yaml.safe_dump({"rule-files": []}) self.rules_list = [] self.run_dir = "/var/run/suricata" self.suricata_socket = None @@ -165,6 +165,8 @@ def suricata_running(self): try: self.suricata_sc.connect() except suricatasc.SuricataException as e: + if "Transport endpoint is already connected" in str(e): + return True self.log.info(f"Suricata not started yet: {str(e)}") return False return True From 3bbadd61c891b81ab74ed96f6d05293628c31fca Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Thu, 13 May 2021 12:10:39 -0700 Subject: [PATCH 2/2] Retry sample on add_extracted exception --- suricata_/suricata_.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/suricata_/suricata_.py b/suricata_/suricata_.py index dd9fa7a..38312b4 100644 --- a/suricata_/suricata_.py +++ b/suricata_/suricata_.py @@ -12,6 +12,7 @@ from pathlib import Path from retrying import retry +from assemblyline.common.exceptions import RecoverableError from assemblyline.common.str_utils import safe_str from assemblyline.common.digests import get_sha256_for_file from assemblyline_v4_service.common.base import ServiceBase @@ -350,7 +351,11 @@ def execute(self, request): if sha256 not in extracted: self.log.info(f"extracted file {filename}") extracted.add(sha256) - request.add_extracted(extracted_file_path, filename, "Extracted by suricata") + try: + request.add_extracted(extracted_file_path, filename, "Extracted by suricata") + except FileNotFoundError as e: + # An intermittent issue, just try again + raise RecoverableError(e) # Report a null score to indicate that files were extracted. If no sigs hit, it's not clear # where the extracted files came from