From f889a3815e0777ae0207d7bd30cbed218c0a6d9d Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Thu, 4 Apr 2024 16:56:39 +0200 Subject: [PATCH] Changes for OpenSearch end-to-end tests (#4860) --- .../end_to_end/extract_and_output.Dockerfile | 2 + .../jenkins/greendale/output_opensearch.ini | 4 +- .../greendale/output_opensearch_ts.ini | 3 +- .../greendale/psort-studentpc1-nsrlsvr.ini | 4 +- ...t-studentpc1-sessionize-unique-domains.ini | 4 +- .../greendale/psort-studentpc1-tagging.ini | 4 +- config/jenkins/linux/run_end_to_end_tests.sh | 74 ------------------- config/jenkins/run_end_to_end_tests.sh | 15 +++- 8 files changed, 25 insertions(+), 85 deletions(-) delete mode 100755 config/jenkins/linux/run_end_to_end_tests.sh diff --git a/config/end_to_end/extract_and_output.Dockerfile b/config/end_to_end/extract_and_output.Dockerfile index 82e180a22e..2a7c64086a 100644 --- a/config/end_to_end/extract_and_output.Dockerfile +++ b/config/end_to_end/extract_and_output.Dockerfile @@ -24,6 +24,8 @@ RUN git clone $GIT_REPOSITORY # Install Plaso dependencies. WORKDIR /home/test/plaso + +# Install Plaso and dependencies. RUN ./config/linux/ubuntu_install_plaso.sh # Configure container for running Plaso. diff --git a/config/jenkins/greendale/output_opensearch.ini b/config/jenkins/greendale/output_opensearch.ini index 1860470350..75873de69d 100644 --- a/config/jenkins/greendale/output_opensearch.ini +++ b/config/jenkins/greendale/output_opensearch.ini @@ -1,6 +1,6 @@ [output_opensearch] -case=output +case=analyze_and_output output_file=studentpc1.csv output_format=opensearch -output_options=--opensearch_mappings=plaso/data/opensearch.mappings +output_options=--opensearch_mappings=plaso/data/opensearch.mappings --opensearch_server=opensearch --opensearch_user=admin --opensearch_password=F0rT3st@nly source=studentpc1.plaso diff --git a/config/jenkins/greendale/output_opensearch_ts.ini b/config/jenkins/greendale/output_opensearch_ts.ini index 4de5822c86..5bd4780d02 100644 --- a/config/jenkins/greendale/output_opensearch_ts.ini +++ b/config/jenkins/greendale/output_opensearch_ts.ini @@ -1,5 +1,6 @@ [output_opensearch_ts] -case=output +case=analyze_and_output output_file=studentpc1.csv output_format=opensearch_ts +output_options=--opensearch_mappings=plaso/data/opensearch.mappings source=studentpc1.plaso diff --git a/config/jenkins/greendale/psort-studentpc1-nsrlsvr.ini b/config/jenkins/greendale/psort-studentpc1-nsrlsvr.ini index 231ec8dbd0..7a22361f12 100644 --- a/config/jenkins/greendale/psort-studentpc1-nsrlsvr.ini +++ b/config/jenkins/greendale/psort-studentpc1-nsrlsvr.ini @@ -1,6 +1,6 @@ [nsrlsvr] -case=output +case=analyze_and_output +analysis_options=--analysis nsrlsvr --nsrlsvr-port=9120 source=studentpc1.plaso -output_options=--analysis nsrlsvr --nsrlsvr-port=9120 output_file=studentpc1.csv output_format=dynamic diff --git a/config/jenkins/greendale/psort-studentpc1-sessionize-unique-domains.ini b/config/jenkins/greendale/psort-studentpc1-sessionize-unique-domains.ini index 2aa954476e..e0993ff758 100644 --- a/config/jenkins/greendale/psort-studentpc1-sessionize-unique-domains.ini +++ b/config/jenkins/greendale/psort-studentpc1-sessionize-unique-domains.ini @@ -1,6 +1,6 @@ [sessionize-unique-domains] -case=output +case=analyze_and_output +analysis_options=--analysis sessionize,unique_domains_visited source=studentpc1.plaso -output_options=--analysis sessionize,unique_domains_visited output_file=studentpc1.csv output_format=dynamic diff --git a/config/jenkins/greendale/psort-studentpc1-tagging.ini b/config/jenkins/greendale/psort-studentpc1-tagging.ini index d84b1a3716..85cd75d20b 100644 --- a/config/jenkins/greendale/psort-studentpc1-tagging.ini +++ b/config/jenkins/greendale/psort-studentpc1-tagging.ini @@ -1,6 +1,6 @@ [tagging] -case=output +case=analyze_and_output +analysis_options=--analysis tagging --tagging-file=plaso/data/tag_windows.txt source=studentpc1.plaso -output_options=--analysis=tagging --tagging-file=plaso/data/tag_windows.txt output_file=studentpc1.csv output_format=dynamic diff --git a/config/jenkins/linux/run_end_to_end_tests.sh b/config/jenkins/linux/run_end_to_end_tests.sh deleted file mode 100755 index 07df627d73..0000000000 --- a/config/jenkins/linux/run_end_to_end_tests.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -# -# Script to run Python 3 end-to-end tests on a Linux Jenkins instance. -# -# This file is generated by l2tdevtools update-dependencies.py. - -# Fail on error. -set -e - -CONFIGURATION_NAME=$1; - -CONFIGURATION_FILE="${CONFIGURATION_NAME}.ini"; - -SOURCES_DIRECTORY="/media/greendale_images"; -REFERENCES_DIRECTORY="/media/greendale_images"; - -RESULTS_DIRECTORY="plaso-out"; - -# Change path to test this script on CI system. -if test ${CONFIGURATION_NAME} = 'ci'; -then - SOURCES_DIRECTORY="test_data"; - REFERENCES_DIRECTORY="test_data/end_to_end"; - CONFIGURATION_FILE="config/jenkins/${CONFIGURATION_NAME}.ini"; -fi - -mkdir -p ${RESULTS_DIRECTORY} ${RESULTS_DIRECTORY}/profiling; - -if ! test -f ${CONFIGURATION_FILE}; -then - CONFIGURATION_FILE="config/jenkins/greendale/${CONFIGURATION_NAME}.ini"; -fi -if ! test -f ${CONFIGURATION_FILE}; -then - CONFIGURATION_FILE="config/jenkins/sans/${CONFIGURATION_NAME}.ini"; -fi -if ! test -f ${CONFIGURATION_FILE}; -then - CONFIGURATION_FILE="config/jenkins/other/${CONFIGURATION_NAME}.ini"; -fi - -PYTHONPATH=. python3 ./utils/check_dependencies.py - -# Start the end-to-end tests in the background so we can capture the PID of -# the process while the script is running. -PYTHONPATH=. python3 ./tests/end-to-end.py --config ${CONFIGURATION_FILE} --sources-directory ${SOURCES_DIRECTORY} --scripts-directory plaso/scripts --results-directory ${RESULTS_DIRECTORY} --references-directory ${REFERENCES_DIRECTORY} & - -PID_COMMAND=$!; - -echo "End-to-end tests started (PID: ${PID_COMMAND})"; - -wait ${PID_COMMAND}; - -RESULT=$?; - -# On CI system print the stdout and stderr output to troubleshoot potential issues. -if test ${CONFIGURATION_NAME} = 'ci'; -then - for FILE in $(find ${RESULTS_DIRECTORY} -name \*.out -type f); - do - echo "stdout file: ${FILE}"; - cat ${FILE}; - echo ""; - done - - for FILE in $(find ${RESULTS_DIRECTORY} -name \*.err -type f); - do - echo "stderr file: ${FILE}"; - cat ${FILE}; - echo ""; - done -fi - -exit ${RESULT}; diff --git a/config/jenkins/run_end_to_end_tests.sh b/config/jenkins/run_end_to_end_tests.sh index 1e373e55bd..4069331fd3 100755 --- a/config/jenkins/run_end_to_end_tests.sh +++ b/config/jenkins/run_end_to_end_tests.sh @@ -26,9 +26,20 @@ RESULTS_DIRECTORY="${PWD}/plaso-out"; mkdir -p "${RESULTS_DIRECTORY}/profiling"; -# Build the extract_and_output end-to-end test Docker image. +# Build the end-to-end test Docker image. docker build -f extract_and_output.Dockerfile --force-rm --no-cache -t log2timeline/plaso . ; docker run log2timeline/plaso ./utils/check_dependencies.py; -docker run -v "${CONFIGURATION_DIRECTORY}:/config:z" -v "${RESULTS_DIRECTORY}:/home/test/plaso/plaso-out:z" -v "${SOURCES_DIRECTORY}:/sources:z" log2timeline/plaso /bin/bash -c "./tests/end-to-end.py --config /config/${CONFIGURATION_NAME}.ini --references-directory test_data/end_to_end --results-directory /home/test/plaso/plaso-out --sources-directory /sources --scripts-directory plaso/scripts" +COMMAND="./tests/end-to-end.py --config /config/${CONFIGURATION_NAME}.ini --references-directory test_data/end_to_end --results-directory /home/test/plaso/plaso-out --sources-directory /sources --scripts-directory plaso/scripts"; + +if test ${CONFIGURATION_NAME} = "output_opensearch"; +then + DOCKER_NETWORK="--network=opensearch-network"; + +elif test ${CONFIGURATION_NAME} = "studentpc1-redis"; +then + DOCKER_NETWORK="--network=redis-network"; +fi + +docker run --name=plaso ${DOCKER_NETWORK} -v "${CONFIGURATION_DIRECTORY}:/config:z" -v "${RESULTS_DIRECTORY}:/home/test/plaso/plaso-out:z" -v "${SOURCES_DIRECTORY}:/sources:z" log2timeline/plaso /bin/bash -c "${COMMAND}"