From b7f61ff12d96b4f45bb16ff8446850ecc50778ac Mon Sep 17 00:00:00 2001 From: Daan Bakker Date: Mon, 4 Mar 2019 19:26:09 +0100 Subject: [PATCH 1/4] Updating to amqp version 2.4.2 --- graypy/rabbitmq.py | 5 +++-- setup.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/graypy/rabbitmq.py b/graypy/rabbitmq.py index 3fb4a3353..8665e0a02 100644 --- a/graypy/rabbitmq.py +++ b/graypy/rabbitmq.py @@ -8,7 +8,7 @@ from logging import Filter from logging.handlers import SocketHandler -from amqplib import client_0_8 as amqp # pylint: disable=import-error +import amqp # pylint: disable=import-error from graypy.handler import BaseGELFHandler @@ -73,7 +73,7 @@ def __init__(self, url, exchange='logging.gelf', exchange_type='fanout', **kwargs ) SocketHandler.__init__(self, host, port) - self.addFilter(ExcludeFilter('amqplib')) + self.addFilter(ExcludeFilter('amqp')) def makeSocket(self, timeout=1): return RabbitSocket(self.cn_args, timeout, self.exchange, @@ -93,6 +93,7 @@ def __init__(self, cn_args, timeout, exchange, exchange_type, routing_key): self.routing_key = routing_key self.connection = amqp.Connection( connection_timeout=timeout, **self.cn_args) + self.connection.connect() self.channel = self.connection.channel() self.channel.exchange_declare( exchange=self.exchange, diff --git a/setup.py b/setup.py index f69f951f4..86ccdc4b1 100755 --- a/setup.py +++ b/setup.py @@ -66,11 +66,11 @@ def run_tests(self): "pylint>=1.9.3,<2.0.0", "mock>=2.0.0,<3.0.0", "requests>=2.20.1,<3.0.0", - "amqplib>=1.0.2,<2.0.0" + "amqp>=2.4.2,<2.5.1" ], extras_require={ 'amqp': [ - 'amqplib==1.0.2' + 'amqp==2.4.2' ], "docs": [ "sphinx>=2.1.2,<3.0.0", From 94a9de94de663e20d373f1222360983f3f74fb2c Mon Sep 17 00:00:00 2001 From: Daan Bakker Date: Fri, 13 Mar 2020 15:04:56 +0100 Subject: [PATCH 2/4] Adding an integration test for the GELFRabbitHandler --- tests/config/docker-compose-rmq.yml | 9 ++++++ tests/config/inputs.json | 26 +++++++++++++++++ tests/config/setup_rabbitmq.sh | 5 ++++ tests/config/start_local_graylog_server.sh | 15 +++++++++- tests/config/stop_local_graylog_server.sh | 1 + tests/integration/test_rmq_logging.py | 33 ++++++++++++++++++++++ 6 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 tests/config/docker-compose-rmq.yml create mode 100755 tests/config/setup_rabbitmq.sh create mode 100644 tests/integration/test_rmq_logging.py diff --git a/tests/config/docker-compose-rmq.yml b/tests/config/docker-compose-rmq.yml new file mode 100644 index 000000000..f402cc43f --- /dev/null +++ b/tests/config/docker-compose-rmq.yml @@ -0,0 +1,9 @@ +version: '2' +services: + rabbitmq: + image: "rabbitmq:3.8-management-alpine" + volumes: + - ./setup_rabbitmq.sh:/usr/local/bin/setup_rabbitmq.sh + ports: + - "5672:5672" + - "15672:15672" diff --git a/tests/config/inputs.json b/tests/config/inputs.json index d1d7b5e4e..4e58d5826 100644 --- a/tests/config/inputs.json +++ b/tests/config/inputs.json @@ -39,6 +39,32 @@ }, "type": "org.graylog2.inputs.gelf.tcp.GELFTCPInput", "global": true + }, + { + "title":"rmq", + "configuration": { + "heartbeat":60, + "prefetch":100, + "exchange_bind":true, + "broker_vhost":"/", + "broker_username":"graylog", + "decompress_size_limit":8388608, + "broker_port":5672, + "parallel_queues":1, + "broker_password":"graylog", + "throttling_allowed":false, + "exchange":"log-messages", + "tls":false, + "override_source":null, + "routing_key":"#", + "requeue_invalid_messages":true, + "broker_hostname":"rabbitmq", + "queue":"log-messages" + }, + "static_fields":{}, + "type":"org.graylog2.inputs.gelf.amqp.GELFAMQPInput", + "global":true, + "extractors":[] } ], "streams": [], diff --git a/tests/config/setup_rabbitmq.sh b/tests/config/setup_rabbitmq.sh new file mode 100755 index 000000000..60b378c11 --- /dev/null +++ b/tests/config/setup_rabbitmq.sh @@ -0,0 +1,5 @@ +rabbitmqctl add_user graylog graylog +rabbitmqctl set_user_tags graylog administrator +rabbitmqctl set_permissions -p / graylog ".*" ".*" ".*" + +rabbitmqadmin declare exchange name=log-messages type=direct -u graylog -p graylog \ No newline at end of file diff --git a/tests/config/start_local_graylog_server.sh b/tests/config/start_local_graylog_server.sh index cf140a3ce..ece59e9f9 100644 --- a/tests/config/start_local_graylog_server.sh +++ b/tests/config/start_local_graylog_server.sh @@ -6,12 +6,25 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd ${DIR} +# remove a potential previous setup +docker-compose -f docker-compose-rmq.yml down +docker-compose -f docker-compose.yml down + +# first setup RabbitMQ, as it requires extra setup before Graylog can connect to it. +docker-compose -f docker-compose-rmq.yml up -d + +# wait for RabbitMQ to start +sleep 30 + +# Configure a user and exchange for graylog to use +RMQ_CONTAINER_ID="$(docker ps -qf "name=rabbit")" +docker exec -ti ${RMQ_CONTAINER_ID} sh /usr/local/bin/setup_rabbitmq.sh + # create ssl certs for enabling the graylog server to use a # TLS connection for GELF input bash create_ssl_certs.sh -h localhost -i 127.0.0.1 # start the graylog server docker container -docker-compose -f docker-compose.yml down docker-compose -f docker-compose.yml up -d # wait for the graylog server docker container to start diff --git a/tests/config/stop_local_graylog_server.sh b/tests/config/stop_local_graylog_server.sh index 1305e1b0a..0cac7f1d6 100644 --- a/tests/config/stop_local_graylog_server.sh +++ b/tests/config/stop_local_graylog_server.sh @@ -6,4 +6,5 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd ${DIR} +docker-compose -f docker-compose-rmq.yml down docker-compose -f docker-compose.yml down diff --git a/tests/integration/test_rmq_logging.py b/tests/integration/test_rmq_logging.py new file mode 100644 index 000000000..4fee28ac8 --- /dev/null +++ b/tests/integration/test_rmq_logging.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""pytests sending logs to Graylog through RabbitMQ""" + +import logging + +import pytest + +from graypy.rabbitmq import GELFRabbitHandler +from graypy.handler import SYSLOG_LEVELS + +from tests.integration import LOCAL_GRAYLOG_UP +from tests.integration.helper import get_unique_message, get_graylog_response + + +@pytest.mark.skipif(not LOCAL_GRAYLOG_UP, + reason="local Graylog instance not up") +def test_rmq_logging(): + """Test that verifies the log message was received by Graylog""" + logger = logging.getLogger("test_rmq_logging") + handler = GELFRabbitHandler(url="amqp://graylog:graylog@127.0.0.1", + exchange="log-messages", + exchange_type="direct", + routing_key="#") + logger.addHandler(handler) + message = get_unique_message() + logger.error(message) + graylog_response = get_graylog_response(message) + assert message == graylog_response["message"] + assert "long_message" not in graylog_response + assert "timestamp" in graylog_response + assert SYSLOG_LEVELS[logging.ERROR] == graylog_response["level"] From a4d964c6573f1313b9b44cdfb1f4b173ec996784 Mon Sep 17 00:00:00 2001 From: Daan Bakker Date: Fri, 13 Mar 2020 16:02:13 +0100 Subject: [PATCH 3/4] Fixing wrong parameter name --- graypy/rabbitmq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graypy/rabbitmq.py b/graypy/rabbitmq.py index 8665e0a02..da9c7d147 100644 --- a/graypy/rabbitmq.py +++ b/graypy/rabbitmq.py @@ -92,7 +92,7 @@ def __init__(self, cn_args, timeout, exchange, exchange_type, routing_key): self.exchange_type = exchange_type self.routing_key = routing_key self.connection = amqp.Connection( - connection_timeout=timeout, **self.cn_args) + connect_timeout=timeout, **self.cn_args) self.connection.connect() self.channel = self.connection.channel() self.channel.exchange_declare( From 4d92ac6a853c098b45e63d5216b5e27ab81d0f9f Mon Sep 17 00:00:00 2001 From: Daan Bakker Date: Fri, 13 Mar 2020 16:54:52 +0100 Subject: [PATCH 4/4] Reformatted with Black --- graypy/rabbitmq.py | 2 +- setup.py | 6 ++---- tests/integration/test_rmq_logging.py | 13 +++++++------ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/graypy/rabbitmq.py b/graypy/rabbitmq.py index a4683ba3d..f156ab7ff 100644 --- a/graypy/rabbitmq.py +++ b/graypy/rabbitmq.py @@ -78,7 +78,7 @@ def __init__( self.routing_key = routing_key BaseGELFHandler.__init__(self, **kwargs) SocketHandler.__init__(self, host, port) - self.addFilter(ExcludeFilter('amqp')) + self.addFilter(ExcludeFilter("amqp")) def makeSocket(self, timeout=1): return RabbitSocket( diff --git a/setup.py b/setup.py index 95c2eccaf..ac90b5fcf 100755 --- a/setup.py +++ b/setup.py @@ -80,12 +80,10 @@ def run_tests(self): "pylint>=1.9.3,<2.0.0", "mock>=2.0.0,<3.0.0", "requests>=2.20.1,<3.0.0", - "amqp>=2.4.2,<2.5.1" + "amqp>=2.4.2,<2.5.1", ], extras_require={ - 'amqp': [ - 'amqp==2.4.2' - ], + "amqp": ["amqp==2.4.2"], "docs": [ "sphinx>=2.1.2,<3.0.0", "sphinx_rtd_theme>=0.4.3,<1.0.0", diff --git a/tests/integration/test_rmq_logging.py b/tests/integration/test_rmq_logging.py index 4fee28ac8..fe1908884 100644 --- a/tests/integration/test_rmq_logging.py +++ b/tests/integration/test_rmq_logging.py @@ -14,15 +14,16 @@ from tests.integration.helper import get_unique_message, get_graylog_response -@pytest.mark.skipif(not LOCAL_GRAYLOG_UP, - reason="local Graylog instance not up") +@pytest.mark.skipif(not LOCAL_GRAYLOG_UP, reason="local Graylog instance not up") def test_rmq_logging(): """Test that verifies the log message was received by Graylog""" logger = logging.getLogger("test_rmq_logging") - handler = GELFRabbitHandler(url="amqp://graylog:graylog@127.0.0.1", - exchange="log-messages", - exchange_type="direct", - routing_key="#") + handler = GELFRabbitHandler( + url="amqp://graylog:graylog@127.0.0.1", + exchange="log-messages", + exchange_type="direct", + routing_key="#", + ) logger.addHandler(handler) message = get_unique_message() logger.error(message)