From f235ba5ad37c599285a298ba96b6838bb662f604 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:24:07 +0100 Subject: [PATCH 01/29] Updated .gitignore to also ignore numbered logs (e.g. *.log.1) --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 830cd8a..3863c8c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ .idea/ __pycache__/ -logs/*.log -logs/*/*.log +logs/*.log* +logs/*/*.log* !logs/alerts/.gitkeep !logs/general/.gitkeep From e5df78dbea3bc6f2a6d6b74c0609d254b057097c Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:24:49 +0100 Subject: [PATCH 02/29] Fixed typo. --- config/internal_config.ini | 2 +- src/utils/config_parsers/internal.py | 2 +- test/test_internal_config.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/internal_config.ini b/config/internal_config.ini index 6a7fe67..e7ec85a 100644 --- a/config/internal_config.ini +++ b/config/internal_config.ini @@ -18,7 +18,7 @@ twiml_instructions_url = https://twimlets.com/echo [redis] redis_database = 10 -redis_rest_database = 11 +redis_test_database = 11 redis_twilio_snooze_key = twilio_snooze redis_github_releases_key_prefix = github_releases_ diff --git a/src/utils/config_parsers/internal.py b/src/utils/config_parsers/internal.py index f213f8f..125485f 100644 --- a/src/utils/config_parsers/internal.py +++ b/src/utils/config_parsers/internal.py @@ -36,7 +36,7 @@ def __init__(self, config_file_path: str) -> None: # [redis] section = cp['redis'] self.redis_database = int(section['redis_database']) - self.redis_test_database = int(section['redis_rest_database']) + self.redis_test_database = int(section['redis_test_database']) self.redis_twilio_snooze_key = section['redis_twilio_snooze_key'] self.redis_github_releases_key_prefix = section[ diff --git a/test/test_internal_config.ini b/test/test_internal_config.ini index b4312ac..115220d 100644 --- a/test/test_internal_config.ini +++ b/test/test_internal_config.ini @@ -18,7 +18,7 @@ twiml_instructions_url = https://twimlets.com/echo [redis] redis_database = 10 -redis_rest_database = 11 +redis_test_database = 11 redis_twilio_snooze_key = twilio_snooze redis_github_releases_key_prefix = github_releases_ From a0c45c2cb87072b830e0af15efa8e6d824b484c7 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:25:21 +0100 Subject: [PATCH 03/29] Split get_full_channel_set into multiple functions. --- src/alerting/alert_utils/get_channel_set.py | 94 +++++++++++++++------ 1 file changed, 68 insertions(+), 26 deletions(-) diff --git a/src/alerting/alert_utils/get_channel_set.py b/src/alerting/alert_utils/get_channel_set.py index e69d62a..aefb609 100644 --- a/src/alerting/alert_utils/get_channel_set.py +++ b/src/alerting/alert_utils/get_channel_set.py @@ -18,55 +18,97 @@ from src.utils.redis_api import RedisApi -def get_full_channel_set(channel_name: str, logger_general: logging.Logger, - redis: Optional[RedisApi], alerts_log_file: str, - internal_conf: InternalConfig = InternalConf, - user_conf: UserConfig = UserConf) -> ChannelSet: +def _get_log_channel(alerts_log_file: str, channel_name: str, + logger_general: logging.Logger, + internal_conf: InternalConfig = InternalConf) \ + -> LogChannel: # Logger initialisation logger_alerts = create_logger(alerts_log_file, 'alerts', internal_conf.logging_level) + return LogChannel(channel_name, logger_general, logger_alerts) + + +def _get_console_channel(channel_name: str, + logger_general: logging.Logger) -> ConsoleChannel: + return ConsoleChannel(channel_name, logger_general) + + +def _get_telegram_channel(channel_name: str, logger_general: logging.Logger, + redis: Optional[RedisApi], + backup_channels_for_telegram: ChannelSet, + user_conf: UserConfig = UserConf) -> TelegramChannel: + telegram_bot = TelegramBotApi(user_conf.telegram_alerts_bot_token, + user_conf.telegram_alerts_bot_chat_id) + telegram_channel = TelegramChannel( + channel_name, logger_general, redis, + telegram_bot, backup_channels_for_telegram) + return telegram_channel + +def _get_email_channel(channel_name: str, logger_general: logging.Logger, + redis: Optional[RedisApi], + user_conf: UserConfig = UserConf) -> EmailChannel: + email = EmailSender(user_conf.email_smtp, user_conf.email_from, + user_conf.email_user, user_conf.email_pass) + email_channel = EmailChannel(channel_name, logger_general, + redis, email, user_conf.email_to) + return email_channel + + +def _get_twilio_channel(channel_name: str, logger_general: logging.Logger, + redis: Optional[RedisApi], + backup_channels_for_twilio: ChannelSet, + internal_conf: InternalConfig = InternalConf, + user_conf: UserConfig = UserConf) -> TwilioChannel: + twilio = TwilioApi(user_conf.twilio_account_sid, + user_conf.twilio_auth_token) + twilio_channel = TwilioChannel(channel_name, logger_general, + redis, twilio, + user_conf.twilio_phone_number, + user_conf.twilio_dial_numbers, + internal_conf.twiml_instructions_url, + internal_conf.redis_twilio_snooze_key, + backup_channels_for_twilio) + return twilio_channel + + +def get_full_channel_set(channel_name: str, logger_general: logging.Logger, + redis: Optional[RedisApi], alerts_log_file: str, + internal_conf: InternalConfig = InternalConf, + user_conf: UserConfig = UserConf) -> ChannelSet: # Initialise list of channels with default channels channels = [ - ConsoleChannel(channel_name, logger_general), - LogChannel(channel_name, logger_general, logger_alerts) + _get_console_channel(channel_name, logger_general), + _get_log_channel(alerts_log_file, channel_name, logger_general, + internal_conf) ] # Initialise backup channel sets with default channels backup_channels_for_telegram = ChannelSet(channels) backup_channels_for_twilio = ChannelSet(channels) - # Add telegram alerts to channel set + # Add telegram alerts to channel set if they are enabled from config file if user_conf.telegram_alerts_enabled: - telegram_bot = TelegramBotApi(user_conf.telegram_alerts_bot_token, - user_conf.telegram_alerts_bot_chat_id) - telegram_channel = TelegramChannel(channel_name, logger_general, redis, - telegram_bot, - backup_channels_for_telegram) + telegram_channel = _get_telegram_channel( + channel_name, logger_general, redis, + backup_channels_for_telegram, user_conf) channels.append(telegram_channel) else: telegram_channel = None - # Add email alerts to channel set + # Add email alerts to channel set if they are enabled from config file if user_conf.email_alerts_enabled: - email = EmailSender(user_conf.email_smtp, user_conf.email_from, - user_conf.email_user, user_conf.email_pass) - email_channel = EmailChannel(channel_name, logger_general, - redis, email, user_conf.email_to) + email_channel = _get_email_channel(channel_name, logger_general, + redis, user_conf) channels.append(email_channel) else: email_channel = None - # Add twilio alerts to channel set + # Add twilio alerts to channel set if they are enabled from config file if user_conf.twilio_alerts_enabled: - twilio = TwilioApi(user_conf.twilio_account_sid, - user_conf.twilio_auth_token) - twilio_channel = TwilioChannel(channel_name, logger_general, redis, - twilio, user_conf.twilio_phone_number, - user_conf.twilio_dial_numbers, - internal_conf.twiml_instructions_url, - internal_conf.redis_twilio_snooze_key, - backup_channels_for_twilio) + twilio_channel = _get_twilio_channel(channel_name, logger_general, + redis, backup_channels_for_twilio, + internal_conf, user_conf) channels.append(twilio_channel) else: # noinspection PyUnusedLocal From 3ee34d8ba87f6a2cd066c44288e3d38e40d70862 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:25:52 +0100 Subject: [PATCH 04/29] Fixed import order. --- src/alerting/channels/console.py | 2 +- src/alerting/channels/log.py | 2 +- src/alerting/channels/telegram.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/alerting/channels/console.py b/src/alerting/channels/console.py index 681a2b4..3f2f26c 100644 --- a/src/alerting/channels/console.py +++ b/src/alerting/channels/console.py @@ -1,8 +1,8 @@ import logging import sys -from src.alerting.channels.channel import Channel from src.alerting.alerts.alerts import Alert +from src.alerting.channels.channel import Channel class ConsoleChannel(Channel): diff --git a/src/alerting/channels/log.py b/src/alerting/channels/log.py index 9e62fb6..ef60743 100644 --- a/src/alerting/channels/log.py +++ b/src/alerting/channels/log.py @@ -1,7 +1,7 @@ import logging -from src.alerting.channels.channel import Channel from src.alerting.alerts.alerts import Alert +from src.alerting.channels.channel import Channel class LogChannel(Channel): diff --git a/src/alerting/channels/telegram.py b/src/alerting/channels/telegram.py index 0590cf8..dc14caa 100644 --- a/src/alerting/channels/telegram.py +++ b/src/alerting/channels/telegram.py @@ -2,8 +2,8 @@ from typing import Optional from src.alerting.alert_utils.telegram_bot_api import TelegramBotApi -from src.alerting.channels.channel import Channel, ChannelSet from src.alerting.alerts.alerts import Alert, ProblemWithTelegramBot +from src.alerting.channels.channel import Channel, ChannelSet from src.utils.redis_api import RedisApi From 944154b23cab897eac75a00c3acd90c077764c0f Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:26:21 +0100 Subject: [PATCH 05/29] Added setup for periodic alerts. --- src/setup/setup_user_config_main.py | 84 ++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 2 deletions(-) diff --git a/src/setup/setup_user_config_main.py b/src/setup/setup_user_config_main.py index 5438f5d..327c5f3 100644 --- a/src/setup/setup_user_config_main.py +++ b/src/setup/setup_user_config_main.py @@ -1,10 +1,12 @@ from configparser import ConfigParser +from datetime import timedelta from src.alerting.alert_utils.telegram_bot_api import TelegramBotApi from src.alerting.alert_utils.twilio_api import TwilioApi from src.setup.setup_user_config_main_tests import test_telegram_alerts, \ TestOutcome, test_email_alerts, test_twilio_alerts, \ test_telegram_commands, test_redis +from src.utils.datetime import strfdelta from src.utils.user_input import yn_prompt @@ -237,7 +239,7 @@ def setup_twilio_alerts(cp: ConfigParser) -> None: cp['twilio_alerts']['phone_numbers_to_dial'] = to_dial -def setup_alerts(cp: ConfigParser) -> None: +def setup_alert_channels(cp: ConfigParser) -> None: print('==== Alerts') print('By default, alerts are output to a log file and to ' 'the console. Let\'s set up the rest of the alerts.') @@ -350,12 +352,90 @@ def setup_redis(cp: ConfigParser) -> None: cp['redis']['password'] = password +def setup_periodic_alerts(cp: ConfigParser) -> None: + print('==== Periodic alerts') + setup_periodic_alive_reminder(cp) + + +def setup_periodic_alive_reminder(cp: ConfigParser) -> None: + print('---- Periodic alive reminder') + print('The periodic alive reminder is a way for the alerter to inform its ' + 'users that it is still running.') + + already_set_up = is_already_set_up(cp, 'periodic_alive_reminder') + if already_set_up and \ + not yn_prompt('The periodic alive reminder is already set up. ' + 'Do you wish to clear the current config? (Y/n)\n'): + return + + reset_section('periodic_alive_reminder', cp) + cp['periodic_alive_reminder']['enabled'] = str(False) + cp['periodic_alive_reminder']['interval_seconds'] = '' + cp['periodic_alive_reminder']['email_enabled'] = '' + cp['periodic_alive_reminder']['telegram_enabled'] = '' + + if not already_set_up and \ + not yn_prompt('Do you wish to set up the periodic alive reminder? ' + '(Y/n)\n'): + return + + interval = input("Please enter the amount of seconds you want to " + "pass for the periodic alive reminder. Make sure that " + "you insert a positive integer.\n") + while True: + try: + interval_number_rep = int(interval) + except ValueError: + interval = input("Input is not a valid integer. Please enter " + "another value\n") + continue + if interval_number_rep > 0: + time = timedelta(seconds=int(interval_number_rep)) + time = strfdelta(time, "{hours}h {minutes}m {seconds}s") + if yn_prompt( + 'You will be reminded that the alerter is still running ' + 'after ' + time + ". Do you want to confirm this (Y/n) \n"): + break + else: + interval = input( + "Please enter the amount of seconds you want to " + "pass for the periodic alive reminder. Make sure that " + "you insert a positive integer.\n") + else: + interval = input("Input is not a positive integer. Please enter " + "another value\n") + + if is_already_set_up(cp, 'email_alerts') and \ + cp['email_alerts']['enabled'] and \ + yn_prompt('Would you like the periodic alive reminder ' + 'to send alerts via e-mail? (Y/n)\n'): + email_enabled = str(True) + else: + email_enabled = str(False) + + if is_already_set_up(cp, 'telegram_alerts') and \ + cp['telegram_alerts']['enabled'] and \ + yn_prompt('Would you like the periodic alive reminder ' + 'to send alerts via Telegram? (Y/n)\n'): + telegram_enabled = str(True) + else: + telegram_enabled = str(False) + + cp['periodic_alive_reminder']['enabled'] = str(True) + cp['periodic_alive_reminder']['interval_seconds'] = interval + cp['periodic_alive_reminder']['email_enabled'] = email_enabled + cp['periodic_alive_reminder']['telegram_enabled'] = telegram_enabled + + def setup_all(cp: ConfigParser) -> None: setup_general(cp) print() - setup_alerts(cp) + setup_alert_channels(cp) + print() + setup_periodic_alerts(cp) print() setup_commands(cp) print() setup_redis(cp) + print() print('Setup finished.') From 4fd96d7811c4e398a8725b7aceb35287dffe1a00 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:26:44 +0100 Subject: [PATCH 06/29] Setup scripts now clear config before adding anything. --- src/setup/setup_user_config_nodes.py | 1 - src/setup/setup_user_config_repos.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/setup/setup_user_config_nodes.py b/src/setup/setup_user_config_nodes.py index fbc1ea7..3beb21d 100644 --- a/src/setup/setup_user_config_nodes.py +++ b/src/setup/setup_user_config_nodes.py @@ -78,7 +78,6 @@ def setup_nodes(cp: ConfigParser) -> None: break # Add nodes to config - cp.clear() for i, node in enumerate(nodes): section = 'node_' + str(i) cp.add_section(section) diff --git a/src/setup/setup_user_config_repos.py b/src/setup/setup_user_config_repos.py index 3cc0100..11c2624 100644 --- a/src/setup/setup_user_config_repos.py +++ b/src/setup/setup_user_config_repos.py @@ -72,7 +72,6 @@ def setup_repos(cp: ConfigParser) -> None: break # Add repos to config - cp.clear() for i, repo in enumerate(repos): section = 'repo_' + str(i) cp.add_section(section) From a659489cce9c0763bbea911102258c7221ea3ed1 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:28:45 +0100 Subject: [PATCH 07/29] Added periodic alive alert. --- config/example_user_config_main.ini | 5 ++ config/internal_config.ini | 1 + doc/DESIGN_AND_FEATURES.md | 24 ++++++ src/alerting/alert_utils/get_channel_set.py | 43 +++++++++++ src/alerting/alerts/alerts.py | 6 ++ src/alerting/periodic/periodic.py | 21 +++++ src/utils/config_parsers/internal.py | 2 + src/utils/config_parsers/user.py | 14 ++++ test/alerting/periodic/test_periodic.py | 86 +++++++++++++++++++++ test/test_internal_config.ini | 1 + test/test_user_config_main.ini | 5 ++ 11 files changed, 208 insertions(+) create mode 100644 src/alerting/periodic/periodic.py create mode 100644 test/alerting/periodic/test_periodic.py diff --git a/config/example_user_config_main.ini b/config/example_user_config_main.ini index afc0675..0306276 100644 --- a/config/example_user_config_main.ini +++ b/config/example_user_config_main.ini @@ -32,3 +32,8 @@ host = localhost port = 6379 password = HMASDNoiSADnuiasdgnAIO876hg967bv99vb8buyT8BVuyT76VBT76uyi +[periodic_alive_reminder] +enabled = True +interval_seconds = 3600 +email_enabled = False +telegram_enabled = True \ No newline at end of file diff --git a/config/internal_config.ini b/config/internal_config.ini index e7ec85a..3a6511b 100644 --- a/config/internal_config.ini +++ b/config/internal_config.ini @@ -25,6 +25,7 @@ redis_github_releases_key_prefix = github_releases_ redis_node_monitor_alive_key_prefix = node_monitor_alive_ redis_network_monitor_alive_key_prefix = network_monitor_alive_ redis_network_monitor_last_height_key_prefix = network_monitor_last_height_checked_ +redis_periodic_alive_reminder_mute_key = alive_reminder_mute redis_node_monitor_alive_key_timeout = 86400 redis_network_monitor_alive_key_timeout = 86400 diff --git a/doc/DESIGN_AND_FEATURES.md b/doc/DESIGN_AND_FEATURES.md index 8600db5..2631260 100644 --- a/doc/DESIGN_AND_FEATURES.md +++ b/doc/DESIGN_AND_FEATURES.md @@ -6,6 +6,7 @@ This page will present the inner workings of the alerter as well as the features - **Alerting Channels**: (console, logging, Telegram, email, Twilio) - **Alert Types**: (major, minor, info, error) - **Monitor Types**: (node, network, GitHub) +- **Periodic Alive Reminder** - **Telegram Commands** - **Redis** - **Complete List of Alerts** @@ -90,6 +91,15 @@ In each monitoring round, the GitHub monitor: 2. Saves its state 3. Sleeps until the next monitoring round +## Periodic Alive Reminder + +The periodic alive reminder is a way for P.A.N.I.C to inform the operator that it is still running. This can be useful to the operator when no alerts have been sent for a long time, therefore it does not leave the operator wondering whether P.A.N.I.C is still running or not. + +The following are some important points about the periodic alive reminder: + +1. The time after which a reminder is sent can be specified by the operator using the setup process described [here](./INSTALL_AND_RUN.md). +2. The periodic alive reminder can be muted and unmuted using Telegram as discussed below. + ## Telegram Commands Telegram bots in P.A.N.I.C. serve two purposes. As mentioned above, they are used to send alerts. However they can also accept commands that allow you to check the status of the alerter (and its running monitors), snooze or unsnooze calls, and conveniently get Cosmos explorer links to validator lists, blocks, and transactions. @@ -214,6 +224,20 @@ The only two alerts raised by the GitHub alerter are an info alert when a new re | `NewGitHubReleaseAlert` | `INFO` | ✗ | | `CannotAccessGitHubPageAlert` | `ERROR` | ✗ | +### Periodic Alive Reminder + +If the periodic alive reminder is enabled from the config file, and P.A.N.I.C is running smoothly, the operator is informed every `P1` seconds that P.A.N.I.C is still running via an info alert. + +The periodic alive reminder always uses the console and logger to raise this alert, however, the operator can also receive this alert via Telegram, Email or both, by modifying the config file as described [here](./INSTALL_AND_RUN.md#setting-up-panic). + +Default value: +- `P1 = interval_seconds = 3600` + +| Class | Severity | Configurable | +|---|---|---| +| `AlerterAliveAlert` | `INFO` | ✓ | + + ### Other (Errors) Last but not least is a set of error alerts, including read errors when gathering data from a node, termination of a component of the alerter (e.g. a monitor) due to some exception, and any problem experienced when using Telegram bots. diff --git a/src/alerting/alert_utils/get_channel_set.py b/src/alerting/alert_utils/get_channel_set.py index aefb609..272a253 100644 --- a/src/alerting/alert_utils/get_channel_set.py +++ b/src/alerting/alert_utils/get_channel_set.py @@ -124,3 +124,46 @@ def get_full_channel_set(channel_name: str, logger_general: logging.Logger, backup_channels_for_twilio.add_channel(telegram_channel) return ChannelSet(channels) + + +def get_periodic_alive_reminder_channel_set(channel_name: str, + logger_general: logging.Logger, + redis: Optional[RedisApi], + alerts_log_file: str, + internal_conf: + InternalConfig = InternalConf, + user_conf: UserConfig = UserConf) \ + -> ChannelSet: + # Initialise list of channels with default channels + channels = [ + _get_console_channel(channel_name, logger_general), + _get_log_channel(alerts_log_file, channel_name, logger_general, + internal_conf) + ] + + # Initialise backup channel sets with default channels + backup_channels_for_telegram = ChannelSet(channels) + + # Add telegram alerts to channel set if they are enabled from config file + if user_conf.telegram_alerts_enabled and \ + user_conf.telegram_enabled: + telegram_channel = _get_telegram_channel(channel_name, logger_general, + redis, + backup_channels_for_telegram, + user_conf) + channels.append(telegram_channel) + + # Add email alerts to channel set if they are enabled from config file + if user_conf.email_alerts_enabled and \ + user_conf.email_enabled: + email_channel = _get_email_channel(channel_name, logger_general, + redis, user_conf) + channels.append(email_channel) + else: + email_channel = None + + # Set up email channel as backup channel for telegram and twilio + if email_channel is not None: + backup_channels_for_telegram.add_channel(email_channel) + + return ChannelSet(channels) diff --git a/src/alerting/alerts/alerts.py b/src/alerting/alerts/alerts.py index c0202b2..336413f 100644 --- a/src/alerting/alerts/alerts.py +++ b/src/alerting/alerts/alerts.py @@ -199,3 +199,9 @@ class ProblemWithTelegramBot(Alert): def __init__(self, description: str) -> None: super().__init__( 'Problem encountered with telegram bot: {}'.format(description)) + + +class AlerterAliveAlert(Alert): + + def __init__(self) -> None: + super().__init__('Still running.') diff --git a/src/alerting/periodic/periodic.py b/src/alerting/periodic/periodic.py new file mode 100644 index 0000000..aa9d0cf --- /dev/null +++ b/src/alerting/periodic/periodic.py @@ -0,0 +1,21 @@ +from datetime import timedelta +from time import sleep + +from src.alerting.alerts.alerts import AlerterAliveAlert +from src.alerting.channels.channel import ChannelSet +from src.utils.redis_api import RedisApi + + +def periodic_alive_reminder(interval: timedelta, channel_set: ChannelSet, + mute_key: str, redis: RedisApi): + while True: + sleep(interval.total_seconds()) + send_alive_alert(redis, mute_key, channel_set) + + +def send_alive_alert(redis: RedisApi, mute_key: str, + channel_set: ChannelSet) -> None: + # If time elapses and periodic alive reminder is not muted, + # inform operator that alerter is still alive. + if not redis.exists(mute_key): + channel_set.alert_info(AlerterAliveAlert()) diff --git a/src/utils/config_parsers/internal.py b/src/utils/config_parsers/internal.py index 125485f..d2430df 100644 --- a/src/utils/config_parsers/internal.py +++ b/src/utils/config_parsers/internal.py @@ -47,6 +47,8 @@ def __init__(self, config_file_path: str) -> None: 'redis_network_monitor_alive_key_prefix'] self.redis_network_monitor_last_height_key_prefix = section[ 'redis_network_monitor_last_height_key_prefix'] + self.redis_periodic_alive_reminder_mute_key = \ + section['redis_periodic_alive_reminder_mute_key'] self.redis_node_monitor_alive_key_timeout = int( section['redis_node_monitor_alive_key_timeout']) diff --git a/src/utils/config_parsers/user.py b/src/utils/config_parsers/user.py index c7360b9..0eb2f75 100644 --- a/src/utils/config_parsers/user.py +++ b/src/utils/config_parsers/user.py @@ -1,4 +1,5 @@ import configparser +from datetime import timedelta from src.utils.config_parsers.config_parser import ConfigParser @@ -80,6 +81,19 @@ def __init__(self, alerting_config_file_path: str, self.redis_port = cp['redis']['port'] self.redis_password = cp['redis']['password'] + # [periodic_alive_reminder] + self.periodic_alive_reminder_enabled = to_bool( + cp['periodic_alive_reminder']['enabled']) + self.interval_seconds = timedelta( + seconds=int(cp['periodic_alive_reminder']['interval_seconds'])) \ + if self.periodic_alive_reminder_enabled else None + self.email_enabled = to_bool( + cp['periodic_alive_reminder']['email_enabled']) \ + if self.periodic_alive_reminder_enabled else None + self.telegram_enabled = to_bool( + cp['periodic_alive_reminder']['telegram_enabled']) \ + if self.periodic_alive_reminder_enabled else None + # ------------------------ Nodes Config # [node_...] diff --git a/test/alerting/periodic/test_periodic.py b/test/alerting/periodic/test_periodic.py new file mode 100644 index 0000000..a9da4f5 --- /dev/null +++ b/test/alerting/periodic/test_periodic.py @@ -0,0 +1,86 @@ +import logging +import time +import unittest +from datetime import datetime, timedelta + +from redis import ConnectionError as RedisConnectionError + +from src.alerting.channels.channel import ChannelSet +from src.alerting.periodic.periodic import send_alive_alert +from src.utils.redis_api import RedisApi +from src.utils.timing import TimedTaskLimiter +from test import TestInternalConf, TestUserConf +from test.node.test_node import CounterChannel + + +class TestPeriodic(unittest.TestCase): + def setUp(self) -> None: + self.alerter_name = 'testalerter' + self.logger = logging.getLogger('dummy') + self.counter_channel = CounterChannel(self.logger) + self.channel_set = ChannelSet([self.counter_channel]) + + self.db = TestInternalConf.redis_test_database + self.host = TestUserConf.redis_host + self.port = TestUserConf.redis_port + self.password = TestUserConf.redis_password + self.redis = RedisApi(self.logger, self.db, self.host, + self.port, self.password) + self.redis.delete_all_unsafe() + + try: + self.redis.ping_unsafe() + except RedisConnectionError: + self.fail('Redis is not online.') + + self.timedelta = TestUserConf.interval_seconds + self.timing = TimedTaskLimiter(self.timedelta) + self.mute_key = TestInternalConf.redis_periodic_alive_reminder_mute_key + + def test_periodic_alive_reminder_can_do_task_no_mute_key(self): + self.timing.did_task() + time.sleep(TestUserConf.interval_seconds.seconds) + self.counter_channel.reset() # ignore previous alerts + send_alive_alert(self.timing, self.redis, self.mute_key, self.channel_set) + self.assertEqual(self.counter_channel.minor_count, 0) + self.assertEqual(self.counter_channel.major_count, 0) + self.assertEqual(self.counter_channel.info_count, 1) + self.assertEqual(self.counter_channel.error_count, 0) + + def test_periodic_alive_reminder_cannot_do_task_no_mute_key(self): + self.timing.did_task() + time.sleep(TestUserConf.interval_seconds.seconds - 2) + self.counter_channel.reset() # ignore previous alerts + send_alive_alert(self.timing, self.redis, self.mute_key, self.channel_set) + self.assertEqual(self.counter_channel.minor_count, 0) + self.assertEqual(self.counter_channel.major_count, 0) + self.assertEqual(self.counter_channel.info_count, 0) + self.assertEqual(self.counter_channel.error_count, 0) + + def test_periodic_alive_reminder_can_do_task_mute_key_present(self): + self.timing.did_task() + time.sleep(TestUserConf.interval_seconds.seconds) + hours = timedelta(hours=float(1)) + until = str(datetime.now() + hours) + self.redis.set_for(self.mute_key, until, hours) + self.counter_channel.reset() # ignore previous alerts + send_alive_alert(self.timing, self.redis, self.mute_key, self.channel_set) + self.redis.remove(self.mute_key) + self.assertEqual(self.counter_channel.minor_count, 0) + self.assertEqual(self.counter_channel.major_count, 0) + self.assertEqual(self.counter_channel.info_count, 0) + self.assertEqual(self.counter_channel.error_count, 0) + + def test_periodic_alive_reminder_cannot_do_task_mute_key_present(self): + self.timing.did_task() + time.sleep(TestUserConf.interval_seconds.seconds - 3) + hours = timedelta(hours=float(1)) + until = str(datetime.now() + hours) + self.redis.set_for(self.mute_key, until, hours) + self.counter_channel.reset() # ignore previous alerts + send_alive_alert(self.timing, self.redis, self.mute_key, self.channel_set) + self.redis.remove(self.mute_key) + self.assertEqual(self.counter_channel.minor_count, 0) + self.assertEqual(self.counter_channel.major_count, 0) + self.assertEqual(self.counter_channel.info_count, 0) + self.assertEqual(self.counter_channel.error_count, 0) diff --git a/test/test_internal_config.ini b/test/test_internal_config.ini index 115220d..c8089ea 100644 --- a/test/test_internal_config.ini +++ b/test/test_internal_config.ini @@ -25,6 +25,7 @@ redis_github_releases_key_prefix = github_releases_ redis_node_monitor_alive_key_prefix = node_monitor_alive_ redis_network_monitor_alive_key_prefix = network_monitor_alive_ redis_network_monitor_last_height_key_prefix = network_monitor_last_height_checked_ +redis_periodic_alive_reminder_mute_key = alive_reminder_mute redis_node_monitor_alive_key_timeout = 86400 redis_network_monitor_alive_key_timeout = 86400 diff --git a/test/test_user_config_main.ini b/test/test_user_config_main.ini index 3a8a6a5..bc19b7b 100644 --- a/test/test_user_config_main.ini +++ b/test/test_user_config_main.ini @@ -32,3 +32,8 @@ host = localhost port = 6379 password = +[periodic_alive_reminder] +enabled = False +interval_seconds = 10 +email_enabled = False +telegram_enabled = True \ No newline at end of file From c9dbef27ee2b4e224348a28228e8a1b83527bf30 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:29:13 +0100 Subject: [PATCH 08/29] Added working directory changer for testing. --- test/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/__init__.py b/test/__init__.py index 0ecb7b4..c66ecd7 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,6 +1,12 @@ +import os + from src.utils.config_parsers.internal import InternalConfig from src.utils.config_parsers.user import UserConfig +# Get path of this __init__.py file and go two steps up +os.chdir(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) +print('Current working directory set to ' + os.getcwd()) + TestInternalConf = InternalConfig( 'test/test_internal_config.ini') TestUserConf = UserConfig( From 8a0f98b14a50bf8b20719184e135188c5a9fd9b3 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:29:35 +0100 Subject: [PATCH 09/29] Minor changes to github monitor. --- src/monitoring/monitors/github.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/monitoring/monitors/github.py b/src/monitoring/monitors/github.py index d745ebb..fdd3858 100644 --- a/src/monitoring/monitors/github.py +++ b/src/monitoring/monitors/github.py @@ -1,8 +1,8 @@ import logging from typing import Optional -from src.alerting.channels.channel import ChannelSet from src.alerting.alerts.alerts import NewGitHubReleaseAlert +from src.alerting.channels.channel import ChannelSet from src.monitoring.monitor_utils.get_json import get_json from src.monitoring.monitors.monitor import Monitor from src.utils.redis_api import RedisApi @@ -56,6 +56,7 @@ def monitor(self) -> None: releases = get_json(self.releases_page, self._logger) # If response contains a message, skip monitoring this time round + # since the presence of a message indicates an error in the API call if 'message' in releases: self.logger.warning('GitHub message: %s', releases['message']) return From d9b2ac6aa8c9ddcc2aba790d3d19d645818bed00 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:29:51 +0100 Subject: [PATCH 10/29] Changed default github period to 3600 seconds (1h) --- config/internal_config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/internal_config.ini b/config/internal_config.ini index 3a6511b..cc887b3 100644 --- a/config/internal_config.ini +++ b/config/internal_config.ini @@ -35,7 +35,7 @@ redis_network_monitor_alive_key_timeout = 86400 [monitoring_periods] node_monitor_period_seconds = 10 network_monitor_period_seconds = 10 -github_monitor_period_seconds = 300 +github_monitor_period_seconds = 3600 # These define how often a monitor runs an iteration of its monitoring loop [alert_intervals_and_limits] From 362729011c9606322a989c61f4e1874d0072a908 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:30:13 +0100 Subject: [PATCH 11/29] Fixed Redis tests. --- test/utils/test_redis_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/utils/test_redis_api.py b/test/utils/test_redis_api.py index 57f4983..53aa4d5 100644 --- a/test/utils/test_redis_api.py +++ b/test/utils/test_redis_api.py @@ -5,7 +5,7 @@ from time import sleep from redis import ConnectionError as RedisConnectionError, DataError, \ - ResponseError + AuthenticationError from src.utils.redis_api import RedisApi from test import TestInternalConf, TestUserConf @@ -80,8 +80,8 @@ def test_set_unsafe_throws_exception_if_incorrect_password(self): self.redis.set_unsafe(self.key1, self.val1) # works try: redis_bad_pass.set_unsafe(self.key1, self.val1) - self.fail('Expected ResponseError to be thrown') - except ResponseError: + self.fail('Expected AuthenticationError to be thrown') + except AuthenticationError: pass def test_set_unsafe_sets_the_specified_key_to_the_specified_value(self): From 36523120ddafe3bdc9c342fa56b14628a44e38f9 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:30:37 +0100 Subject: [PATCH 12/29] Removed extra set_as_down error argument. --- src/monitoring/monitors/monitor_starters.py | 19 +++++++++---------- src/node/node.py | 3 +-- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/monitoring/monitors/monitor_starters.py b/src/monitoring/monitors/monitor_starters.py index ff0e7b9..34f0f06 100644 --- a/src/monitoring/monitors/monitor_starters.py +++ b/src/monitoring/monitors/monitor_starters.py @@ -25,12 +25,10 @@ def start_node_monitor(node_monitor: NodeMonitor, monitor_period: int, logger.debug('Reading %s.', node_monitor.node) node_monitor.monitor() logger.debug('Done reading %s.', node_monitor.node) - except ConnectionError as conn_err: - node_monitor.node.set_as_down(node_monitor.channels, - conn_err, logger) - except ReadTimeout as read_timeout: - node_monitor.node.set_as_down(node_monitor.channels, - read_timeout, logger) + except ConnectionError: + node_monitor.node.set_as_down(node_monitor.channels, logger) + except ReadTimeout: + node_monitor.node.set_as_down(node_monitor.channels, logger) except (urllib3.exceptions.IncompleteRead, http.client.IncompleteRead) as incomplete_read: logger.error('Error when reading data from {}: {}. ' @@ -61,9 +59,9 @@ def start_network_monitor(network_monitor: NetworkMonitor, monitor_period: int, except NoLiveFullNodeException: network_monitor.channels.alert_major( CouldNotFindLiveFullNodeAlert(network_monitor.monitor_name)) - except (ConnectionError, ReadTimeout) as conn_err: + except (ConnectionError, ReadTimeout): network_monitor.last_full_node_used.set_as_down( - network_monitor.channels, conn_err, logger) + network_monitor.channels, logger) except (urllib3.exceptions.IncompleteRead, http.client.IncompleteRead) as incomplete_read: network_monitor.channels.alert_error(ErrorWhenReadingDataFromNode( @@ -78,8 +76,9 @@ def start_network_monitor(network_monitor: NetworkMonitor, monitor_period: int, network_monitor.save_state() # Sleep - logger.debug('Sleeping for %s seconds.', monitor_period) - time.sleep(monitor_period) + if not network_monitor.is_syncing(): + logger.debug('Sleeping for %s seconds.', monitor_period) + time.sleep(monitor_period) def start_github_monitor(github_monitor: GitHubMonitor, monitor_period: int, diff --git a/src/node/node.py b/src/node/node.py index f8b182e..3733ee2 100644 --- a/src/node/node.py +++ b/src/node/node.py @@ -145,8 +145,7 @@ def save_state(self, logger: logging.Logger) -> None: self._redis_prefix + '_no_of_peers': self._no_of_peers }) - def set_as_down(self, channels: ChannelSet, error: Exception, - logger: logging.Logger) -> None: + def set_as_down(self, channels: ChannelSet, logger: logging.Logger) -> None: logger.debug('%s set_as_down: is_down(currently)=%s, channels=%s', self, self.is_down, channels) From ea4fa391a896c8ebad7a77a51ff8967d7157e73f Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:31:34 +0100 Subject: [PATCH 13/29] Added validator peer safe boundary. --- config/internal_config.ini | 1 + doc/DESIGN_AND_FEATURES.md | 16 ++++++++++---- src/alerting/alerts/alerts.py | 9 ++++++++ src/node/node.py | 33 +++++++++++++++++++--------- src/utils/config_parsers/internal.py | 18 +++++++++++++++ 5 files changed, 63 insertions(+), 14 deletions(-) diff --git a/config/internal_config.ini b/config/internal_config.ini index cc887b3..8f5a3e6 100644 --- a/config/internal_config.ini +++ b/config/internal_config.ini @@ -43,6 +43,7 @@ downtime_alert_interval_seconds = 900 max_missed_blocks_interval_seconds = 120 max_missed_blocks_in_time_interval = 10 validator_peer_danger_boundary = 1 +validator_peer_safe_boundary = 5 full_node_peer_danger_boundary = 15 missed_blocks_danger_boundary = 5 github_error_interval_seconds = 3600 diff --git a/doc/DESIGN_AND_FEATURES.md b/doc/DESIGN_AND_FEATURES.md index 2631260..a3f00e8 100644 --- a/doc/DESIGN_AND_FEATURES.md +++ b/doc/DESIGN_AND_FEATURES.md @@ -181,15 +181,23 @@ Voting power change alerts are mostly info alerts; voting power increase is alwa ### Number of Peers -Alerts for changes in the number of peers range from info to major. Any increase is positive and is thus an info alert. As for peer decrease alerts: -- For validator nodes: any decrease to `N` peers inside a configurable danger boundary `D1` is a major alert (i.e. `N <= D1`). Otherwise, any other decrease is a minor alert. -- For non-validator nodes: any decrease to `N` peers inside a configurable danger boundary `D2` is a minor alert (i.e. `N <= D2`). Otherwise, any other decreases raises no alert. +Alerts for changes in the number of peers range from info to major. +#### For Validator Nodes +- Any decrease to `N` peers inside a configurable danger boundary `D1` is a major alert (i.e. `N <= D1`). +- Any decrease to `N` peers inside a configurable safe boundary `S1` is a minor alert (i.e. `D1 < N <= s1`). +- Any decrease to `N` peers outside a configurable safe boundary `S1` raises no alerts (i.e. `N > S1`). +- Any increase to `N` peers inside a configurable safe/danger boundary `S1`/`D1` raises an info alert (i.e. `N <= S1/D1`) +- Any increase to `N` peers outside a configurable safe boundary `S1` raises no alerts (i.e. `N > S1`). +#### For Non-Validator Nodes +- Any decrease to `N` peers inside a configurable danger boundary `D2` raises a minor alert (i.e. `N <= D2`). Otherwise, any other decreases raises no alert. +- Any increase to `N` peers inside a configurable danger boundary `D2` raises an info alert (i.e. `N <= D2`). Otherwise, any other increase raises no alert. Non-validator nodes typically have much more peers where not each one of them is important. Thus, once `D2` is exceeded (`N > D2`), a special *'increased outside danger range'* info alert is issued and no further peer increase alerts are issued, to reduce alert spam. Default values: - `D1 = validator_peer_danger_boundary = 1` -- `D2 = full_node_peer_danger_boundary = 15` +- `D2 = full_node_peer_danger_boundary = 10` +- `S1 = validator_peer_safe_boundary = 5` | Class | Severity | Configurable | |---|---|---| diff --git a/src/alerting/alerts/alerts.py b/src/alerting/alerts/alerts.py index 336413f..ef3ae93 100644 --- a/src/alerting/alerts/alerts.py +++ b/src/alerting/alerts/alerts.py @@ -132,6 +132,15 @@ def __init__(self, node: str, danger: int) -> None: ''.format(node, danger, danger)) +class PeersIncreasedOutsideSafeRangeAlert(Alert): + + def __init__(self, node: str, safe: int) -> None: + super().__init__( + '{} peers INCREASED to more than {} peers. No further peer change' + ' alerts will be sent unless the number of peers goes below {}.' + ''.format(node, safe, safe)) + + class PeersDecreasedAlert(Alert): def __init__(self, node: str, old_peers: int, new_peers: int) -> None: diff --git a/src/node/node.py b/src/node/node.py index 3733ee2..cc190d6 100644 --- a/src/node/node.py +++ b/src/node/node.py @@ -11,7 +11,8 @@ VotingPowerDecreasedAlert, VotingPowerIncreasedAlert, \ VotingPowerIncreasedByAlert, VotingPowerDecreasedByAlert, \ IsCatchingUpAlert, IsNoLongerCatchingUpAlert, PeersIncreasedAlert, \ - PeersDecreasedAlert, PeersIncreasedOutsideDangerRangeAlert + PeersDecreasedAlert, PeersIncreasedOutsideDangerRangeAlert, \ + PeersIncreasedOutsideSafeRangeAlert from src.alerting.channels.channel import ChannelSet from src.utils.config_parsers.internal import InternalConfig from src.utils.config_parsers.internal_parsed import InternalConf @@ -49,6 +50,8 @@ def __init__(self, name: str, rpc_url: Optional[str], node_type: NodeType, self._validator_peer_danger_boundary = \ internal_conf.validator_peer_danger_boundary + self._validator_peer_safe_boundary = \ + internal_conf.validator_peer_safe_boundary self._full_node_peer_danger_boundary = \ internal_conf.full_node_peer_danger_boundary self._missed_blocks_danger_boundary = \ @@ -310,21 +313,31 @@ def set_no_of_peers(self, new_no_of_peers: int, channels: ChannelSet, # Variable alias for improved readability if self.is_validator: danger = self._validator_peer_danger_boundary + safe = self._validator_peer_safe_boundary else: danger = self._full_node_peer_danger_boundary + safe = None # Alert if number of peers has changed if self.no_of_peers not in [None, new_no_of_peers]: if self.is_validator: - if new_no_of_peers > self.no_of_peers: # increase - channels.alert_info(PeersIncreasedAlert( - self.name, self.no_of_peers, new_no_of_peers)) - elif new_no_of_peers > danger: # decrease outside danger range - channels.alert_minor(PeersDecreasedAlert( - self.name, self.no_of_peers, new_no_of_peers)) - else: # decrease inside danger range - channels.alert_major(PeersDecreasedAlert( - self.name, self.no_of_peers, new_no_of_peers)) + if new_no_of_peers <= self._validator_peer_safe_boundary: + # beneath safe boundary + if new_no_of_peers > self.no_of_peers: # increase + channels.alert_info(PeersIncreasedAlert( + self.name, self.no_of_peers, new_no_of_peers)) + elif new_no_of_peers > danger: + # decrease outside danger range + channels.alert_minor(PeersDecreasedAlert( + self.name, self.no_of_peers, new_no_of_peers)) + else: # decrease inside danger range + channels.alert_major(PeersDecreasedAlert( + self.name, self.no_of_peers, new_no_of_peers)) + elif self._no_of_peers <= self._validator_peer_safe_boundary \ + < new_no_of_peers: + # increase outside safe range for the first time + channels.alert_info( + PeersIncreasedOutsideSafeRangeAlert(self.name, safe)) else: if new_no_of_peers > self.no_of_peers: # increase if new_no_of_peers < danger: # increase inside danger range diff --git a/src/utils/config_parsers/internal.py b/src/utils/config_parsers/internal.py index d2430df..b4c3b89 100644 --- a/src/utils/config_parsers/internal.py +++ b/src/utils/config_parsers/internal.py @@ -1,4 +1,5 @@ import configparser +import sys from datetime import timedelta from src.utils.config_parsers.config_parser import ConfigParser @@ -74,6 +75,9 @@ def __init__(self, config_file_path: str) -> None: section['max_missed_blocks_in_time_interval']) self.validator_peer_danger_boundary = int( section['validator_peer_danger_boundary']) + self.validator_peer_safe_boundary = int( + section['validator_peer_safe_boundary']) + self._check_if_peer_safe_and_danger_boundaries_are_valid() self.full_node_peer_danger_boundary = int( section['full_node_peer_danger_boundary']) self.missed_blocks_danger_boundary = int( @@ -102,3 +106,17 @@ def __init__(self, config_file_path: str) -> None: self.tx_mintscan_link_prefix = section['tx_mintscan_link_prefix'] self.github_releases_template = section['github_releases_template'] + + # Safe boundary must be greater than danger boundary at all times for + # correct execution + def _peer_safe_and_danger_boundaries_are_valid(self) -> bool: + return self.validator_peer_safe_boundary > \ + self.validator_peer_danger_boundary > 0 + + def _check_if_peer_safe_and_danger_boundaries_are_valid(self): + while not self._peer_safe_and_danger_boundaries_are_valid(): + print("validator_peer_safe_boundary must be STRICTLY GREATER than " + "validator_peer_danger_boundary for correct execution. " + "\nPlease do the necessary modifications in the " + "config/internal_config.ini file and restart the alerter.") + sys.exit(-1) From db0551e47503b102a0b3239f8e3ef1d6e00966eb Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:31:55 +0100 Subject: [PATCH 14/29] Reduced default full_node_peer_danger_boundary to 10 for less full-node alerts. --- config/internal_config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/internal_config.ini b/config/internal_config.ini index 8f5a3e6..b2a0184 100644 --- a/config/internal_config.ini +++ b/config/internal_config.ini @@ -44,7 +44,7 @@ max_missed_blocks_interval_seconds = 120 max_missed_blocks_in_time_interval = 10 validator_peer_danger_boundary = 1 validator_peer_safe_boundary = 5 -full_node_peer_danger_boundary = 15 +full_node_peer_danger_boundary = 10 missed_blocks_danger_boundary = 5 github_error_interval_seconds = 3600 # These limit the number of alerts of a specific type received using either From 1f081ed853a3e75e19c848fa52593f4845400b6a Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:32:21 +0100 Subject: [PATCH 15/29] Updated tests. --- test/node/test_node.py | 194 +++++++++++++++++++++++----------- test/test_internal_config.ini | 4 +- 2 files changed, 135 insertions(+), 63 deletions(-) diff --git a/test/node/test_node.py b/test/node/test_node.py index fd54283..ae8e752 100644 --- a/test/node/test_node.py +++ b/test/node/test_node.py @@ -6,8 +6,8 @@ import dateutil from redis import ConnectionError as RedisConnectionError -from src.alerting.channels.channel import ChannelSet, Channel from src.alerting.alerts.alerts import Alert +from src.alerting.channels.channel import ChannelSet, Channel from src.node.node import Node, NodeType from src.utils.redis_api import RedisApi from test import TestInternalConf, TestUserConf @@ -99,6 +99,13 @@ def setUp(self) -> None: self.peers_more_than_validator_danger_boundary = \ self.peers_validator_danger_boundary + 2 + self.peers_validator_safe_boundary = \ + TestInternalConf.validator_peer_safe_boundary + self.peers_less_than_validator_safe_boundary = \ + self.peers_validator_safe_boundary - 2 + self.peers_more_than_validator_safe_boundary = \ + self.peers_validator_safe_boundary + 2 + self.peers_full_node_danger_boundary = \ TestInternalConf.full_node_peer_danger_boundary self.peers_less_than_full_node_danger_boundary = \ @@ -140,65 +147,65 @@ def test_status_returns_three_values(self): 'number_of_peers=999') def test_first_set_as_down_sends_info_alert_and_sets_node_to_down(self): - self.validator.set_as_down(self.channel_set, self.dummy_exception, self.logger) + self.validator.set_as_down(self.channel_set, self.logger) self.assertEqual(self.counter_channel.info_count, 1) self.assertTrue(self.validator.is_down) def test_second_set_as_down_sends_major_alert_if_validator(self): - self.validator.set_as_down(self.channel_set, self.dummy_exception, self.logger) + self.validator.set_as_down(self.channel_set, self.logger) self.counter_channel.reset() # ignore previous alerts - self.validator.set_as_down(self.channel_set, self.dummy_exception, self.logger) + self.validator.set_as_down(self.channel_set, self.logger) self.assertEqual(self.counter_channel.major_count, 1) self.assertTrue(self.validator.is_down) def test_second_set_as_down_sends_minor_alert_if_non_validator(self): - self.full_node.set_as_down(self.channel_set, self.dummy_exception, self.logger) + self.full_node.set_as_down(self.channel_set, self.logger) self.counter_channel.reset() # ignore previous alerts - self.full_node.set_as_down(self.channel_set, self.dummy_exception, self.logger) + self.full_node.set_as_down(self.channel_set, self.logger) self.assertEqual(self.counter_channel.minor_count, 1) self.assertTrue(self.full_node.is_down) def test_third_set_as_down_does_nothing_if_within_time_interval_for_validator( self): - self.validator.set_as_down(self.channel_set, self.dummy_exception, self.logger) - self.validator.set_as_down(self.channel_set, self.dummy_exception, self.logger) + self.validator.set_as_down(self.channel_set, self.logger) + self.validator.set_as_down(self.channel_set, self.logger) self.counter_channel.reset() # ignore previous alerts - self.validator.set_as_down(self.channel_set, self.dummy_exception, self.logger) + self.validator.set_as_down(self.channel_set, self.logger) self.assertTrue(self.counter_channel.no_alerts()) self.assertTrue(self.validator.is_down) def test_third_set_as_down_does_nothing_if_within_time_interval_for_non_validator( self): - self.full_node.set_as_down(self.channel_set, self.dummy_exception, self.logger) - self.full_node.set_as_down(self.channel_set, self.dummy_exception, self.logger) + self.full_node.set_as_down(self.channel_set, self.logger) + self.full_node.set_as_down(self.channel_set, self.logger) self.counter_channel.reset() # ignore previous alerts - self.full_node.set_as_down(self.channel_set, self.dummy_exception, self.logger) + self.full_node.set_as_down(self.channel_set, self.logger) self.assertTrue(self.counter_channel.no_alerts()) self.assertTrue(self.full_node.is_down) def test_third_set_as_down_sends_major_alert_if_after_time_interval_for_validator( self): - self.validator.set_as_down(self.channel_set, self.dummy_exception, self.logger) - self.validator.set_as_down(self.channel_set, self.dummy_exception, self.logger) + self.validator.set_as_down(self.channel_set, self.logger) + self.validator.set_as_down(self.channel_set, self.logger) self.counter_channel.reset() # ignore previous alerts sleep(self.downtime_alert_time_interval_with_error_margin.seconds) - self.validator.set_as_down(self.channel_set, self.dummy_exception, self.logger) + self.validator.set_as_down(self.channel_set, self.logger) self.assertEqual(self.counter_channel.major_count, 1) self.assertTrue(self.validator.is_down) def test_third_set_as_down_sends_minor_alert_if_after_time_interval_for_non_validator( self): - self.full_node.set_as_down(self.channel_set, self.dummy_exception, self.logger) - self.full_node.set_as_down(self.channel_set, self.dummy_exception, self.logger) + self.full_node.set_as_down(self.channel_set, self.logger) + self.full_node.set_as_down(self.channel_set, self.logger) self.counter_channel.reset() # ignore previous alerts sleep(self.downtime_alert_time_interval_with_error_margin.seconds) - self.full_node.set_as_down(self.channel_set, self.dummy_exception, self.logger) + self.full_node.set_as_down(self.channel_set, self.logger) self.assertEqual(self.counter_channel.minor_count, 1) self.assertTrue(self.full_node.is_down) @@ -210,7 +217,7 @@ def test_set_as_up_does_nothing_if_not_down(self): def test_set_as_up_sets_as_up_but_no_alerts_if_set_as_down_called_only_once( self): - self.validator.set_as_down(self.channel_set, self.dummy_exception, self.logger) + self.validator.set_as_down(self.channel_set, self.logger) self.counter_channel.reset() # ignore previous alerts self.validator.set_as_up(self.channel_set, self.logger) @@ -219,8 +226,8 @@ def test_set_as_up_sets_as_up_but_no_alerts_if_set_as_down_called_only_once( def test_set_as_up_sets_as_up_and_sends_info_alert_if_set_as_down_called_twice( self): - self.validator.set_as_down(self.channel_set, self.dummy_exception, self.logger) - self.validator.set_as_down(self.channel_set, self.dummy_exception, self.logger) + self.validator.set_as_down(self.channel_set, self.logger) + self.validator.set_as_down(self.channel_set, self.logger) self.counter_channel.reset() # ignore previous alerts self.validator.set_as_up(self.channel_set, self.logger) @@ -228,14 +235,14 @@ def test_set_as_up_sets_as_up_and_sends_info_alert_if_set_as_down_called_twice( self.assertFalse(self.validator.is_down) def test_set_as_up_resets_alert_time_interval(self): - self.validator.set_as_down(self.channel_set, self.dummy_exception, self.logger) - self.validator.set_as_down(self.channel_set, self.dummy_exception, self.logger) - self.validator.set_as_down(self.channel_set, self.dummy_exception, self.logger) + self.validator.set_as_down(self.channel_set, self.logger) + self.validator.set_as_down(self.channel_set, self.logger) + self.validator.set_as_down(self.channel_set, self.logger) self.validator.set_as_up(self.channel_set, self.logger) self.counter_channel.reset() # ignore previous alerts - self.validator.set_as_down(self.channel_set, self.dummy_exception, self.logger) + self.validator.set_as_down(self.channel_set, self.logger) self.assertEqual(self.counter_channel.info_count, 1) self.assertTrue(self.validator.is_down) @@ -388,8 +395,10 @@ def test_set_voting_power_raises_no_alerts_first_time_round(self): self.assertTrue(self.counter_channel.no_alerts()) def test_set_voting_power_raises_no_alerts_if_voting_power_the_same(self): - self.validator.set_voting_power(self.dummy_voting_power, self.channel_set, self.logger) - self.validator.set_voting_power(self.dummy_voting_power, self.channel_set, self.logger) + self.validator.set_voting_power(self.dummy_voting_power, + self.channel_set, self.logger) + self.validator.set_voting_power(self.dummy_voting_power, + self.channel_set, self.logger) self.assertTrue(self.counter_channel.no_alerts()) @@ -397,8 +406,10 @@ def test_set_voting_power_raises_info_alert_if_voting_power_increases_from_non_0 self): increased_voting_power = self.dummy_voting_power + 1 - self.validator.set_voting_power(self.dummy_voting_power, self.channel_set, self.logger) - self.validator.set_voting_power(increased_voting_power, self.channel_set, self.logger) + self.validator.set_voting_power(self.dummy_voting_power, + self.channel_set, self.logger) + self.validator.set_voting_power(increased_voting_power, + self.channel_set, self.logger) self.assertEqual(self.counter_channel.info_count, 1) @@ -407,7 +418,8 @@ def test_set_voting_power_raises_info_alert_if_voting_power_increases_from_0( # This is just to cover the unique message when power increases from 0 self.validator.set_voting_power(0, self.channel_set, self.logger) - self.validator.set_voting_power(self.dummy_voting_power, self.channel_set, self.logger) + self.validator.set_voting_power(self.dummy_voting_power, + self.channel_set, self.logger) self.assertEqual(self.counter_channel.info_count, 1) @@ -415,14 +427,17 @@ def test_set_voting_power_raises_info_alert_if_voting_power_decreases_to_non_0( self): decreased_voting_power = self.dummy_voting_power - 1 - self.validator.set_voting_power(self.dummy_voting_power, self.channel_set, self.logger) - self.validator.set_voting_power(decreased_voting_power, self.channel_set, self.logger) + self.validator.set_voting_power(self.dummy_voting_power, + self.channel_set, self.logger) + self.validator.set_voting_power(decreased_voting_power, + self.channel_set, self.logger) self.assertEqual(self.counter_channel.info_count, 1) def test_set_voting_power_raises_major_alert_if_voting_power_decreases_to_0( self): - self.validator.set_voting_power(self.dummy_voting_power, self.channel_set, self.logger) + self.validator.set_voting_power(self.dummy_voting_power, + self.channel_set, self.logger) self.validator.set_voting_power(0, self.channel_set, self.logger) self.assertEqual(self.counter_channel.major_count, 1) @@ -467,24 +482,32 @@ def test_set_catching_up_raises_info_alert_if_from_true_to_false(self): def test_set_no_of_peers_raises_no_alerts_first_time_round_for_validator( self): - self.validator.set_no_of_peers(self.dummy_no_of_peers, self.channel_set, self.logger) + self.validator.set_no_of_peers(self.dummy_no_of_peers, self.channel_set, + self.logger) self.assertTrue(self.counter_channel.no_alerts()) def test_set_no_of_peers_raises_no_alerts_first_time_round_for_full_node( self): - self.full_node.set_no_of_peers(self.dummy_no_of_peers, self.channel_set, self.logger) + self.full_node.set_no_of_peers(self.dummy_no_of_peers, self.channel_set, + self.logger) self.assertTrue(self.counter_channel.no_alerts()) - def test_set_no_of_peers_raises_info_alert_if_increase_for_validator(self): + def test_set_no_of_peers_raises_no_alerts_if_increase_for_validator_if_outside_safe_range( + self): increased_no_of_peers = self.dummy_no_of_peers + 1 - self.validator.set_no_of_peers(self.dummy_no_of_peers, self.channel_set, self.logger) + self.validator.set_no_of_peers(self.dummy_no_of_peers, self.channel_set, + self.logger) self.counter_channel.reset() # ignore previous alerts - self.validator.set_no_of_peers(increased_no_of_peers, self.channel_set, self.logger) + self.validator.set_no_of_peers(increased_no_of_peers, self.channel_set, + self.logger) - self.assertEqual(self.counter_channel.info_count, 1) + self.assertEqual(self.counter_channel.minor_count, 0) + self.assertEqual(self.counter_channel.major_count, 0) + self.assertEqual(self.counter_channel.info_count, 0) + self.assertEqual(self.counter_channel.error_count, 0) def test_set_no_of_peers_raises_info_alert_if_increase_for_full_node_if_inside_danger( self): @@ -517,7 +540,7 @@ def test_set_no_of_peers_raises_info_alert_if_increase_for_full_node_if_inside_t self.channel_set, self.logger) self.counter_channel.reset() # ignore previous alerts self.full_node.set_no_of_peers( - self.peers_full_node_danger_boundary, + self.peers_more_than_full_node_danger_boundary, self.channel_set, self.logger) self.assertEqual(self.counter_channel.info_count, 1) @@ -525,27 +548,39 @@ def test_set_no_of_peers_raises_info_alert_if_increase_for_full_node_if_inside_t def test_set_no_of_peers_raises_info_alert_if_increase_for_validator_if_inside_danger( self): self.validator.set_no_of_peers( - self.peers_less_than_full_node_danger_boundary, + self.peers_less_than_validator_danger_boundary, self.channel_set, self.logger) self.counter_channel.reset() # ignore previous alerts self.validator.set_no_of_peers( - self.peers_less_than_full_node_danger_boundary + 1, + self.peers_less_than_validator_danger_boundary + 1, self.channel_set, self.logger) self.assertEqual(self.counter_channel.info_count, 1) - def test_set_no_of_peers_raises_info_alert_if_increase_for_validator_if_outside_danger( + def test_set_no_of_peers_raises_info_alert_if_increase_for_validator_if_outside_danger_inside_safe( self): self.validator.set_no_of_peers( - self.peers_more_than_full_node_danger_boundary, + self.peers_validator_danger_boundary, self.channel_set, self.logger) self.counter_channel.reset() # ignore previous alerts self.validator.set_no_of_peers( - self.peers_more_than_full_node_danger_boundary + 1, + self.peers_validator_danger_boundary + 1, self.channel_set, self.logger) self.assertEqual(self.counter_channel.info_count, 1) + def test_set_no_of_peers_raises_info_alert_if_decrease_for_validator_if_outside_danger_inside_safe( + self): + self.validator.set_no_of_peers( + self.peers_validator_safe_boundary, + self.channel_set, self.logger) + self.counter_channel.reset() # ignore previous alerts + self.validator.set_no_of_peers( + self.peers_validator_safe_boundary - 1, + self.channel_set, self.logger) + + self.assertEqual(self.counter_channel.minor_count, 1) + def test_set_no_of_peers_raises_minor_alert_if_decrease_for_full_node_if_inside_danger( self): self.full_node.set_no_of_peers( @@ -573,7 +608,7 @@ def test_set_no_of_peers_raises_no_alerts_if_decrease_for_full_node_if_outside_d def test_set_no_of_peers_raises_major_alert_if_decrease_for_validator_if_inside_danger( self): self.validator.set_no_of_peers( - self.peers_more_than_validator_danger_boundary, + self.peers_validator_danger_boundary, self.channel_set, self.logger) self.counter_channel.reset() # ignore previous alerts self.validator.set_no_of_peers( @@ -582,18 +617,48 @@ def test_set_no_of_peers_raises_major_alert_if_decrease_for_validator_if_inside_ self.assertEqual(self.counter_channel.major_count, 1) - def test_set_no_of_peers_raises_minor_alert_if_decrease_for_validator_if_outside_danger( + def test_set_no_of_peers_raises_minor_alert_if_decrease_for_validator_if_outside_danger_inside_safe( self): self.validator.set_no_of_peers( - self.peers_more_than_validator_danger_boundary, + self.peers_validator_safe_boundary, self.channel_set, self.logger) self.counter_channel.reset() # ignore previous alerts self.validator.set_no_of_peers( - self.peers_more_than_validator_danger_boundary - 1, + self.peers_validator_safe_boundary - 1, self.channel_set, self.logger) self.assertEqual(self.counter_channel.minor_count, 1) + def test_set_no_of_peers_raises_no_alerts_if_decrease_for_validator_if_outside_safe( + self): + self.validator.set_no_of_peers( + self.peers_more_than_validator_safe_boundary, + self.channel_set, self.logger) + self.counter_channel.reset() # ignore previous alerts + self.validator.set_no_of_peers( + self.peers_more_than_validator_safe_boundary - 1, + self.channel_set, self.logger) + + self.assertEqual(self.counter_channel.minor_count, 0) + self.assertEqual(self.counter_channel.major_count, 0) + self.assertEqual(self.counter_channel.info_count, 0) + self.assertEqual(self.counter_channel.error_count, 0) + + def test_set_no_of_peers_raises_info_alert_if_increase_for_validator_outside_safe_for_first_time( + self): + self.validator.set_no_of_peers( + self.peers_less_than_validator_safe_boundary, + self.channel_set, self.logger) + self.counter_channel.reset() # ignore previous alerts + self.validator.set_no_of_peers( + self.peers_more_than_validator_safe_boundary, + self.channel_set, self.logger) + + self.assertEqual(self.counter_channel.minor_count, 0) + self.assertEqual(self.counter_channel.major_count, 0) + self.assertEqual(self.counter_channel.info_count, 1) + self.assertEqual(self.counter_channel.error_count, 0) + class TestNodeWithRedis(unittest.TestCase): @@ -615,6 +680,8 @@ def setUpClass(cls) -> None: def setUp(self) -> None: self.node_name = 'testnode' + self.network_name = 'testnetwork' + self.redis_prefix = self.node_name + "@" + self.network_name self.date = datetime.min + timedelta(days=123) self.logger = logging.getLogger('dummy') @@ -633,11 +700,13 @@ def setUp(self) -> None: self.non_validator = Node(name=self.node_name, rpc_url=None, node_type=NodeType.NON_VALIDATOR_FULL_NODE, - pubkey=None, network='', redis=self.redis) + pubkey=None, network=self.network_name, + redis=self.redis) self.validator = Node(name=self.node_name, rpc_url=None, node_type=NodeType.VALIDATOR_FULL_NODE, - pubkey=None, network='', redis=self.redis) + pubkey=None, network=self.network_name, + redis=self.redis) def test_load_state_changes_nothing_if_nothing_saved(self): self.validator.load_state(self.logger) @@ -651,12 +720,13 @@ def test_load_state_changes_nothing_if_nothing_saved(self): def test_load_state_sets_values_to_saved_values(self): # Set Redis values manually - self.redis.set_unsafe(self.node_name + '_went_down_at', str(self.date)) - self.redis.set_unsafe(self.node_name + '_consecutive_blocks_missed', + self.redis.set_unsafe(self.redis_prefix + '_went_down_at', + str(self.date)) + self.redis.set_unsafe(self.redis_prefix + '_consecutive_blocks_missed', 123) - self.redis.set_unsafe(self.node_name + '_voting_power', 456) - self.redis.set_unsafe(self.node_name + '_catching_up', str(True)) - self.redis.set_unsafe(self.node_name + '_no_of_peers', 789) + self.redis.set_unsafe(self.redis_prefix + '_voting_power', 456) + self.redis.set_unsafe(self.redis_prefix + '_catching_up', str(True)) + self.redis.set_unsafe(self.redis_prefix + '_no_of_peers', 789) # Load the Redis values self.validator.load_state(self.logger) @@ -670,7 +740,7 @@ def test_load_state_sets_values_to_saved_values(self): def test_load_state_sets_went_down_at_to_none_if_incorrect_type(self): # Set Redis values manually - self.redis.set_unsafe(self.node_name + '_went_down_at', str(True)) + self.redis.set_unsafe(self.redis_prefix + '_went_down_at', str(True)) # Load the Redis values self.validator.load_state(self.logger) @@ -692,13 +762,13 @@ def test_save_state_sets_values_to_current_values(self): # Assert self.assertEqual( dateutil.parser.parse(self.redis.get_unsafe( - self.node_name + '_went_down_at')), self.date) + self.redis_prefix + '_went_down_at')), self.date) self.assertEqual( self.redis.get_int_unsafe( - self.node_name + '_consecutive_blocks_missed'), 123) + self.redis_prefix + '_consecutive_blocks_missed'), 123) self.assertEqual( - self.redis.get_int_unsafe(self.node_name + '_voting_power'), 456) + self.redis.get_int_unsafe(self.redis_prefix + '_voting_power'), 456) self.assertTrue( - self.redis.get_bool_unsafe(self.node_name + '_catching_up')) + self.redis.get_bool_unsafe(self.redis_prefix + '_catching_up')) self.assertEqual( - self.redis.get_int_unsafe(self.node_name + '_no_of_peers'), 789) + self.redis.get_int_unsafe(self.redis_prefix + '_no_of_peers'), 789) diff --git a/test/test_internal_config.ini b/test/test_internal_config.ini index c8089ea..3eba9a8 100644 --- a/test/test_internal_config.ini +++ b/test/test_internal_config.ini @@ -35,6 +35,7 @@ redis_network_monitor_alive_key_timeout = 86400 [monitoring_periods] node_monitor_period_seconds = 10 network_monitor_period_seconds = 10 +network_monitor_max_catch_up_blocks = 500 github_monitor_period_seconds = 300 # These define how often a monitor runs an iteration of its monitoring loop @@ -43,7 +44,8 @@ downtime_alert_interval_seconds = 3 max_missed_blocks_interval_seconds = 4 max_missed_blocks_in_time_interval = 10 validator_peer_danger_boundary = 1 -full_node_peer_danger_boundary = 15 +validator_peer_safe_boundary = 3 +full_node_peer_danger_boundary = 10 missed_blocks_danger_boundary = 5 github_error_interval_seconds = 3600 # These limit the number of alerts of a specific type received using either From 59c38499e9e6bc699dd34e0dfb11df0afd601caa Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:33:01 +0100 Subject: [PATCH 16/29] Added periodic alert muting and network monitor block height in telegram. Also added general Commands class. --- src/commands/commands.py | 52 +++++ .../handler_utils/telegram_handler.py | 14 +- src/commands/handlers/telegram.py | 183 ++++++++++++------ 3 files changed, 187 insertions(+), 62 deletions(-) create mode 100644 src/commands/commands.py diff --git a/src/commands/commands.py b/src/commands/commands.py new file mode 100644 index 0000000..cd7d2e7 --- /dev/null +++ b/src/commands/commands.py @@ -0,0 +1,52 @@ +import logging +from typing import Optional + +from src.utils.config_parsers.internal import InternalConfig +from src.utils.config_parsers.internal_parsed import InternalConf +from src.utils.config_parsers.user import UserConfig +from src.utils.config_parsers.user_parsed import UserConf +from src.utils.redis_api import RedisApi + + +class Commands: + + def __init__(self, logger: logging.Logger, redis: Optional[RedisApi], + redis_snooze_key: Optional[str], redis_mute_key: Optional[str], + redis_node_monitor_alive_key_prefix: Optional[str], + redis_network_monitor_alive_key_prefix: Optional[str], + internal_conf: InternalConfig = InternalConf, + user_conf: UserConfig = UserConf) -> None: + self._logger = logger + + self._redis = redis + self._redis_enabled = redis is not None + self._redis_snooze_key = redis_snooze_key + self._redis_mute_key = redis_mute_key + self._redis_node_monitor_alive_key_prefix = \ + redis_node_monitor_alive_key_prefix + self._redis_network_monitor_alive_key_prefix = \ + redis_network_monitor_alive_key_prefix + + self._internal_conf = internal_conf + self._user_conf = user_conf + + def snooze(self) -> None: + pass + + def unsnooze(self) -> None: + pass + + def mute(self) -> None: + pass + + def unmute(self) -> None: + pass + + def add_node(self) -> None: + pass + + def remove_node(self) -> None: + pass + + def current_nodes(self) -> None: + pass diff --git a/src/commands/handler_utils/telegram_handler.py b/src/commands/handler_utils/telegram_handler.py index 2f3609f..b8c8f28 100644 --- a/src/commands/handler_utils/telegram_handler.py +++ b/src/commands/handler_utils/telegram_handler.py @@ -1,8 +1,8 @@ from typing import Optional, List -from telegram import Bot, Update +from telegram import Update from telegram.ext import Updater, CommandHandler, \ - Handler + Handler, CallbackContext from src.alerting.alert_utils.telegram_bot_api import TelegramBotApi @@ -15,7 +15,7 @@ def __init__(self, bot_token: str, authorised_chat_id: Optional[str], self._authorised_chat_id = authorised_chat_id # Set up updater - self._updater = Updater(token=bot_token) + self._updater = Updater(token=bot_token, use_context=True) # Set up handlers ping_handler = CommandHandler('ping', self._ping_callback) @@ -34,7 +34,7 @@ def start_handling(self, run_in_background: bool = False) -> None: if not run_in_background: self._updater.idle(stop_signals=[]) - def authorise(self, bot: Bot, update: Update) -> bool: + def authorise(self, update: Update, context: CallbackContext) -> bool: if self._authorised_chat_id in [None, str(update.message.chat_id)]: return True else: @@ -43,11 +43,11 @@ def authorise(self, bot: Bot, update: Update) -> bool: api = TelegramBotApi(self._bot_token, self._authorised_chat_id) api.send_message( 'Received command from unrecognised user: ' - 'bot={}, update={}'.format(bot, update)) + 'update={}, context={}'.format(update, context)) return False - def _ping_callback(self, bot: Bot, update: Update) -> None: - if self.authorise(bot, update): + def _ping_callback(self, update: Update, context: CallbackContext) -> None: + if self.authorise(update, context): update.message.reply_text('PONG!') def stop(self) -> None: diff --git a/src/commands/handlers/telegram.py b/src/commands/handlers/telegram.py index f8afdee..f55fb53 100644 --- a/src/commands/handlers/telegram.py +++ b/src/commands/handlers/telegram.py @@ -1,11 +1,13 @@ import logging -from datetime import datetime, timedelta +from datetime import timedelta, datetime from typing import Optional from redis import RedisError -from telegram import Update, Bot -from telegram.ext import CommandHandler, MessageHandler, Filters +from telegram import Update +from telegram.ext import CommandHandler, MessageHandler, Filters, \ + CallbackContext +from src.commands.commands import Commands from src.commands.handler_utils.telegram_handler import TelegramCommandHandler from src.utils.config_parsers.internal import InternalConfig from src.utils.config_parsers.internal_parsed import InternalConf @@ -14,33 +16,27 @@ from src.utils.redis_api import RedisApi -class TelegramCommands: +class TelegramCommands(Commands): def __init__(self, bot_token: str, authorised_chat_id: str, logger: logging.Logger, redis: Optional[RedisApi], - redis_snooze_key: Optional[str], + redis_snooze_key: Optional[str], redis_mute_key: Optional[str], redis_node_monitor_alive_key_prefix: Optional[str], redis_network_monitor_alive_key_prefix: Optional[str], internal_conf: InternalConfig = InternalConf, user_conf: UserConfig = UserConf) -> None: - self._logger = logger - - self._redis = redis - self._redis_enabled = redis is not None - self._redis_snooze_key = redis_snooze_key - self._redis_node_monitor_alive_key_prefix = \ - redis_node_monitor_alive_key_prefix - self._redis_network_monitor_alive_key_prefix = \ - redis_network_monitor_alive_key_prefix - - self._internal_conf = internal_conf - self._user_conf = user_conf + super().__init__(logger, redis, redis_snooze_key, redis_mute_key, + redis_node_monitor_alive_key_prefix, + redis_network_monitor_alive_key_prefix, internal_conf, + user_conf) # Set up command handlers (command and respective callback function) command_handlers = [ CommandHandler('start', self._start_callback), CommandHandler('snooze', self._snooze_callback), + CommandHandler('mute', self._mute_callback), + CommandHandler('unmute', self._unmute_callback), CommandHandler('unsnooze', self._unsnooze_callback), CommandHandler('status', self._status_callback), CommandHandler('validators', self._validators_callback), @@ -63,21 +59,21 @@ def formatted_reply(update: Update, reply: str): # Adds Markdown formatting update.message.reply_text(reply, parse_mode='Markdown') - def _start_callback(self, bot: Bot, update: Update): - self._logger.info('Received /start command: bot=%s, update=%s', - bot, update) + def _start_callback(self, update: Update, context: CallbackContext): + self._logger.info('Received /start command: update=%s, context=%s', + update, context) # If authorised, send welcome message - if self.cmd_handler.authorise(bot, update): + if self.cmd_handler.authorise(update, context): update.message.reply_text("Welcome to the P.A.N.I.C. alerter bot!\n" "Type /help for more information.") - def _snooze_callback(self, bot: Bot, update: Update): - self._logger.info('Received /snooze command: bot=%s, update=%s', - bot, update) + def _snooze_callback(self, update: Update, context: CallbackContext): + self._logger.info('Received /snooze command: update=%s, context=%s', + update, context) # If authorised, snooze phone calls if Redis enabled - if self.cmd_handler.authorise(bot, update): + if self.cmd_handler.authorise(update, context): if self._redis_enabled: # Expected: /snooze message_parts = update.message.text.split(' ') @@ -104,12 +100,11 @@ def _snooze_callback(self, bot: Bot, update: Update): update.message.reply_text('Snoozing is not available given ' 'that Redis is not set up.') - def _unsnooze_callback(self, bot: Bot, update: Update): - self._logger.info('Received /unsnooze command: bot=%s, update=%s', - bot, update) - + def _unsnooze_callback(self, update: Update, context: CallbackContext): + self._logger.info('Received /unsnooze command: update=%s, context=%s', + update, context) # If authorised, unsnooze phone calls if Redis enabled - if self.cmd_handler.authorise(bot, update): + if self.cmd_handler.authorise(update, context): if self._redis_enabled: # Remove snooze key if it exists if self._redis.exists(self._redis_snooze_key): @@ -122,12 +117,62 @@ def _unsnooze_callback(self, bot: Bot, update: Update): update.message.reply_text('Unsnoozing is not available given ' 'that Redis is not set up.') - def _status_callback(self, bot: Bot, update: Update): - self._logger.info('Received /status command: bot=%s, update=%s', - bot, update) + def _mute_callback(self, update: Update, context: CallbackContext): + self._logger.info('Received /mute command: update=%s, context=%s', + update, context) + + # If authorised, mute the periodic alive reminder if Redis enabled + if self.cmd_handler.authorise(update, context): + if self._redis_enabled: + # Expected: /mute + message_parts = update.message.text.split(' ') + if len(message_parts) == 2: + try: + # Get number of hours and set temporary Redis key + hours = timedelta(hours=float(message_parts[1])) + until = str(datetime.now() + hours) + set_ret = self._redis.set_for( + self._redis_mute_key, until, hours) + if set_ret is None: + update.message.reply_text( + 'Muting unsuccessful due to an issue with ' + 'Redis. Check /status to see if it is online.') + else: + update.message.reply_text( + 'The periodic alive reminder has been muted for' + ' {} hours until {}'.format(hours, until)) + except ValueError: + update.message.reply_text('I expected a no. of hours.') + else: + update.message.reply_text('I expected exactly one value.') + else: + update.message.reply_text('Muting is not available given ' + 'that Redis is not set up.') + + def _unmute_callback(self, update: Update, context: CallbackContext): + self._logger.info('Received /unmute command: update=%s, context=%s', + update, context) + # If authorised, unmute the periodic alive reminder if Redis enabled + if self.cmd_handler.authorise(update, context): + if self._redis_enabled: + # Remove mute key if it exists + if self._redis.exists(self._redis_mute_key): + self._redis.remove(self._redis_mute_key) + update.message.reply_text( + 'The periodic alive reminder has been unmuted.') + else: + update.message.reply_text('The periodic alive reminder was ' + 'not muted.') + else: + update.message.reply_text('Unmuting is not available given ' + 'that Redis is not set up.') + + def _status_callback(self, update: Update, context: CallbackContext): + self._logger.info('Received /status command: update=%s, context=%s', + update, context) # If authorised, send status if Redis enabled - if self.cmd_handler.authorise(bot, update): + if self.cmd_handler.authorise(update, context): if self._redis_enabled: status = "" @@ -165,6 +210,17 @@ def _status_callback(self, bot: Bot, update: Update): else: status += '- Twilio calls are not snoozed.\n' + # Add periodic alive reminder mute state to status + if redis_running: + if self._redis.exists(self._redis_mute_key): + until = self._redis.get( + self._redis_mute_key).decode("utf-8") + status += '- The periodic alive reminder has been ' \ + 'muted until {}.\n'.format(until) + else: + status += '- The periodic alive reminder is not ' \ + 'muted.\n' + # Add node monitor latest updates to status if redis_running: node_monitor_keys_list = self._redis.get_keys( @@ -206,11 +262,22 @@ def _status_callback(self, bot: Bot, update: Update): if len(net_monitor_keys_list) == 0: status += '- No recent update from network monitors.\n' + for name in net_monitor_names: + redis_last_height_checked_key = \ + self._internal_conf. \ + redis_network_monitor_last_height_key_prefix + \ + name + last_height_checked = self._redis.get( + redis_last_height_checked_key).decode('utf-8') + status += '- *{}* is currently in block height {}' \ + '.\n'.format(name, last_height_checked) + # If redis is not running if not redis_running: status += \ '- Since Redis is not accessible, Twilio calls are ' \ - 'considered not snoozed and any recent update from ' \ + 'considered not snoozed, the periodic alive reminder ' \ + 'is not muted, and any recent update from ' \ 'node or network monitors is not accessible.\n' # Send status @@ -220,12 +287,12 @@ def _status_callback(self, bot: Bot, update: Update): update.message.reply_text('Status update not available given ' 'that Redis is not set up.') - def _validators_callback(self, bot: Bot, update: Update): - self._logger.info('Received /validators command: bot=%s, update=%s', - bot, update) + def _validators_callback(self, update: Update, context: CallbackContext): + self._logger.info('Received /validators command: update=%s, context=%s', + update, context) # If authorised, send list of links to validators - if self.cmd_handler.authorise(bot, update): + if self.cmd_handler.authorise(update, context): update.message.reply_text( 'Links to validators:\n' ' Hubble: {}\n' @@ -239,12 +306,12 @@ def _validators_callback(self, bot: Bot, update: Update): self._internal_conf.validators_mintscan_link, self._internal_conf.validators_lunie_link)) - def _block_callback(self, bot: Bot, update: Update): - self._logger.info('Received /block command: bot=%s, update=%s', - bot, update) + def _block_callback(self, update: Update, context: CallbackContext): + self._logger.info('Received /block command: update=%s, context=%s', + update, context) # If authorised, send list of links to specified block - if self.cmd_handler.authorise(bot, update): + if self.cmd_handler.authorise(update, context): # Expected: /block message_parts = update.message.text.split(' ') if len(message_parts) == 2: @@ -272,12 +339,12 @@ def _block_callback(self, bot: Bot, update: Update): else: update.message.reply_text("I expected exactly one value.") - def _tx_callback(self, bot: Bot, update: Update): - self._logger.info('Received /tx command: bot=%s, update=%s', - bot, update) + def _tx_callback(self, update: Update, context: CallbackContext): + self._logger.info('Received /tx command: update=%s, context=%s', + update, context) # If authorised, send list of links to specified transaction - if self.cmd_handler.authorise(bot, update): + if self.cmd_handler.authorise(update, context): # Expected: /tx message_parts = update.message.text.split(' ') if len(message_parts) == 2: @@ -296,28 +363,34 @@ def _tx_callback(self, bot: Bot, update: Update): else: update.message.reply_text("I expected exactly one value.") - def _help_callback(self, bot: Bot, update: Update): - self._logger.info('Received /help command: bot=%s, update=%s', - bot, update) + def _help_callback(self, update: Update, context: CallbackContext): + self._logger.info('Received /help command: update=%s, context=%s', + update, context) # If authorised, send help message with available commands - if self.cmd_handler.authorise(bot, update): + if self.cmd_handler.authorise(update, context): update.message.reply_text( 'Hey! These are the available commands:\n' ' /start: welcome message\n' ' /ping: ping the Telegram commands handler\n' ' /snooze : snoozes phone calls for \n' ' /unsnooze: unsnoozes phone calls\n' + ' /mute : mute periodic alive reminder for \n' + ' /unmute: unmute periodic alive reminder\n' ' /status: shows status message\n' ' /validators: shows links to validators\n' ' /block : shows link to specified block\n' ' /tx : shows link to specified transaction\n' ' /help: shows this message') - def _unknown_callback(self, bot: Bot, update: Update) -> None: - self._logger.info('Received unrecognized command: bot=%s, update=%s', - bot, update) + def _unknown_callback(self, update: Update, + context: CallbackContext) -> None: + self._logger.info( + 'Received unrecognized command: update=%s, context=%s', + update, context) # If authorised, send a default message for unrecognized commands - if self.cmd_handler.authorise(bot, update): + if self.cmd_handler.authorise(update, context): update.message.reply_text('I did not understand (Type /help)') + + # TODO: Need to add callbacks for add_node, remove_node, current_nodes From bec3f4d71daa26f303de6693af7e519612ac0c73 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:33:31 +0100 Subject: [PATCH 17/29] Fixed peer change bug. Catching up now shows up for full nodes as well. --- src/node/node.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/node/node.py b/src/node/node.py index cc190d6..71a400d 100644 --- a/src/node/node.py +++ b/src/node/node.py @@ -293,12 +293,11 @@ def set_catching_up(self, now_catching_up: bool, '%s set_catching_up: before=%s, new=%s, channels=%s', self, self.catching_up, now_catching_up, channels) - # Alert if catching up has changed for validator - if self.is_validator: - if not self.catching_up and now_catching_up: - channels.alert_minor(IsCatchingUpAlert(self.name)) - elif self.catching_up and not now_catching_up: - channels.alert_info(IsNoLongerCatchingUpAlert(self.name)) + # Alert if catching up has changed + if not self.catching_up and now_catching_up: + channels.alert_minor(IsCatchingUpAlert(self.name)) + elif self.catching_up and not now_catching_up: + channels.alert_info(IsNoLongerCatchingUpAlert(self.name)) # Update catching-up self._catching_up = now_catching_up @@ -340,7 +339,8 @@ def set_no_of_peers(self, new_no_of_peers: int, channels: ChannelSet, PeersIncreasedOutsideSafeRangeAlert(self.name, safe)) else: if new_no_of_peers > self.no_of_peers: # increase - if new_no_of_peers < danger: # increase inside danger range + if new_no_of_peers <= danger: + # increase inside danger range channels.alert_info(PeersIncreasedAlert( self.name, self.no_of_peers, new_no_of_peers)) elif self.no_of_peers <= danger < new_no_of_peers: From bbd6e66db6384619858bbf5ed14f4d259cd8183d Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:34:44 +0100 Subject: [PATCH 18/29] Added max catch up blocks so that network monitor syncs up faster and for less old and extra alerts. --- config/internal_config.ini | 1 + doc/DESIGN_AND_FEATURES.md | 17 ++++- doc/IMG_TELEGRAM_COMMANDS.png | Bin 16021 -> 21864 bytes doc/IMG_TELEGRAM_STATUS_COMMAND.png | Bin 16768 -> 27525 bytes src/monitoring/monitors/network.py | 106 +++++++++++++++------------ src/utils/config_parsers/internal.py | 2 + 6 files changed, 76 insertions(+), 50 deletions(-) diff --git a/config/internal_config.ini b/config/internal_config.ini index b2a0184..383defe 100644 --- a/config/internal_config.ini +++ b/config/internal_config.ini @@ -35,6 +35,7 @@ redis_network_monitor_alive_key_timeout = 86400 [monitoring_periods] node_monitor_period_seconds = 10 network_monitor_period_seconds = 10 +network_monitor_max_catch_up_blocks = 500 github_monitor_period_seconds = 3600 # These define how often a monitor runs an iteration of its monitoring loop diff --git a/doc/DESIGN_AND_FEATURES.md b/doc/DESIGN_AND_FEATURES.md index a3f00e8..5a6ad2f 100644 --- a/doc/DESIGN_AND_FEATURES.md +++ b/doc/DESIGN_AND_FEATURES.md @@ -69,16 +69,27 @@ The network monitor deals with a ***minimum* of one validator node and one (non- An important note is that the full node(s) should be a reliable data source in terms of availability. So much so that if there are no full nodes accessible, this is considered to be equivalent to the validator losing blocks and thus a `MAJOR` alert is raised. +If the alerter is not in sync with the validator with respect to block height, the maximum number of historical blocks checked is `MCUB`, which is configurable from the internal config (`network_monitor_max_catch_up_blocks`). + In each monitoring round, the network monitor: 1. Gets the node's abci info from `[RPC_URL]/abci_info` 1. Gets the latest block height *LastH* -2. For each height *H* from the last block height checked until *LastH*: +2. Sets *H* = *LastHChecked* + 1 where *LastHChecked is the height of the last block checked by the network monitor +3. If *LastH* - *LastHChecked* > `MCUB`: + 1. Sets *H* = *LastH* - `MCUB` + 2. Gets the block at height *H* from `[RPC_URL]/block?height=H` + 3. Checks whether our validator is in the list of participating validators + 4. Increments or resets (depending on the outcome) the missed blocks counter for our validator +4. Otherwise if *H* <= *LastHChecked* : 1. Gets the block at height *H* from `[RPC_URL]/block?height=H` 2. Checks whether our validator is in the list of participating validators 3. Increments or resets (depending on the outcome) the missed blocks counter for our validator -3. Saves its state and the nodes' state -4. Sleeps until the next monitoring round +5. Saves its state and the nodes' state +6. Sleeps until the next monitoring round if it is not syncing (*LastH*-*LastHChecked* > 2). + +Default value: +- `MCUB = network_monitor_max_catch_up_blocks = 500` ### GitHub Monitor diff --git a/doc/IMG_TELEGRAM_COMMANDS.png b/doc/IMG_TELEGRAM_COMMANDS.png index 80755b4d9d20ec0436b454752566636d7c0621f7..022793cd401d8949521d9702a0218f2e6d025515 100644 GIT binary patch literal 21864 zcmb5VbyQrJ-EBOySux))4035=KFp3j&sKw z@11l1=&{G{ZB?~b&6;zrwF70OM3LZe;Gv+Pki>op$w5KAtAPCYK7N4w!$=^21lhc^ zl@k?!DjmfGLw4R9@k{bUK~;n!Jn6tf_CNjlscH)ah1~u2dly56j0**&@GK_8ui&J4 z+=AewIOn|g41f0Q_j4=#2kq<*-?UmNb^nfbpEbN?L?>Igsr(FD-TS3uS$`AIw9%7u z$oDmgo%XE=Y?FO1E@F3v&%M=wKsgq{QCUPc5(W_yq)OBrMSj%EtRHeKSOZ~J^X>Pu zO*e~=#Vup41S!OJPtsVrQq|>zCEsoSf>k`Kj^sbV3I-+3;`&;vwNioHzO8Ux zS`3_MY$zx%(8rSQ*5_lZDfiozys(R*y4Gu&#@kibnJwyYs! zv;()5hMZ?CPQ6PJs*n7*Qz+)tw51QNB|S0VcDjv^fz>m|&CIR&I|(sG`c zIKg3}fS%SbbfQ|DcVWk5#uBh%q*CKPc%>F7H{`xny%xMe(uEi1=a8Yi5Q#Ysm*X&pVSumn|BA=L`4Z(>;63lU3OeJFp4)!A(b1 zC0C$Q-+FgxiN-8e+*i=u=uanyIIoj65RW>0rY6kFFj#q<;IK)BWJROTGtXE-S=Ru= zZccZVBFCnHoD6KhO#lVeUR`1G_2s)QZT1vqLAE~6@lz7ow5@LR%U)VpEgyVPWJXzdg@jaD+Bg&Zwhv3 z-FOV`Lu|pXtct2DMk-4F9bLU@dn1d6rzGIbgvohbE2}Fh5f!}>h^uNM`5QE32JTiw zW0m>tBaxwHQCz1L1crRV2VibfcO!sN8_6%uUWEImU5LXA5JR7&cJ{lqYL_``Z=sI= zW|b&b=gfZ0THNW?M-k&#vG8lsCT|TPg3Jl}Diz@Hnz<1v6oFpn47EvD2;@hJQU{omxEDx?k zwi0IL;NTFXa*pQ>+FgrcY4AXSfpms5Lht(x+%6n0s%l4h;{8)$`1q7G^LQNL3-ijw z3vP_ug%S@oBCL{PZ9wYE)7j4L^S@>r>W$WkrMpW3d1w!%hDYoM%}h zI1$!vF!$J$|In)Fn(GD$;LayaM)H1SQi41#)dp8O@Y{<=?~;{5b6V^9t^3t2 zFF{u2r0|u{-rCfor3&9%kj-;zAOk-mBvRsCf0Eoxiqj3iczzvsX8EG$5qO?dc1dyx zUT)gUNr({DE$+us63qqv)zRG&BLLfC8h7~K6Y-IAzYs?l?M!C z*5?hRwBl^e1Ir9o-j?W-mlBezOXbnY8(uZxvNngX)z)8{+p;+0K7813QNpo()P37> z>i%7+7?YUh{Pek~2yA;E4+3v`x<9^eA?tj-a*Uh8K=?X-l8On|(gaW`SG~^Qo{q0w zBTtAHS?2OS9f^WV*w%8}o|N*mgBt6ZUU3h^xI07J{(xL>QcmY z)~ld>RN$PdpiQz#_iCIl;?-O|ZOePl@jRgX_o~;-YBFuD?Kdm$nKe!lySV3M>|gkh zss3pq6y+iK5KqC_4o~W@)Wnc=v%zngDD7#RY{#3o*BIPv^L`}wMH5}8`FEf5j-|oO zg?ho$tngL%>yzz6Ux(JXm6ChJ$wLXs5WAE-QFJWUT_CX_#RW>18^}R z86-#Ln3OXU9#J+UxS)%7rmi__qylHu3>94XuuW(YzK{V}weNUW&lXHwE>mvzty+1h z!6iC?1Q2(?g zqJm3h*U^=5+zZWy4t6*G9;4}sp`neIDW~1`y3{7d`O;;I^SX0*MY!aW4ZVdOlzgb6 z#&@c@O{XoSJlys>tl7!$MbA|(WOOy-^(OCDW-Cv3l6y#4mVm1KcFnFv<_{a?Wu~`m z)4QP$un>CKKJfHPw4QQU)|T3OnR43Cr^nKDui!F*an-Z-_U%mXQ5@p;H`$zF+m~k%-{*a50yGxgD5+g^_+E_}X=XUml_Ng&jI0-FJyXui z?2MVrsYqj5;Lv9E7HYlGmfVUZ&nS;W7YuB8%^&P>Zjje<5y--Q_VReD>{=Y65~`8@ z`XK5!#d+E+h@1MrdzHr7CT+$m%l}#cMytM5Cj+;=&hU;eu#FB7GUM(Fzu+&pT^|q2 z@HUUFO?7*TcSO=as2stt?7mL;kl_0yx|uP>`Fg>q=(^f7L1m4*{ZQS+{E2CDr=Q`W#-~1Y`h0F zzPJc(U?2ME4>4=>K8$@{(bUn(Xnl>a`#Z4Y!~HNwV$RCk>Fipm1-rD)tWn+8R|Hux zpnM^u$*KFY#L8XVqigegrh+hhQu*bos#+KP2+m-sn^?SKWI)9=aLqCx(x;_VrX+M( zv!8yrKAj2`xF1RLB(Zos+^XJ&VH;$hTrSD>MHLV3I(OYdA%WK?o3VNu(ZVwJ>tpUOlBWq zmv|&oK!@PeZykBv;g;J~@!ORqDXG~zmoe%6f!Aj|mQ__6k)Lzpgs*izS6NJDXZ1k$ zYexF(soxOnI57JWoDIo2A!h7jzd-}~9#K?Q)jl0i87(B6+ z;wioM@)8k{EPcD;SQbrER-cH9IoZn+(`#t~J2Um8Wf(qSDlBfQ%PhK69mHQWBm&Mq zG#*y%MGqASdRt7$xKDKKP%5Hma^B;bo`g90Dsju7&( zV0_dSYypxUT1I|kwiY}GmP3-OoIhbU&+o+B-)aTcLb7h<+^XuR)t>rCWQssROq@p2&;d`x<@Ubw~C0(R1Yl8(U!6YAZMe zJ&RGUEeoNn*BL>$O4eOix1GvB^La0BEuoM!CT;K#bv3kJ*iA<_UtZNtA#F8Gf&Y_t6lIBim= zu?Qto%?F7>C~~#*ZpanrG%~u7P0&|z2FOMj;lIBu_c&;$vmy9umg;o!P;YO4C3r2 z;P$~811GG=62uqG>p@JT=0Y>llquv3Sx2EJ0<*hSoI~5r0FnVTgw!wdp$ep|+^^hyq&O7)a^9cyRP+YwtcGE(785L$ou+E;stW(GrTlOFZ>aiYajTDvsa=kLAPV( za=#YI09Hny_-dP2lsqR)B9^r{i`(pL0WsT^+s@VWsO!E|!>PdQ_eQ7?`EvTa^y*h{;F$d6f)c0Zzh<7sJlSuDE<_d|Rfv66kk7SVP9Fo}q7t^Zftt*9~ zEv3iWmyBW+9sG2Stp{^q$g4=|C*-_DNqT&p8&e!V%D$22`4hthiP`xGHScHd5r-gt z5y&1>FkayE)14WI)Ioh};Y)F}rJU&B0&NUuWp zfBvl0x*6w`Kqe=YEnY#ll;PJKI~(K=CD@t+_w{=IV#DO7=(^r2&0Ym{GJF zx+nsJUa7)l=WvgqagyNeaBlt7jt!FO`xsq<{Wpwu;B7aWP!zQs`K({5eyP> zTG*C<>sB0`bNor2m!)N+Z|rMA1x-k zR6Wo`Q*4R{je++z}VW6PayC)GLD3`UE(i>ht4H;V!|+$R3{1ulmT)wK945%acWh>{?D-VvMRkCxx|H zU+LmNcrcQF?l{KZdVuVFDC^8TIIIQl)PW%kFw$N7rG>WWZ_CPee+Lcy?V!Zjk4$yN zFE+7y8GAcsd|ihwipQ6yxKPcI`A%@I;BhiHJ(1&YkkB9-e}0Bv`)8WHiIR8oaP5LVr zJi}#E4u(w)>T%RLK9T)=Ws&iewfpvfa{}ICf^XdL^>qB_ozwnp9G-dMmOzm&Xwy<` z0#~6WG>01KI?)c;CY)$uihG}HCMdp8Mhg`Cn*Ifb>4!QooJ9ym(-9S9tMjrx#(zp; z61wS{8aQ zg|l+<%~6d#_-ZvKR3t7<`{`~nIr&XYQj%uEjfDmZ#AwoR=jEQ$PQ&uO|X%brf0o#Bs+z<{SXlaSg4qTwghxO5hGU zN;QGZg%55^U}3(s`NhmM1mS3hfoipcC!+E14?Ht8rDN|^M9**Y`$t&{BKe4H<1 zyYq9DdjgZf)~=dVg3o6O_OOSd0 z{t9`aRHeqLI-1zyf#{0Zk^Ibeas#=gI$Zg555*t6#S_8C4a<5b6 z5@j{RV~JSDX4Gn5PLWsVs|KjS1$jg}Bw2f6u}c?N?T7zRYWXg!seVV#g0KsOF@fRNA7O4?Rb{%&^(~0B_gcR=Q zuaZw&hju5|mz2zSPbp6ax<$|)S?LpVE5xrDZMe{_8jOgBFOf?p~emL zvs1z-H_3{kacIo~&fc>w@1eZJp`um38iC>HP)ka$ZDE-Nq+H8~8?He0d15+=?6>uDunl_I&&ToIpQb?A7t# zkH=FrG3mb1ZyU4&o=pvx4HL0O7ydD3QVF#>@7eAVLbw~0 zSAvi75fo`zZrZ}l$Q9Gp!LKLZD?d8ZKJbR!VXDqn8O^fy@U8g+hjY6L=Ok5KOTNRq2qs%TRE$}iY;Q|qbzM?ip#h5! zOkQ`qL_0DZn!1VB^*4NG=r^RDIWmaQxV$wD(NeMFk81r9nOjJb2P9*{?6kk?qevVE06+I}_$uJ2F1sHMK zSA50g5r(ne`DkM-!xg|;(7|Zm)7_;Kt1X&alwm+SX3W@f0@PJFK6G65DYp^oDFbP0oN2t{PZ#OWyN?wP1XjjT%4W zNlrlurR+Ruf?Z?2in;ChGG9D2I=gQfKN=N9(NwL-c~DYDjwhN)efFbtW5z8Xg}Bdm zoZ7xLMGAg~6^ai%y_=fU#I89yc0afKMet@V zn1(hd$ABo5YkW8auQUlIM-D-NO&;Ej%mJ&rJHj}atYVpqufIV;t)#PyHTkDgYNF}UzD05 z;4`hwn1AT8mBQEvBW{C7$~3iz!ot4-%4dqX>j}J{F$U%avEoYyeNh``OzCMXM3T=! zBG$;P`mTox@wci|0e*`C!%s*s)2z|KY=>_zB@e{GIP&KWPfgZH3oqcy7KI-5=7NEWXVL!g!3*xm=iJ{Ku%CK;HcjcMNM|OmrBRQ?tU%eFF zlImWJRmtLuY)xI_P*5LHqT|t&({wErCSy}1ulTeZ1>~gh!9*$_#O{GxX+LmIq#*1w zTJ`yvebyQpYWrh!JWP``1Iyw8hdvE-G6T~AF+-c(??AMxba*yai}#rIE-g6N92&QW z))(V+DeZGJtpoHmW#;Kp2a3U)rbD#&SLDjGNfdKd7%$qLu{%+tRVjs@a)w-WBbs3? z)G@0IX%uB1zc$I2sOL%3JdiV|FDIVHLc|)S+x^e%LncQK`Jh6^h4K;saPd$OEvOP4d!O5ObS=)S+*ss}j}rIdQFw0-auxM=pOxwtkHmHXI$?mZVp&U>f^)dP9uek;-YA`~%k1uNt# zVa%8OA%)x;jF0O(rk;T9DM`=y%HNa1rewTAUClG1LW3%TxoL8^toJrtagqHMp_NDk zXTfnNW|HO}vqFyW-{Pd^4?m%E%jka9F(j$HHLAh&wPv7{Z<;a+_+ZH~~VXN<~#{=B4@a)bMv z;6zG(>+?`Kq0gGSSGsRQ~Nwz{eEzhi&7<7}~GKuhG3P6X}=5S*%B2V{m_A$*UM zdV5}xH!73Pj+`>TV__E}B!8Whs95j%m({Yhk0g{Zrg|sC26(Jw@X_GwbHYrKAwrO( zhfMgx_r}$}nbs&5mui~Q$qy|Wd;bjI)1W}RXxj?z-XT~kfYFH9HYWvYSc4ulDr9`3HNdd*Bi@qhPsuvWvw$+{;qZdX zA&_+qukzw2D0(l{sIvoH*^xi8SUG*egn(00mxFT*9;R;yzO3>^-sH|tHYu81Z>MT& zOTFptY6z13PhH7)1U+sIhrHN0jOL&oVDWclbo8x;NrO>Yj}hzq6>P(dc5c~WzCeR7 zhAgg7d^%XMVUm)48GRL+SRbK)!+}M5!$kfX1EP+13iNU#4R>kKwlEITXn66=enn|@2mh4dCx@3$TpH&~sy7p=(nG)d=dcT{u3hgIjN z3?Ebw(2_i{gW6v{XO+hwO`AAd$jr&gaHh7=nM#rS zw0>IMWjZ7iNY$NSv%Yv#5Vy#kAm}$5O^pq}5SLyHJ4Z;h-I==>T>)QWc>629oB)_0Ez5CR3Zg}es(5GPTEw>d^kGh=$fp zefhCnbeB4$T8YCg`By`(xxufM#wSx=IZyUU*qW(pfaABrTVT;316APkn3U>uE$0Ek z0$l3t`(Loc4EW0Oo9Llx#09{`$_%+6cBpFHB-|wt&1Xi4}@mtz=&;dy7M2 zQx%JPQ=Np~A@Byv0q)8syqd=DaFq85D#>BsO$C|ySdZ1#^i94CLQEJ=jI97zTgG;B z-;NO+OwKfv7kvV%N(F$H4=!-?>ybn2#$%2*H9#G-XTtSuWSV6cH8)~pLEjfK3y(LC z0kZ;`8GdGQ8$Q!BMb2bOqx4M9zn84mxd>faL1QtZgzpE7zfYO&l!uohU*cW<6bvO5 zO1nBhrse*$M2Mrah`l~^RM|l0MJ1z)qng{@LuDL(GR{~`QC1uso3D{6M@c8&6rz>9 zKEFyFU8pYXDv`s&AeMKnct*&<*VG$I&bEzIZXcnWSU`lziqgVRhsv17C`*XmW9mx!Cs6#2ufvSU2m>4!VQvOd2x^|i$EdiXaJMn(n9F6D zw01OOVP-Mbi0l4gEu_LvV}GH&QPJYyHVD0u5Q?YpSv(r$#GiJ=LW%3O+%gQb*}6gg zd!0$nTUoRMqTgAUTy0LU`dW*t*zNC3G1i&qCjGRyht{{&g9AW|9!6F8lfali0{k27 zCi}p(XS+jOB~`2zLIRS>LNjB=j7@$Jj0iWgDP2ah{R|a?+KAUBHl2QUtZG>)y zRUW?0+fd?r*KrBasJ}wh^Az#X3bHZ1?WxXsm9g}G-8#~p;=z~eys+&#SRSg;O*?Z@a*jV=KdC*hv4Kbiq|B{ zFjUD}#1GOUU7nRvzE6JNKeT-?x>_Ycm{3)oH*!_+m6Y|B5k2Dp#%tkIbbM%jZa2ju zi>%Jw-6BXdn#<%4N6)wFZ9!*!_AHqMIXP|X_*RvKQxeKK6Hw-8EJ>^xtXkIDQiXlc_C-(AYo1PK4XS@PWffKCCdKR=_36Mu zwT$A25h%WvUekS@*!HW4n!`B7RIY!1s8ry%%Rxk+=_QTemjYg>cz3=)un;RrBD`RE zeKe)XU(H$(X?bSTW97-OLlZfo1S@q|S#II@I}=e5jLq23*J0sqzpzz@Ags=m;ME(z z=I-dwRc&U(G@TNSq%@tpXm0NOULg*j5{KgLV-4zjWDUP$%p9fcml7QiTM2}%_Zk^B6CkD`bRI+sq!S$ypBzbYEc*O z8~Pn*9ou?U;EsW5+<7n>gp>S{voA6ZFQ_IZ1y{@1g{$s%!&Hmk(rb|0*b7?OM(qE3 zcZx~&lLa^H?^Z7KnQcc}RnpT(OEAo~eb2v;CqFXM4ljW_;)|`gvrGszj0_R97hF^o zH>n{B(v;PobDRI6oPK#GHA8pZ=Z+P94})L|vuY!vP6_O3R^pdQ8*9PSJ?K@ku(n_p zJ5url7MC;Y4hRy+E>G?qjjoCs;)7KeU|Ha+MQn8j-bNgsFHp5y_j4A05=Epa7k1UU z%Gkg!HpkcRkAFgYa;m3U|Jv{;d`x4#1d|H2)k6eR&X0-S5oBe?njsUtgjiAfSzB@@ zRUt{9GmpcteV3IVm~2;r+q(zQt8`M;I^61C`~gC*#&-?tW5PVWCZt!L(we+a`VN^A zp-8g>xVbmG4mh!cl1zkyjWGQ+koJ)91tyg1<`@0CLsu6v7_r3Tm#|kt<=@)=Kmy9NWGr1u($ji9Zg;yxvS4$0`}5l8_n{R>;boNr)~~@i zUs`V!Cvr=klq}=pXXT@u(Ea(iy3N#dV@(Qw%nD0B4xh4UPRsizZ>L`j1b)cXy$BOO zUbo_DAp*|yECJ8yjTXYmENr?P3r%PczKHRZm?uei+|~4;K&B)8Bq;z_VG*lloUyR* zO=ZsjcP$B)U?ZmPp|x+IJavQq$qLB1D5cQ#}9_9hr}Ir z8rRgkSFb{Q&PA}Bg=0H9i|}H`X0>u9CnO7#eAFqqInIR=8>1MBm9_O8PV(Wyg zZIOb4wK!OhEEX~>ajZLf8A&7TDn*w6e77VIB(GAOi91PmiREwnDr>;O1$Wsqx-r$X zUgHo?D|we&8#dzLoUt4&0~#2xlnH^0*g02;W?DbIn^=fDoEafh2+L|y$oF1z{uO-) zj#TCTi6$0f8L+q?{n5TN>#HJ)mG^N;Uhkg>cTv)wvkuw=N#@;I$6B6`6Gj{(q9znWnWCBcW-bb0=B zN@L^Wd^Hgk)xN~3zH&su>{88Oa2!tzoagpDCJQB-(INKBgGk&mNaE?X?H2wQ$~mIeTey# zW_f7}4mh0oxeE1}98#zhoo~^K%yGyW0wQ;I(%7et55KW}${#QjU(C@W}|wWmj$n#4{8 zQ+7)2_#I0qt9Wh_FSNINw2|tCwbjOI>Q{(q5ycG91d7|F#;8D6EP3WaiC{f$vJ!;$ zZ)l=#NWMVMUGn_ZqIgfHp5pESmT*^Bnbc3|5}|D5eTS7LHd^N|*P?*S?=$AF`NJrP z*QEWJ&g-zH=C2LSzYNiSG#TQ^W+4`#yze?5SroM*yn0EL;_%qxPwZl5iU_ggPiVs_ zM#js%F~p1tbBi!DWG}nGQz)^>bTXnUi33O!l!L1i5-eZx@2(A64bEHYY|`UTkHsMl zj@T<`LMmk5&q+)u3u#Gpp>3m z%KBmYQgg1MM-{Kzyax_@ujIws{>c7^-p&%WVxwh?g4(1PivftIV-z>kEh8ZI`49-A zAB(L-8m?2GkJcGJs~p!o*84g3!Zt=44eNCkS;o3LVp~ObsHY#5UNC6xV$vx ztCC0Sn9>qP%jg9AQF}*tq=Y2jh3awq3CAf_m@7@ayq?Ivq2MwGj4?OdJVVJO+j%OSL= zURPi3WRpetXv&`RbZ2vAY^EhlLRoMKI<2y}y))#9Iht5hyda1nb^6Qy1~((MufJ5^ zkr*;08InB6N=a8x$Swsf!Ksx(hOu-$Z^L1AsK+(E$W?gW&U6}+y|PMH*eq#Nmp&F1 zAkhe#75On`Uz28}{Os}mW&&{#nwIK+r*GyZ0&^5JS%tF(kJx5y<*KIkg3|$v)(TC@ ztg)e!Q;Syfy)MWGq`vQ^jso^8xyXJW^mj`RX*64YNQU4J?P?B(1e4hSJ7)BKQOCjE zo94xQ1w^3Oj3DkQrur1tNTQ#b=UmJAQ(&-aC_mmi%n<(3(i*L%Uay1(4rjk2<{VJs z@$`J1fVtj@UbKimZ56_Ry~{sR7A0}d(8&ne`h?f39}Z0Yktq>$+opoEfvYJLx5Kfa zEc(A-gm)P?io3HZ_*9Vzw`bR`C$(zn-h~kJlxTDcGO)6qnN|EVgjrwJNCof}EHd3I z3)3&~0^HQmS~9OC)sk|j3=JFavo`1Am}n5{lfC|zOIkZGX&cA%#Qi3I&ys#X@qGyC z{y}V4U6y>}2PyFJ%7g|=$cd?MpO!;#109W?3{sFtu15bquekgVe3%$iWF^Dv;sc=C z4L(A$$2v+?jxKsP#K6UdpXekAzKBrekgV~KO1-z5^*DHP1~}C)hDQ`ts@@@s({N)* zU9_$Qy>L@)LebErvA8&KEm=<8P8EU4b!5U8dr|8bVy^L#pl>tgCD62N+~*w<%RqPX ztU;E|VOiv+;JQ3!0C$=BSQZUU=zPc~BIeD)e{L2H zqLgmR@~m#IeX;2`qM9t?nfq1dKxw|`%VxyMeesTdZrvcyYoG{hRO`<4JX{aJ^Zeej z5xi*a!oJdIG=FD?;P~y5JIOFJg4PqE5+s`50b|A^XM0ublxzqod zLAHHB-j&swAq5cZA1m3i54Ulzzn5_|N-m^Hd(pfGd`gnSHR)Cpq8*9U|Is;{cO7oGy3eI_jq(roVyd$7 zGvRQ#DjuU+&Ppl3pU#}iLL-MX41t^>lv;ihB6jyOLb`yfh)fx1O4_2v**f?T$L)Kq z;-+j2Ova(DcTDn$os4t~u?P50^Q=?Gx383!9D})A1Qe z4W-+~RN@SmC%kdmSdzRpG4?I9EN7~gbPmF+U8M(4?~mJ~qsD+9h$G&Uj4;j^A{93u z*dm~q<{dY)pGUs);`k69-}56hEN&8YC3@bo|HZWeB8ICavLoRV7s9*jFEbQO0gZv> z(St*beGnqWWb%A~kT@R+$+qwXD9SG%TC55sps0_yAz$iBAP#*{Q!ike$QOzA(>1fe zPfs*B*ubsTT(e7T2z|R(w3sN{$az15E!Q%wK8u6xVfeQL!kpUrS*_n2H->thzRK>J z8y?5g!;Prw$>ESe5$;yU)7ErjZJA6K(&>nz1x!+v&@h2F^9rmA>GhyRcU8b616m%~HzYybuyCZ7D8Im781!P}(Q& z^5XxabS-|@P8UKx8bo?}E`BrUi7>S?)V_Z0K0=B-x2Ir;g@Q%w&^L zQ09>QOFQ)+PtdBi`^T+7Jp6_cC)jmHxPME=|26Oae+$S#Scl+Bx>xj1d2RG`ow-S& zXW>@%uS#)2pzU~?N`I2#=AKzvs0$bfkP#rf+Y{nJS7e}SO3=?G*o>1?-OjQk=ltdT zj%2rcCQqgG-*mF(AoR<$yy%0CR;&qa^%pN=zCe!tJ9Jy@cM$Iuj!z_2Rc_YHR=q`6&;AN(HOKgo1u)9JMshr~ zFDNecXLSv;Y-qhHj+n$(R3|!dVhuDGARz^Ih0oYlk5>?4XPe$txjK_{Q{(k<= zH_sV%>Y%8#uY2uPT)omB#G|!9k9`klZNv7T5`8+r=BnD{{L>H{;K15oZhmX69KnK! z^wPtA^@;Q_60Pgl74 zw#(n&QPZ;BpnKLzD7`-B8L(SZQ5-T2_#jmv8~!tH9{5-Y7nmY^sqXm?YWizwUi%40 zm6>AC?6)?(_u*8MGrRtbU$?MplRuT$B6Wc43*0|q6=BN6M0U6ahA(_r+^3i&`cW~I z)N{svaByu)fK%DbcQ4F)vJ`FYMVWBY@a~%@U!aM;L}1Y3PIZL@#m9p6gM9=!=kiXD zH{&%Lb60Uw30YfLj+~T^DAtH$SATvWchO$&9Db3QTZjslg4J;<522y)u8t#3zc0o# zjxb_S&$>{Z3(ylKzmNGZ#16+S+Lo{}w@fd9&%nGXfR@EWA&%;3_}?*~Ya%m@R5zH% z`IaIsrjyZwet2c@PryT-g{kC`yT13~Ey7jnM}QzmxqfQ|q}tyBHcv!wHH#86`_>30 zGnS23#Kd&&>}QEXOMZ(eg=mGCZ*FUh?{`ehSJt@)L1tra3#75}6$%3Tf;!KPc~dn9 z86a15fr-U#C~usLAh8Ihpy2d0MisA@ZOK$si&UYpg**tcS5Cm+-e9rl-zOOvL!N}G z<{`{Kh6FX?1xbq=d=p_0OZZbatu#K_8g)gY(IMuWo$#2A3eUG)Fg@bolG=X3Y@5zV z$Yb#mCP~33t!4b|f7Ir@)i@jp8t+D?n|nAI$1n)1T;AgxR%AF&Ff115L|~9ii3GC&EVgQVt||XTp4KQ+{uo&%8~VtX_>-k7EtRai zd3h%-!4`(kDIHS(hdr%0o^mj*dk*hXZy6+YICUaCuf-gZ$Ux4T5#M}rj~G)e%ytNN zb{i_nrehA6NFVMaCYK$3EAx0qe0(CdSbv{t-XQsBvWR)=q84Irjy@RfdgGDwP*=%{ z6$d9)wuZA8u(EQ*PyJE0%yM?>@-CK5^4{oR8R|3Y z9hyAe(vp~{8Co^$YRWsvhdPi+_~uZ$N!UPGo*~J<>hu4CO#b`T&p)+3YF08F33j%D zo4A;M;N3adQG^!#2IQrfCv>s)|LZ4#EMD zup@mZ=?WBkv+=M6wmxBV&0m1&ak!N^`oH@C?BVlR`a!qJ%yx%3Z;p+m3U6CNy!TZNr=Ih zlnA@RgSaaS{ue09DHaJAxFXoAi;O)Hj%C=9k?dGD=m@bQcko|tToQYOz+fnj^g$Nv zBw_JY=DXk6;Xs28e=T0%5|X6QGCbox$B654Yv=yZitL>`IT;x1w)Z{7%F3_4?+gl~ z_TuY9d0`_$>5D(Qb7m>C4#;nJZ|Qn0=iws_I+ET1{timOEA{~vy+I5lPMk#0N~;nt zz8A#&Wonc}*or@_)d73;P?06KL#lqj#yM`bmxf9bD{$>3xv_xGySAZ`!;z^7*^zUU zp_+4n2wH#1Ccuxj;J4!RYRkzwpj>dKDh%Iq#?a(k`}dGNtQRd67U6K!pJDGE89q&uf}ys;JNMu*EmyH(&F-rk)nua$kLz;r{K;g zV&};&PAmD9-X-%ujRCE9C854Zm}u?FIzOgimb9^4oQ>Vf;G{Zdh~5gqGJpav5o<$9 zfcTm`nD#R$q4d(Z3|F0!2VIOgRT={LRyQ`5dh+P?0|O_lr~h=^tK;N( z&c`S5E^39o3AVLqRh^npAy#=#YL1(o$VN57;izT{smKOfXN z?YAg46N80H7yrp)!YpO<`v#$8vs_HEZ0DUf_quy!KYvRv$c}OCt}bR!!%w-Sq4%B5 z+VfS#l)i5~iQbl~P)j}*{`D}p_PW}ta8sO}9bJ|1W*|)^pk6Y<%yMIYDeguFUR~#< z$J;JtaC8{O6X?y7m(Y;I37y+it-W+U9uz_U2{WO~_;x@4l1OsOOJ4|xxdB2Gjlf+v zW-C8aGTSzao!d5b&q?!i(}*cD6!>u@Yv#FUHa-m&&Gfm|WWwoIF=ck;9Lz|9dT&g- z`ynzw0`((vm#U5|xkE`&F~xrpDgI6;y8GC(RM(dz$=KNd+mFHk%M|P>Y18Eayzxfu zK&wc#n|@7ga%$1Id)q0Gt?QP@(=>=z+8&{R+Afj+sO=8tC|zmU`B$=>fa02PMCSr6 zdULbr@~O(+`Ywjb$kT=$L-d&sv;99s<%65OW|fy9UQ4$s2FnYLmB$JmcxW}xL%dwt zbnIQC(&`)u$$8E=>r=u_m1pDKEC@_s&3J4;oxrZ;%N;_3G_ zSmCO{Bk9{Y@BU7~g5Bgg0An32>tjU}e?WPhmVe|?FRHYsWz2)>{j5*2td3^@DCSr4 zdB4gtLcMYh73fEI!{{U~@5FwtJY9JIFfu3J3uexU?=-9R6_p{M%$PVsK!(x|A$ zGMU8gBQo_H5({$;5BEkB$R`;w#74H2wJ<&AknI$x4&f3p1@jhWZAtef7kK%so(j?tsGO8L>3KyC6G2 z%=7S0`5Z9gyya{pbeT^{58r8hemVDTi2A$^fnF)`)?5-RuN*65(a_b?J_@qGemKx%b2WJG8~g2ohgQ^2V?i#un|Om=bKoootz|0( zq;%xb_0SOZToCVpZ{=c++z#Bies^|{= zc+|$uMR(wcrQSRt6G{&li*W-NysTP`b&`1#exy6RSgFAK3O$AXH)~0x zIH#Iv1r05?iM8r1jFD!Ehi3X6JKPI>j19`|VnmFMh31l?d`77BMvhD+3zBlfTk6md zNidvPhdUoMTg1g#@!gUk1>H^@+~r^RRQZ`#bB$B$=2fTi_qEozH4Zn~E6`7J{1WrGMl>)b{(v9ocHf0E6pw=Il?i-x~<`j&EHX zbOffNvr_PSlssODZmnNpj26k-4ugL~cY3adjLaDghni8+J%6SMU}R+uXirO7MMuXZ zNOrHrlB-w#vaLVp$*Z?M5MEb4=wjFesJK>p^5V1YV+CE@t2f&N5qj*X*x+F=zI-mS zls`ILu^&kQRaXs|K*s&d(Qy}yWal{AIX){spk=ILEL7I@z9)VQSGwCiU!jCQ(fna( zb7@!SZ`+<>z30cePzhrnZSJ0CTo2UZ@%X4ye;~P`h_QZr{~*TY41Dz7TzD zGltI4rRpqBI1A-YFsYCueEk}{%xnd{W}D9@>5pf6?lYG4{9Z8}fEA-n=c+Kj&Op`G zJbPbv-)n87{xY7cDx-6^#P&1bm&6{-o1&GS8)Xh!A)W6Po|eLIjve%r)my{kefZVY zf#m{~PuPH1zJde(Yg4!B_Frym34yvuth;Qz!p*N`xks{IV$l0o(G1T`E+{eWMnPWd zx+*$}KS?p~J~lzIR4@g-90Iiu!%Mn2YRf#5+=@Y3{Xhk*^0~l?Zh0mTVseHptr!a0 zgWvR3^b%wZntyn%R{QdOk-6Y7N9rzkoIEmZxlaW=8dx>XW179xUJ?1;^tX0FrHiDd zGohsS0=6OZuuQL}fKKi?<)=>p(%%0#0`iERDa$csJ3JyBK+}EHW(9oDH~U}4U2aaj zL~u!ulyo*Od*SN~uxlZ>@tJoz1*`I2uvl)qSkmm}+NMq1oqB6hQ0rS$<2m-!^d*kt z$(ht9AaxMX_&=!wC;m%HSS2}V6HZ+2*m_G0|7+AMI`=Ad%Z{xt?>pKI;+qRu7M63# zoK5fLNthU7S%Xx1$n;a-NJhf4_EQ%IFNgYQU5n4Q|CoiE9-c6|Zi!w`FQECj^0et& z6=dgJXywWItG{QyozgfXwMOPPaCq}yVOQ#b?H@2g$c~070U=#A!qde6yhuPMX2*~4 zOvU-r#3>c-b_$&1!+i9Kvf;@ZDEW;e7EcG1n}nV37MykcuEuivfqF9yXcN%ClGHj- z7ykkxs>#Mc{_7M25Ep=TppGLKV>9a9tRS&QL1u4EzKXvC`LuVXu6tKkiGDi$m0IL z6_50n_eG6bYYX9+3$Kax$uUL#v~6eBr%bCW?NtWWu;cHQ6MT+H-AsSNYXhR%hZSD~ zb}bgY2zCQ@jK@o}vN_(ahcm{{7{ag1aq-+KH4I)>FkO?3;?wZqAu@&|t$A|riUhp8 zjh)|e$Nl{XNVW1V)zTG~4-3ikSuY@*WkuU^kSZI)HAS@f+qLOIY7_r_jE|4}vEhQw zM_SN$k8JoL0%9Qn(ULBYzKZcZ;)_-4{=)0=Uh>gK`PMG!&5b!$W|(A!34eUiF~a}L zJKi5k8CzY~S}hy~-}B^#nML$Z`@O(al_3&_i<=4uilpPCZFlIW{)BNjbd&H&ahw?) zMMuAVuWROUOwyvUnLB~uZbA|pkS}gK-G=c`ywNk2TipzW z!$AMc^FTdu2_V^2#h)K>=I0t)e4W8x-&aom#qg$$7L_*iN};jy&4d_R{?R`fH7w79 zMBuo$Y$y7!b3!MHgjk;UhdQqMjNHAdXY7`A1yIN69xtEIYe+T;O{F#ZA#$(b7?%LR zX=^H+c4M>8?TN(PSB(q=i$uU_F_?kZE^HF9;QwYKt2#SN^R)@H{iT0{XwT+{L6zXC zmSWYihd+wZ>Tm&xD_bRP91DPkLH*FzE@oFVGhvT`Q$&HNsy=?k6n%_PoBOE9s-`RAl=(Wnx(hwpZ*uWQe>q7 literal 16021 zcma*ObyO5__dgB@(!GQrAib0l(z!H9EG|Jk!AJTR8H$ zsKgmCeXa+v|AwURh}%wmis z%NwG{U{;bZkb!>Ze)gv%y=+oT(s!7-{rt1A(7_hL(wslZagnDiDvQ&44C}=T(HPA}=1BgqRRq&2n^CIuP@niE;Hf6gw^1!e2y?I1`bzRZt?+GlVznWFtXcVA3DG{-;X&mTGdqm1DBeck6yR zUysF``Zpf~hW;#AlecZ~ym|14TZ{H#9yVppNc!zHJ3_AaKKk+5b;4lMLA*qzlwxdP zC0>zJ^!FO=#%UQTi*1g3=f0zlOP5CH&Fy~EX%`vo{(JRBk&*dc=r-lHUU8s_j&IUU zmCqsGckfnugC*NEq4Aa)=|qY2LkYW>EYoj1PWu8ANP)-B_qcXW(R$!cF|4Adu97~M zkgrS%&I!)r(&WS+zF6FVF>KfD3tAj)YHb5} zqQGFqAP1Otd(}geLfv=eu1&cVU9e@xE7fBTz^lmR7{UE9H1_#k15;D^H<*=t?=or* zgcs^Ryw5SN`bhvvRGpmOeeYtvpj*73lDgc*9?CTBQt)~D`k)ll-xbF+pge{(7)&TJ z`|~h9QYqJL=qraw%dKm>^Kf|J4cur!a8zY<3O9xT66b2=V>ffbMR&84yyCeKYjVCe z+2A;*9M58c`%RZl7p+3pEAJa{y-BP=gYi@Et$Jr~`hp6|-7)~lsP{{=FNjN|afi*LS?XJJF5lKxFM9l0P-dNRVZk1k;Q z{POSPv0lqjqjuwUShC+8g>jwh?NIt*!5R=nO3PndCMwSg)rTE>16#wvuEq>%Hth}y5xDHtLMl2 z;WT6a6@`E^$BO1JyXXCNKu}G%lDBN67E6$)re}Q1$UrPNdHdM8RQv1>d`;a+mN-<9 zxcyWTurvJ1dtvXD&n@-KySz-lfkXe*;$*}rV-4RVx{8__sTf~tQn7YLAk9pfzV|=^ zQ>$4P;rkL?vJK=7{~!p?tv`U<}v9I#pOFe&=ct5?`TjX`Kw zU3-iuw_t*iGk$fCNitD2ei?8?=>`n7={HHcdY9+wQ|!^Ry!@xxt{!xENi%oJv4*ZI z3gyCA9D1!+*Ub$o!@9`=q#^W@Nl!JZZpv+fzDWf*rZcr>5hq?tI+V!e_{^AFRvV`} z4QpcKpz(+bUoRSoe!NfR2)L!V{5#irf1qd5u~)jZ5V%KSd~Yv#3g7xW&!Z*&-6w2* zI#1mwSx=|ohEi@oy)s+U;Z3|Hzb!iR!&yMJfr++OyK(GKr&g`|ivaWLCLU`SyM9-z zk6s5Dm-qAhuY~OTV;s{2)zy}+c5y$noOI0%6lv8v2K-JfcsLGU@minWI+$%swd0%S zJLcdiIz;)#Os~I{m}ll5;FHNTz&?hBbLFa^|HzaO=;C8%XE)&fMy@ ze%RN203J{<7- zn)qE@l)cLr93D|o5x2hmx+JPj|GNhrN5@~Sr2gw^a8d(E5T81`>!|s)sqR6lJ0-51r5DB&!*h&KRMPtq#zK&w`T{3O`E;#{*z?MZKjv8 z?S6zxe_Q^zA3j8W2Cm=wE72;ny0)JpqPhJ{Fu>3P0N_%Le->1RF#S|f^S^1;Yrj9I z$Z)@5qsVBz6 zPUJVxXzjO~&mnnoq1R?QYU}Ia7)z2H7rFL)sS z*!nSmCs|r&=XuTK|R$Z*zI`_bX5D?;bvn_{oNkgQtqs zTeXf9`@#q65%A{X1=}yOUDXy`cQ3XO z9)lRZ9)7#YCJp|lErG&cJ|jzvZ(7t_tPkIni#-m$8M(V^YZYw2maCb*lpDISeho!6 zIYik-@+;PQ$HKcyh(MvJW{rwU@l{Y9oc)D{kLHs<_xH~}kBG0wq#2(+P`?s)ow-OY z+6|tSXSJ&zRSP^3uV}t%F|y8RLIhD{xb7kj@5c|)wjO0nrZFpPtpCp63<(-u90QOz zeyW~{Kzy5h-TE_do5T1}N*i--g23qTs+bi>sk+|?Ad!!to&Bi|JU#Gqc;)s%)ne6?6_=! zher%r09G-a0P@L#^`Xj58~JeH_c^jCl527@In(tQ-Q6zr!>XQ1;B}+5V@5`F`&-;} zMd|rJkMYshuZ)N^XdFma32pSp{T{L^+;QNNW}RyBmZ@&IINS#%?l?D2(hQrC7|C(~ z&i)wOOf9Rac~=scP@tJ$9?GkiHzL#Zrv#fM=%pfC$aGevzR$J~Yk;LasSA50vg|^|yp{i~KKnnG-tzLpNTNG2s-0f5mlJ);J4SbS78aA~DJdEmxO7L33%i?8~xfQ(d$w`8oMga|+UtO>l0O@jHY zsZ#_1zXBR%m*t;EQH91xc2DFaMvWV5tLQ7m<9BtzPWD$G52YEu9x*5-yi|&R;MIl( z9PRT3KInzM=Dd~z1WD>YK@yDdXJ`HKu>2KScI8hB?9{{ z@UInY4WWUlDKq~OBu2t?A}1+mB9D2U3bct+KF zgyn$}tNaT12JMAcZfh~}#25ashUxKag`$VTGlkE%n~n%qZvL8M$UPI8W>*qmImP&M z#n)SFGCmU@m`Q@cJWqV5i8+9prrPzy5j|WkK6t>wF6Atrip+gvA_L1FlG8!Sys-g?XKa`CJ(GF zlh(p*9ydSnCNK0Yc$VkiKcG$!2HeX_cv^2SsU7iOY6JLMLCRLiMR7(Z;MlKh(BGdO zdKOn$bsZW$?#w1E$w_*7^rXLeC}&lQuajp4PuI*CXtkc|3_l*y1#XT$Ql(we(FV$Y zJK2BqNf&Do-Op@0H=64 z|1u3oXO?PE`L>4dt(gZaXm8SR61P&kF~HllB^{`lz%9kuy^jRU zfwN!#L&NTN{NtI1tks~uriVY3j2`e0_swsPE6}xG%-ef^tk)x^^6wX2cHW<@vhtO< zs%ZY(^eXUrXxL-;h&DtL^dZ)3-kc)i1K%GPiJLx_@4Ps^)RmOtb65boAYeLC`vtMFQRj&`zWTUhj`9BS;XaCzS8D@XN^n>1C(Ed{C(5Fd zGOl`09#TGOqvO+){ZQTpYF=#(;<4_Efb)`lD)&FmpZ})BM5V`NQ};rSh(8Y(@d+{G z2sja}u%++5EzL%OKRq8qMdmN)%$I%d4sOywSESl-Hdqd))5*e1v`b(>+o)RlxQRW; z{z~&}y%_4#gi)|l1{0VHhkcEPnlS9CiJP6DO=;~oczl(}a%>i^N`%xPSzi)NClotE z>5GVlF(yF;9!A0#w9h2sGt2EJ$t)U{lxdbfJu#p6;5sCz#pwN>E%GYuNA^oKI97~K zA_UxV|LJX>kZf^Gf_@o$NHirhFC&;CX<`(jprbYi@{V=lMX_{aIb;T6HhgbAb9@!_ zD-cBuIY&FnTpwPFm+6vJAKunEx}%p{7^WlsA$z!4z6RkO5;TdXVo{z{W3o|8ZC0P6d8xaePuKZmR=IRyE_wvTq5D%_ z=9z%}eEp|MUv%Rc2PtL9r@BW%e;)6(T1z7CN}jt0dVC}m?W)9j3jKrhTk<@P;%#|Z7f)kvfp?X%uA55cE2P2Ux!Tx{{E-xru~ zV)!ZsoxEN+6<4EuM;}ZW8cc!Lk#C_Yxm%*>L(S;e&pywX_s6jC!>!7B1_9*y_qWFa zMPYSk0~h&ciRq?CK`}%>2R6~XdPtx5rz5lX<~PMXa)LCe{MRB`#YRN3MpbAryn@JS zD3^(##V=8@7i8!}Vx#DhOZaogW47}@JXjH8RNbC~lNCnE6KYsT0g-Qe2P`&v+IVLweY(uar^1FRKCe1S$ zm1gSM-AF>jNufo{sX~h9j3YGN&}?`k*ks zR4GX-;*-anzRPqi>MmA> zi?F@}n1{Y9IU{&;XMI${SUjc4XN3CZWI$$mul#ndM_+t}DO2$a6Lma*tpMCp{6DNE zCmeS2V}0de)#y^PE=-cL%|FZ6HQo|2Ap>c=eG}MSGKb~2<*RIIxw4=vhD$*@nNJI$K{ zKO9}C6WXxh6`n*`Zm9Gf@}g}g=Q+W;&#zW)#&1E9NF**OF}US~`hi-t%R$Hax#Hxzm;z4>8ovpZh&Yq`SY9S#KiQI5mSJiy1~=TChI-!Dfi=EHDTS zDweg4s1w;Koo~oTYL4pM-UZaaD1c*(k|abHt4p z*uk1+tFXo7%?u~w>L+sylsQE6t7#U@CkY2teB=YsL$61rgFPMlIH<~GowBJ;UXV@@ zJA9>YnTMhAWL)zpLgCp$IkiRZZFRwo(XNU`V(}=wLT1c(T(&_T8H(0rLIa;r-hUCW zE!fcTTgY*6ShLp1SDREqfI#ea^& zm#dJFQTl~;Ji(I8O8viR0v|9qw)9#->Lq-1_;KlAcDzDPBoH+ni(mv?usVJeTlk7J zGtu4C131r<&tRnZuji0l}jG1(Z0xllIRgbr~A`Za#m!-}cuV4*Zex>s8A-Q;Mc!9=_kC=^0;% zCcibnTJAg0%yZ3vB!6Y}i~n~WjsUMKoew!%PLcF3HG5ur4CmarjS~)Eu5&2%25Ajgfm4;?eMeD ztQrC0mc;Wb!{LkilZ6g%Aa^}u@rYGCnS{g9Q*~PTG|PtA(z3gtE9nZp^v!mr_+uOp zwsKES(jY&ZI1-HBA@T*q$k$lbphU>kl)0~^>5qZ$d;Di2tk znwCWTXypRdit<$Fm}(PVR!wQrdQLnYNr|Sl+ot#)US+;-wn#(z1f4cqI0+x4*0pcr zFFhaxy9CB}e%LIac!phNXU5I8+E8CM@=ZKdbC30+zFM#jS0&LjcyB`ee?Wm?#!_}G z|AG|4+d`stLNCxGRA>8E)vKJlU<~a7X(Id^$6rGG-sQB93W@qsZ&CL=)Lx3SL^fN@ zk8C@$Qwxc@_8I)=w$)2+uQ{DSwxYVI^P2N-m-1LVDz=M@gKid|J-bxI|6PTu3a3IK zyySmjwUU4~q|E7YAaUY`(6?Ef3Q)_1?dO84lKjC>(JkiJ`31}tRKZq6|NgwrYisNM zlxI`#R9i%H=bcSxzK1&ioT7-tv%d%BQ@@$odl}S&be=}vA1rF{4Q;94+LpE6dTc!- zSYpfXNt>;yBbaClZuch`Cc+Jt7?O`@=j?324O>&r|5< ztHQk!wVqk_`?ChYwSmqC_uzhhmW_SS|JlUc<?~(^Zu0nIk;WWIE~M=y$gr;!^F{ zG4q2HQ)<#=9uG-7A)0SLZ-{5z2bsJv2~qnH07Z&Ffw$KQKY45qQ5lmE3x{LD9cHNh zyrboUK%Ni_by5W)*5y%E5#KI?UPS#?BNn!mo;y-l;8}In9TQ&tQDiXBNT?47RZM&r zJ;9RG+(mhhjCi<$eCd6-+3q!psP*hWX!;8iws5(GE@`PXbTMHH(>!yWW{CGXGq7FH z8gq{5`74(eQVh$IAcVUG(8vo+IV?KVUseG_MpXy2#2b!@IQ_E(WOlI*_ZmAhdr zJ1(V&rZVt+fH912-=n*~lGkh<p{fLAu3e9m0Xv+uT0rT|{A6fqG8iTg|G zbq9*B2N3>p<~{fkqrq3ROHTJ+2#z{VJB9ZE$kRhE~~QuSU&m4 z?5H{4uin0jenVfA`5uWv9BhTdw+>4JJhFpqX-uH}F1yyOu__IzUZH<_3 zEI#5+jgCAav!9E(#X9#Y+V!Aqn&o*9CQyV}t5l$4utlyMa!;@SQ7S;nvMNGx+#_Jp zqyhP#c8jrF0RkaX^zhD3MBUEKBc{^E*DvVZrwgGMrIRc92v&~)54WbwGkDv$OnOzO z6y~Bx*a@yqmdN0cRl!5w=AyFOe2khWr$=8s-Z=!Cj6&+!zv_@JGl~mA#p?wLceZvJ zS2iFD3CqFeTnlo=J}ps9V5F{8Gh4w6E16Pg%IEo5ch^n*adHmNTSuotS+Q!W-9e9M&TR6;CL2n!IX+9+P;gw7H^otO_09I!%vP5vf zl~hT`i@);ieRuf?YmNJjGrNv&@2r!=K$UvS(mLExSjNEEGtH> zi&A6PWB+Ywo~DlB7a#-dB2zQZVhbq-4D$c6$S9ZIxKag8=r7#X8$9M64mR*!=>Jeu z28dxV7i?hOEZXn(_RmqmuR8&smNO{yuQ6+E&|*+7@+_kb8wWRm{{ibqvxXxjn53@Lm<0bA_*YtG_t?z8OwU1KS?|!-%YvGwx#cyD=FTSIMMipp&OF1# zoTrVg`IY?i3xwjiub2Dz!FVza8{>t4W%k_u!CZLL41v~`g$nwpsGWJzxM=L&y#NX7uN+L`{AZ4qUJb=l;KRW4es&<=4gL0%&|eicL6G3|Ln)~Es0*3{ zBD;LPK)NTAB_FueW9lbkM!KVKW}Qy<+26*Zfb`vJL1GH!CIC!ZiRBgH0u#Sp1Xo_j zP$!J5ZiAE}&hV?d1lBBhk4NEFFM;~Lv~ zfiQDUGK2gEL={_ne^ZVr;Kf@Zr84X(wBHsGW*K5a;vYp3RpbIal>%pFMpS&?Z?+3YQZBp{3@8EXD2^0|62n5CCHO2))bZ%gwC?6>r z9*wLD&AR7srQIYU*HxOAqgEGFwQ_{pLBY=W_m11dS25`&i~KD6@%74-VWJ$(BRslR zi+TS8wOTW;W*pt1d3Uj};xB{YBVpzp_{?vRlCUoAWJ@OdPe3&F{YR78ILDP@XhH>8 z{soi1XfD!0<$vhbY-b_Gs>A3|y?s?{;+R?i_Th3Fq#zaWe|3)=+q)MZa4#_{&{^f6 z-?k>8&^1i zkH9O*=)EfShw^wk}SZ=lR#w5lo5Vgn5hu#<=}ZyGE`%AInIF_2b|EKLQ$AB@Abs zDg1h#rU-X8~+1`l^5o z_<4GDPmmyKoCbn1&}g7k5MyAZQR;LSKEdqzQBMBHjF^@+*g>6J`17-_l>^bkmV~C+ zm8(8Mv4f(*pOnRvY|6M0=L%Dx2aZgfQ%f2VKKx+O4EKz_d!0)jdeLxt^&KAWI+X{Wt`$j*lZ9lBD zyuc^R9`ucS59GSq@#+5d(tZL*)di#0xk%F%*2~@!`w|?cdR{qK?%YsiBqypC7X`W3 zE-J?IBq=rd0Q+(An?W<|ZeSoA24SH*XQ*ZaQN!ozv2c^d1Fzq65vN1YPfj z?_)j=dUYK&iKZe_lot_%em&(Ec5O%y>MngWdA~w{0GsCMoHkcpJdM^|Omj!RZ(1qu zJu%2i2p503?pcLRElFpj-H%|fqC(|3hOvgBld|sfM}S$|hpwR*7o+{eBy{d(Q7h{( zG8nDXgj#Y;&~|tLg_zf)WEDG9JDfTJ16mGmVA3dYE)ya1UHR^k{u6bE%p|PBamCq; z(6i2G^Zf-&uKlYs4HjPeAQuv`yGo`KnLu=m=%Y1;xfd^pc!!yvv*CkT&tC8@X{iT&J zO37p@hPPWs^pv!uw(t&O5enp9XgEFEcpnTl{!n({1j#aQ9>l2{1a_}Ee5*o67HL21 zQd5wF@K!1;!*Frx-1?z4h^SxyL-+#67++vkrIRVgV&u)yb{LQ*0*xp(aEbZ~dhd*= z$~hR_z`rt5tKnkjuNj$lsQLR&DCMg^wli|}aLOg*KFZ0DQ(%GDMqa1i zh!^VOz{Q+ag92Eo0-4}tihm-rgIMQ_|6J93d@skpKQ1uR(IDN8+GM9~l1)kH`B}*( zSx9)%Lqp6XT==AAU4snDm2RB%R^MDu${XIaPvJY`5RSQKJBjFj;Q%zJ<{_7<6UZ}&nYie{X_10eWU|9oV5j}AEn-;t;YeG-BCd$8ggryt+$=)#n+4C%W zi=LfV&1IQw8rffjvMJB`J|2J^9AcHAH8yd-t4h3eP3kG;U>0t*Vde%oX>TyIyl9uz zEl=D_j4kJi4MDG|YzNP`UYJ=#AZBf&x!ODfnBp{Z3OnUVGuhodzi=Gk+y}Jg`;bA| ze zEE3&=q>tq@F3Z#UO~QI+B!1oN3I}NKiRC+*7mZp)8e;C>2*BAp_0d^jwQP%DF7wz5 zHTHzC!RWnY*wn@p-7rnr3Gt2y|D0MOMx`_$wQDs26+EkoBs6Zf-rt{N3K=QAjzlGJ zl$&2>fW^r_Ltxn(gR3l0J5zq4y$fF8qn{tgK9{wzLVFs3w~6xkf>9yHX3+KoTw`6A z1<(HyEhOhyV#IqCQ`U5q>LMh{N_(&2x*L?%ca+N?Lm%4|JsFMTv#m)jw8YHZIjK(( z3n&M8$PM@_Q1~e>d2Krhh%k7ce(~H;QfP&L%52+Ldd+Momh$Tk*n~b{J?|h+Z;wN_ zSr4BiKdRWaW54i|bJ@Z^5!$<1ZPup{GekCnjEe7O34@LU5w{{Vg5Ema#r(@81Id9J z1l7G`i6uyN?gJtN1T*i}!P+bRYzA>=5+EvC9cj~$v(BAULj%f0n;tywsYj(4Vq0uU zM2hwRCpJB7ixyMjrlFb|B91t)E+?SFq_a&WO#0B zqWpp6FwC$leu>93Xcb&Z4`T@4QD90iagE)4P0x7FSK9H*pD#s2sH}Dlhr-na?5( z0fLy|r#VfJrSgoyUg*pcAsURDX~?XiT10T<9U|adUug=J!Cjpu!`uXvPfW+kS0*~q zwCNvWz6n6zizlLQ4Mk1A3&Z^)q;wK98PJ?aIMt(G%Oxjf)eJhZ*0o&^f2>XmyiN`* zdV*p3u&dWTnD1GSV7hn)??%%;?wm18oP5TuO;I3bS`hOjt$2^dVNEv_!12=+pU#bB z@C(cc*d2Ld_+~~p=#BG@yxFJBiNLF0p4c9nXwlkgv;~#U)7vWci%!PwcJAMBSr4gR zUm^M#WvVi34_ZxYW>rH6WQqYmFr^ej zz}67wE-G`CJMqMAP0<79b#VJ7+|LJiH*E$bZx$XHkltPFs*c#@j^OI)*wi z(m#&SZGE33*q$o>A;8$7Mo3IZg;&V#iLabqUo_1ImVV-Y<;B2vy_6n7Q#stM)u`t( zDCBp=eW?6H4gVuT!V?FFj4fI~SZRA0O|Rc$l0BQ@2e~qJGl*K#{UGKPH1c-F?0{(& zp|oA5ZjKkx^Fd2a;LvXVsp*$bVmhJMzRJYm7|&b20cgH9@gml|rJ z*rFD6px6RL&q#g%zS7ZpE?EB8fErQr!Fm+tf_PM(y4bO-8;-z|&ANfyKlXVobbt$q za`5zY(q3PYNShp7LN4JfCe340NDLGiBx}R(y0s=~wm6HsUvtd^Y1l;q30fpKE}G zSK}_`&3-AN@D53rKwGUGPr%zLi(JYVo@*(wrCzOp>ZG(6uuE?X+Z?dyegiU+0nt{= zG`)lXeym$hW>(?U$jB&kj4CR-kW?OTT|Q^UT715jXv%_ATvw#zOPLll9h2Lz3jcZ` z&!L`f{w@A!BdT56My=duJ8h#p24V5($YT?^E}(KAfol0>lBgXFiOV9;KHoaZ8nY5J zJZO|%0+x|}8L{HMp@%Xui+Ug`#}kQ|Jz#?Oydvy}d+sZ?HcvnF)$VdVP1}`;D~7%k zj2>s};ZP1lm!sfp5nQPA*tQEzPLxx#4d2fOfs?d=_ffJUDm7mBUQLnL%lox*I$9wy zqo3x%rF0@b;=*0nOT}N~{y6-_c$EZU)S-2dW5sO|put>cDJ+`lo;n85q{12?G* z;c>x5BNbLGg$^YOkQg3Xm~>66cJZ?o1K5NEp_P}l{Ai5}=Ye*) zGvq%#{hK8^LLCxITsJgi@*HLyo3)tn2bqU#F`SD_Yq5oEryxFa7 z7*8x>>ip0gn^KzsG{?pf;(qga!z<$X&^iqbT>{oj9V$XJ^!Fwho}j8Tg)WCan={Pv z63JA&#;6qFZGKYEC!kQ|Q@-%H!E~D3cE(JXFf!7^gnaVBuP2CQmPvR^`We=T!OfW1 z>N-M$sx+?_(N%d1>krKb0*6@JRY@^!?sWKKh+s=uU^*cpRF4_Zsn#GdpwWf{T z@$RpDsCRn4kke_c&Ew@Z0$2Q!Ny#95f8bt?P|m}1xXB9X-s8=HPpYhF=gpPdvi85w zWPF=PCDQ}WLO9RNo}p%m0Bx4NPnfh3;;6~O_tydM zwCCuhEb9Qj(Vh<(Lc9Sm&)sn~s!UcSwniZOU6d-t$oDj2G%z!Zhx}^psN5wyU_v}X z-YizJ;-5B|cL)rF^%Dx(=Ey8QOm?zy5^&HWTC7A@Cs%CWcuJ`Obqj|BuemUuPD@>< zbt;v(=jzYts_TXhKB*buJv{Z0SdN1aMrFang3Q+-TWZ2eRk2;uRoF)_d+>ov}CMY9rZATIEqgXypDIyj@XQ~z-#m$7=J~3O#3aRH^DO}*n6KRPK3m8 z7mi<`hF^>n(|g0&m2p4tJk?S@B7)fV>nU|#)0}xJ9HQxDBZKyKg({z=)h{ipDz}PK zhT!kgbJ_6)43U!FXp>a1hl(=*==+PsIF^YzU5Gwm8Zcke@Q{f*6_l9ppLqtyh8e2V&&EqfW1@7f@60X+|9<5T}}Ov4H$GiWQ2`VIn=C{@vl}hbof& zW5im>oI}MM>*C(%SeM6Px(h0XT}Fds(av31j`%29Ls&osKR%H)H(HLR6)!eDhnUXP zscV0{9Ojvi^LpE2U&SzcXlniD*DGM!1Jr4cZ@D)A?hs-=Mqw%U5FR7OlD8U~wGfz% zEQF_PJ|7;qaIlo%=BTAL)5Y@+V|4$cjiO`DR--Y23_-W>#pm7N`8@BSmt4ag!K$M} zzuL4*X$lk>ZvSMgYlmlTzXSDmhZ7DFDVJ!sa7flq_MwP8(W$ds0zwJ^BvUycL^Pan zSPuho)A9;%4x|$xlR7;+X75f7)y8V861WBQr@mmOH4aAgR(O}QV>l}ZlaLD>GHG+1 zxM_y#<-w=g+%6VT588e+E#F7>)PBG{0*d<%yiVqXg9)V+0EIgC!7QbyuhSksS8Lh* z?T-yZ`-#f6%1W!j5-G`BVL+_}lfjY+y|*3*G!g=RTTu8zk-$)oQyGd!JN(`-a4*iZ%qI>+5RZSWL zCCvPaw2WbyCg<}Dhi)rB8yjZqC+MN%#B8d10y?0gZcH4Udf%8bW9-nPkKu%Ly4GIF z6LDHaN0sf>7VHaETmnlAz?dFT=wkd51`bmB_hQ9^9Wa&>CKYU4!_YNmt!q=Z zoI%^G?VNl1qv#s%r}$9FrWr9^7P*`;fl#F5vJSC~_coqUagG`^oRAa*=B>Sz&lP~w zJr8aR>RN-K&{R?4Aw<(k8E+WDoZ_6QZ~8bAtwi;o5fNbPPw)W;3zb@y^{Iejm6Rbq zvv>}mP&qR}iAjvc^BX6*75`6{Ka%d@=<$g!Gr6GXpXZ#wtK=w3a%!^W(x$Ng544H9 A9{>OV diff --git a/doc/IMG_TELEGRAM_STATUS_COMMAND.png b/doc/IMG_TELEGRAM_STATUS_COMMAND.png index f3767597ddd7fa7f4c43446a9396fc86bba6580f..795f1d6e2f57af74500cd6a35be3690048c820a7 100644 GIT binary patch literal 27525 zcmce;WmFtdv@VEi@BqOzxFyiRrE!7>hsJ{kXxt&VTY?Ao;7&u3;0YSsJ-EBW6uI}! zytn4goA=h5H9uIrs=B&P)!zH;^L_il`iI#gE22@Vds`}q$(krkT^4$g*D zP7m_ zSGy?1@jU18|vDF{gijWwcuD z4Ce8={4`f>8cIkSRaVB9_Qn(?niLd~#$#m?{BIO@S|ScKqVC<_j9GQ!*1hLDk6?HVftw|uYSM+`QV$fw}zuSrSZ#6;raI~T37t0yhxW@t`; z@0Fq1`N(4Ei}~TI78aH=x(BP#Tx8q%26pVV`smD*d-z0cy}=8KqkLl2#;OKy+*N!e zFzm1U?a4|_U;&|e-EKiVjV`g({%5=MWBn&3&X(2?1HUJN+Z|XN&sx%6qU9tvTO~EM zVm^gvqcL~d`FMDp1BOt##%uaQwM zPVF@{Q3q|x6*(hi)II(nXU@+0jirL5UuSKM%SDW3*W#C$z<8?bERa{i*MPfq^*W6F z)>9)e&nHuzCUWKItf%s6GK73GISg8gFEvoOVV~kq|GsXk%eJ9`puFCT`Qzuk=&YPn z#?4+ef=#a4KJC{^pr1Fzv+|awtReC2ZXDOtY2eGo`5e4cvdI%(f3wzddy-~~7Z8;} zOy3zG*MjAiAuUDBkpo=suru=e4sRGJgdHL{AbiTAt%Av}J~xSTGKrZenT)?MPhTom z*s5J{9He#dvc~7s9EAKLFKmBYFRmVqvO8LJN1ZApZnbRRYL2lx2w%>tlOV2W$k-O0 zGb-RWV|EY{|8f5C90}d(YWVjK4m|3bnku@w0z*&hq`J-Ks1KWd%SuiRK9LckOA75z zM;8fpXk<(NQWA9@82uH9xQtyT*5eD;$x@c6K6TE5?929Lmo2HiOz;?HJ@OiMKdF-c zBhVn|cpTdAR@=hh2O`Q@v%Q|E6?QJhcG}Wy5`aJ;ti_VHbgxSl~c1&HP``rij}-;WynbzZs;r{V7pt?ezbP2fOkj7Oyi;` z>w>GRxNm6lGnN$5V*e^fiiy^1-)D<>1CQ*NoX#u~mlWk`r9Se9BkaQ@np;diZpnLz zv?wT>cBR0t`yPh2VtDxOEI0TW)AzW-VdX!LnS}4oSctP~(!?J*O09pQT8=M6*#e9dS21poGN2Ik?^vVIcKgR1_aE9!-S(`vw6=Q32Im?khW z!R`~JtT^8R(LR5@Xm~VB^V6HoT@qtl5sBk5mT1#Ss|jl1z+nTPwa;-R#)K?b_9p_4 ze`VQyR0E$>Nc@fxw{Csq0RzE!W{8M4$4?QJ#KSQ1Hx3{5x39QE#`d4?8T6et2$OI< zFqH@kEzKHj!R;+i?>v2baY=8e_@%i_uAE+`1r3-EYBw(4H-`wbr~r2>W}WE_mZN@< z%ZobqJdc!XRCNta;@L?a*sjdl&oxxqReoAR_A`3XnDe72-wX3@VJ0&(<(Qp?dGY;M zzYEHX{jp4QpQmpjuEE4@#~x95!YCMO^H$aUyO+=3$nQK*1K)+@4 z`V1j*2lEa2SC4I9mmcoC9(zH=?W9bOi9 zAP$3@=*2{_kXQU$jzTi{kUy(^Vm&(H;VSp&NixrCMrw8N-b}UWR2pi?F}bkMFz$B@ zPiKheT>TX;*6w6{$kp#cJuiLv#$|7IeVDc}+bd?vwo388+A`Ntbl#Rnd&Tx6HkRGI zl|_2?%}JNO4k?-UzJ9m6Gdco3CjkyI1)h{aLwP_pk2{~q!_aH^M6x1(93o|QFSvVI zaelBse$f-YMO2w{CVh?5=EOZY@++G-=`rx#flyf+zTbT}iRX{H^k$33LN64TA~iPj5}owwQQ|1o@q}r6gv1Jq@Af_ZRy zg+^yH&J}h;O1kE>na)q1*|gE?%42Tn#xj4-uaPb)!8l@&pEQ&<7Cmhs*KSyo`Pk-> z<&rPAetWSqadXz2Mq)8pDVXxn@$T~A-x?MdF;meQLm{YZV^C#jEWPgezK%;u^fB%g zhluyo&bso|*h4!+oG~UN|4F!n_`!LQce@rbbmuVu|FtfkG(~ygS!IzVrr)|oaQhIU zNbK|EtT8Q>K3Zxm2Z<__h83;$g}Sgtae2$F2Y_7Up1uiw(Ci~R7k!$Of2wt$T8#4n zJ@hFaOp1_yHAhRd`kN{qCaP%;=6Skc(Eb+~AA4O1*X;A{+8W&I(OS$Y+}#gbIE*nB&Z-tz7(khDb7z`V$M65N}k!9 z9LSEqc=2oH<#{8-JnZYnK=LzaC})3;9m12FziNr@Op*P!N@~p)(cpWHH=hd46*==rpE){( zEyJhu8yCf#C}A0wRQC~-Y}zG~m~@y9Q~dqePK54xQtG0uB0k5l2j?p-uR{o3LmrQC zQ2re2IZmiE5=wjHma}q`k*>J~ZwTR?`CBBoZx)YyWu9)bo~&#q(?^ns#`By7Tlr?*)K&x2Nu>zL<_c3YbyjLNm}44p zXMTKf{a57qhELz4bS>WHJ~1wKJb?YWMfJ7il&Xvo_%O}Ct^$CRAd&lwQtj}IU@?mSn57`+n z($m`NCuzY^I9fU&l7ln#Nh?=Fv`a`oJP9nkE*dt~=u}w*e#(?uzA?wf z+MoQ5#$CoZ5Mnyo*_3s79q?H5$nAO4K0QttO{R9mabV?pLRwk>>ZVBpH0l&bxy;qz~mB6PNu8EZ6JH30iTB ze%D?cEtP5B7M#0|F08k+n0|!=Z^tYzeIBeP3x#)a;!dvgUWr%6@Rsq=3p;s>>Yi1$ z*8TN!RgsoPaj~2XDvW+a5mq^F^YQ*v&Q8p`+I;o>2vcy-<?Au^sFWsq>tjqf8<5eFIK|mG;QY1gEb$5^xlxNM zuRo$boT3I6PhV(_`2^bM8#iTOMGjaxM`t9VFte0d$<5V&60bFQ#MB?ne^XFoD6QPq z7E#fZ>!^=NzA$qO0ee@_37r zJMX2P>tckpc-M*dj8`|+8JIg4|`fC~b zX!w|*`?$N0DNp+_eVKax^@o$y z0ntD#(Qcx85p@OdY~u@}_tAyK97T(UkNb1mbK&P=+7Tg9Tg22&$Yi0AZj#YB0FR6}R~Y+mMY!v&YeD((s(t8OIME zsy?6H7NSQsP0?w0;jyPGieCBAs zl_ok10-05(D)dV?&vjcpK?e&>UxB5+@Y2lOyu|n-1qt8DYSH^_=rqyod01-`S_tsF z02MvS^O$F(ipv4|cSc;Crt=w=(7*iOIs`pB_V&~gI`2k0olja=?EHPKbv`VObUsPv zkCwYNyV$N_9s4dfkK)_|!YX#y8(?bnw=4k0#}!sZ*Xp@>FVM2_lHxk6a(iEkxa^M+ z4~#Z&&U+oh+QG-PM9WZd=J}D({KI8o)Y^AXkAxIlN!oulmn*UxdT9|DfbWoAS3xNj zJ<}Qa3-_0}%u{QCq#e5$l9p29LPI}^Kp8Rzp% zBr{y?{23YTcP+Gbr9B#OWvGAD_8tZLJZPpn2C?Mcc=F|VF!v@}#~f{*v2gQ=m#tLG z^ZKg=nS?(CYwOmt>%v*l+WrMjfpODe7{p6flSy8GumD2*e3O8$)}6tlfhUw6Z#>8JOyeK z`R)_dR#E+br~V6nMh0>X2dTs!CYmw#mWtXq_f<+I4!_BY?t$yp3TST~bS4|Hk4ifQ zAD`Ff;@2|e?6?e^zzA|-whjP)b4i@pDko9A-I3Jge^N@DNU|LG!n?CnAQA|kvYqa> z132ceE|$daX4|PW-iJ2ccp_|DbGy0&IZWXoG=w6_QVT!2<@76x^Y!G;A{qchD*!|V zXLCwpExD2CoXU__%5=b=Zg)DGaL))#V&Z0~+`OcXNh>??Jb`SB2&c@Wxyty)S-9EC zj`WKH;;!C|18Lm`h)Ws`6~H9#+7h~!yZl>dVv|E`&+#PlY#DE3DKoZRyTtQq>H#`m zKOU;&Y%{Vll}eRWU3|I!x2m@yFKbpD2#q*WrqxbwU6+jOzYb)rpIW&|$GY_4MD&_9 zo6swtJQElL66KbfML+qxK zPllpYq8o;`rTgB@^6K8vQr`+SJ0xU2c;H>sqd;I-Xa(r`pmpkmH zJ?65W=YNXTfpHy{)3M@N_BagNfpM5A)V>VN%d(e?F*R~#d>p*`ftzYaV9IHmv8e--g` z!VqKX`6Ao!ygNc9;+U)Tk{)Li2fql&SB^&5!+gX){Bg?L9QZ+ep8iU8wvwqs9Dl0W ztE&@CQu~b5-3Z<`{DA(RI~Ip`%c@94cr_J|> z8rT@8ny4!NZH#<)k|GMI^Z4WyV7TXM;O~GBfddt?T4Zm`o~#61c>>>7+8ZHHM8W>5 z0#jF(F98a&A|rD^-(d4;xO@TH9-Vu0-^y$z7&?ebE#mPKa2Gz-Yls$vfKm1OTwYht zJDv1$`YEQAe{}Ab3}G+(Kb9n(gJQAb^nNdM-UfM6(Q{!rW{m8Y7YDE=sLL?JBjt1u zlCWfR1hj<1K%yfI(19ECN^50U!Xpc9l~B=|EYXZc-L_W7&2(Chlpd4YhiPF zH`6RosK@-$6i6U6BxoKE6AjoHTKa>}=>xih_FU|l_W6-m@3PZz>`~qDpLqshsD@1S zEM5Ld={v{Y=M+`!rB1*R2P*D)O+gu37mxaU4!XtyvD&ekxHv4GAfB}$%MY)M4{Zjc zZxmhRc<7ZY(!h@%?GhW?n_Bkro8A-Av_2f@spD zD>--*#=mt#v2$jBL0Ebz(tMLF#20&}J-ndWnIWvNPy!wEh}d>mc%Tn(|##LoInGH$n#Tz*HSa1&p`k$L+!nq z1s=OfAK|Uilf4Ofy911*(m|NVCB{Ygr+rq;w1aeN)VKH2y+c06m6NQL5)jA24~?(G zy0nv=^S<*p6Yf7hq9fA*dn3$PSocKtuH^7L*(?tmXY%{V<@(@~zvZ<3LuBUrct|-M z1o4L0erl5y%7ysfZ0uw0{?ZJ0G=(8LNZq>W>TO+&dQKX&`Wo@L^J}yoHHMHy16I4Y z_eqYBXB<iT_hhku?9VbSJ#9Uwiy=bJ{TSYOXv&UhB8Mjn0_rcN4w2v8HWV5yl~|> zy;r|NnzpXOQHkmM?ebhmKN(R=H!@Hble`c|HyPgzVQ^k=wViaPn~sHj&{0lqw9gJg zr$ekZqA;EbHHESPZ!-F5>~NW&q0wr+e8nw|Sj0v)E-;hbXtY5VEnOZK;z*aAWXdym z+XnYhIY&c2%G`~q4{sP`hC9UX61-56U|K#E7U%9*Y*YNyfFf3hpxsaNB4?FA{`;$b zWYwV|x>0s;@!fd_kDY?XN4tp&+hc^+ZO39g8sX<*4eH?B@dsPg@*rA)GFVbZGes^Y3WFM0n zA&0GsK&5emqzbL~>h9=0GgS`K~V>7_|!lMq7=m$pG zmJ?a+OkVMr!v!%Cj0~7@l+xb9mT*~|GtLG8MFt5R#>>3T!c$cChrRO4{U2MT%?Ja}55ls9!ACIr&kG_SQ&gzf73a zo-PN|Q;xc%kVJi-_s5E9B1KOk$~$q3i3Xy(fQNqYa_Al~cN=8EtNn$>9!=`jBU$!J zvr1&kpfvTbr_lvGZ;rLJqEXl^SX}f>bX??{Nw8|B|Hf(N2|I+F!6&J~+rASw?$FQ) zPTWAQqgg=uY8f_JN9~$I_J?%=z zH$@5GYl8F(K_{&Zvg2pV#BcK&@C-RlL0J5!tf+l$Vzum;0l2LAN-%M+Uoi$QVl{$p zU64oVGt;V zD>hCm$p$Gc34gyORXt!&RFY4K033)A@IF$?1yQSoww9*y0&cAxRFEVZe*>$uMl8T6 zJX?<6_S|}M;{23Dn+V)3d`%Oa^ok_XdRrgjsoEnPp9F20!k?s%1V)Iap5E-{6wdOg zS_>t;Fp{f}JZD0!yki1pKwNHTa06rVyTLTG^BNBh2DL%+p*gX|l5dhr+IB@c`?b!^ z1o$7w5sk0}jKE(gFNehfC4pvFp)kC}Dt8oBwdfzt6vtG5nWVSN zetyxF%_hmJ$j0B3P}E*(LK`zbm`CZRXZ)J22Tu-p1$uo9y{zJ^x52!~YiP{I4&L2h96jB=_4u z=^@^Tja;E4hIom-y)CU7Yj@0Ff{Hlixl&FVLyAhZ?zIjgJnrw99_2$q6Pmi=>Zm~< z8E%3wzv9lFF&d9`LrT&R+G?zGe+u;BvP8L>D16iIUX>DBj{&*RREDb?1jK3QUkX;Z zoe8tOIcF;^<2m}CSVp8u3%-Q|?>%=5on%;IYo<)>*{AdK^BN@M(+})N0-lsr zI*Dz%LPx>M2KswQn@{3x9@a!zEs_w|&yB5P9lqSyY&ppVM@~uQ)RM{=n8_z_cI0-t zCh`m)n9W5sS2q!@m0b)C#K^oYmR)+eIr70LF@1dYK$_Subsk|Oi$xO_4FX4LRaY7_ z-0VSES+tL}Z+YXnZCj{P`j*GSUR=$KIO37JEx3TZO}MV=Vdr%Bjmc6aZKsymOytTI zNj=vnJpo^{zgi6eD%?P};TjUSIxNwnjW-hdJYFUP8H%$R!f(0IXW^JEY;H7XZ?O1L z9>Ps&W4>H<@@mUNF=jiReI~6#D`PKe8q*~`iI+zx*x>6MYhlwo2ewHI1y^_iyAPv! zu80?Q)gT%R9MjFW&Q#R=?40;n$CDbA$_^ zw7B?7w?|?zUeyBW+Gcs9roH$y{apjIO7W%``O`SSP_mOlP=z%y7Og z;yFKPZ4lU+N?;Ac-!_VIbe|opr+o4_Q1vJGtspWuXaM=_wKH!^O9O;i!oN#pq13v>BboB=M;#@C zo(mcs{soEe!=RIIl(Iw=ua^Bp%OOo;7~weS*lpHa?#D&~&%z+%o6$J_pBtKnb<~t%mY{d_^7q#Nz9cS(%!dJx?S_U2k(iACYp|PXw z(jp3GyvvGN=OabqdagQNVXmS#0T_L!8as@dmXrL$>#Ryy1-4)Pc06Nm0R*-bx^d9Btl1tm@;eP=qb>sH&0KGA4zNEq_~i+N`|S8UsU?r$J+`)JwEGgsI0`66KNc#oQYFU-Jq*lVAGZ&>K; zY#oJ7-x=HJ!gI9NLOIcXA`mF@6ZVdfSBNsnBqe^h3++=KsrSo%N6;p5=X(}{|CN)7 zL*ou_>EK~`>~V{jRkfq$VHKg@)A}t4)=(}GGRu7gYHk@(yyQ1*YvU=K+1~nle_2#_ z_nml>AGh`F4(CHGA!mtCd7IadejEDye5lLeLh4kxp2Zc1+u_1@7BtacBSL3UbB$Kx z4AwwH6HwIF1BuK?^NFs+AKz?cmG+Wue>A#3c1;_*6dTi%}ic`ua z=vY~O?zwLZ$LnNo1~=)UjJw z5lf%_^7Dc=%NIwK4F|%#Cb_`IKrYJ55JqJdd;mRJhIV1OHo5Dh!u%FeXejcp7Mw=D zdmU(DzgVLX@rK>H1hn|xpY-i|dS2|B1(UWK&>c>eYL#Bh+x<{{c#B(!PQVbW(~=g0B17EiFQXn=+-E0E&sr5ZujkAK>l=yzt12qF`?^4%Va zmJ6e3d0-j32;@DqE>Jac&X!InnVEKT`jt|Kz;{EKTRLE3J!zdxi1nGN3*qsFW$)QQQ-rG}rpgtNBG;L)%|F=5hS$@5O6FMUH`6}fD2eh-Tu z>BUmUzP;1D$XmN+?zfy&&=C(Ih;xrZo zXCv5AA{DAEioY6$lpiRKeMLig!VN~^!&gy?=c)0$UHC(X9i~>6^-ViJcq>g%7Ix*k z-C1ip|L$V%^);-IoT`!JK1y1*@B|6Ly#vX+I~!_-i@irX&8 zI=kC!Ya~&bb^KAzNT1^ zAY!d}0+%tmf_5=8=jE|TebJPrzKW`Ih-izgY~PhK?-AoLs|HjBA!?2(QoSs3`@o0u za|2aCpgFF!7t1)g?y+L7(Azg>O!r5tv@gvl^Gf@H;rndg+bG(c*DI6P0HzuqrXuYcdhgUeO z&kTE^;x+k+LZqOq0NS{I{Ac|?Xze?N-zJ2ouJ&9MI7NlresPN37-the;zv7opm%|= zlXS^bU8JdjOny$*Qj`EVYL-`A{>XcLo88M#R_*N=ogIx&Wo3*p<5*$z-QQ=+%_)Sd z?YJ`*?~p;e-sPTI$atm$`-2reyBQ>>1PS>>96bB`A6AfgDIcxRkEfCo z5Ziz@;yY~^ymZtnV=YhDu9L33KvZ-b+#oc~m|B|Nl^4U9Ec_^1*=uxmi^Ay1B%*Qm zM$l%y*maiyg0bM4H38fZ6an2F@SMBf&yAqvqAXwC0y3LxUFSbPeAlCS)z`z{_WWBW zNQurJX~<_nX%@FP-u@D0W2NNsxB0pDj)jdBSM+HJYZ=k^H&uCJX)HsRaF8Gw!j2X; zSj>Q?&=YYmo1DhHSj(G2mJDkvU+!z?5^8o7S3lc0jw~SWk}vKC z5+dj5P9GW6<9Jw!zOJ#57OBz0%-#ZdoPJCb_48fnmBuwe-=&V7kys~*#SX(I`-2`V zmPZgcjEBTSz7>;IH}!?!=2r!EKJ_-2F%kpY#K)ViwU0if4@)M)YJ$y^t6W(;FXCD+S6w&oDZmYdnpfb&Um3S!Caiah59YU~$Nk z49K(nC)SR97OG{L@W;ejNYlY6!&220fNa#i6GYS z?=WbGE}*c+cJ2|_@Jk@ zn?zmg`CibyANgcWaex!4ac8wY!HF0zQ1c1o;`P+XLIJWL%*82decb1SNJck1tJi+s zg{G(a1P+x&o&9Kg%-_4`!R>n(;cT!VA_T@j2FB2hCH=&^mLea5j9@I3s-too?%TbE z^fgCc$TdL8{#|)l9#wfxh>ljN%TLY3@i~8YleW|;55#g@f=uh|gCM-!+D9eoY&)Vf zV%)w>L@%|kcC(scfT>TvzC3~fC_tUyn-r}k_~*i#lsVE`r%sAcxzPnArO}c``S(Zi
@$W*he!=PP%sB`vyhh z*cNB#a3+TCqK8-ZtR(y?qT6e>_Ht48evER2qanV^sI$_3z~$jeow^X*cwr|4SHf~_ z)pj!meLkS4J5waDVT4LRnnaOI%`-1#vtgZrSkLGajj;+ciqgXMp|>@!M;%2BazNt4 zx2&MBq-8;lAh6^~3|57-Fvkm)j2Qca|=p7Aqz270i?yb|q?|EYUmi`d+JSlKLpyE}~GiaDg5F*jk$0f0cccedeaR5QB zg^wW5+S+0Eub?I;CyotBzQFaw)iB^AC)Za^nAGG zkrpEbgJoO)%4z7>Y~||WL8UPr%S4bS#{!^Pf-j@4j)zppor^-i`Pbanl9oq$%QQk3NrYXZejFDp7mlQD7rrrpc$OrhuXUZlq0Z;qqjer< z=B)IdCL~&A#a5ZXzKb$n9@doUv~+$CvYrUTz`$S<70sgb zIUfz$`K*u8z4l(U!G2lkV6I*bXQ03YAeib9$S2zZN^Xl``l#qm`xbCuljSHaDUW3w z{%e%ghr27SfCiHujKr)!RGbnY+~$vP(?b;b@SA;r>%88zehtSTh!abfF3L1~3{cV&W2ZsLs_XwqNQVt#m%3d3YdX5N^Z!go zS*DRVw=-z|J38vFU3KihhjSy5+0ki-Wa0XKS)N%DLI?NNK!K@yu2| z{YrcU@z1zZ5?Nj{DpBRzRTxyC3Gu<5t!3!y<%*VQQmoq{8OS=J)vz69n9Mi+5oom5 z;HyBK?y|Z`L_F#`y_~l%H3aew7J~qW<;&S->4oi8I)K|y6}9=?l|c9VTc&+#8S1`kN->FQNbdSxgzG#3Ajssm-iDJf08%cWV z*L_repOV2NnZg=vsyO}q{pznoonVjIIL`Af9Z~TyE9h z0O5c5026RKRM?v;uk}h6bURch45IpLk?A3QFkfdyP;WUZuEE3eKO>_KoKdULVVvQ` zRJs3+>@uxyEqaJYR7UsWu1e^7C9p)n zp?uhJ9R+o5FbS!G_~zW}+Sp?y4Ucin{LSih(;M(T6YHnY$fpo8yQ+bnf&T7?F5l12 zaZ64k!bOMmhWuP%Zr8`FyVK~W5g~Zf}(*j|{ z88QvI&}dX_a+Q$;Jx@fG7ibOxU1zJ3#5{G1qXEouTy9jb77MKsHxJViKBpulyP5Y| zG5eLBoa939E!+-@*3HN!d6fQxgN7)5!L0t_zLpSyXtV?#?Dh3@N<`j3LO^z~7U7y+ z5OG#eM2y^y7eEXBT?tjd6nE{RZj$+x5adH zbSiRJdVa;zO@a{rKJPauNk%;v;!Y6`8q zLI7G$cm9tmfPNZ((1-IMyjjd+IZzH&EG+edCbL(8px~wWZYWw#`*R_yh=f`>`w6&7Vb=0?cd)l6UdPCRs3)M)ZU386KM05m9lc6ukMM&N&ueTJ~A{Y@% zj#Ak>f%~5jQ7#9L|AL2Zy*MBB*7(XKNTxzR0{C~nhTiwX8n5cMi1=x~cIH2e(bY1LGxKeTsj-OL60z)o< z5*W_6wx#foyriUdMJ@zE?u9)R^sU^XUTCUg9l{^0*KVo0Jx#Ro+E$ue5X~aD@I7Q6 z{<;~Qz+s6ZG4B1QVj-qtqlTV8OeS1Jh9)q9wL9TicM5SZCIy+(QCCh8DGHa1I;+L9 zaT(c~$G^FbHF08>KHzf0u;G_1KnE8fGc%f{7-gYi$(McGJ4Pop-9XZW`?k6N0pY$1 zVL#9>or*!5$~TcXnvl{LjdBfqG>j=^nVOnfvih?t5O%X_N}PcGBYOoy+|1%!h^8IJ z;6slO!PCx#_mH6EH@qLPtJV&X?qUAbJb2QLN%;+y(1`1px{0YJ#*XlKNr%Q2U%OMr9(FR>cXnYO( z3wEco7QGQMPp5tJ4uebV#e*ls_yEob#VMwW@2hPjt$YIVuH4}JryXU#4>!<%-1O}D z|8H(;)ag`U!AmYGB;X!*3bnPyj( zB{qrN6MIFvZ5fPWwf+E~OyHa4duufvT2^>mzn$&&&`V}t+)H5guLvf>!?wMV3;$E( z#f~&*GWgM0=j1aw1d*sw)p+#t^^LvVHRZAVqh&e7hlk{lW?>drw%rj8;S}S|B$*M) z2HG2m*0nvcIZHl9#k7}~B5)5ve<`NolIvGxTE-pss8?sl_t1Q+qoE08nQG}>cgr4I z6nhE;`Oz_$Td8Lu{z`Vo%VY3uBYdcsjjXsLEsV&c&Tw#&@EolGCLZq{9hZREmmgjy zk>dK>?%Rmkw&2sij=6Yzx;~>$h+*uTPE!r9^BLuIL|~vozIttPsBCR(Qk5~GO#-mE z#OIg%9do=e_i6m1E}U>Bu=RKR{^ksk_gw(4yb7vEH>H66$@XWoYIp<`e`!XP3_3JR zzF`VctMb>3r}6q=S@0j@PBew{%dZw6Fk1K9C^Po6nNuk#O$i+_pt)$w_657p_{Hu!HUk@hnV z7Z;C@BhsqDc_?Fs5>E{LQaMEYM@88x=#gr3)}+`CW`Gt%dNNvwoI+19=2Y4<5u@%* z?EZNQh)S~WB>w}f(;5dyilT+b@|Zjld2ghw^6OVo^&!K5&7k2JWY6nbwcEMwxEbFn z8eMmkAHZrOA~&?HAd3}x-b})c&I(BxM<$3j9!>%(`gv_ZgNAQh_-=@j>+o^bdiV0tUPHioDMgA(+v{Q05Ee%2vQumhKM5uknpS{uvPs;#ieBWV zCBgNXvIGd94Ju!_Z|DFb)~uZNs=5Ac1oqItaMVqs1ze97Q36^Y^ZLS%rY1W(Ij6_L$*;>3Ds+fOo0(**fe86X+hzep_P1&83eTmn zhHNE@&)p!O{Hrx4FD@9r->ir@um3nJ-M|Z!hwDR5KY~%)rTTak8-Dhuao-{gmo<%{ zQh<-Ox(7n_i+|8dphWhH-yJ9knN9v|?m)nd3XP?^|9i_`XZ#bN3Qx z54qZZ9><~h!HHNUwC^;}H#x)`YVeG#YJ5-P>>;>J+Is0CDBg8hZ>0~;`fCA8)X-7Q zDmtbUQ~~lRy@1@Cl7G4=1yEs0QBTFvzzv!Z*KMFjadLeA{yEgq*vijQl7CBn0ghjz z^YckO7jg9en~wUApngb++!`*O9oy^dt_44N%lu=jFOhRyPWGFEzOn276HcW7;WX@7 zIK7?^VAZOq6JJ9t0`|>I12nBrA{1N43!mV@b4}m18eutB(${8B@C&RDHZ6NrKRt*p zhwS#wen|mfmnxW38S_dUW)BjcveT^gzVvAYJ@nImPv>slv1sjHLRRavqMk_N z(q%wS8XYT;|86#rWG%R~9bNR8y-Ga)n>u$C;SIPc8(vc4l})D4rH(%z8mSMpg z(?NR!gon)IFTbw*2Sz=xAbq)3E2hx~n;^nwfGdqf5VBf6b5Io(+~j7pY1~Lwso6Fk z!FrqN7*de+$KVMO_doxad>T}(3dpCH#l-Bol7T2#AFvn#j~^fISiutOq4(rM?kWd? ze_qqVL?Db6Rshx0WEy=I3^Vo?fAG3&|LMTMlHMB5uv%<(vF`BH&$AfGOvuVo2EB?>Bj~m2Iv}ucKxcgE2Ez}|Y1zST!S-I7|jjd56K+*LSC}l8~;I8amM~n?X-=Q*RG6(s>=?}U`10r-N3`-ihzjI4Y!lZ z=Ulb^zj4)~k}dh80>}SKIxQqsiYy8d# zKy)aD_MJ7)>L||#pr!S4ko&ABU;-%BUJQtxb74w9i=|F{Y(#)i>Tne9m0Y~AU9e)l z4))zlwU`h&pVF326dXkQ|U#h zlP~n}aPf!!SOJ8R806T(6MRDyt*{@ zIPO5zNL|y-*yzRK?@nb$hj5<$#_}h#RDceJ5jfPQK<^jwr2f}=sWjPt@={A^6ilnq zpnS^y$HBXcf$WnkE@SbTyU)Eu(}NzpE@(eCij3A7p&(`xgupXcq4c=9|~^~Sba-?oP^ky~a1 z(xv6{r+Y_EEQC;JlmDZ!vkq#jU)wcOG)SPuf&{7HQlMCHDHYt^-2#;2t}WgoEybb5 zAyC{MiUgNZ+}$bO_gm@Pz4v>*vuDovGc(K()`YCz_1xF}tTqiT0}FBPd}FjkszeVU z^Grxi&TGH1o+wPLH-rr<);ddJ2F_FHGyDfD9RzTX-FByWCiF19(y0_dmk>dCrY|A$ z5d9*oGmtL(BxSfZheR$oMlQj7yyFt&I@c~)dwQuOsgpB+4l%k-zr!)6o!dn(n%m!#?Z-gZ)V#T| z>=_6m>8TYpQf!|P2ix4lkF&gTftQ_~kIg303c_0zG1Eq^7fhs!(H=I0;_?sF&%A!{ zMZo6XYixJ4Kup1d{;5GccQZ*;1dOToQu~x*P*MWR_`ROMk+bX3a87uzZ~TYpks$oA zf0N>_Asonj8)#?sDdGcsXoXxlXWl~Dqu{Qg(iC*$&^%`DUU3Ls0!1I8MZ4&&d4 zb&wW-cpeypq7wHO>)rCrhhnTEo#3U-rp_HKx0CfjjnADyZUV|GZUT%Diqr1d#d-1u z%4{;XZZ4zpJA?$qr9!Uj`Ep^-cHmh^Bs^LG;N36Dc=~XD+Jtt(xBm#HyXavZ-&s4oWHL8zzgNhmjN8}S`YX{O=h$b7yD z%uoZ;1&1l)4!g21zC+|}BS9iZ?QJ=)mO@%YNQ2zxZk~=`mwXaq&h=7&uoe;08v-gj zc_$UhhzT$nu6TiW*ny8A&e^{(2|8PvVw3G^?}){J>#_Lj7=u{X((;R~pt25$fVj^t z=kO8^m^i@EdCODE&spIrQU_N2AC@EbW0x%Z&eA*EAooipT^kX@{X1P`c5gabQ_)0* z5%x3?8b;i6JS7otP^!}!a${3f4o?kX3#W(Y$+st9YGuozPa(}Ehm<) zAGvE4=dZ1K<3c?ZfVD;j0*hlGFB%?+wn~6kS#;U7oUEORmymL1>%X(Zk;5#8iG@h? z1SW{iI9oGOZZtN0H20aw0p{?K^t=fn!#wI!<1-hv|FBaC5dO~|(JXBn)n3F4l%Dt~ zY*tR@DmwR&?}7UsL4HzjX7h@ik4!>g77a(-xwIHqc6I=RsZ;6)?B#v%TxK}%UT4n? zcH*ot@CUc~NBd|yNE3(*ra`nf+ZmwWORyH7e1LgA9?TFIhnq3smWLQd`SdmosoS-o{|mm!fCo)JIGL zxdXor4a8ez|`UiYH;_JYn;VA<(iwzjF+Ju<3J*7J;e=2@ji`aDSI1Fb@v$<$< z9TXf}hb!Vh^9V3DhW2#hrZSz6${XnwI|nz{$hU3;0UoT9ZO5NV_be;0$>K?VfpqeJ z-jLy?!oR&;)cFrhN`zKN(>84l`Nr+jUM=d(FvIH}_!cDf^&i@mTcxP~NouZHJ1c zJlTh!%*)pK@m#%XGF2>g1pVYg2?&K!sw!_1K*8f~!0+e2wruLBE|P;1o+g)ZRh1A$ zC?F5$G)`KK>#bjk(!M!ce8^m$-En%Ipn<7Hb7w$d@zwNmI}sBZ{v-4L$ExK^moBf| z=|iB#x{e_@WH11>Ur0(3BQKVTAH1^c5qp5Pr7-wrY4PjPNzjk@5R^h~oqjk_ z7w@zre-Dl7+in^a(%>%ZK_~F z+Q~OpKZp9MlGZi%2{U;f_-bS^<;i%yU6O7%Th;fyzZ{Lu*F+mwr%Z-Xg9F?50Qcb% zp$v+q@h6bKzIwMPALI?=N_Wobqxgt<+oPh#z%IRP~yQwEh#k_8rFauRGNao zxm83+&c7%0rv1t^tIIfMpjmX>>C^Q-qNBVrnc-&lyQ*aF;MmeE0KD^!%#wi7ckcel&_>x z#gdda%;hJ=2o$OVE)wbsAczZ5IZCbn@a{AGAZ080-Zm=K>}H$sU&SNT_evb2KFr;4 z(9%%rVOaD}CHuc!Ui80R-nZWeWPvN5Pm-sU@#XrUC1Z4S)EZcGr)h1PV{}){m7EaZ zQ}vCWtGhNdPO)^N)+d(dPHFY%FW7XzONC~Kn8AHiN~bt#&7zc4)~P!#OtfX#W03%H zs+pKDhyso$pX>TdHuZcw{T`1I*+Qp<=1L%t)NSz-CA)Vo-%1Ggo3RbgGd6t>HABO6GS=}s2`s`;@FN0A<9IK?nB2AcgNkLs1Qr$+tn7*Z4O1y20n3+zK{NAd z40hQQ;rpMSIAo&?V!0TaXK5nt)lv$JrmkNhuk11?=4kWW9X$fEvO$LcE0R{kmb_#~ zB4fwAMmQ|WDqqb5b42e9&R;ZK8*}}mqVPquIp6cS*EGnBxr>olzubMM&C$;wU_XU? zB5xn{VqMcly7#!SFp*>l=c5Z_K`|O6bYZ;?gznExvxJO|T`yK;SO`bd zEcoe_G$gVl*BQzY0M5??&1jAT=`^WE9|EJC^n>GLm#GS==F{V)&@!ck@arfyD?0Ra zV)ze-8eoLyNdthd3sg#`JPInFj!YGjM(M9^sT}w;!DjJKrX2W28==u|z+j-{!ypmy z_E1|ejs{gG9TSv>0vT$Jxr%vh1nAyX(3WXAkGRTKLV;Q1jL6E!oK1Vwz5RZRPuZ-5 zm04>5`XqV53B8}9_V8=K;DtRxn+o*~wF6;%5vXsB6>78p>ExGf8Qoe;sr`Fe7)5an z9{_H9-=)G?J_B|}16d@VOzfk+E+HCnVjxTy=GCupOe>7azgK(2UE}<9i5CUCyU%B{ zWkD&T-cNw-jn5sJQoW`A_JP9*H&z%Vv80(=iHXnOsh}@m7z?JJ1iJ!R{9ei<*u_6= z;B0Qo0m`!g*PI{+1i-oz`HNZM0gOhQ6HAl{Z}tn(n7@vWswO_-gUxy@qHpY+ti*`y zX96DeFfcYLj$@5G0ER0?ZUE^D{w!_SSKUz(zpZ3dVz1SlRjO4&G_niyo8R1d2?L;3wj|dI3UdYbAbguhyCmtC^9~vQ-@kmFL zkk198UMoPB#Nw?hRp$>N!qs>RaAisiB0rYBn{F^Rs z8$S*ic7RvpaUi|(kHFK~;I_z5i^kLrq}#EEmH}l@H}W%92sh!qhtpUeh7ghsleV{~ z>jvY@dBLK4q?Re-%8qQTe47v*p?!yQAmn)Pt9zkg37#~i_8OOPIY=5*rcN*;_=NG4k zqivY1UwXvHwTEH{aXe)AG9$eMOW4Jhlx`=(OvDhNYf{{Gb&c+Xz*gl!#0k{$(91=$ z;r#lxY!Im3(_rME6x_9yNrE$*xjEHA3wq?C%^-!=Lr)ZOhn>g%V5aH?zcx6}uCz{$ zHZS;u0lSrrkXAe2{+Z`oi*}|!buHHs#ug$#(EC?>>!n-p*l-(v7L9WmJmZL9xlS(4 z@&Hqq3b_|HI<3zHymoKn!X!|8YaG7Tq1CW@J_ORCLV!)Krd_PywrS5!8uy=7Z z_G)djZQN^D8D(sXG=H|t=(#RI<--r>%4&XoI6Aq0%nmf-zT_6c_h;+2DH1ix5CPtj zxIMwAyI+TNy-i;L)I1R!jJ4yKc{Ds1;juX^{W{qDc=zYh1638Pn`5#U!#V*ISp8Vh z$#-8{VK+UjRy5{sl?sgWJ^8et9!)mOE01Xc&nlPaVRotwMAMV?Lo|>%%ROfQ9!nqX zz%Bvv!yv8fiFw>9B$(VLDw`(jQT|g5*CNDSU$H1@?o}<9$NnOztPUe_J~f(W5I;Gd zm)P`4rrvQY8&Fgn;*{Ap58c__oO~rn9LcHc@nvos)`xF1S!pPt-Uq!9s>*mW|C;bg z<4@a}bNg~qj#?~~zCfXPp?3m?)sh_f?Eu+of#LhGK3mKP3ZS;$H8?@}o3rPnue`z9 z0s!7O>h_4i67+>kd^d^n?yA35XH z^?3vDvWuU=z)@lZZuSOymC$q5=csJoES3yWpBfZ}Lrq-kdWa2y=)c#)Bo?Hxh^T1y zUE-*6Z;Lc!0?{P-tte#;PX0h21oouYN&f)@ur*PwXZvt4 zyxIQZS@a_dGqT^YspA-7L;?43a0<+`0z$eoHAWgD6EdeO7fdy+OJa-7-X+h``8R6g z{6_8M5E3T2eC&Jss($|Loy6oE9eo_}b;#A!MnVg;T8&tk*QU($sFK}i4#GCD(Wl^sTuCIwvHsYVsoV@nZG9Hq<2DI+kHWl%4PN~^===!)U zp16pM{yM8?EtL&>gOSJz;&(X9i>wq%9lJC&LA;q1^!M`VW$FpWx^~tVJgpchQ;Q|; zL3Q;s08#emfVYCe-RL?47r$$@ncMNmLV&WOz72R4d#LQ$oz4B&>a9@SVRu(7{$byP zQaXpz>p<%I;FLXL3gt<)_XTZL4kbCal2`sm3;b4;o^2g_b*pYC|1q z$QkmRoGdT?@YifXQpxK9N1L}G(0bbv-eRM1*6QiGc#F0H({FoALRke+c=H`lkUb4R zcIm$%d;I@`Y*jvSZeoQUmGzg6AN&E)_kD(&-x5zB_c8(gvddkPe?c~~JXh)cVC$%} zt|FLhz|QrI_^A#{Mfqiy@W;=MzGKFi?psT--R=9uGy znzu0Gi3+YniYMgDQDDT)kK0?1T>JyZ-=$EyMbHSGIvz9|&!I9=(gwI=jkUxT!hEs& zj^Yk}`^P?C$CIWWvdSpZ_n< z9Nx^?Y^7gukDV^-$yfm#J(rhX3Yvg>jVTJBcbQB~fAt0}`f83D8mSSA6mvDINl8bW z4S#SoJFp)g*_2+LQKBwGM)HH7tsQ@rDZjnvn&Q%EI!wZZBwge zBOw%r`11WixNu*Xt!!?Ygr`Y^xmRg>mT6K!l0mqwGg5}%gW4O4tV=}Z_TISkdiq)N z*JQreboY6Yh=Y{;HC^913QT{1wO-8Pbj7IYz*Jo9E`H#f*P{Z!tVa9Ya4;`$fkMWC zW-PAESNjw;L;2Xf;$`o-0*T9G9kG_m*+oD#S|D$~Szh4#>C9JCH>OwxB zeSE4{l36eDtU3PCDqBk@(69Z&>PDaW?IzjT_{vAh@D_Zc!V=lE3Qx>#o8)6^fP+z` z=hy4-jQ5{N`;>O1Pi)srPF@s!xy1GIYb+CJK6+FIcSxJ+u{V$Y9Lw5P!dDhsM_W8s zB0LW>!hPVk_SNIal$D)57tmQ#Es?1O(nY;h{rsA@xf2o-3lQ4`S?zrqd=GxiytGj*XW#DT z&kKXEt*bhY=!GF0(9g=YLrqQW>~|WS`KreCxzbY1_J;Cwk{Y5)2!2$2RjjP}K3L;M zcc!v&oPGYJaI|OZ3ma#Pu3=GEfms9D3Clu7zyM#R-Pq7ce{9oGMC4phx2c|bYEt=M z=X7g1$A-an^MFSS*xKHxrH9+IbzgyZOrfWds-YRYAGRXv-jn4V=B=GwQ$A*2(82kbXrJBkol-8kNCJ{4C@ zBKzk!fDe$q_v<$3s$yb-8z$B^IewhA>B0=CVm4mVZbYz||Ip|q`1S7yytO-ae=!F`bX(#J!fwVYvyB(E#P(EY5U2FB=D zMh!awe|dSSys$Bwse3cgF9uQVT<*(hd?S0Bbkbb-DL^V!{)MrudXkzXr^<-(iJXO! zPG+^Fqhy_pjXL@0W~n8;p7R=Y$)tCOrG})3h=3EN#Y*@3OMb_@dF7qj9SW|RQY)uA zzSh?9Ras`8Jc=wKBt3;JtKYw~%t;tfsH@iX@r`;!hUrb2GlK@(&Tx>i@$pODKvwfu z397}>qUD`16bCUp0YdtGzdz}zGtjdHh65=!8lynDbPMRi@wsiQTd*Kx1A!CX9W=tj zC@NlcrornGZ2^aw@wXon&d@;zKq0$O(*JbisfL&!XKDj5NOEI84!kXEzABsntb8Vz zyzU~(H6D_k>FVm5OkW)MJtzPRaRJKnGr#~wRLiH~9J#g!Rihr`;Kn*&o?Sa6Eubt} z=gBp2y&E>6tpKc)uo{2%ID^bTO4w}k2*WK#rA)tLpjH5W%e%c3mQ@d?>T#lRJjf|< zjq?4@XF+zHVz-vIg8lFHS0`G>n(>$HdOzlCKd*(p-X&exf_pZrUz@|8&Lz>u3YLC4 zA~>oF_KjJFjTm`-K<+D{S(BrzS0^LRp{iYXIRj~ZfLZg3!21Afs{rIC~2IKHy>;CDIcm9KdF zdnHv6;iu?XY~+v(BnGoWL`{H~o*P(Uz;}ZhWiXBwYOtM9Vro-ce@NSx#+?K<7$;a~ zwaIpXv`@u;U6=_20|DCz^hic_q11l*dBY|zm6h)30<)eN)&Z8(A2m)|Nh83MEOuAW z0ax#4@B7zB_PUwCWN0!4uk`y|yE8DQdQ#{^FK;i)AUgNF6N7D>imUpF4ir?7OtrWZ zo-aAuuJ&BR@PeV+7haI1xA1vcHRqS&im%*?#^xCMS01F!A`D6LWjS4*_q%(dXw3K8{S_#48b3fCHnAaXvZmn6X94Bq77j1y^SBmXG zYG0@~PP23j`fU=+l!iXyu(g=F9M;^=Eayalc{L>GQSl`uJ`xR`+j~E3a2or~3RdEMTIrh%5zt zrjBlk`u*;cVnzayY)Iz=`8$lpAH!D5e#$c}J!Z>^**j)SA=|cP|5jsozd~Z^SI9hF zOwqmP`7(XNlhbl@Dc0Z5HEa$S!DCesePjc-#^WlroCs6xun{ep_u8v@_~*w3&mah* znJT+HnWyd=HOi%L5wLnhx?uP z#WiY^lnBQb-CwYa5wKIy;kKP`uO%C4E@J}A)yB$<#K~fQ)49hvWTM z)(&lI9-N|SJg%Hr{e#z{DY?t|NMT z{K)!a%HF#@RzJ{dU9c2z^E4OZ%t8hRQi5q}7v?=y2p6z*B51)}zOO8!=OXY34Y8FNi9@D|OyQF)W6geoLzfMxic9Go2hHVf zpFN$O4f%Gbg@`*TYxOVpFLW|$-M{Onz!~#j%{i2N2$W9$#KThm{Uxb(?P2oXX5b*3eb(l@YgDg6{tbLXLoNJ~Hb z;_b4Q!S5bN4o%x0`(rI}CFHKh-P*nm2B-*|QPv<-1KL*AV(!S33vM^_6`TxQXa=_{ zB2aseWnQ#{k{9m3-oGC3teftexpH)l65p#xTpwi3o(-}&uA>Cb&w_ES?V(BsT6G(J z6IREUbMF_#r0cnj0+0UE3(GX=HcJ;_dl-RQl?>K;*RAoPG7eNbJQyUDeKB}~9C?+hCE?$Kx-i9>yJlpO>TZtv470p>n~wvu4yMsdO1GCn=~8re4h4oMzw*gr-fu` zxs$0p5sD$0<1hc%(`WO4Z|;fye{2r?KlTdV8TkwIcrx$D&%NVV;L%xV&tw&4N`X}I Fe*o9;-N^s| literal 16768 zcmc({by!qy_y4OXq97pB2oBvL5<^Hx58WL~OAG=tAgv%MFmx$HcS!e8f;7?{14sxA zCEf7b_<5e^`}>~XIoCPYb?T3~hCOlbeebpIwO;GJ?jcxRRURLg3isBnTlf$K8O>X_ zZZ`rC>iby0E3*$T^?|3`uA1^vw~G6z*MWn(HZN6P-nvy5fp_s58#sROR>8pa)-8e` zH;>z$PI;EMZoMRh$h_2kXS$V)S88PiUp*H;4@3{*6H!HdO7mA_m=3w0s9Z`X_n=>f zPSN}+#bDQ{a`aK$S}cM}(=LXWSsARut4yV(`fXix?5;H{tGyz#=I#(%SWY`T7PdwH zNM1Vy%8<5yC+WKBy8dkDJV|_YRzB5r`|Q;D;^=I=W_;Sq?|lAz)^(t%&^MY95`7?z zu?C*~9_Ar&2%k2rrg({;v=Ev)x0B|kcw)4lh&Y&A-v87i0ExD2SW^Ro{4RE9oSJET zjK}WO{`whf=Mv4>tv^@@rT3dv&<&YnQC2=wj<$4IgMg2>(MFI3ELsIpZs5zY!XeP7 zBCxe-`;wyAlK$n#-ldxJ!ZCHx)JL^|k`>Cp;qWrBmNK|q%P-*pC0zS|V-uC=~vwbp0 z=tshJpZfW6-6bJeCLSG}4-&0V0o{Y55(xVyRMi?+%J-pHhmPb)hsx8Si|J~!rVDKQ zT+t>q@LjWoCvN$S_`#OMJ~B*2$UOs2O>hijF#m>iP3oI_{Nx(;qj2Ke`jI#~GDA`D(Xfi?8;6XPHL77YG(yn^0{>wE&Kha~=Td2&SJXEl znEqZ*0QBf;{d9V2D&qRL?10aBhZfjT$Z@}ePt~A#pC*ACENX-g3JdgZ4Ty?ITqovgZZA1i@v!&NCu6<_m$?o`4#6h;KVqx z-wnj||IRrUH8sIgqo$@@Y*up=;%MW$NJH+oSi|kRRn0e1VgeT6)xbZ#wO-KXqsdC^ z{yDliaP7DB{ON77&9ZDLGeq<4N2pufl*6UDbz#0)}E z&cZVN=h~67t?ODvSw*$^DueG0%&_@%?o6#2i)Z>Weq(vLAgJaKW6|)jbOb7RNb@sh zsAMB&TkLCS#o1jwWx;)4z#se>U=^qC5scA}4`j%5jS7Uv$fVG+f@{|2eth?>R^|nJ z3_oS*ocXS4fHg!8oqrz|9N}yvwi|h!@z27VA$K>URcl837$La7;rk0M0ddiB$|^;6 ze)zW5eSG6$7U=0JhsqrZ8&S;fg7xvnwdFQcmGaaxzQ!OgRsTw;DN;Yh=@CS~C;}4A z7{FVCJf3w`W3pf(j$vI$6d)J#+fSKq-W1a_t9^5c`MUn~Xv-o1+9Sj#R`upG(cSEF zQFQLQf?VRd4Q@aFI4eXiU{r@l>HQAJ8@5D9G-r|rdh-Bs)>g2-G+uN$IAgjybA9Gt zQRC3GjLmYL;BMx2wT@)bE&9z}UX0uV+4QGeZ}$1^rc4;0WoY#a!}pv0evC&LABQZ} zYHa=KiufR*qZZ9*;0sJOgrhjm^4lwyF}5cYS91b0f=>GNrzcYa753F)HY4_CZXE^V z^@ju~VS!WEnf~wit1*K8(7D)Y}W^!kT0sOdOBb(a%w37If0r#Kt|VTu0G!lB9vT@(;l~Ui;WADWzmP>}ydE zkA~ENkj4Aj6f7!sY3pKczJwA0U5z<9T?djwjhnYyE7Z>R=NmVV+?&t*`P7o2m(MCj zRS-r4LhA$bmjyHK7n@2c=gK+8ek*w57xp|ewZ1AWXhyh>3^44VD-7*04}@pr*Q<7= z{-&kIlB*x{wQGpJ4b>OFC_xu-3BG@dN`$ONrrEZ+c|-|tS(9qPmfgbv9qX^<%7|KgUy5Ku@SA9n4(;V zVqo?Ds_@0cfK4xuC0;F0%;5b}eX zWqv`Hp)rh}Xp|yJHB9!thzt*nrsSNwfEWeKBeo zfxn^0=a_Ri!EKZ{b)|dGc{^ByC5^OqU#u6%tFnNDwSe%`U1Dr!0c(wp=Gc6jDTVeO zv>lAL{XByg)`%TfHlHM?)-7Y_I{J1@f9*ElclNZ~>j_QSN9iRA2%tU9!goL7+nb)Y za{2AnCpdatD{l<>Ng>9NG&yy$F9#Onl#YBa?o`uAxI zBtC`#HZl-pKmSTKwbmJ3ay?t;cMz_0eK2pj$4BnqG2|+E*#hUeiknEPS;<dD zJ&|zV4t_FGlU;$Dr`{BHSsd6tE5FK-Z8`Qm$+VZ))627~Lbm~_R&8k(xG&q(=}2EF zZx5JH7~VU><1td$FTT>Y)GrR--)UZF0$qoN4u~$b!3%R|2p1eKOEr6h#ROc(m{uEp z#n}uf_f^kFDm^lic-Vg0Frs#L0w-_WQQ`6Fk0-8nFucxD8o0`=F>4h)J9=_NcIP_( zA(RiuCM2CVQ8zxE0Y-d~e>PgsJvPGUeld46qB{HbR~<_#tQEDzBeoL6F@Q{YvH0Zo z2(gy^c)#f$aNjTbmBdf`mAE~I^6e-7#9!P`{n{PuZVaDm_S}uPff~C1+E4K~gFLz3 zJFX@+K1Sl5GKw`pa!Wor3}1E5`)maLS>+Z7AM5dsMk-w_Nk=oL`8HT@@-e1*p8(jE zGunrVcQtqG8(sFhyqrk#=-D+AESiDMq9&lC`&%Vm;yd$3nXW%o)y&QlUFWZVg!u5fFLAF9s8;&uUGr2P@^0lWU+THx4F+b6s!Z0q^XydFu-HJ~ zUDkn`j)5BNUKRw2aQZCmHOJ1y25?50-eT=N`Y;UQSgrd}W-_?0C;L#buv8GKS?9^MvBiTb-PzQ(Vi@MI@SI5?!DktO328)@=OKy7iAld4;Tx_g3^K&J z1Ii}H`$qG}y+#9Kj!Sg|b0;Y^Nq(0b+>(xkzdXJO35y9sSRILzNOa0SNOoZinX{!I zWkRPF%&?|2p?3HG%+Fm_$d0*1GjbrU(H{eF2>Zq89+cVpX_R-gjkYDOuj8qTu2j!^ z96C1d7vQN%SREcXTm&9{_kPsMeL;@a5bFk|1*OiOV;^P*X@Ihkn0=}pFjG$Xxm;X0 zY#%UQVr8_R_7^%~7D#z3z`yTNRofy_EgnoSq=-FuB!q=OcowG^h>F%%AJ&W*Wf;sl z^{c$;q+4i}C4L}1&1tDA*O-1(M2}&^OJuDZ%{wF!7?uB%m&@?XbKmTW2YZ(@F$Y+= zl3?`RITbUnEuI?CPv?*3xZ35H?fYN5j}lmOb&KBkEX;W~2g5_u@w~mb)p_UHnE|Tc zlOIwkbKjX$yd?WBB`(Gz$-jT^86?_5Rt@zaLgMOV=qdjM5i|GlF+UKUgF;NGb~^-6p-jRg@PDJTt3K1l#5^rQPd>MJ9p z+#GVle(Mo8Yd!kvF9`lWeee>#Klm#bbT+5ucOIeUd*ladI_&=enwhbynd@ej3woRP z0+>ffC=ks4#Hd~W_`#KavF0c0S%0XS^i{J@C7O{E&vU%*hnfq60lS4$Sa5y{BpQJ) zw*Gr${>+Uo@D9Jf_AdZ+}=;KjYBVI!Pu^%QS)M<9b?FX0NWoFKo)S2rv>NV}0?-QW2 z?IU-e5RHs7$lb+PE!;Rnu4D2o&ZAeG#XNd>^)#E0st#hSHy;N(iJACif%cvK0S_?e; z1Sf8nIaeR>g=-xF);G|IMG61msUaLUYp3KKpD*T{-_ly~Lpr_y@PCNj zPb}tw+xvWK^rM3FVbZ8MZ61pw;6;^Gb|J6|3U(OZy@^OCVK&HR#(TgKf*4@?ZzBjd z^=I8Sl7Ye?k{v$W_o->$r;Ve@XZ%t8`PHx2c;*V(U~%ie3R#ry%EFugOkjZFN5f3TTu&3ZshqF~;MK7niL z`@qNR$+?2hz(6hzR4pTA7$UAiaSy3gZF(rqZu=QNJ)^pc-|&j#d9R%;WaYT}b#wFA=HnE`?)yNcbNUyl zOz)$FH3ts$VJYtxgEvZ=ubR%c3#((=7jlr$=?WXQr!$s6g`~jpYr-zcLht@8tR$E< z*%6%S7nBvBOpCu=_83WcwUff6VG@@>$-L^k_58S?x^;=Q7ei(ke+Y*c& zS@=Z(U(wRfREJ&2DIIxW0YUhmL7q#+qUwIuB!zlaJTTrGS1~C@Js(R@KEV`;%AjGN zhU#~kjA%yckuZ6rhR|$;I0-G$T@kmzfxE_xDJM!5mZQZUo>9I0zwHpWxYNFTW?5wQ zlT&@SBISJ znyZ24Yy9zj){PB}lng@9x|$$wJ;zx$_9oSxbven;fGjyZt@Hr_UHqk#n=g``L*+cj4d9!|E2D-xljj!g4^D$+A;$YDp_~ zQW|`=*j2mxO3?`FKGbPUZpzhekvpkk&F8R%-EP{@`{Y@$81a&|Om4licwyqOrs|Cp z-tcvFpUWMMUf`B(6Hk?|_>!bmYd=j#V)PMzC)2>wPm1@s|A1HVd`%QDf9$phyYyjK zhB|J&)4cd|c0=G?G%#WUn(fO6kjtuwEfbU(Y_%Lg&PZMach&C_|t8g&t4$+-3n`F8D3#yq@wn?GT1=B4q+z<16* ziXw2I>Uo)dvr6NiEM;jNwKh zyE#~8YZ@>AOnu*sTQiqRA^cuv2t#~xP3%Kbi)`dsM~bf(@9-s<@{(h5yifKzdzWsa z@3rp!XrJYM%7OE~mXA*Db51#c6lM19J{o~~)J&}cs_A^{rR)M=lc8jt$(WBl3te>p zJ)Z_L%?K@rW=!;^YW}wMB~9^5z1T%JK4G{Pt1!^E zRZ22C=~YOj>Y7POV3clAz~mI0q7iEt_Ro5m3z{~>qRBqzN4xAGGZb2s01JaB%s+lh z7Y8p=l)|eatW`_#qw9q=I8Ml^@!4m&V~;~l5t0G!TJ|ZG#xDJU^M&;WxtCcZDSEnV zeL@LMSe^;Idm(o^#w!^q33Ph~z;z^LoAb?nfZrL195kQyIbr+{NKJS2u}RwW>e;A2 zBBJ51MllNn(2vp5L|V_3GHHok7Yitn>i%%Vyfm!pryS!U=mk{8FQiMIRI|CGtga%o z+m#vA1`p*{bj)@GEvEy2NhSdXo787Rbq98?e6qzT{-&C%vY!a25&aX(cf?JD9ZcRF zDXq*HI{S^Lr$0y3il$&fEKs`Zl$FfE#a@)u);>~F=&?j5&`c#Lto zi@;Ktyprs{WVf#IeGIsav7`Xb!4^6Xcd>GDXTUN0bx4q@VT{4T5mK0_q+H!oBr`1{ z*nV}_n+o0nJ%~cep)FDS=Cd_Ushb^|q~|HvFxcIYt{jskB3U`L!jPYva<;5wh!dYBpBX3T^kR48gB? zoW@6iP(`Zcaf)%?byiL-T)0JscNS+HoPb%%!mPsKHf5;LA-x5@tXT@L)aBr5*L}I& zp}NR+CXb1Cp|st)mHMvH)Et0jd_xD}|5q@1nPPwR7wGyVHC5>1BqFFzr>gn}vX}M! zZOL#2l8V@CRBl#cXY{Y;5$(br#~4(db{h{37q#E9+&oRew;vzujlLz*JoH(4Do8Q0($ z_av9{pKwGs4DXnveNqP>3iJhnc-s*f>pz%+wo-ZhB*y%8<^#7M<{%FZ8cRa{H5vr{ zjpCg-OC^*{32boL^q4Gx^{ygs28z;v1pN(!W6+Q*0#kyS-ur=C_rqq3=*u2avH57O z8~6=R`)F@upWQ2 z$e9><<0I`8+?fplsa@WFXfoWtrX(A0Qt$Q#=j$Stac&e}N}KlGw9Ew|Tdk3mv|#7l zj{#n0X-%Su3f`c*-jY5b%pxmWc$l7#FB&!i0si=3Od6dY0!l*P$bxhe zId}tKh?SpLXo`sd?GSk!lOO&WK8NIfReu zu<@Je$HANP`8o|oDaY2^Jg%=UO_*JB8fj-IJcsFN!{cLH?Swl>f($%^)T6kAxSo14 zeqPs6?IcH(lUUuN!EWnn9W$O?9t?T@Dd|Dp`pKiODjpq8V!S=YO?HDTMCW%|czoh) zw4=RAb|A#veo z$`sxJ2zMqGh++A|O*v`mCr-kdS&M0+TGm_5o1Ug{j%lGN-MATRM)*i!UvAr5C~7Sy zo>>c%TiPo#+dw>ALCZr z$!nn=zZFYhy{AtNCjpF)uaMrj^&TC>F^?4t4Q_$N?+qoz?To26{MS7dzKBswSmXhIm zIdm3;yDkvn|9KCClaR=b_CARhn-mB^%EW}z+h%00r;sT9vxbudlQpTeHTkghcrx=-v@n!i^d|y_=8UYwndI!~=!+k3}gdq-@FRbp$M@DP~(WlSpy|r2v7|*9IQ{{BA?MssG zZ*r)FMD4Y8MV?Vbv!rh_c+DD|u|If*GSo;d{Nh}EF2omjKM-B748n6eWlhlR>bK5q zj5P6?rB~E`$i|t+ej2IajM-)5@wB5K%om;cWZ;ACpdov`vXB}$8!E?NVDHAb97lV& zT?<`i5b01CsE|(3ewK~`2!hRN%CQhras@ zzDyxk0G~JUH3`P&EfcD;bd8pkGrkE%to=odMk1#aiO5RJL`Zs2JuWl+j-R>MxhdW+ z)4`R(z>1jG#@8?9$aiV4qjH?Zqii%tTTryrE?ty7&dj`(zG;Oj;lvQjEBQa-Nji9HCo3M@sqWklwtI|A=(f^it(u1t@1 zUdJ@CLIvy9yG9193uXoLk$0j|b&vC!2UI*;GNyVZginnlZp5*!)h;HW^{*mcBLvj_ zKmR41MUD0?UU@iBIqAInp}A!VC~JzC--A`L{fv|@n&3~x&JzY(HqngI5X|pq)pim< z<46&BxlHH$D`U6%^8(b=$%?m#1BQ~NoWEM$&9C}oubES}vLmOVJ>gowM2$MHW>Eus zNJWkQ6^hGl%oou1Vi5(<41veABED3($vb{m9A=K!eUI$bha#172EQTiXXif3I{%?f zA@+8ukAEX4Cq@L<+~J*|IDRBT*^(6ie!0*nztr%jV>;i?FPXV8IZ(Q6Pt7sghz#pD zZgQsCUz8ERq8v3A4@2b}+qJ7#r9z?~GBK1OKZlb>3ciO9Gy1h7*QCHxjL&y7HAM+Z zOki3qZv++n*NI!$W(zKXdW4nI5iR2rl4?zmh8z~)3{)q>_ro^;TNF!yMFnPU<6m*M zeY5#?1k@hsmLK87$pwB0zW30x@d)4C^=NH0UqRGa3n7|}i@$Sow2LW_iy3vR>YsZj zrvoOcU5tp#1B5YW#ZmP)R+rX~u1|<{1oB_ca%M$#)=!bj;)w#rz>pJd{0raU%0|{*xQ^=nLiPKwwAtKcBIXG3bixab)2aCqsCp|d*UX(i5Too=8;CMG2#z=pCY)kRL{lwwvYeCrO_75tJjd_I zGI!^^cE;Gq+Yfi*h|ZPgn|#iJx+H{6BS_MXi?&7N2$y(o=q%{$ac2fyS_F$naGKII zrB8lWx%~w97Oj@?`wf1Vj+ieYw_qgHIF5EGYJT4B*tc(Y@J8L`iGz0n-V^5SNSjbI zwU_N;3QOYAOl)i=vafL`YssIt0_S>X#VwZ^9ZQ}hf^z=swlk-a%Pan_S@P(qd-L)8 za=fs|A5RLjK4I5WSF$xQw5JDkJa1s62h3L!)?ZvlP#BmV_ghh!PAy{e3PwRAHuq3= z=Nqn%5*>o4M^yFAwx~v!W9ZF3hVUh2C{W?ec>+X|NrPm)b8w&#@UlcqamT$74rn; zw$OpRKKU_#WWN{L|V z;~#bF8A!uK<^0rEZer(s&9@*myrhgbbu)R!6h?HvOKp_N#Y|q$_&n9kiiZ!eU^#!5 zleniRPr&^E&rSG=@J2af3G*2)9TCpddS&pF$>PK*OZu0KW2ESZP>Pj1{o6u6L&Q^B z*DNB$tpD$mp8)^Rm6Y6$0z=@7y7Y*y)Hnj+C6$8!5UMB!YPeTQTN1X{8Tm9t>Cwe{ zZyh5ypXnf{SE8S~2F_*B` zrpDI8vG~R(0?f-(-OOon1wxN;7Qr3K*(@0OlyD-M`(0kGOMW*_V~XZe>Ch)OL_wB3 z$Y)f1QYT`ukkJ%R6zSZR>A?Q2+`cRC8Tl<2Yx=5gb5v(84@uDt)~0!X6dUVo;T6!U zqkv=Tw^5;bpENyw?n!81AEI$&Ek-+yZ>T6rDrGb)y6Y=!f5v^byYQe`1N?mvgK@Nx1Pz0sLTln%|J+!==pt>X+lQ?Rv+{Y{j zx753Icno@>r-hLLy>;uP6Gx(Xti4MMUvT~mW0I{03*6mvPyBW`nOA%!$rw#Zz0F$~_D&IqyNb=_)f)VO6%{mx z2HED%*5xz{I++zGAYk+##LkpOjEo{b=~v^sv4I^quB10{2~ZZ2y^2VS=q`okGewS* ztSK2Ns%V!p3!~J>8FG4AaaM*ye(6jE(9f`{QMM2*Thym36&C|V^;50Mz8+|*}MJ0TJ@zrB8%=fyzfXug5b7Q?@EYAQ82 z>VFpXIPwLd@+)kG)C>a;qO`v+R$+4*xkXP>eA{@h+t zwwzrQ_HWHo8GDaiAI+|pR5%1w+(Srg++rCyYz_jLgks1mn78Nb<=h>)<%K)NC;AQK zyQ8CM$>zMYfDU!eJI2Amx5IGvpT5yD0REiL@{(*6J@Q(%ja^=^9un6S3#y!w<$sg% z#4oWsmZGP>>>pXgSWsVVJkydh%2Hv0U2B-|jJ);nZ18`C4(iC!zPR_$9B;Gu16AjW zslfmKj^w2ebQq7U&-ALhd({(wzXN%3d0<0kmeXEcJJ3b4P`y!bAbyA}DB1CV#mG&% z?^cLJ3B$Ou`5-}n&X#UJ3)lV2c>fI~UOW5;;RJn!Qwfls9l@MoTT$p#uBHnfC8DA- zHYab-5N5L5p*gL8g4OWz5`O9a<;U(k40A>%OXlG2i29oM^}gAu-MLecen3c&DsP~5 zi`3|gKw6XKh>o=nPn>cLz;A}q=RR-dng}~JA}0UEWT(gLYf(DNNHkoF3pXC1zSigp z+hgXwVQ+0TW4IGnCfP`E&5yGGeBJ}inPqbeokjPyi+U$9JH;*#TAn}CkvEFKsCe$) zOI;XO@wQdmUHAlD61c(2#G!E&OYXU*p?iZWY*g`5tf4{8K^aig7>BdP_>Y=RI&5*F zn;vE!Un?s3;h01Rr6f2wARdAvs?zDti5V~7y;t$5ZSr%3M9D$u2>?>hLjcsJmuv^o zX>2fh%ZB=s=E~r=55GGP_Pg{Y2CHQez33>*GLMj&f`AWCTD^^-(GOV|@{wyqiX=N% zgQCmI%^R8HdGh>OJ>vZ6d(#y2_Qx4W%b_;b#B%visBor{1ts+6JU($&`xfs_vR^8cP z!5DhiMu0+>KApGaV%8`{+FiT&%SJ~0Y;}4FfTqEL3rD-5VKjjHkEQq`oph+4*+y zNEfA=IXl*%##`#j{B@VXwpy*$mPQ`+c^VbQCPFOnv?fBhb1X?`M`1~mp<6;$Rs&fe z+p8thuKVXlTT5|u6k&>*rf)H89dy%;Uf|@8lZ8vHDX*LO)WZ#X^e27CzbcdkC9s?wcz=Bzk$W8bIBvd}Q$#^2%f~H0mm$ z2r&!P+B)vc_I~M}w@#-x(PwG?Gu~Pem$wl8(q2FVD@<)j?8c6y@x* z9-Jh!^A}Q|7lA`SkLjoQBq<2b)gphW-K0`>c!L95vA_WEJB@`bWUN!cD_FT+T zI26G3-#rR?Q`Jfbg-7b{mMjErFuyBe{V$SZ)g>+U_qpBX`!i1nx0s+q-v%b=G9In- z{|3n+ev5oAK1 zke^sC0JnKo_9ykH9Ei5SWLeL6Eglw!Q+rOR2u`((XFsc1tD@AZ2-cvP(vd*6;S7w$ ziVWQt=JP;KF@O6Ii5yP-{Af;)WJ-U=zvNa`@%B_CoT3~%rKy6c+wz)H*X^lE+RFpVkViA^XR!BP79jBmz8QP zS?KAq|EBfQJXwEvFj4qZQln+njXbx8Cn>$h80%vhA*y7?bX!i31LI1{*AEBT_>hK)n%ZQ;$ti2Wh>+@UfiZQcO29|4p@f^7f zUoTYf#85EVI} zR}2~xgD)G4bVnRBp~7$O=ZW)nLP1mnkB!fE12R2<4ESmm<~sGt(I6taOh#n*-EEf8 z&+Xc0Gl5dm@s&Dkq5wP5<|8>iydy+%61DePljFWI8U|veBA^%lJ0wTPXP+Mj2ehi0yeK9y`j!PXItT43Jd@M?KJS}i#x55@}w+ixSG9w6?N+5Ph(r4AZFI5aYHYHL3$-86r2k+YW{Zo; z8)Lc$x;A($Acr%kvZ!>G0VsQ*v2M!0ncef`OacpX#M&xFd2{&hnReMCf9yZ@k# zNr(hR^C|F$%1TkPTpPo0OQ9WX+&@z6%9ZNTPp>jXag`G6#wb=&ab2QX>$`Kd>$m=8 zAEGN`1^V{FZKid79}L(WIy{T|4rqVFGJ)(lF^$}TC$C10!HU`KeT(Dzef5-hc{|KF zxg+xw3elEj6gp@bp+TbU(M2^rqNvQdrYU*+b!XNpRo8AQ?s`aR2Cv<&`8qQa_h}eo zbXkgjKn$SaDfQ)HtgAcz7xw^sCN+NGP~jg}GQ6n=;y;b{IoqDjE;1UZ%KhC?5l(lb z{{fx}4AlLn+`;c80dQh~p8y?^M5ZFLn#oaT{OQd$OG^cVYEhLcc6-7sC^xCvi`}-v z2{XTWpJSaFJn13bBEzc3P0}6!1(xMD=+5+Fv)B(sl&a0{?Wpq9Hi&s0Y3F2?{Q-1y zK-&`oWIrp6WV*c)y5O7GGbpONuHkp0eYmGM=09CwPWud^ifPjqc^~a*i!Dso-J72+ zFR_L8K3n^3zD3Kj{bf@rg0l*W(GGd{jC46q2{l-i4gGtme|y4YdSC-TCD@imFpN0X zzd(PHLmlA0@&J))C0Hz3Ak&!2RulIJ^Z0!?l%!0${ijwf%1a6w2_qKq-*OQQM7L8-c+r z8{B)c7n%Hbg@l?A4%;&|5#er;;W>FF4Gy?E_riYwN-9E|p=8M#&gV>4yL9+o)#q?buQxB6`epk`*Q1v{ zvqJBS9#qlr3jt$og228>fA_wRwRJ*S2q2QYq-7CB@+Ee z>-E_2qHcV~!?euz6piK1G1448)Go|vIZM)(f-98gM{TV2XLo~= zW@-5oYYi?M)AbPzeH{*STpcFAAbxVMA-%jc;s~|PP8xSiEdy`)mXxdWBbN~py3n5Y z$)fI8m-E*?$6e)G_xkasc01k+vVTqvrBzM`4J3pk?A3pA*SdJw6`PksMA*}xhJ{E? z%tk7~BK6iY5wOK2vP|f3FcpSP_QuqJ^geKVt%0ilB{_PO{dGvcLDGyBJybk%d+E&w zEq*c?es;aGl>ch<>R=!KuPB&s+e08q#l!p;El0AQ1u;(2?$9@Lg_n~b<)3=Uv=k9c zSFI7kVvBVKXGxW&lw%_{nXMzzDTa+rLvgF}IgeVNR&@IEHnM6Wmwh(NIv0IqwZAaj zat(gOHORJ?86h4L)r27_EAVxn1rPD21?2fALpdi&)sOToHA?q;xJlCE(!wg<#fKGF zL;*APzGi(nD)t+Y|5VJ`Q>DP!aeKz;s447zMUyMp z-5e+T4uaQ&7q{S9E9#@?f0O*|PYX)Gn|q8JI$H1d-gd}Wkl;L0_x2Ky>`vM{}zy7elNQsZr(_oT_`LPT#vG~r@^JyzVwg*YYG;gjnwQ*xFruV@v6MBGk2&|aFMKPsEO*M?AEg3qT#$_{#jDPl-SGSRYvLsbaVEE| zc3`m-AvhI8?H?B(L(!|J1=6DW(pFeiW0G7nZ1JeQzxcu*FgN zrYL_4wD@efZu@+J#O=`HXj0EE`+E^D9?SmG_;wJY--s zSy9jQ4KXG?NM>*OIx>|Vk%dK8BK&6x0}J0$wj@Jc&}g|Odkq#~`79Ty3!1kWbkHY{ zi?rDWY2+QZ#JxMk{8#gKWstilBX8a{KSV`sG>4GqS(#X7AlmYP**4>-Y&R=1JgaA; zg5i_U1Ty299iKH1JDb~J@ZHRl$vl7^VWz)I2brD=s6(t0MG=3)20=zvfr|8yS!#7> zCW*#vxVGa8|0jRG0-?Dqg=3X;dxi8l-9iW%ycZ%Zc^O>iQ!6aX!Bp$eK^Ts?nblGb zviF{St7j}@n1UyxYaH7fi^lD&x{Uwir5gz^i+qvB*h+-E* z&+DJk!03lIR|H?AZbaR5=1!^W041-f8_;fm?T~l=Nz}a2D1%uOo8D$1CG`S&&q*cF zxixsI3GU1VI=BB@MEsw)QDdy{M6wZ7o!Tb9e|%c+?KMyQ#J(@16Ezo|Kd%V=UG~;2~QseChETj+VkeMr@ZW%ys{_1 z5}JQWXb(@4T%V_2honj~v9+m!@O-A8Jb^fu5>{Xe^qhW1KdI18IVR5}^#3<(X@WBq z=p<3((&`5!YpgaVh7U9Pwo&8O{&_+3e|WK6pv-nHZ3ptz7ii$o~j_y+7ajF@^&3>W`elx1>*Gmo8!jc!U z9vU(hJTx`R*a?=YfKG#O5WsdALAl^>>iSQctRlh#a`pb<*pKiLKNd#C2Q+d~s=-k* zVb;8wKRW#~WWt7S!?aq%Tx8QZ6WhWGD!w71hl;by*IT07+Lnu~BxG@lN&qRX*l5q^ zW(GWF(7t2vl8H-=|1U|_0VGLOVTML%83&1tSsoW<8^V6v-m?E$ST{#>q&mF*1ak(} z{nrIZkVl{WDe&rMrPq)SW$p{>vGQt0v6*P)nTt%}$!7iv&~QZ?Hw)%BOlA$yl|z-@ zpgOhP!cfj{kN7X64uGFPMrn+x#A5sWn&Z0SC$1XCH!-Q0-OLQy7K$~@t+S*{Kxg$1 zQB^k`NuY%#m%~o8+qAfiX z{?a8-&fF9#)P^^E4B}&ey#;{BS}6P@Hs45!fDrxnK;mMV0saO9016dD|7ytkO diff --git a/src/monitoring/monitors/network.py b/src/monitoring/monitors/network.py index 6278f71..12fb66d 100644 --- a/src/monitoring/monitors/network.py +++ b/src/monitoring/monitors/network.py @@ -1,6 +1,5 @@ import logging from datetime import datetime, timedelta -from time import sleep from typing import List, Optional import dateutil @@ -17,15 +16,20 @@ class NetworkMonitor(Monitor): def __init__(self, monitor_name: str, channels: ChannelSet, - logger: logging.Logger, redis: Optional[RedisApi], - all_full_nodes: List[Node], all_validators: List[Node]): + logger: logging.Logger, + network_monitor_max_catch_up_blocks: int, + redis: Optional[RedisApi], all_full_nodes: List[Node], + all_validators: List[Node]): super().__init__(monitor_name, channels, logger, redis) + self.network_monitor_max_catch_up_blocks = \ + network_monitor_max_catch_up_blocks self._all_full_nodes = all_full_nodes self._all_validators = all_validators self.last_full_node_used = None self._last_height_checked = None + self._monitor_is_syncing = False self._redis_alive_key_timeout = \ self._internal_conf.redis_network_monitor_alive_key_timeout @@ -38,6 +42,9 @@ def __init__(self, monitor_name: str, channels: ChannelSet, self.load_state() + def is_syncing(self) -> bool: + return self._monitor_is_syncing + def load_state(self) -> None: # If Redis is enabled, load the last height checked, if any if self.redis_enabled: @@ -75,6 +82,41 @@ def node(self) -> Node: return n raise NoLiveFullNodeException() + def _check_block(self, height: int) -> None: + self._logger.info('%s obtaining data at height %s', + self._monitor_name, height) + + # Get block + block = get_cosmos_json(self.node.rpc_url + '/block?height=' + + str(height), self._logger) + + # Get validators participating in the precommits of last commit + block_precommits = block['block']['last_commit']['precommits'] + non_null_precommits = filter(lambda p: p, block_precommits) + block_precommits_validators = set( + map(lambda p: p['validator_address'], non_null_precommits)) + total_no_of_missing_validators = \ + len(block_precommits) - len(block_precommits_validators) + + self._logger.debug('Precommit validators: %s', + block_precommits_validators) + self._logger.debug('Total missing validators: %s', + total_no_of_missing_validators) + + # Call method based on whether block missed or not + for v in self._all_validators: + if v.pubkey not in block_precommits_validators: + block_time = block['block']['header']['time'] + v.add_missed_block( + height - 1, # '- 1' since it's actually previous height + dateutil.parser.parse(block_time, ignoretz=True), + total_no_of_missing_validators, self.channels, + self.logger) + else: + v.clear_missed_blocks(self.channels, self.logger) + + self._logger.debug('Moving to next height.') + def monitor(self) -> None: # Get abci_info and, from that, the last height to be checked abci_info = get_cosmos_json(self.node.rpc_url + '/abci_info', @@ -87,47 +129,17 @@ def monitor(self) -> None: # Consider any height that is after the previous last height height = self._last_height_checked + 1 - while height <= last_height_to_check: - self._logger.info('%s obtaining data at height %s', - self._monitor_name, height) - - # Get block - block = get_cosmos_json(self.node.rpc_url + '/block?height=' + - str(height), self._logger) - - # Get validators participating in the precommits of last commit - block_precommits = block['block']['last_commit']['precommits'] - non_null_precommits = filter(lambda p: p, block_precommits) - block_precommits_validators = set( - map(lambda p: p['validator_address'], non_null_precommits)) - total_no_of_missing_validators = \ - len(block_precommits) - len(block_precommits_validators) - - self._logger.debug('Precommit validators: %s', - block_precommits_validators) - self._logger.debug('Total missing validators: %s', - total_no_of_missing_validators) - - # Call method based on whether block missed or not - for v in self._all_validators: - if v.pubkey not in block_precommits_validators: - block_time = block['block']['header']['time'] - v.add_missed_block( - height - 1, # '- 1' since it's actually previous height - dateutil.parser.parse(block_time, ignoretz=True), - total_no_of_missing_validators, self.channels, - self.logger) - else: - v.clear_missed_blocks(self.channels, self.logger) - - self._logger.debug('Moving to next height.') - - # Move to next block - height += 1 - - # If there is a next height to check, sleep for a bit - if height <= last_height_to_check: - self.logger.debug('Sleeping for 0.5 second between heights.') - sleep(0.5) - - self._last_height_checked = last_height_to_check + if last_height_to_check - self._last_height_checked > \ + self.network_monitor_max_catch_up_blocks: + height = last_height_to_check - \ + self.network_monitor_max_catch_up_blocks + self._check_block(height) + self._last_height_checked = height + elif height <= last_height_to_check: + self._check_block(height) + self._last_height_checked = height + + if last_height_to_check - self._last_height_checked > 2: + self._monitor_is_syncing = True + else: + self._monitor_is_syncing = False diff --git a/src/utils/config_parsers/internal.py b/src/utils/config_parsers/internal.py index b4c3b89..e3e583c 100644 --- a/src/utils/config_parsers/internal.py +++ b/src/utils/config_parsers/internal.py @@ -62,6 +62,8 @@ def __init__(self, config_file_path: str) -> None: section['node_monitor_period_seconds']) self.network_monitor_period_seconds = int( section['network_monitor_period_seconds']) + self.network_monitor_max_catch_up_blocks = int( + section['network_monitor_max_catch_up_blocks']) self.github_monitor_period_seconds = int( section['github_monitor_period_seconds']) From ddf8eb28eb02fbec4e2e3f2e519479937d05ee63 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:35:04 +0100 Subject: [PATCH 19/29] Documentation update. --- doc/DESIGN_AND_FEATURES.md | 6 +++--- doc/INSTALL_AND_RUN.md | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/DESIGN_AND_FEATURES.md b/doc/DESIGN_AND_FEATURES.md index 5a6ad2f..3fbf7b7 100644 --- a/doc/DESIGN_AND_FEATURES.md +++ b/doc/DESIGN_AND_FEATURES.md @@ -24,7 +24,7 @@ P.A.N.I.C. currently supports five alerting channels. By default, only console a - **Console**: alerts printed to standard output (`stdout`). - **Logging**: alerts logged to an alerts log (`logs/alerts/alerts.log`). - **Telegram**: alerts delivered to a Telegram chat via a Telegram bot. -- **Email**: alerts sent as emails using a personal SMTP server. +- **Email**: alerts sent as emails using an SMTP server, with option for authentication. - **Twilio**: alerts trigger a phone call to grab the node operator's attention. Instructions on how to set up the alerting channels can be found in the [installation guide](./INSTALL_AND_RUN.md). @@ -113,11 +113,11 @@ The following are some important points about the periodic alive reminder: ## Telegram Commands -Telegram bots in P.A.N.I.C. serve two purposes. As mentioned above, they are used to send alerts. However they can also accept commands that allow you to check the status of the alerter (and its running monitors), snooze or unsnooze calls, and conveniently get Cosmos explorer links to validator lists, blocks, and transactions. +Telegram bots in P.A.N.I.C. serve two purposes. As mentioned above, they are used to send alerts. However they can also accept commands that allow you to check the status of the alerter (and its running monitors), snooze or unsnooze calls, mute or unmute periodic alive reminders, and conveniently get Cosmos explorer links to validator lists, blocks, and transactions. telegram_commands -For example, the `/status` command returns the following, if Redis is running along with three node monitors and one network monitor, and with calls not snoozed: +For example, the `/status` command returns the following, if Redis is running along with three node monitors and one network monitor, with calls not snoozed, and periodic alive reminder not muted: telegram_status_command diff --git a/doc/INSTALL_AND_RUN.md b/doc/INSTALL_AND_RUN.md index 08a82d0..bf7cf04 100644 --- a/doc/INSTALL_AND_RUN.md +++ b/doc/INSTALL_AND_RUN.md @@ -23,11 +23,12 @@ The only major requirement to run P.A.N.I.C. is Python 3. However, to unlock the 2. To install **pip** package manager: - On Linux, run: `apt-get install python3-pip` - On Windows, it should come included in the installation. -3. To install **pipenv** packaging tool, run `pip install pipenv`. +3. To install **pipenv** packaging tool, run `pip install pipenv`. + (If 'pip' is not found, try using 'pip3' instead.) **At the end, you should be able to:** 1. Get the Python version by running `python --version`. - (If multiple versions of Python are installed, the `python` executable may be `python3.6`, `python3.7`, etc.) + (You may have to replace 'python' with 'python3.6', 'python3.7', etc.) 2. Get the pip version by running `pip --version`. 3. Get the pipenv version by running `pipenv --version`. From 3547780d134b1918b2e9fd432d4fdab3e7249251 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:35:30 +0100 Subject: [PATCH 20/29] Updated alerter runner and added update util. --- run_alerter.py | 152 ++++++++++++++++++++++------------- run_util_update_to_v1.1.0.py | 39 +++++++++ 2 files changed, 136 insertions(+), 55 deletions(-) create mode 100644 run_util_update_to_v1.1.0.py diff --git a/run_alerter.py b/run_alerter.py index 991a0fc..3b8c3c8 100644 --- a/run_alerter.py +++ b/run_alerter.py @@ -3,7 +3,10 @@ from typing import List, Tuple from src.alerting.alert_utils.get_channel_set import get_full_channel_set +from src.alerting.alert_utils.get_channel_set import \ + get_periodic_alive_reminder_channel_set from src.alerting.alerts.alerts import TerminatedDueToExceptionAlert +from src.alerting.periodic.periodic import periodic_alive_reminder from src.commands.handlers.telegram import TelegramCommands from src.monitoring.monitor_utils.get_json import get_cosmos_json, get_json from src.monitoring.monitors.github import GitHubMonitor @@ -91,19 +94,21 @@ def run_monitor_nodes(node: Node): node.name, InternalConf.logging_level, rotating=True) # Initialise monitor - node_monitor = NodeMonitor(monitor_name, channel_set, + node_monitor = NodeMonitor(monitor_name, full_channel_set, logger_monitor_node, REDIS, node) - # Start - log_and_print('{} started.'.format(monitor_name)) - sys.stdout.flush() - try: - start_node_monitor(node_monitor, - InternalConf.node_monitor_period_seconds, - logger_monitor_node) - except Exception as e: - channel_set.alert_error(TerminatedDueToExceptionAlert(monitor_name, e)) - log_and_print('{} stopped.'.format(monitor_name)) + while True: + # Start + log_and_print('{} started.'.format(monitor_name)) + sys.stdout.flush() + try: + start_node_monitor(node_monitor, + InternalConf.node_monitor_period_seconds, + logger_monitor_node) + except Exception as e: + full_channel_set.alert_error( + TerminatedDueToExceptionAlert(monitor_name, e)) + log_and_print('{} stopped.'.format(monitor_name)) def run_monitor_network(network_nodes_tuple: Tuple[str, List[Node]]): @@ -132,25 +137,29 @@ def run_monitor_network(network_nodes_tuple: Tuple[str, List[Node]]): return # Initialise monitor - network_monitor = NetworkMonitor(monitor_name, channel_set, - logger_monitor_network, REDIS, - full_nodes, validators) + network_monitor = NetworkMonitor(monitor_name, full_channel_set, + logger_monitor_network, + InternalConf. + network_monitor_max_catch_up_blocks, + REDIS, full_nodes, validators) except Exception as e: msg = '!!! Error when initialising {}: {} !!!'.format(monitor_name, e) log_and_print(msg) raise InitialisationException(msg) - # Start - log_and_print('{} started with {} validator(s) and {} full node(s).' - ''.format(monitor_name, len(validators), len(full_nodes))) - sys.stdout.flush() - try: - start_network_monitor(network_monitor, - InternalConf.network_monitor_period_seconds, - logger_monitor_network) - except Exception as e: - channel_set.alert_error(TerminatedDueToExceptionAlert(monitor_name, e)) - log_and_print('{} stopped.'.format(monitor_name)) + while True: + # Start + log_and_print('{} started with {} validator(s) and {} full node(s).' + ''.format(monitor_name, len(validators), len(full_nodes))) + sys.stdout.flush() + try: + start_network_monitor(network_monitor, + InternalConf.network_monitor_period_seconds, + logger_monitor_network) + except Exception as e: + full_channel_set.alert_error( + TerminatedDueToExceptionAlert(monitor_name, e)) + log_and_print('{} stopped.'.format(monitor_name)) def run_commands_telegram(): @@ -161,21 +170,24 @@ def run_commands_telegram(): if not UserConf.telegram_cmds_enabled: return - # Start - log_and_print('{} started.'.format(monitor_name)) - sys.stdout.flush() - try: - TelegramCommands( - UserConf.telegram_cmds_bot_token, - UserConf.telegram_cmds_bot_chat_id, - logger_commands_telegram, REDIS, - InternalConf.redis_twilio_snooze_key, - InternalConf.redis_node_monitor_alive_key_prefix, - InternalConf.redis_network_monitor_alive_key_prefix - ).start_listening() - except Exception as e: - channel_set.alert_error(TerminatedDueToExceptionAlert(monitor_name, e)) - log_and_print('{} stopped.'.format(monitor_name)) + while True: + # Start + log_and_print('{} started.'.format(monitor_name)) + sys.stdout.flush() + try: + TelegramCommands( + UserConf.telegram_cmds_bot_token, + UserConf.telegram_cmds_bot_chat_id, + logger_commands_telegram, REDIS, + InternalConf.redis_twilio_snooze_key, + InternalConf.redis_periodic_alive_reminder_mute_key, + InternalConf.redis_node_monitor_alive_key_prefix, + InternalConf.redis_network_monitor_alive_key_prefix + ).start_listening() + except Exception as e: + full_channel_set.alert_error( + TerminatedDueToExceptionAlert(monitor_name, e)) + log_and_print('{} stopped.'.format(monitor_name)) def run_monitor_github(repo_config: RepoConfig): @@ -196,7 +208,7 @@ def run_monitor_github(repo_config: RepoConfig): # Initialise monitor github_monitor = GitHubMonitor( - monitor_name, channel_set, logger_monitor_github, REDIS, + monitor_name, full_channel_set, logger_monitor_github, REDIS, repo_config.repo_name, releases_page, InternalConf.redis_github_releases_key_prefix) except Exception as e: @@ -204,16 +216,37 @@ def run_monitor_github(repo_config: RepoConfig): log_and_print(msg) raise InitialisationException(msg) - # Start - log_and_print('{} started.'.format(monitor_name)) - sys.stdout.flush() - try: - start_github_monitor(github_monitor, - InternalConf.github_monitor_period_seconds, - logger_monitor_github) - except Exception as e: - channel_set.alert_error(TerminatedDueToExceptionAlert(monitor_name, e)) - log_and_print('{} stopped.'.format(monitor_name)) + while True: + # Start + log_and_print('{} started.'.format(monitor_name)) + sys.stdout.flush() + try: + start_github_monitor(github_monitor, + InternalConf.github_monitor_period_seconds, + logger_monitor_github) + except Exception as e: + full_channel_set.alert_error( + TerminatedDueToExceptionAlert(monitor_name, e)) + log_and_print('{} stopped.'.format(monitor_name)) + + +def run_periodic_alive_reminder(): + if not UserConf.periodic_alive_reminder_enabled: + return + + name = "Periodic alive reminder" + + while True: + log_and_print('{} started.'.format(name)) + try: + periodic_alive_reminder( + UserConf.interval_seconds, + periodic_alive_reminder_channel_set, + InternalConf.redis_periodic_alive_reminder_mute_key, REDIS) + except Exception as e: + periodic_alive_reminder_channel_set.alert_error( + TerminatedDueToExceptionAlert(name, e)) + log_and_print('{} stopped.'.format(name)) if __name__ == '__main__': @@ -240,10 +273,16 @@ def run_monitor_github(repo_config: RepoConfig): # Alerters initialisation alerter_name = 'P.A.N.I.C.' - channel_set = get_full_channel_set( + full_channel_set = get_full_channel_set( alerter_name, logger_general, REDIS, log_file_alerts) - log_and_print('Enabled alerting channels: {}'.format( - channel_set.enabled_channels_list())) + log_and_print('Enabled alerting channels (general): {}'.format( + full_channel_set.enabled_channels_list())) + periodic_alive_reminder_channel_set = \ + get_periodic_alive_reminder_channel_set(alerter_name, logger_general, + REDIS, log_file_alerts) + log_and_print('Enabled alerting channels (periodic alive reminder): {}' + ''.format(periodic_alive_reminder_channel_set. + enabled_channels_list())) sys.stdout.flush() # Nodes initialisation @@ -280,11 +319,14 @@ def run_monitor_github(repo_config: RepoConfig): monitor_network_count = len(unique_networks) monitor_github_count = len(UserConf.filtered_repos) commands_telegram_count = 1 + periodic_alive_reminder_count = 1 total_count = sum([monitor_node_count, monitor_network_count, - monitor_github_count, commands_telegram_count]) + monitor_github_count, commands_telegram_count, + periodic_alive_reminder_count]) with concurrent.futures.ThreadPoolExecutor(max_workers=total_count) \ as executor: executor.map(run_monitor_nodes, node_monitor_nodes) executor.map(run_monitor_network, nodes_by_network.items()) executor.map(run_monitor_github, UserConf.filtered_repos) executor.submit(run_commands_telegram) + executor.submit(run_periodic_alive_reminder) diff --git a/run_util_update_to_v1.1.0.py b/run_util_update_to_v1.1.0.py new file mode 100644 index 0000000..19f71df --- /dev/null +++ b/run_util_update_to_v1.1.0.py @@ -0,0 +1,39 @@ +import os +from configparser import ConfigParser + + +def main(): + if not os.path.isfile('config/user_config_main.ini'): + print('User config does not exist, so there is no need to update it.') + print('To create this file, you can run the setup (run_setup.py).') + return + + cp = ConfigParser() + cp.read('config/user_config_main.ini') + + # Create periodic_alive_reminder section + if 'periodic_alive_reminder' in cp: + print('Periodic alive reminder config was ALREADY UPDATED.') + else: + cp.add_section('periodic_alive_reminder') + cp['periodic_alive_reminder']['enabled'] = str(False) + cp['periodic_alive_reminder']['interval_seconds'] = '' + cp['periodic_alive_reminder']['email_enabled'] = '' + cp['periodic_alive_reminder']['telegram_enabled'] = '' + print('Periodic alive reminder config UPDATED.') + + # Set new SMTP user and pass to blank + if 'user' in cp['email_alerts'] and 'pass' in cp['email_alerts']: + print('User and pass in email_alerts config were ALREADY UPDATED.') + else: + cp['email_alerts']['user'] = '' + cp['email_alerts']['pass'] = '' + print('User and pass in email_alerts config UPDATED (set to blank).') + + with open('config/user_config_main.ini', 'w') as f: + cp.write(f) + print('Update process finished.') + + +if __name__ == '__main__': + main() From 65111b5476f529189f129a77bf269b9a705584d7 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 13:35:44 +0100 Subject: [PATCH 21/29] Updated Pipfile.lock --- Pipfile.lock | 101 ++++++++++++++++++++++++++------------------------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index bf1aae3..058f836 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -16,47 +16,48 @@ "default": { "certifi": { "hashes": [ - "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", - "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" ], - "version": "==2019.9.11" + "version": "==2019.11.28" }, "cffi": { "hashes": [ - "sha256:00d890313797d9fe4420506613384b43099ad7d2b905c0752dbcc3a6f14d80fa", - "sha256:0cf9e550ac6c5e57b713437e2f4ac2d7fd0cd10336525a27224f5fc1ec2ee59a", - "sha256:0ea23c9c0cdd6778146a50d867d6405693ac3b80a68829966c98dd5e1bbae400", - "sha256:193697c2918ecdb3865acf6557cddf5076bb39f1f654975e087b67efdff83365", - "sha256:1ae14b542bf3b35e5229439c35653d2ef7d8316c1fffb980f9b7647e544baa98", - "sha256:1e389e069450609c6ffa37f21f40cce36f9be7643bbe5051ab1de99d5a779526", - "sha256:263242b6ace7f9cd4ea401428d2d45066b49a700852334fd55311bde36dcda14", - "sha256:33142ae9807665fa6511cfa9857132b2c3ee6ddffb012b3f0933fc11e1e830d5", - "sha256:364f8404034ae1b232335d8c7f7b57deac566f148f7222cef78cf8ae28ef764e", - "sha256:47368f69fe6529f8f49a5d146ddee713fc9057e31d61e8b6dc86a6a5e38cecc1", - "sha256:4895640844f17bec32943995dc8c96989226974dfeb9dd121cc45d36e0d0c434", - "sha256:558b3afef987cf4b17abd849e7bedf64ee12b28175d564d05b628a0f9355599b", - "sha256:5ba86e1d80d458b338bda676fd9f9d68cb4e7a03819632969cf6d46b01a26730", - "sha256:63424daa6955e6b4c70dc2755897f5be1d719eabe71b2625948b222775ed5c43", - "sha256:6381a7d8b1ebd0bc27c3bc85bc1bfadbb6e6f756b4d4db0aa1425c3719ba26b4", - "sha256:6381ab708158c4e1639da1f2a7679a9bbe3e5a776fc6d1fd808076f0e3145331", - "sha256:6fd58366747debfa5e6163ada468a90788411f10c92597d3b0a912d07e580c36", - "sha256:728ec653964655d65408949b07f9b2219df78badd601d6c49e28d604efe40599", - "sha256:7cfcfda59ef1f95b9f729c56fe8a4041899f96b72685d36ef16a3440a0f85da8", - "sha256:819f8d5197c2684524637f940445c06e003c4a541f9983fd30d6deaa2a5487d8", - "sha256:825ecffd9574557590e3225560a8a9d751f6ffe4a49e3c40918c9969b93395fa", - "sha256:8a2bcae2258d00fcfc96a9bde4a6177bc4274fe033f79311c5dd3d3148c26518", - "sha256:9009e917d8f5ef780c2626e29b6bc126f4cb2a4d43ca67aa2b40f2a5d6385e78", - "sha256:9c77564a51d4d914ed5af096cd9843d90c45b784b511723bd46a8a9d09cf16fc", - "sha256:a19089fa74ed19c4fe96502a291cfdb89223a9705b1d73b3005df4256976142e", - "sha256:a40ed527bffa2b7ebe07acc5a3f782da072e262ca994b4f2085100b5a444bbb2", - "sha256:b8f09f21544b9899defb09afbdaeb200e6a87a2b8e604892940044cf94444644", - "sha256:bb75ba21d5716abc41af16eac1145ab2e471deedde1f22c6f99bd9f995504df0", - "sha256:e22a00c0c81ffcecaf07c2bfb3672fa372c50e2bd1024ffee0da191c1b27fc71", - "sha256:e55b5a746fb77f10c83e8af081979351722f6ea48facea79d470b3731c7b2891", - "sha256:ec2fa3ee81707a5232bf2dfbd6623fdb278e070d596effc7e2d788f2ada71a05", - "sha256:fd82eb4694be712fcae03c717ca2e0fc720657ac226b80bbb597e971fc6928c2" - ], - "version": "==1.13.1" + "sha256:0b49274afc941c626b605fb59b59c3485c17dc776dc3cc7cc14aca74cc19cc42", + "sha256:0e3ea92942cb1168e38c05c1d56b0527ce31f1a370f6117f1d490b8dcd6b3a04", + "sha256:135f69aecbf4517d5b3d6429207b2dff49c876be724ac0c8bf8e1ea99df3d7e5", + "sha256:19db0cdd6e516f13329cba4903368bff9bb5a9331d3410b1b448daaadc495e54", + "sha256:2781e9ad0e9d47173c0093321bb5435a9dfae0ed6a762aabafa13108f5f7b2ba", + "sha256:291f7c42e21d72144bb1c1b2e825ec60f46d0a7468f5346841860454c7aa8f57", + "sha256:2c5e309ec482556397cb21ede0350c5e82f0eb2621de04b2633588d118da4396", + "sha256:2e9c80a8c3344a92cb04661115898a9129c074f7ab82011ef4b612f645939f12", + "sha256:32a262e2b90ffcfdd97c7a5e24a6012a43c61f1f5a57789ad80af1d26c6acd97", + "sha256:3c9fff570f13480b201e9ab69453108f6d98244a7f495e91b6c654a47486ba43", + "sha256:415bdc7ca8c1c634a6d7163d43fb0ea885a07e9618a64bda407e04b04333b7db", + "sha256:42194f54c11abc8583417a7cf4eaff544ce0de8187abaf5d29029c91b1725ad3", + "sha256:4424e42199e86b21fc4db83bd76909a6fc2a2aefb352cb5414833c030f6ed71b", + "sha256:4a43c91840bda5f55249413037b7a9b79c90b1184ed504883b72c4df70778579", + "sha256:599a1e8ff057ac530c9ad1778293c665cb81a791421f46922d80a86473c13346", + "sha256:5c4fae4e9cdd18c82ba3a134be256e98dc0596af1e7285a3d2602c97dcfa5159", + "sha256:5ecfa867dea6fabe2a58f03ac9186ea64da1386af2159196da51c4904e11d652", + "sha256:62f2578358d3a92e4ab2d830cd1c2049c9c0d0e6d3c58322993cc341bdeac22e", + "sha256:6471a82d5abea994e38d2c2abc77164b4f7fbaaf80261cb98394d5793f11b12a", + "sha256:6d4f18483d040e18546108eb13b1dfa1000a089bcf8529e30346116ea6240506", + "sha256:71a608532ab3bd26223c8d841dde43f3516aa5d2bf37b50ac410bb5e99053e8f", + "sha256:74a1d8c85fb6ff0b30fbfa8ad0ac23cd601a138f7509dc617ebc65ef305bb98d", + "sha256:7b93a885bb13073afb0aa73ad82059a4c41f4b7d8eb8368980448b52d4c7dc2c", + "sha256:7d4751da932caaec419d514eaa4215eaf14b612cff66398dd51129ac22680b20", + "sha256:7f627141a26b551bdebbc4855c1157feeef18241b4b8366ed22a5c7d672ef858", + "sha256:8169cf44dd8f9071b2b9248c35fc35e8677451c52f795daa2bb4643f32a540bc", + "sha256:aa00d66c0fab27373ae44ae26a66a9e43ff2a678bf63a9c7c1a9a4d61172827a", + "sha256:ccb032fda0873254380aa2bfad2582aedc2959186cce61e3a17abc1a55ff89c3", + "sha256:d754f39e0d1603b5b24a7f8484b22d2904fa551fe865fd0d4c3332f078d20d4e", + "sha256:d75c461e20e29afc0aee7172a0950157c704ff0dd51613506bd7d82b718e7410", + "sha256:dcd65317dd15bc0451f3e01c80da2216a31916bdcffd6221ca1202d96584aa25", + "sha256:e570d3ab32e2c2861c4ebe6ffcad6a8abf9347432a37608fe1fbd157b3f0036b", + "sha256:fd43a88e045cf992ed09fa724b5315b790525f2676883a6ea64e3263bae6549d" + ], + "version": "==1.13.2" }, "chardet": { "hashes": [ @@ -101,9 +102,9 @@ }, "future": { "hashes": [ - "sha256:858e38522e8fd0d3ce8f0c1feaf0603358e366d5403209674c7b617fa0c24093" + "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" ], - "version": "==0.18.1" + "version": "==0.18.2" }, "idna": { "hashes": [ @@ -127,11 +128,11 @@ }, "python-dateutil": { "hashes": [ - "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", - "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" ], "index": "pypi", - "version": "==2.8.0" + "version": "==2.8.1" }, "python-telegram-bot": { "hashes": [ @@ -166,10 +167,10 @@ }, "six": { "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" ], - "version": "==1.12.0" + "version": "==1.13.0" }, "tornado": { "hashes": [ @@ -185,17 +186,17 @@ }, "twilio": { "hashes": [ - "sha256:e78a2006b9449fb9fad5050537e0998c181c7d3a62eaa9eed434e59dbaf58324" + "sha256:da282a9c02bd9dfb190b798528b478833d8d28cb51464e8c45da0f0794384cde" ], "index": "pypi", - "version": "==6.32.0" + "version": "==6.34.0" }, "urllib3": { "hashes": [ - "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", - "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" + "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", + "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" ], - "version": "==1.25.6" + "version": "==1.25.7" } }, "develop": { From b38a33a5b06d2d94b9d496b22f0d22b179fe31fe Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 14:01:34 +0100 Subject: [PATCH 22/29] Aliased ConnectionError to avoid clashes with the keyword. --- src/monitoring/monitor_utils/live_check.py | 4 ++-- src/monitoring/monitors/monitor_starters.py | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/monitoring/monitor_utils/live_check.py b/src/monitoring/monitor_utils/live_check.py index 66103b4..4dbb902 100644 --- a/src/monitoring/monitor_utils/live_check.py +++ b/src/monitoring/monitor_utils/live_check.py @@ -1,7 +1,7 @@ import logging import requests -from requests.exceptions import ConnectionError +from requests.exceptions import ConnectionError as ReqConnectionError def live_check_unsafe(endpoint: str, logger: logging.Logger) -> None: @@ -14,5 +14,5 @@ def live_check(endpoint: str, logger: logging.Logger) -> bool: try: live_check_unsafe(endpoint, logger) return True - except ConnectionError: + except ReqConnectionError: return False diff --git a/src/monitoring/monitors/monitor_starters.py b/src/monitoring/monitors/monitor_starters.py index 34f0f06..314ccd4 100644 --- a/src/monitoring/monitors/monitor_starters.py +++ b/src/monitoring/monitors/monitor_starters.py @@ -4,7 +4,8 @@ from json import JSONDecodeError import urllib3.exceptions -from requests.exceptions import ConnectionError, ReadTimeout +from requests.exceptions import ConnectionError as ReqConnectionError, \ + ReadTimeout from src.alerting.alerts.alerts import CouldNotFindLiveFullNodeAlert, \ ErrorWhenReadingDataFromNode, CannotAccessGitHubPageAlert @@ -25,7 +26,7 @@ def start_node_monitor(node_monitor: NodeMonitor, monitor_period: int, logger.debug('Reading %s.', node_monitor.node) node_monitor.monitor() logger.debug('Done reading %s.', node_monitor.node) - except ConnectionError: + except ReqConnectionError: node_monitor.node.set_as_down(node_monitor.channels, logger) except ReadTimeout: node_monitor.node.set_as_down(node_monitor.channels, logger) @@ -59,7 +60,7 @@ def start_network_monitor(network_monitor: NetworkMonitor, monitor_period: int, except NoLiveFullNodeException: network_monitor.channels.alert_major( CouldNotFindLiveFullNodeAlert(network_monitor.monitor_name)) - except (ConnectionError, ReadTimeout): + except (ReqConnectionError, ReadTimeout): network_monitor.last_full_node_used.set_as_down( network_monitor.channels, logger) except (urllib3.exceptions.IncompleteRead, @@ -100,7 +101,7 @@ def start_github_monitor(github_monitor: GitHubMonitor, monitor_period: int, # Reset alert limiter github_error_alert_limiter.reset() - except (ConnectionError, ReadTimeout) as conn_err: + except (ReqConnectionError, ReadTimeout) as conn_err: if github_error_alert_limiter.can_do_task(): github_monitor.channels.alert_error( CannotAccessGitHubPageAlert(github_monitor.releases_page)) From 431121b7a56d965136e614669c80aab35dfadda3 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 14:24:59 +0100 Subject: [PATCH 23/29] Documentation fix. --- doc/DESIGN_AND_FEATURES.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/doc/DESIGN_AND_FEATURES.md b/doc/DESIGN_AND_FEATURES.md index 3fbf7b7..d61dd08 100644 --- a/doc/DESIGN_AND_FEATURES.md +++ b/doc/DESIGN_AND_FEATURES.md @@ -245,13 +245,10 @@ The only two alerts raised by the GitHub alerter are an info alert when a new re ### Periodic Alive Reminder -If the periodic alive reminder is enabled from the config file, and P.A.N.I.C is running smoothly, the operator is informed every `P1` seconds that P.A.N.I.C is still running via an info alert. +If the periodic alive reminder is enabled from the config file, and P.A.N.I.C is running smoothly, the operator is informed every time period that P.A.N.I.C is still running via an info alert. The periodic alive reminder always uses the console and logger to raise this alert, however, the operator can also receive this alert via Telegram, Email or both, by modifying the config file as described [here](./INSTALL_AND_RUN.md#setting-up-panic). -Default value: -- `P1 = interval_seconds = 3600` - | Class | Severity | Configurable | |---|---|---| | `AlerterAliveAlert` | `INFO` | ✓ | From ca46fd1b9c80a749d836bbbb04d504d60538e783 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 14:28:44 +0100 Subject: [PATCH 24/29] Minor change. --- src/alerting/periodic/periodic.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/alerting/periodic/periodic.py b/src/alerting/periodic/periodic.py index aa9d0cf..b228c91 100644 --- a/src/alerting/periodic/periodic.py +++ b/src/alerting/periodic/periodic.py @@ -15,7 +15,6 @@ def periodic_alive_reminder(interval: timedelta, channel_set: ChannelSet, def send_alive_alert(redis: RedisApi, mute_key: str, channel_set: ChannelSet) -> None: - # If time elapses and periodic alive reminder is not muted, - # inform operator that alerter is still alive. + # If reminder is not muted, inform operator that alerter is still alive. if not redis.exists(mute_key): channel_set.alert_info(AlerterAliveAlert()) From 8d0b9fd0edb76004a98703252496fe3b452d0471 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Thu, 5 Dec 2019 15:11:51 +0100 Subject: [PATCH 25/29] Updated CHANGELOG.md --- doc/CHANGELOG.md | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/doc/CHANGELOG.md b/doc/CHANGELOG.md index 7a2847a..5a14bce 100644 --- a/doc/CHANGELOG.md +++ b/doc/CHANGELOG.md @@ -1,9 +1,51 @@ # Change Log +## 1.1.0 + +Released on December 05, 2019. + +### Update Instructions + +To update an instance of P.A.N.I.C. to this version: +```shell script +git fetch # Fetch these changes +git checkout v1.1.0 # Switch to this version + +pipenv update # Update dependencies +pipenv run python run_util_update_to_v1.1.0.py +``` + +The `run_util_update_to_v1.1.0.py` script updates `user_config_main.ini` so that it becomes compatible with the v1.1.0 `user_config_main.ini` file. + +P.A.N.I.C. can now be started up. If the alerter was running as a Linux service, the service should now be restarted: + +```shell script +sudo systemctl restart panic_alerter +``` + +### Features +* Add **authenticated SMTP**, so that email channel can use public SMTP servers, such as smtp.gmail.com, by supplying a valid username and password. +* Add **periodic alive reminder** as a way for the alerter to inform that it is still running. It is turned on through the setup process and can be muted/unmuted using commands from Telegram. +* Add **validator peer safe boundary** (`validator_peer_safe_boundary`, default: 5) to limit peer change alerts up to a certain number of peers. +* Add **max catch up blocks** (`network_monitor_max_catchup_blocks`, default: 500) to limit the number of historical blocks that the network monitor checks if it is not in sync, so that it focuses on the more important present events. +* Add **current network monitor block height** to Telegram status message. + +### Changes and Improvements +* Email channel now supports multiple recipients. +* Internal config + * Changed default GitHub monitor period to 3600 seconds (1h). + * Changed default `full_node_peer_danger_boundary` to 10 for less alerts. +* Other: + * Updated Telegram bot to use new context-based callbacks. + * Now .gitignoring numbered log files (e.g. `*.log.1`) + +### Bug Fixes +* Fixed validator peer change alert not sent if new number of peers increases to exactly the value of the danger boundary. +* Setup processes now clear config file before adding new entries. + ## 1.0.0 Released on August 23, 2019. ### Added - * First version of the P.A.N.I.C. alerter by Simply VC \ No newline at end of file From f9b8ba5bc6a34b5581842e52d1b7ea3303e28944 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Mon, 9 Dec 2019 10:50:58 +0100 Subject: [PATCH 26/29] Added new lines at end of files. --- config/example_user_config_main.ini | 2 +- config/example_user_config_nodes.ini | 2 +- config/example_user_config_repos.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/example_user_config_main.ini b/config/example_user_config_main.ini index 0306276..c577df4 100644 --- a/config/example_user_config_main.ini +++ b/config/example_user_config_main.ini @@ -36,4 +36,4 @@ password = HMASDNoiSADnuiasdgnAIO876hg967bv99vb8buyT8BVuyT76VBT76uyi enabled = True interval_seconds = 3600 email_enabled = False -telegram_enabled = True \ No newline at end of file +telegram_enabled = True diff --git a/config/example_user_config_nodes.ini b/config/example_user_config_nodes.ini index e0b5f3f..acb82c7 100644 --- a/config/example_user_config_nodes.ini +++ b/config/example_user_config_nodes.ini @@ -24,4 +24,4 @@ node_name = Sentry 2 node_rpc_url = http://11.22.33.44:26657 node_is_validator = false include_in_node_monitor = false -include_in_network_monitor = false \ No newline at end of file +include_in_network_monitor = false diff --git a/config/example_user_config_repos.ini b/config/example_user_config_repos.ini index 25a664f..3a2ab27 100644 --- a/config/example_user_config_repos.ini +++ b/config/example_user_config_repos.ini @@ -6,4 +6,4 @@ include_in_github_monitor = true [repo_2] repo_name = Gaia repo_page = cosmos/gaia/ -include_in_github_monitor = false \ No newline at end of file +include_in_github_monitor = false From 97ed1bda9ed1ee9e857d5bab3722adcb94c571b8 Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Mon, 9 Dec 2019 10:51:47 +0100 Subject: [PATCH 27/29] Minor change to docs. --- doc/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/CHANGELOG.md b/doc/CHANGELOG.md index 5a14bce..e276129 100644 --- a/doc/CHANGELOG.md +++ b/doc/CHANGELOG.md @@ -25,7 +25,7 @@ sudo systemctl restart panic_alerter ### Features * Add **authenticated SMTP**, so that email channel can use public SMTP servers, such as smtp.gmail.com, by supplying a valid username and password. -* Add **periodic alive reminder** as a way for the alerter to inform that it is still running. It is turned on through the setup process and can be muted/unmuted using commands from Telegram. +* Add **periodic alive reminder** as a way for the alerter to inform the user that it is still running. It is turned on through the setup process and can be muted/unmuted using commands from Telegram. * Add **validator peer safe boundary** (`validator_peer_safe_boundary`, default: 5) to limit peer change alerts up to a certain number of peers. * Add **max catch up blocks** (`network_monitor_max_catchup_blocks`, default: 500) to limit the number of historical blocks that the network monitor checks if it is not in sync, so that it focuses on the more important present events. * Add **current network monitor block height** to Telegram status message. From cb5ce67d503beb3ead7e446579fc847aeeab46dc Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Mon, 9 Dec 2019 10:55:47 +0100 Subject: [PATCH 28/29] Fixed changelog. --- doc/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/CHANGELOG.md b/doc/CHANGELOG.md index e276129..628a6e3 100644 --- a/doc/CHANGELOG.md +++ b/doc/CHANGELOG.md @@ -40,7 +40,7 @@ sudo systemctl restart panic_alerter * Now .gitignoring numbered log files (e.g. `*.log.1`) ### Bug Fixes -* Fixed validator peer change alert not sent if new number of peers increases to exactly the value of the danger boundary. +* Fixed full node peer increase alert not sent if the new number of peers is equal to the danger boundary. * Setup processes now clear config file before adding new entries. ## 1.0.0 From 18bcc92eda6de67ab6dab8a4715192735f72951f Mon Sep 17 00:00:00 2001 From: Miguel Dingli Date: Mon, 9 Dec 2019 10:57:10 +0100 Subject: [PATCH 29/29] Added newlines. --- test/test_internal_config.ini | 2 +- test/test_user_config_main.ini | 2 +- test/test_user_config_nodes.ini | 2 +- test/test_user_config_repos.ini | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_internal_config.ini b/test/test_internal_config.ini index 3eba9a8..54aa1b6 100644 --- a/test/test_internal_config.ini +++ b/test/test_internal_config.ini @@ -73,4 +73,4 @@ tx_mintscan_link_prefix = https://www.mintscan.io/txs/ github_releases_template = https://api.github.com/repos/{}releases # This is a Python template string, where {} is replaced with (for example) cosmos/cosmos-sdk/ -# so that the complete link becomes: https://api.github.com/repos/cosmos/cosmos-sdk/releases \ No newline at end of file +# so that the complete link becomes: https://api.github.com/repos/cosmos/cosmos-sdk/releases diff --git a/test/test_user_config_main.ini b/test/test_user_config_main.ini index bc19b7b..9624b16 100644 --- a/test/test_user_config_main.ini +++ b/test/test_user_config_main.ini @@ -36,4 +36,4 @@ password = enabled = False interval_seconds = 10 email_enabled = False -telegram_enabled = True \ No newline at end of file +telegram_enabled = True diff --git a/test/test_user_config_nodes.ini b/test/test_user_config_nodes.ini index e0b5f3f..acb82c7 100644 --- a/test/test_user_config_nodes.ini +++ b/test/test_user_config_nodes.ini @@ -24,4 +24,4 @@ node_name = Sentry 2 node_rpc_url = http://11.22.33.44:26657 node_is_validator = false include_in_node_monitor = false -include_in_network_monitor = false \ No newline at end of file +include_in_network_monitor = false diff --git a/test/test_user_config_repos.ini b/test/test_user_config_repos.ini index 25a664f..3a2ab27 100644 --- a/test/test_user_config_repos.ini +++ b/test/test_user_config_repos.ini @@ -6,4 +6,4 @@ include_in_github_monitor = true [repo_2] repo_name = Gaia repo_page = cosmos/gaia/ -include_in_github_monitor = false \ No newline at end of file +include_in_github_monitor = false