Closes #18627: Proxy routing (#18681)

* Introduce proxy routing

* Misc cleanup

* Document PROXY_ROUTERS parameter

Author: jeremystretch

docs/configuration/system.md

@@ -64,7 +64,7 @@ Email is sent from NetBox only for critical events or if configured for [logging
 
 ## HTTP_PROXIES
 
-Default: None
+Default: Empty
 
 A dictionary of HTTP proxies to use for outbound requests originating from NetBox (e.g. when sending webhook requests). Proxies should be specified by schema (HTTP and HTTPS) as per the [Python requests library documentation](https://requests.readthedocs.io/en/latest/user/advanced/#proxies). For example:
 
@@ -75,6 +75,8 @@ HTTP_PROXIES = {
 }
 ```
 
+If more flexibility is needed in determining which proxy to use for a given request, consider implementing one or more custom proxy routers via the [`PROXY_ROUTERS`](#proxy_routers) parameter.
+
 ---
 
 ## INTERNAL_IPS
@@ -160,6 +162,16 @@ The file path to the location where media files (such as image attachments) are
 
 ---
 
+## PROXY_ROUTERS
+
+Default: `["utilities.proxy.DefaultProxyRouter"]`
+
+A list of Python classes responsible for determining which proxy server(s) to use for outbound HTTP requests. Each item in the list can be the class itself or the dotted path to the class.
+
+The `route()` method on each class must return a dictionary of candidate proxies arranged by protocol (e.g. `http` and/or `https`), or None if no viable proxy can be determined. The default class, `DefaultProxyRouter`, simply returns the content of [`HTTP_PROXIES`](#http_proxies).
+
+---
+
 ## REPORTS_ROOT
 
 Default: `$INSTALL_ROOT/netbox/reports/`

netbox/core/data_backends.py

@@ -7,13 +7,13 @@
 from urllib.parse import urlparse
 
 from django import forms
-from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.utils.translation import gettext as _
 
 from netbox.data_backends import DataBackend
 from netbox.utils import register_data_backend
 from utilities.constants import HTTP_PROXY_SUPPORTED_SCHEMAS, HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS
+from utilities.proxy import resolve_proxies
 from utilities.socks import ProxyPoolManager
 from .exceptions import SyncError
 
@@ -70,18 +70,18 @@ def init_config(self):
 
         # Initialize backend config
         config = ConfigDict()
-        self.use_socks = False
+        self.socks_proxy = None
 
         # Apply HTTP proxy (if configured)
-        if settings.HTTP_PROXIES:
-            if proxy := settings.HTTP_PROXIES.get(self.url_scheme, None):
-                if urlparse(proxy).scheme not in HTTP_PROXY_SUPPORTED_SCHEMAS:
-                    raise ImproperlyConfigured(f"Unsupported Git DataSource proxy scheme: {urlparse(proxy).scheme}")
+        proxies = resolve_proxies(url=self.url, context={'client': self}) or {}
+        if proxy := proxies.get(self.url_scheme):
+            if urlparse(proxy).scheme not in HTTP_PROXY_SUPPORTED_SCHEMAS:
+                raise ImproperlyConfigured(f"Unsupported Git DataSource proxy scheme: {urlparse(proxy).scheme}")
 
-                if self.url_scheme in ('http', 'https'):
-                    config.set("http", "proxy", proxy)
-                    if urlparse(proxy).scheme in HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS:
-                        self.use_socks = True
+            if self.url_scheme in ('http', 'https'):
+                config.set("http", "proxy", proxy)
+                if urlparse(proxy).scheme in HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS:
+                    self.socks_proxy = proxy
 
         return config
 
@@ -98,8 +98,8 @@ def fetch(self):
         }
 
         # check if using socks for proxy - if so need to use custom pool_manager
-        if self.use_socks:
-            clone_args['pool_manager'] = ProxyPoolManager(settings.HTTP_PROXIES.get(self.url_scheme))
+        if self.socks_proxy:
+            clone_args['pool_manager'] = ProxyPoolManager(self.socks_proxy)
 
         if self.url_scheme in ('http', 'https'):
             if self.params.get('username'):
@@ -147,7 +147,7 @@ def init_config(self):
 
         # Initialize backend config
         return Boto3Config(
-            proxies=settings.HTTP_PROXIES,
+            proxies=resolve_proxies(url=self.url, context={'client': self}),
         )
 
     @contextmanager

netbox/core/jobs.py

@@ -5,6 +5,7 @@
 from django.conf import settings
 from netbox.jobs import JobRunner, system_job
 from netbox.search.backends import search_backend
+from utilities.proxy import resolve_proxies
 from .choices import DataSourceStatusChoices, JobIntervalChoices
 from .exceptions import SyncError
 from .models import DataSource
@@ -71,7 +72,7 @@ def send_census_report():
                 url=settings.CENSUS_URL,
                 params=census_data,
                 timeout=3,
-                proxies=settings.HTTP_PROXIES
+                proxies=resolve_proxies(url=settings.CENSUS_URL)
             )
         except requests.exceptions.RequestException:
             pass

netbox/core/plugins.py

@@ -11,6 +11,7 @@
 from netbox.plugins import PluginConfig
 from netbox.registry import registry
 from utilities.datetime import datetime_from_timestamp
+from utilities.proxy import resolve_proxies
 
 USER_AGENT_STRING = f'NetBox/{settings.RELEASE.version} {settings.RELEASE.edition}'
 CACHE_KEY_CATALOG_FEED = 'plugins-catalog-feed'
@@ -120,10 +121,11 @@ def get_catalog_plugins():
     def get_pages():
         # TODO: pagination is currently broken in API
         payload = {'page': '1', 'per_page': '50'}
+        proxies = resolve_proxies(url=settings.PLUGIN_CATALOG_URL)
         first_page = session.get(
             settings.PLUGIN_CATALOG_URL,
             headers={'User-Agent': USER_AGENT_STRING},
-            proxies=settings.HTTP_PROXIES,
+            proxies=proxies,
             timeout=3,
             params=payload
         ).json()
@@ -135,7 +137,7 @@ def get_pages():
             next_page = session.get(
                 settings.PLUGIN_CATALOG_URL,
                 headers={'User-Agent': USER_AGENT_STRING},
-                proxies=settings.HTTP_PROXIES,
+                proxies=proxies,
                 timeout=3,
                 params=payload
             ).json()

netbox/extras/dashboard/widgets.py

@@ -17,6 +17,7 @@
 from extras.choices import BookmarkOrderingChoices
 from utilities.object_types import object_type_identifier, object_type_name
 from utilities.permissions import get_permission_for_model
+from utilities.proxy import resolve_proxies
 from utilities.querydict import dict_to_querydict
 from utilities.templatetags.builtins.filters import render_markdown
 from utilities.views import get_viewname
@@ -330,7 +331,7 @@ def get_feed(self):
             response = requests.get(
                 url=self.config['feed_url'],
                 headers={'User-Agent': f'NetBox/{settings.RELEASE.version}'},
-                proxies=settings.HTTP_PROXIES,
+                proxies=resolve_proxies(url=self.config['feed_url'], context={'client': self}),
                 timeout=3
             )
             response.raise_for_status()

netbox/extras/management/commands/housekeeping.py

@@ -11,6 +11,7 @@
 
 from core.models import Job, ObjectChange
 from netbox.config import Config
+from utilities.proxy import resolve_proxies
 
 
 class Command(BaseCommand):
@@ -107,7 +108,7 @@ def handle(self, *args, **options):
                 response = requests.get(
                     url=settings.RELEASE_CHECK_URL,
                     headers=headers,
-                    proxies=settings.HTTP_PROXIES
+                    proxies=resolve_proxies(url=settings.RELEASE_CHECK_URL)
                 )
                 response.raise_for_status()
 

netbox/extras/webhooks.py

@@ -3,10 +3,10 @@
 import logging
 
 import requests
-from django.conf import settings
 from django_rq import job
 from jinja2.exceptions import TemplateError
 
+from utilities.proxy import resolve_proxies
 from .constants import WEBHOOK_EVENT_TYPES
 
 logger = logging.getLogger('netbox.webhooks')
@@ -63,9 +63,10 @@ def send_webhook(event_rule, model_name, event_type, data, timestamp, username,
         raise e
 
     # Prepare the HTTP request
+    url = webhook.render_payload_url(context)
     params = {
         'method': webhook.http_method,
-        'url': webhook.render_payload_url(context),
+        'url': url,
         'headers': headers,
         'data': body.encode('utf8'),
     }
@@ -88,7 +89,8 @@ def send_webhook(event_rule, model_name, event_type, data, timestamp, username,
         session.verify = webhook.ssl_verification
         if webhook.ca_file_path:
             session.verify = webhook.ca_file_path
-        response = session.send(prepared_request, proxies=settings.HTTP_PROXIES)
+        proxies = resolve_proxies(url=url, context={'client': webhook})
+        response = session.send(prepared_request, proxies=proxies)
 
     if 200 <= response.status_code <= 299:
         logger.info(f"Request succeeded; response status {response.status_code}")

netbox/netbox/settings.py

@@ -9,6 +9,7 @@
 from django.contrib.messages import constants as messages
 from django.core.exceptions import ImproperlyConfigured, ValidationError
 from django.core.validators import URLValidator
+from django.utils.module_loading import import_string
 from django.utils.translation import gettext_lazy as _
 
 from netbox.config import PARAMS as CONFIG_PARAMS
@@ -116,7 +117,7 @@
 FIELD_CHOICES = getattr(configuration, 'FIELD_CHOICES', {})
 FILE_UPLOAD_MAX_MEMORY_SIZE = getattr(configuration, 'FILE_UPLOAD_MAX_MEMORY_SIZE', 2621440)
 GRAPHQL_MAX_ALIASES = getattr(configuration, 'GRAPHQL_MAX_ALIASES', 10)
-HTTP_PROXIES = getattr(configuration, 'HTTP_PROXIES', None)
+HTTP_PROXIES = getattr(configuration, 'HTTP_PROXIES', {})
 INTERNAL_IPS = getattr(configuration, 'INTERNAL_IPS', ('127.0.0.1', '::1'))
 ISOLATED_DEPLOYMENT = getattr(configuration, 'ISOLATED_DEPLOYMENT', False)
 JINJA2_FILTERS = getattr(configuration, 'JINJA2_FILTERS', {})
@@ -131,6 +132,7 @@
 METRICS_ENABLED = getattr(configuration, 'METRICS_ENABLED', False)
 PLUGINS = getattr(configuration, 'PLUGINS', [])
 PLUGINS_CONFIG = getattr(configuration, 'PLUGINS_CONFIG', {})
+PROXY_ROUTERS = getattr(configuration, 'PROXY_ROUTERS', ['utilities.proxy.DefaultProxyRouter'])
 QUEUE_MAPPINGS = getattr(configuration, 'QUEUE_MAPPINGS', {})
 REDIS = getattr(configuration, 'REDIS')  # Required
 RELEASE_CHECK_URL = getattr(configuration, 'RELEASE_CHECK_URL', None)
@@ -201,6 +203,14 @@
             "RELEASE_CHECK_URL must be a valid URL. Example: https://api.github.com/repos/netbox-community/netbox"
         )
 
+# Validate configured proxy routers
+for path in PROXY_ROUTERS:
+    if type(path) is str:
+        try:
+            import_string(path)
+        except ImportError:
+            raise ImproperlyConfigured(f"Invalid path in PROXY_ROUTERS: {path}")
+
 
 #
 # Database
@@ -577,6 +587,7 @@ def _setting(name, default=None):
         sample_rate=SENTRY_SAMPLE_RATE,
         traces_sample_rate=SENTRY_TRACES_SAMPLE_RATE,
         send_default_pii=SENTRY_SEND_DEFAULT_PII,
+        # TODO: Support proxy routing
         http_proxy=HTTP_PROXIES.get('http') if HTTP_PROXIES else None,
         https_proxy=HTTP_PROXIES.get('https') if HTTP_PROXIES else None
     )

netbox/utilities/proxy.py

@@ -0,0 +1,55 @@
+from django.conf import settings
+from django.utils.module_loading import import_string
+from urllib.parse import urlparse
+
+__all__ = (
+    'DefaultProxyRouter',
+    'resolve_proxies',
+)
+
+
+class DefaultProxyRouter:
+    """
+    Base class for a proxy router.
+    """
+    @staticmethod
+    def _get_protocol_from_url(url):
+        """
+        Determine the applicable protocol (e.g. HTTP or HTTPS) from the given URL.
+        """
+        return urlparse(url).scheme
+
+    def route(self, url=None, protocol=None, context=None):
+        """
+        Returns the appropriate proxy given a URL or protocol. Arbitrary context data may also be passed where
+        available.
+
+        Args:
+            url: The specific request URL for which the proxy will be used (if known)
+            protocol: The protocol in use (e.g. http or https) (if known)
+            context: Additional context to aid in proxy selection. May include e.g. the requesting client.
+        """
+        if url and protocol is None:
+            protocol = self._get_protocol_from_url(url)
+        if protocol and protocol in settings.HTTP_PROXIES:
+            return {
+                protocol: settings.HTTP_PROXIES[protocol]
+            }
+        return settings.HTTP_PROXIES
+
+
+def resolve_proxies(url=None, protocol=None, context=None):
+    """
+    Return a dictionary of candidate proxies (compatible with the requests module), or None.
+
+    Args:
+        url: The specific request URL for which the proxy will be used (optional)
+        protocol: The protocol in use (e.g. http or https) (optional)
+        context: Arbitrary additional context to aid in proxy selection (optional)
+    """
+    context = context or {}
+
+    for item in settings.PROXY_ROUTERS:
+        router = import_string(item) if type(item) is str else item
+        if proxies := router().route(url=url, protocol=protocol, context=context):
+            return proxies