Merge pull request #215 from artines1/PR_CMP

Add the ConsentManager category

Author: artines1

constants.py

@@ -55,11 +55,10 @@
     # email trackers added in 2022
     "tracking-protection-email-content",
     "tracking-protection-email-base",
+    # consent manager trackers added in 2025
+    "tracking-protection-consent-manager",
 )
-DNT_EMAIL_SECTIONS = (
-    "tracking-protection-email-content",
-    "tracking-protection-email-base",
-)
+
 DNT_CONTENT_SECTIONS = (
     "tracking-protection-content",
     "tracking-protection-contenteff",
@@ -104,7 +103,6 @@
 ]
 
 VERS_LARGE_ENTITIES_SEPARATION_STARTED = 74
-VERSION_EMAIL_CATEGORY_INTRODUCED = 97
 
 WEBKIT_LISTS_DIR = 'webkit-lists'
 WEBKIT_BLOCK_ALL = "block"

lists2safebrowsing.py

@@ -13,22 +13,21 @@
 from publicsuffixlist.update import updatePSL
 
 from constants import (
-    DNT_EMAIL_SECTIONS,
     DNT_SECTIONS,
     PLUGIN_SECTIONS,
     PRE_DNT_SECTIONS,
     LARGE_ENTITIES_SECTIONS,
     STANDARD_ENTITY_SECTION,
     TEST_DOMAIN_TEMPLATE,
-    VERSION_EMAIL_CATEGORY_INTRODUCED,
     VERS_LARGE_ENTITIES_SEPARATION_STARTED,
     ENTITYLIST_SECTIONS
 )
 
 from utils import (
     get_blocked_domains,
     add_domain_to_list,
-    load_json_from_url
+    load_json_from_url,
+    should_skip_section_for_version,
 )
 
 from publish2cloud import (
@@ -442,10 +441,8 @@ def get_versioned_lists(config, chunknum, version):
         version_configurations(config, section, version)
         ver = p_version.parse(version)
         if (section in PRE_DNT_SECTIONS or section in DNT_SECTIONS):
-            skip_section = (
-                section in DNT_EMAIL_SECTIONS
-                and ver.release[0] < VERSION_EMAIL_CATEGORY_INTRODUCED
-            )
+            skip_section = should_skip_section_for_version(config, section, ver.release[0])
+
             if skip_section:
                 # import ipdb; ipdb.set_trace()
                 continue

publish2cloud.py

@@ -33,6 +33,8 @@
     shared_state
 )
 
+from utils import should_skip_section_for_version
+
 from kinto_http import Client, BearerTokenAuth, KintoException
 
 try:
@@ -170,7 +172,7 @@ def new_data_to_publish_to_remote_settings(config, section, new, version=None):
     # Check to see if update is needed on Remote Settings
     record = get_record_remote_settings(record_name)
 
-    if version is None:
+    if version is None and record:
         # We need to check if the filter_expression needs to be updated for the
         # nightly records. The filter_expression needs to be updated if the
         # latest supported version has changed
@@ -356,6 +358,11 @@ def publish_to_cloud(config, chunknum, check_versioning=None):
                 continue
 
             version = p_version.parse(config.get(section, 'version'))
+            skip_section = should_skip_section_for_version(config, section, version.release[0])
+
+            if skip_section:
+                continue
+
             skip_large_entity_separation = (
                 version.release[0] < VERS_LARGE_ENTITIES_SEPARATION_STARTED
                 and section in LARGE_ENTITIES_SECTIONS

rs_dev.ini

@@ -185,8 +185,16 @@ versioning_needed=true
 categories=Email
 output=base-email-track-digest256
 versioning_needed=true
+min_supported_version=97
 
 [tracking-protection-email-content]
 categories=EmailAggressive
 output=content-email-track-digest256
-versioning_needed=true
+versioning_needed=true
+min_supported_version=97
+
+[tracking-protection-consent-manager]
+categories=ConsentManagers
+output=consent-manager-track-digest256
+versioning_needed=true
+min_supported_version=137

rs_prod.ini

@@ -185,8 +185,16 @@ versioning_needed=true
 categories=Email
 output=base-email-track-digest256
 versioning_needed=true
+min_supported_version=97
 
 [tracking-protection-email-content]
 categories=EmailAggressive
 output=content-email-track-digest256
-versioning_needed=true
+versioning_needed=true
+min_supported_version=97
+
+[tracking-protection-consent-manager]
+categories=ConsentManagers
+output=consent-manager-track-digest256
+versioning_needed=true
+min_supported_version=137

rs_stage.ini

@@ -185,8 +185,16 @@ versioning_needed=true
 categories=Email
 output=base-email-track-digest256
 versioning_needed=true
+min_supported_version=97
 
 [tracking-protection-email-content]
 categories=EmailAggressive
 output=content-email-track-digest256
-versioning_needed=true
+versioning_needed=true
+min_supported_version=97
+
+[tracking-protection-consent-manager]
+categories=ConsentManagers
+output=consent-manager-track-digest256
+versioning_needed=true
+min_supported_version=137

utils.py

@@ -5,6 +5,7 @@
 from trackingprotection_tools import DisconnectParser
 import sys
 import json
+from packaging import version as p_version
 from publicsuffixlist import PublicSuffixList
 from publicsuffixlist.update import updatePSL
 from urllib.request import urlopen
@@ -198,4 +199,18 @@ def get_domains_from_filters(parser, category_filters,
         print(" * found %d rule(s) with filter %s. Filtered output to %d." %
               (len(result), tag_filters, len(output)))
 
-    return output
+    return output
+
+def should_skip_section_for_version(config, section, version):
+    """
+    Checks if the section should be skipped for the given version.
+    """
+    skip_section = False
+
+    min_supported_version = (config.has_option(section, 'min_supported_version')
+                             and p_version.parse(config.get(section, 'min_supported_version')))
+
+    if min_supported_version and version < min_supported_version.release[0]:
+        skip_section = True
+
+    return skip_section