Merge pull request #213 from mozilla-services/feat/add-rs-script

[WebkitLists] P3. Add `publish2rs` script to push changes to RS

Author: issammani

Dockerfile

@@ -20,4 +20,4 @@ RUN python -m pip install --no-cache-dir -r requirements.txt
 # Copy the rest of the application code
 COPY --chown=app:app . .
 
-CMD ["python", "lists2safebrowsing.py"]
+CMD ["sh", "enrypoint.sh"]

entrypoint.sh

@@ -0,0 +1,3 @@
+# Build and publish the tracking lists for safebrowsing and webkit
+python lists2safebrowsing.py
+python lists2webkit.py

lists2webkit.py

@@ -1,6 +1,7 @@
 import json
 import os
 from settings import config
+from publish2rs import publish2rs
 from constants import (
     DNT_SECTIONS,
     PRE_DNT_SECTIONS,
@@ -149,5 +150,10 @@ def main():
 
     print("All content blocker rules have been generated successfully.")
 
+    # Push to remote settings
+    publish2rs()
+
+    return os.EX_OK
+
 if __name__ == "__main__":
     main()

publish2rs.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+
+import os
+import kinto_http
+import hashlib
+import pathlib
+import uuid
+
+from settings import config
+
+from constants import WEBKIT_LISTS_DIR
+
+def get_config_if_env(env_var, config_section, config_option, fallback=""):
+    """
+    Return the config value if the environment variable exists; otherwise, return the fallback.
+    """
+    if env_var in os.environ:
+        return os.environ[env_var]
+    return config.get(config_section, config_option, fallback=fallback)
+
+def get_file_hash(list_path):
+    with open(list_path, "rb") as f:
+        return hashlib.sha256(f.read()).hexdigest()
+
+SERVER = get_config_if_env("SERVER", "main", "remote_settings_url")
+BUCKET = "main-workspace"
+COLLECTION = "tracking-protection-lists-ios"
+AUTHORIZATION = get_config_if_env("AUTHORIZATION", "main", "remote_settings_authorization")
+ENVIRONMENT = os.getenv("ENVIRONMENT", "local").lower()
+DRY_RUN = os.getenv("DRY_RUN", "0") in "1yY"
+
+def publish2rs():
+    client = kinto_http.Client(
+        server_url=SERVER,
+        auth=AUTHORIZATION,
+        bucket=BUCKET,
+        collection=COLLECTION,
+        dry_mode=DRY_RUN
+    )
+
+    remote_attachments = {
+        r["name"]: {"id": r["id"], "hash": r["attachment"]["hash"]}
+        for r in client.get_records()
+    }
+
+    local_attachments = {
+        file.stem: get_file_hash(file)
+        for file in pathlib.Path(WEBKIT_LISTS_DIR).iterdir()
+    }
+
+    # Determine records to create, update, or delete
+    # by comparing the attachment sha256 hashes.
+    to_create = []
+    to_update = []
+    for name, hash in local_attachments.items():
+        remote_attachment = remote_attachments.pop(name, None)
+        if remote_attachment is None:
+            to_create.append({"name": name})
+        elif remote_attachment["hash"] != hash:
+            to_update.append({"id": remote_attachment["id"], "name": name})
+    # Remaining records in `remote_attachments` are to be deleted.
+    to_delete = [{"id": record["id"]} for _, record in remote_attachments.items()]
+
+    # Print changes
+    print("Changes to apply:")
+    print(f"To create: {to_create}")
+    print(f"To update: {to_update}")
+    print(f"To delete: {to_delete}")
+
+    has_pending_changes = (len(to_create) + len(to_update) + len(to_delete)) > 0
+    if not has_pending_changes:
+        print("Records are in sync. Nothing to do ✅.")
+        return os.EX_OK
+
+    # Batch delete operations.
+    # NOTE: Attachment deletion is implicit when deleting a record.
+    with client.batch() as batch:
+        for record in to_delete:
+            batch.delete_record(id=record["id"])
+
+    # Adding, updating attachments on client
+    # since batch operations are not supported.
+    for record in to_create:
+        id = str(uuid.uuid4())
+        filepath = f"{WEBKIT_LISTS_DIR}/{record['name']}.json"
+        client.add_attachment(id=id, data={"name": record["name"]}, filepath=filepath)
+    for record in to_update:
+        filepath = f"{WEBKIT_LISTS_DIR}/{record['name']}.json"
+        client.add_attachment(id=record["id"], filepath=filepath)
+
+
+    if ENVIRONMENT == "dev":
+        # Self approve changes on DEV.
+        # message arg is not supported as of now.
+        client.approve_changes()
+        print("Changes applied to dev server ✅")
+    else:
+        # Request review.
+        print("Request review...", end="")
+        client.request_review(message="r?")
+        print("✅")

requirements.txt

@@ -3,4 +3,4 @@ publicsuffixlist==0.7.5
 requests==2.32.0
 trackingprotection-tools==0.6.1
 packaging==20.4
-kinto-http==11.1.0
+kinto-http==11.7.0