Add GH Action to create tranco_top_10k.csv and traco_top_50k.csv (#361)

Author: zoomequipd

.github/workflows/process-tranco.yml

@@ -0,0 +1,114 @@
+name: Process Tranco CSV
+
+on:
+  push:
+    paths:
+      - 'tranco.csv'
+  pull_request:
+    paths:
+      - 'tranco.csv'
+  workflow_dispatch: {}  # Simple manual trigger with no inputs
+
+jobs:
+  process-tranco:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1  # Only fetch the latest commit to speed up checkout
+      
+      - name: Set configuration
+        id: config
+        run: |
+          # Configuration is defined here
+          CONFIG='[{"count": 10000, "filename": "tranco_top_10k.csv"}, {"count": 50000, "filename": "tranco_top_50k.csv"}]'
+          echo "CONFIG=$CONFIG" >> $GITHUB_ENV
+          echo "Using configuration: $CONFIG"
+      
+      - name: Validate manifest.json
+        id: validate
+        run: |
+          # Check if manifest.json exists
+          if [ ! -f "manifest.json" ]; then
+            echo "Error: manifest.json file not found"
+            exit 1
+          fi
+          
+          # Create a temporary file to store validation results
+          TEMP_FILE=$(mktemp)
+          
+          # Check each output file in the configuration
+          echo $CONFIG | jq -c '.[]' | while read -r config; do
+            filename=$(echo $config | jq -r '.filename')
+            
+            # Check if the filename is in manifest.json
+            if ! grep -q "\"file\": \"$filename\"" manifest.json; then
+              echo "Error: $filename is not defined in manifest.json"
+              echo "VALIDATION_FAILED=true" >> $TEMP_FILE
+            else
+              echo "✓ $filename is defined in manifest.json"
+            fi
+          done
+          
+          # Exit if any file is not defined in manifest.json
+          if grep -q "VALIDATION_FAILED=true" $TEMP_FILE; then
+            echo "One or more output files are not defined in manifest.json. Please update manifest.json first."
+            rm $TEMP_FILE
+            exit 1
+          fi
+          
+          rm $TEMP_FILE
+      
+      - name: Process Tranco CSV
+        id: process
+        run: |
+          echo "Processing Tranco CSV with configuration: $CONFIG"
+          
+          # Check if tranco.csv exists
+          if [ ! -f "tranco.csv" ]; then
+            echo "Error: tranco.csv file not found"
+            exit 1
+          fi
+          
+          # Parse the JSON configuration and process each output
+          echo $CONFIG | jq -c '.[]' | while read -r config; do
+            count=$(echo $config | jq -r '.count')
+            filename=$(echo $config | jq -r '.filename')
+            
+            if [ -z "$count" ] || [ -z "$filename" ]; then
+              echo "Skipping invalid configuration: $config"
+              continue
+            fi
+            
+            # Get exactly the requested number of lines from the file
+            head -n $count tranco.csv > "$filename"
+            
+            lines=$(wc -l < "$filename")
+            echo "Successfully created $filename with $lines rows"
+          done
+      
+      - name: Configure Git
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "Tranco Process Bot"
+      
+      - name: Commit and push changes
+        run: |
+          # Parse the configuration to get the filenames
+          echo $CONFIG | jq -c '.[]' | while read -r config; do
+            filename=$(echo $config | jq -r '.filename')
+            # Add each generated file individually
+            git add "$filename"
+          done
+          
+          # Check if there are any changes to commit
+          if git diff --staged --quiet; then
+            echo "No changes detected in the output files. Nothing to commit."
+          else
+            echo "Changes detected. Committing and pushing..."
+            git commit -m "Update Tranco Top Domain [automated]" && git push
+          fi

manifest.json

@@ -160,6 +160,15 @@
         "entry"
       ]
     },
+    {
+      "file": "tranco_top_50k.csv",
+      "identifier": "tranco_50k",
+      "display_name": "Tranco Top 50,000 Domains",
+      "headers": [
+        "x",
+        "entry"
+      ]
+    },
     {
       "file": "umbrella_top_1m.csv",
       "identifier": "umbrella_1m",