feat(nginx): Add extra agents

szinn · Mar 4, 2025 · 5c00475 · 5c00475
1 parent 3565398
commit 5c00475
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/kubernetes/main/apps/networking/nginx/external/helmrelease.yaml b/kubernetes/main/apps/networking/nginx/external/helmrelease.yaml
@@ -59,14 +59,14 @@ spec:
       config:
         allow-snippet-annotations: true
         annotations-risk-level: Critical
-        block-user-agents: "AdsBot-Google,Amazonbot,anthropic-ai,Applebot-Extended,Bytespider,CCBot,ChatGPT-User,ClaudeBot,Claude-Web,cohere-ai,Diffbot,FacebookBot,FriendlyCrawler,Google-Extended,GoogleOther,GPTBot,img2dataset,omgili,omgilibot,peer39_crawler,peer39_crawler/1.0,PerplexityBot,YouBot," # taken from https://github.com/ai-robots-txt/ai.robots.txt
+        block-user-agents: AI2Bot,Ai2Bot-Dolma,Amazonbot,anthropic-ai,Applebot,Applebot-Extended,Brightbot 1.0,Bytespider,CCBot,ChatGPT-User,Claude-Web,ClaudeBot,cohere-ai,cohere-training-data-crawler,Crawlspace,Diffbot,DuckAssistBot,FacebookBot,FriendlyCrawler,Google-Extended,GoogleOther,GoogleOther-Image,GoogleOther-Video,GPTBot,iaskspider/2.0,ICC-Crawler,ImagesiftBot,img2dataset,ISSCyberRiskCrawler,Kangaroo Bot,Meta-ExternalAgent,Meta-ExternalFetcher,OAI-SearchBot,omgili,omgilibot,PanguBot,PerplexityBot,PetalBot,Scrapy,SemrushBot-OCOB,SemrushBot-SWA,Sidetrade indexer bot,Timpibot,VelenPublicWebCrawler,Webzio-Extended,YouBot
         client-body-buffer-size: 100M
         client-body-timeout: 120
         client-header-timeout: 120
         enable-brotli: "true"
         enable-ocsp: "true"
-        force-ssl-redirect: "true"
         enable-real-ip: "true"
+        force-ssl-redirect: "true"
         hide-headers: Server,X-Powered-By
         hsts-max-age: 31449600
         keep-alive: 120

diff --git a/kubernetes/main/apps/networking/nginx/internal/helmrelease.yaml b/kubernetes/main/apps/networking/nginx/internal/helmrelease.yaml
@@ -58,6 +58,7 @@ spec:
 
       config:
         allow-snippet-annotations: true
+        block-user-agents: AI2Bot,Ai2Bot-Dolma,Amazonbot,anthropic-ai,Applebot,Applebot-Extended,Brightbot 1.0,Bytespider,CCBot,ChatGPT-User,Claude-Web,ClaudeBot,cohere-ai,cohere-training-data-crawler,Crawlspace,Diffbot,DuckAssistBot,FacebookBot,FriendlyCrawler,Google-Extended,GoogleOther,GoogleOther-Image,GoogleOther-Video,GPTBot,iaskspider/2.0,ICC-Crawler,ImagesiftBot,img2dataset,ISSCyberRiskCrawler,Kangaroo Bot,Meta-ExternalAgent,Meta-ExternalFetcher,OAI-SearchBot,omgili,omgilibot,PanguBot,PerplexityBot,PetalBot,Scrapy,SemrushBot-OCOB,SemrushBot-SWA,Sidetrade indexer bot,Timpibot,VelenPublicWebCrawler,Webzio-Extended,YouBot
         annotations-risk-level: Critical
         client-body-buffer-size: 100M
         client-body-timeout: 120
@@ -66,6 +67,7 @@ spec:
         enable-ocsp: "true"
         enable-real-ip: "true"
         force-ssl-redirect: "true"
+        hide-headers: Server,X-Powered-By
         hsts-max-age: 31449600
         keep-alive: 120
         keep-alive-requests: 10000