diff --git a/plugins/pencilblue/templates/robots.html b/plugins/pencilblue/templates/robots.html index 2c4f7c644..c2f50b9d4 100644 --- a/plugins/pencilblue/templates/robots.html +++ b/plugins/pencilblue/templates/robots.html @@ -6,30 +6,33 @@ User-agent: trovitBot Disallow: / - + User-agent: Jobrapido Disallow: / - + User-agent: sogou spider Disallow: / - + User-agent: JobBot Disallow: / - + User-agent: JobCrawlerBot Disallow: / - + User-agent: Exabot Disallow: / - + User-agent: WijuBot Disallow: / - + User-agent: bingbot Crawl-delay: 5 User-Agent: * Disallow: Disallow: /admin +Disallow: /jobs +Disallow: /it-jobs +Disallow: /careers Sitemap: ^site_root^/sitemap