fix: Dynamic robots.txt (#11166)

* dynamic robots.txt

* Update app/robots.txt/route.ts

---------

Co-authored-by: Stephanie Anderson <[email protected]>

Author: a-hariti

app/robots.txt/route.ts

@@ -0,0 +1,11 @@
+import {isDeveloperDocs} from 'sentry-docs/isDeveloperDocs';
+
+export const GET = (_request: Request) => {
+  return new Response(
+    `
+Sitemap: ${isDeveloperDocs ? 'https://develop.sentry.dev/sitemap.xml' : 'https://docs.sentry.io/sitemap.xml'}
+User-agent: *
+`.trim(),
+    {headers: {'content-type': 'text/plain'}}
+  );
+};