feat: add robot.txt

Chasen-Zhang · Chasen-Zhang · commit 55a088e6182c · 2024-11-29T14:24:00.000+08:00
diff --git a/package.json b/package.json
@@ -12,12 +12,14 @@
     "block_after:cn": "node generate-language-block.js block_after:cn",
     "start:cn": "site=cn site_env=development docusaurus start",
     "start:en": "site=en site_env=development docusaurus start",
+    "generate:robots:cn": "echo 'User-agent: *\nDisallow: /404\nDisallow: /tags/\nSitemap: https://www.databend.cn/sitemap.xml' > ./static/robots.txt",
+    "generate:robots:en": "echo 'User-agent: *\nDisallow: /404\nDisallow: /tags/\nSitemap: https://www.databend.com/sitemap.xml' > ./static/robots.txt",
     "dev": "npm run start:en",
     "dev:cn": "npm run start:cn",
     "dev:en": "npm run dev",
     "build": "npm run build:en",
-    "build:cn": "npm run generate-release-index && npm run block_before:cn && site=cn site_env=production docusaurus build && npm run block_after:cn",
-    "build:en": "npm run generate-release-index && npm run block_before:en && site=en site_env=production docusaurus build &&  npm run block_after:en",
+    "build:cn": "npm run generate:robots:cn && npm run generate-release-index && npm run block_before:cn && site=cn site_env=production docusaurus build && npm run block_after:cn",
+    "build:en": "npm run generate:robots:en && npm run generate-release-index && npm run block_before:en && site=en site_env=production docusaurus build &&  npm run block_after:en",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",
     "clear": "docusaurus clear",
diff --git a/src/theme/Footer/index.tsx b/src/theme/Footer/index.tsx
@@ -25,6 +25,9 @@ function Footer() {
   } = useDocusaurusContext() as any;
   return (
     <footer className={clsx("footer", styles.footer)}>
+      <Head>
+        <meta name="robots" content="index, follow"></meta>
+      </Head>
       {!isChina && (
         <Head>
           <script async src={useBaseUrl("/Koala/index.js")}></script>
