From f4532e3f978e29188a851c0e9f9ad60da955b47e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93scar=20Bustos?= Date: Wed, 25 Dec 2024 21:47:54 +0100 Subject: [PATCH 1/4] feat: analyzer beta --- src/app/analyze-robots/page.tsx | 9 + src/app/api/analyze-robots/route.ts | 111 +++++++++ src/app/page.module.css | 230 ------------------ src/app/page.tsx | 1 - src/components/analyzer/analyzer.module.css | 42 ++++ src/components/analyzer/index.tsx | 151 ++++++++++++ src/components/content/content.module.css | 1 - src/components/faqs/faqs.module.css | 9 +- src/components/generator/index.tsx | 4 +- src/components/header/index.tsx | 8 + src/components/hero/index.tsx | 6 +- src/components/showcase/index.tsx | 73 ++++++ src/components/showcase/showcase.module.css | 89 +++++++ src/components/{code => terminal}/index.tsx | 17 +- .../terminal.module.css} | 0 src/components/validate/index.tsx | 4 +- 16 files changed, 503 insertions(+), 252 deletions(-) create mode 100644 src/app/analyze-robots/page.tsx create mode 100644 src/app/api/analyze-robots/route.ts delete mode 100644 src/app/page.module.css create mode 100644 src/components/analyzer/analyzer.module.css create mode 100644 src/components/analyzer/index.tsx create mode 100644 src/components/showcase/index.tsx create mode 100644 src/components/showcase/showcase.module.css rename src/components/{code => terminal}/index.tsx (90%) rename src/components/{code/code.module.css => terminal/terminal.module.css} (100%) diff --git a/src/app/analyze-robots/page.tsx b/src/app/analyze-robots/page.tsx new file mode 100644 index 0000000..35eca22 --- /dev/null +++ b/src/app/analyze-robots/page.tsx @@ -0,0 +1,9 @@ +import RobotsAnalyzer from "@/components/analyzer"; + +export default function AnalyzePage() { + return ( +
+ +
+ ); +} diff --git a/src/app/api/analyze-robots/route.ts b/src/app/api/analyze-robots/route.ts new file mode 100644 index 0000000..57d2885 --- /dev/null +++ b/src/app/api/analyze-robots/route.ts @@ -0,0 +1,111 @@ +import { NextResponse } from 'next/server'; + +export async function POST(request: Request) { + try { + const { url } = await request.json(); + + const response = await fetch(url); + if (!response.ok) { + throw new Error('No robots.txt found'); + } + + const content = await response.text(); + + const cleanedContent = cleanRobotsTxt(content); + + // Parse and analyze robots.txt + + return NextResponse.json({ + content: cleanedContent, + analysis: analyzeRobotsTxt(content) + }); + } catch (error) { + return NextResponse.json( + { error: 'Failed to analyze robots.txt' }, + { status: 500 } + ); + } +} + +function cleanRobotsTxt(content: string): string { + const lines = content.split('\n'); + const cleanedLines = []; + + for (const line of lines) { + const [directive, ...valueParts] = line.split(':'); + const value = valueParts.join(':').trim(); + + if (!directive || !value) continue; + + const cleanDirective = directive.trim(); + if (cleanDirective.toLowerCase() === 'user-agent') { + cleanedLines.push(`${cleanDirective}: ${value}`); + } else if (['disallow', 'allow', 'sitemap'].includes(cleanDirective.toLowerCase())) { + const cleanValue = value.startsWith('*') ? '/' + value : value; + cleanedLines.push(`${cleanDirective}: ${cleanValue}`); + } + } + + return cleanedLines.join('\n'); +} + +function analyzeRobotsTxt(content: string): string { + const lines = content.split('\n'); + let analysis = []; + let currentAgent = '*'; + + const rules: {[key: string]: {allow: string[], disallow: string[]}} = {}; + let sitemaps: string[] = []; + + for (const line of lines) { + const trimmedLine = line.trim(); + if (!trimmedLine || trimmedLine.startsWith('#')) continue; + + if (!trimmedLine.includes(':')) continue; + + const [directive, value] = trimmedLine.split(':').map(part => part.trim().toLowerCase()); + + switch(directive) { + case 'user-agent': + currentAgent = value; + if (!rules[currentAgent]) { + rules[currentAgent] = { allow: [], disallow: [] }; + } + break; + case 'disallow': + if (value) rules[currentAgent].disallow.push(value); + break; + case 'allow': + if (value) rules[currentAgent].allow.push(value); + break; + case 'sitemap': + if (value) sitemaps.push(value); + break; + } + } + + // Generate human-readable analysis + Object.entries(rules).forEach(([agent, ruleSet], index) => { + analysis.push(`\n\nAgent rules ${index + 1}: ${agent}`); + + if (ruleSet.disallow.length) { + analysis.push('Blocked from crawling:'); + analysis.push('PATHS_START'); + ruleSet.disallow.forEach(path => analysis.push(path)); + analysis.push('PATHS_END'); + } + + if (ruleSet.allow.length) { + analysis.push('Explicitly allowed:'); + analysis.push('PATHS_START'); + ruleSet.allow.forEach(path => analysis.push(path)); + analysis.push('PATHS_END'); + } + }); + + if (sitemaps.length) { + analysis.push(`\n\nSitemaps declared: ${sitemaps.join(', ')}`); + } + + return analysis.join('\n'); +} \ No newline at end of file diff --git a/src/app/page.module.css b/src/app/page.module.css deleted file mode 100644 index 5c4b1e6..0000000 --- a/src/app/page.module.css +++ /dev/null @@ -1,230 +0,0 @@ -.main { - display: flex; - flex-direction: column; - justify-content: space-between; - align-items: center; - padding: 6rem; - min-height: 100vh; -} - -.description { - display: inherit; - justify-content: inherit; - align-items: inherit; - font-size: 0.85rem; - max-width: var(--max-width); - width: 100%; - z-index: 2; - font-family: var(--font-mono); -} - -.description a { - display: flex; - justify-content: center; - align-items: center; - gap: 0.5rem; -} - -.description p { - position: relative; - margin: 0; - padding: 1rem; - background-color: rgba(var(--callout-rgb), 0.5); - border: 1px solid rgba(var(--callout-border-rgb), 0.3); - border-radius: var(--border-radius); -} - -.code { - font-weight: 700; - font-family: var(--font-mono); -} - -.grid { - display: grid; - grid-template-columns: repeat(4, minmax(25%, auto)); - max-width: 100%; - width: var(--max-width); -} - -.card { - padding: 1rem 1.2rem; - border-radius: var(--border-radius); - background: rgba(var(--card-rgb), 0); - border: 1px solid rgba(var(--card-border-rgb), 0); - transition: background 200ms, border 200ms; -} - -.card span { - display: inline-block; - transition: transform 200ms; -} - -.card h2 { - font-weight: 600; - margin-bottom: 0.7rem; -} - -.card p { - margin: 0; - opacity: 0.6; - font-size: 0.9rem; - line-height: 1.5; - max-width: 30ch; - text-wrap: balance; -} - -.center { - display: flex; - justify-content: center; - align-items: center; - position: relative; - padding: 4rem 0; -} - -.center::before { - background: var(--secondary-glow); - border-radius: 50%; - width: 480px; - height: 360px; - margin-left: -400px; -} - -.center::after { - background: var(--primary-glow); - width: 240px; - height: 180px; - z-index: -1; -} - -.center::before, -.center::after { - content: ""; - left: 50%; - position: absolute; - filter: blur(45px); - transform: translateZ(0); -} - -.logo { - position: relative; -} -/* Enable hover only on non-touch devices */ -@media (hover: hover) and (pointer: fine) { - .card:hover { - background: rgba(var(--card-rgb), 0.1); - border: 1px solid rgba(var(--card-border-rgb), 0.15); - } - - .card:hover span { - transform: translateX(4px); - } -} - -@media (prefers-reduced-motion) { - .card:hover span { - transform: none; - } -} - -/* Mobile */ -@media (max-width: 700px) { - .content { - padding: 4rem; - } - - .grid { - grid-template-columns: 1fr; - margin-bottom: 120px; - max-width: 320px; - text-align: center; - } - - .card { - padding: 1rem 2.5rem; - } - - .card h2 { - margin-bottom: 0.5rem; - } - - .center { - padding: 8rem 0 6rem; - } - - .center::before { - transform: none; - height: 300px; - } - - .description { - font-size: 0.8rem; - } - - .description a { - padding: 1rem; - } - - .description p, - .description div { - display: flex; - justify-content: center; - position: fixed; - width: 100%; - } - - .description p { - align-items: center; - inset: 0 0 auto; - padding: 2rem 1rem 1.4rem; - border-radius: 0; - border: none; - border-bottom: 1px solid rgba(var(--callout-border-rgb), 0.25); - background: linear-gradient( - to bottom, - rgba(var(--background-start-rgb), 1), - rgba(var(--callout-rgb), 0.5) - ); - background-clip: padding-box; - backdrop-filter: blur(24px); - } - - .description div { - align-items: flex-end; - pointer-events: none; - inset: auto 0 0; - padding: 2rem; - height: 200px; - background: linear-gradient( - to bottom, - transparent 0%, - rgb(var(--background-end-rgb)) 40% - ); - z-index: 1; - } -} - -/* Tablet and Smaller Desktop */ -@media (min-width: 701px) and (max-width: 1120px) { - .grid { - grid-template-columns: repeat(2, 50%); - } -} - -@media (prefers-color-scheme: dark) { - .vercelLogo { - filter: invert(1); - } - - .logo { - filter: invert(1) drop-shadow(0 0 0.3rem #ffffff70); - } -} - -@keyframes rotate { - from { - transform: rotate(360deg); - } - to { - transform: rotate(0deg); - } -} diff --git a/src/app/page.tsx b/src/app/page.tsx index ededd9f..8e19cb6 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -1,5 +1,4 @@ import Hero from "@/components/hero"; -import Faqs from "@/components/faqs"; import Content from "@/components/content"; import Generator from "@/components/generator"; diff --git a/src/components/analyzer/analyzer.module.css b/src/components/analyzer/analyzer.module.css new file mode 100644 index 0000000..fddfc22 --- /dev/null +++ b/src/components/analyzer/analyzer.module.css @@ -0,0 +1,42 @@ +.container { + display: flex; + align-items: center; + justify-content: center; + padding-top: var(--space-9); + position: relative; +} + +.analysis { + padding: var(--space-5) 0; + margin: var(--space-5) 0; +} + +.box { + z-index: 1; + position: relative; + padding: var(--space-4) 0; +} + +.pre { + font-family: var(--font-mono); + font-size: 0.875rem; + background-color: hsl(var(--gray-3)); + border-radius: var(--radius-3); +} + +.code { + font-family: inherit; + display: block; + line-height: 1.5; + background-color: transparent; + white-space: pre-wrap; +} + +.wrapper { + display: grid; + align-items: center; + gap: var(--space-2); + justify-content: center; + padding: var(--space-8); + grid-template-columns: 1fr auto; +} \ No newline at end of file diff --git a/src/components/analyzer/index.tsx b/src/components/analyzer/index.tsx new file mode 100644 index 0000000..3ad4f00 --- /dev/null +++ b/src/components/analyzer/index.tsx @@ -0,0 +1,151 @@ +"use client"; + +import { JSX, useState } from 'react'; +import { TextField, Button, Card, Box, Text, Container, Code, Grid, Heading } from '@radix-ui/themes'; +import Terminal from '@/components/terminal'; +import GridDecoration from '@/components/grid-decoration'; +import Showcase from '@/components/showcase'; + +import styles from './analyzer.module.css'; + +const RobotsAnalyzer = () => { + const [url, setUrl] = useState(''); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(''); + const [analysis, setAnalysis] = useState(''); + const [robotsContent, setRobotsContent] = useState(''); + + const validateUrl = (url : string) => { + try { + new URL(url); + return true; + } catch { + return false; + } + }; + + const fetchRobotsTxt = async () => { + if (!validateUrl(url)) { + setError('Please enter a valid URL'); + return; + } + + setLoading(true); + setError(''); + + try { + const robotsUrl = new URL('/robots.txt', url).href; + const response = await fetch('/api/analyze-robots', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ url: robotsUrl }) + }); + + if (!response.ok) throw new Error('Failed to fetch robots.txt'); + + const data = await response.json(); + setAnalysis(data.analysis); + setRobotsContent(data.content); + } catch (err) { + setError('Could not fetch or analyze robots.txt'); + } finally { + setLoading(false); + } + }; + + const renderPaths = (lines: string[]) => { + let content: JSX.Element[] = []; + let pathsContent: JSX.Element[] = []; + let inPathsBlock = false; + let blockId = 0; + + lines.forEach((line, j) => { + if (line === 'PATHS_START') { + inPathsBlock = true; + blockId++; + } else if (line === 'PATHS_END') { + if (pathsContent.length) { + content.push( + + {pathsContent} + + ); + } + inPathsBlock = false; + pathsContent = []; + } else if (inPathsBlock) { + pathsContent.push( + 
+            {line}
+
+ ); + } else { + content.push( + + {line} + + ); + } + }); + + return content; + }; + + const onClickSuggestion = (url: string) => { + setUrl(url); + fetchRobotsTxt(); + } + + return ( + + + + + Analyze Robots.txt of any website + +
+ setUrl(e.target.value)} + /> + + +
+ {error && ( + {error} + )} + + {!url && } + + {analysis && (
+ +
)} + + {analysis && ( + + Result + + Below is the analysis of the robots.txt file from from {url}. + + + + + {analysis.split('\n\n').map((section, sectionIndex) => ( + + {renderPaths(section.split('\n'))} + + ))} + + + + + )} + + + ); +}; + +export default RobotsAnalyzer; \ No newline at end of file diff --git a/src/components/content/content.module.css b/src/components/content/content.module.css index 6794245..8732f13 100644 --- a/src/components/content/content.module.css +++ b/src/components/content/content.module.css @@ -1,7 +1,6 @@ .container { padding: var(--space-6) 0; border-top: 1px solid var(--gray-4); - border-bottom: 1px solid var(--gray-4); min-height: 60vh; display: flex; align-items: center; diff --git a/src/components/faqs/faqs.module.css b/src/components/faqs/faqs.module.css index 2b6ee37..2530806 100644 --- a/src/components/faqs/faqs.module.css +++ b/src/components/faqs/faqs.module.css @@ -1,7 +1,8 @@ .container { position: relative; width: 100%; - padding-top: 3rem; + padding-top: var(--space-8); + border-top: 1px solid var(--gray-4); } .pre { @@ -18,7 +19,7 @@ font-family: inherit; display: block; line-height: 1.5; - padding: 1rem; + padding: var(--space-4); white-space: pre-wrap; } @@ -28,7 +29,7 @@ } .title { - margin-bottom: 1rem; + margin-bottom: var(--space-4); font-weight: 600; color: hsl(var(--gray-12)); } @@ -40,7 +41,7 @@ } .list { - margin: 0.75rem 0 1.5rem 1rem; + margin: 0.75rem 0 1.5rem var(--space-4); list-style-type: disc; } diff --git a/src/components/generator/index.tsx b/src/components/generator/index.tsx index b2d6da8..ab6e556 100644 --- a/src/components/generator/index.tsx +++ b/src/components/generator/index.tsx @@ -14,7 +14,7 @@ import { import { PackageOpen, AppWindow } from "lucide-react"; import { defaults, apps, aiBots } from "@/configs"; -import Code from "@/components/code"; +import Terminal from "@/components/terminal"; import Sitemap from "@/components/sitemap"; import Aibots from "@/components/aibots"; @@ -193,7 +193,7 @@ export default function Generator() { Your raw file, feel free to create it by your own or download it - setValue(value)} onClear={onReset} diff --git a/src/components/header/index.tsx b/src/components/header/index.tsx index 97fa831..e057e53 100644 --- a/src/components/header/index.tsx +++ b/src/components/header/index.tsx @@ -35,6 +35,14 @@ export default function Header() { > Validator + + + Analyze Robots + FAQs diff --git a/src/components/hero/index.tsx b/src/components/hero/index.tsx index 44260f4..40324ae 100644 --- a/src/components/hero/index.tsx +++ b/src/components/hero/index.tsx @@ -5,15 +5,13 @@ import Robot from "../robot"; import GridDecoration from "../grid-decoration"; interface HeroProps { - feature: string; + feature: string; } - + export default function Hero({ feature }: HeroProps) { return (
- - diff --git a/src/components/showcase/index.tsx b/src/components/showcase/index.tsx new file mode 100644 index 0000000..94f1cc1 --- /dev/null +++ b/src/components/showcase/index.tsx @@ -0,0 +1,73 @@ +"use client"; + +import { Heading, Text } from '@radix-ui/themes'; +import styles from './showcase.module.css'; + +const websites = [ + { + name: "Tinybird", + robots: "Best data solution infrastructure for software teams", + color: "var(--grass-9)", + url: "https://www.tinybird.co/" + }, + { + name: "Topdelmes", + robots: "TV shows and movies reviews and rankings site", + color: "var(--amber-9)", + url: "https://www.topdelmes.com" + }, + { + name: "Marketgoo", + robots: "Empower web hosts, agencies, & SMB providers", + color: "var(--blue-9)", + url: "https://www.marketgoo.com/" + }, + { + name: "Faedo Digital", + robots: "Digital solutions for small and rural businesses", + color: "var(--grass-9)", + url: "https://faedodigital.com/" + } +]; + +interface ShowcaseCardProps { + name: string; + robots: string; + color: string; + url: string; + onClick: (url: string) => void; +} + +interface ShowcaseProps { + onClickWebsite: (url: string) => void; +} + + +const ShowcaseCard = ({ name, robots, color, url, onClick }: ShowcaseCardProps) => ( +
onClick(url)} + > +
+
+
+ {name} + {robots} +
+
+); + + +const Showcase = ({ onClickWebsite } : ShowcaseProps) => ( +
+
+
+ {[...websites, ...websites, ...websites].map((site, i) => ( + onClickWebsite(url)} /> + ))} +
+
+
+); + +export default Showcase; \ No newline at end of file diff --git a/src/components/showcase/showcase.module.css b/src/components/showcase/showcase.module.css new file mode 100644 index 0000000..e4221cb --- /dev/null +++ b/src/components/showcase/showcase.module.css @@ -0,0 +1,89 @@ +.container { + position: relative; + padding: var(--space-9) 0; +} + +.wrapper { + position: relative; + overflow: hidden; + padding: 1rem 0; + mask-image: linear-gradient( + to right, + transparent, + black 20%, + black 80%, + transparent + ); +} + +.track { + display: flex; + gap: 2rem; + animation: scroll 40s linear infinite; + width: fit-content; +} + +.card { + flex-shrink: 0; + width: 300px; + border: 1px solid var(--gray-4); + border-radius: var(--space-2); + cursor: pointer; + transition: all 0.3s ease; +} + +.card:hover { + transform: scale(1.02); + box-shadow: 0 10px 30px -15px rgba(0,0,0,0.2); +} + +.cardContent { + background: white; + padding: var(--space-5); + position: relative; + overflow: hidden; + border-radius: inherit; +} + +.cardDots { + position: absolute; + bottom: -20px; + left: -20px; + width: 120px; + height: 120px; + opacity: 0.1; + background-image: + radial-gradient(var(--card-color) 1px, transparent 1px), + radial-gradient(var(--card-color) 1px, transparent 1px); + background-position: 0 0, 10px 10px; + background-size: 20px 20px; + filter: blur(1px); +} + +.cardGlow { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + background: radial-gradient( + circle at 50% 0%, + var(--card-color), + transparent 70% + ); + opacity: 0.15; + transition: opacity 0.3s; +} + +.cardContent:hover .cardGlow { + opacity: 0.3; +} + +@keyframes scroll { + from { transform: translateX(0); } + to { transform: translateX(-33.33%); } +} + +.track:hover { + animation-play-state: paused; +} \ No newline at end of file diff --git a/src/components/code/index.tsx b/src/components/terminal/index.tsx similarity index 90% rename from src/components/code/index.tsx rename to src/components/terminal/index.tsx index 235c58e..47d381d 100644 --- a/src/components/code/index.tsx +++ b/src/components/terminal/index.tsx @@ -2,17 +2,18 @@ import { useState, useEffect } from "react"; import { Container, Button, Flex, Text, TextArea } from "@radix-ui/themes"; import { RotateCcw, CloudDownload, ShieldCheck, ShieldX } from "lucide-react"; -import styles from "./code.module.css"; +import styles from "./terminal.module.css"; -interface CodeProps { +interface TerminalProps { readonly value: string; - readonly onClear: () => void; - readonly onChangeValue: (value: string) => void; + readonly onClear?: () => void; + readonly onChangeValue?: (value: string) => void; } -export default function Code({ value, onClear, onChangeValue }: CodeProps) { +export default function Terminal({ value, onClear, onChangeValue }: TerminalProps) { const [isValid, setIsValid] = useState(false); const [error, setError] = useState(null); + function validateRobotsTxt(robotsTxt: string) { const lines = robotsTxt.split("\n"); @@ -124,7 +125,7 @@ export default function Code({ value, onClear, onChangeValue }: CodeProps) {