# ============================================================ # robots.txt — GavelGrow (https://gavelgrow.com) # Law Firm Marketing Platform + Done-for-You Agency # Last updated: 2026-10-13 # ============================================================ # ------------------------------------------------------------ # TRADITIONAL SEARCH ENGINES (Allow all) # ------------------------------------------------------------ User-agent: Googlebot Allow: / User-agent: Googlebot-Image Allow: / User-agent: Googlebot-News Allow: / User-agent: Googlebot-Video Allow: / User-agent: AdsBot-Google Allow: / User-agent: Bingbot Allow: / User-agent: MSNBot Allow: / User-agent: Slurp Allow: / User-agent: DuckDuckBot Allow: / User-agent: Baiduspider Allow: / User-agent: YandexBot Allow: / User-agent: Applebot Allow: / # ------------------------------------------------------------ # AI RETRIEVAL & CITATION BOTS (Allow — power live AI search) # These bots answer live user queries and cite sources. Allowing # them ensures GavelGrow is discoverable inside ChatGPT Search, # Perplexity, Claude, Gemini, and Copilot answers. # ------------------------------------------------------------ User-agent: OAI-SearchBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: GPTBot Allow: / User-agent: ClaudeBot Allow: / User-agent: Claude-Web Allow: / User-agent: anthropic-ai Allow: / User-agent: PerplexityBot Allow: / User-agent: Perplexity-User Allow: / User-agent: Google-Extended Allow: / User-agent: Applebot-Extended Allow: / User-agent: Meta-ExternalAgent Allow: / User-agent: Meta-ExternalFetcher Allow: / User-agent: Amazonbot Allow: / User-agent: DuckAssistBot Allow: / User-agent: YouBot Allow: / User-agent: cohere-ai Allow: / # Common Crawl. Its open dataset is a major source of LLM training data, # so allowing it puts GavelGrow's own pages into what AI models learn # from (Abram, 2026-10-13). User-agent: CCBot Allow: / # ------------------------------------------------------------ # TRAINING-ONLY CRAWLERS (Block — no attribution, no citation) # ------------------------------------------------------------ User-agent: FacebookBot Disallow: / User-agent: Diffbot Disallow: / User-agent: Bytespider Disallow: / User-agent: ImagesiftBot Disallow: / User-agent: Omgilibot Disallow: / User-agent: Omgili Disallow: / User-agent: magpie-crawler Disallow: / # ------------------------------------------------------------ # DEFAULT RULE (All other crawlers — allow with path hygiene) # ------------------------------------------------------------ User-agent: * Allow: / # Explicitly allow rendering assets so Googlebot can see CSS/JS Allow: /*.css$ Allow: /*.js$ Allow: /*.mjs$ Allow: /*.json$ Allow: /*.xml$ Allow: /*.txt$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ Allow: /*.ttf$ Allow: /*.otf$ Allow: /*.ico$ # Block noisy query-string crawl variants to preserve crawl budget Disallow: /*?utm_ Disallow: /*?gclid Disallow: /*?fbclid Disallow: /*?msclkid Disallow: /*?ref= Disallow: /*?session= Disallow: /*?category= # Block internal/dev artifacts if ever deployed by accident Disallow: /dist/ Disallow: /node_modules/ Disallow: /.git/ Disallow: /.env Disallow: /*.map$ # ------------------------------------------------------------ # AI AGENT DIRECTIVES # llms.txt is the AI-optimized site index (llmstxt.org spec) # llms-full.txt contains expanded content for deeper context # ------------------------------------------------------------ # llms-txt: https://gavelgrow.com/llms.txt # llms-full-txt: https://gavelgrow.com/llms-full.txt # ------------------------------------------------------------ # SITEMAPS & FEEDS # ------------------------------------------------------------ Sitemap: https://gavelgrow.com/sitemap.xml # Atom feed for the blog (also linked on-page via ) # feed: https://gavelgrow.com/blog/feed.xml # ------------------------------------------------------------ # CRAWL RATE # Polite default; Googlebot ignores this but other bots honor it. # ------------------------------------------------------------ Crawl-delay: 1 # ------------------------------------------------------------ # HOST (canonical host preference) # ------------------------------------------------------------ Host: https://gavelgrow.com