User-agent: * Allow: / Disallow: /node_modules/ Disallow: /*.bak$ Disallow: /submit.php Disallow: /LaundryDaddy/ # Server-side PHP partials (header/footer/head). Pulled in by pages, never URLs. # .htaccess also 403s them; this stops crawlers asking. Disallow: /includes/ # Per-customer Daddy POS short URLs — redirected to pos.laundrydaddy.com. # Must never appear in search results. Disallow: /i/ Disallow: /t/ # ============================================================= # AI POLICY # # Laundry Daddy wants to be discovered via AI-powered search # (Google AI Overviews, ChatGPT, Perplexity, Copilot, Claude answers) # because these crawlers cite the source, which drives traffic. # # But Laundry Daddy does NOT permit its content or photos to be # used for training generative AI models, or scraped in bulk # for commercial datasets. Those crawlers are blocked below, # and .htaccess enforces the block at the server level for # operators that ignore robots.txt. # ============================================================= # === BLOCKED: AI model training crawlers ==================== # These fetch content to train LLMs with no citation back. # OpenAI model training User-agent: GPTBot Disallow: / # Anthropic model training User-agent: ClaudeBot Disallow: / User-agent: anthropic-ai Disallow: / # Common Crawl, feeds most LLM training datasets User-agent: CCBot Disallow: / # ByteDance / TikTok / Doubao training User-agent: Bytespider Disallow: / # Meta AI training User-agent: Meta-ExternalAgent Disallow: / User-agent: FacebookBot Disallow: / # Amazon (Alexa + AI training) User-agent: Amazonbot Disallow: / # Other training / bulk-scraper bots User-agent: cohere-ai Disallow: / User-agent: Diffbot Disallow: / User-agent: Omgilibot Disallow: / User-agent: Omgili Disallow: / User-agent: ImagesiftBot Disallow: / User-agent: Timpibot Disallow: / User-agent: YouBot Disallow: / User-agent: MistralAI-User Disallow: / User-agent: AI2Bot Disallow: / User-agent: Ai2Bot-Dolma Disallow: / User-agent: Kangaroo Bot Disallow: / User-agent: PanguBot Disallow: / User-agent: Scrapy Disallow: / # === ALLOWED: AI search and citation crawlers =============== # These fetch to answer user queries and link back to the source. # ChatGPT Search (SearchGPT), cites sources User-agent: OAI-SearchBot Allow: / # ChatGPT user browsing, cites sources when users click User-agent: ChatGPT-User Allow: / # Claude user-initiated browsing, cites sources User-agent: Claude-User Allow: / User-agent: Claude-Web Allow: / # Perplexity, cites sources User-agent: PerplexityBot Allow: / User-agent: Perplexity-User Allow: / # DuckDuckGo AI answers, cites sources User-agent: DuckAssistBot Allow: / # Meta link unfurling (when users share URLs) User-agent: Meta-ExternalFetcher Allow: / # Google Gemini, explicitly allowed for AI Overviews presence User-agent: Google-Extended Allow: / # Apple Intelligence, explicitly allowed for AI answer presence User-agent: Applebot-Extended Allow: / # (Googlebot, Bingbot, Applebot, etc. inherit Allow from User-agent: *) Sitemap: https://www.laundrydaddy.com/sitemap.xml