# robots.txt for https://perfectpuppy4u.com
#
# Named AI crawlers are allowed explicitly. A crawler that matches a named group
# follows only that group and ignores "User-agent: *", so every named group repeats
# the site's crawl rules.
#
# Crawl rules (all groups):
#   Disallow: /404   literal GoDaddy 404 path (the app serves the 404 body via /p404)
#   Disallow: /p404  the private 404 body route (also X-Robots-Tag: noindex)
#   Disallow: /api/  the lead-form endpoint, never a page
#
# This file is served the same on every host. Preview hosts (*.vercel.app) are kept
# out of search by the "X-Robots-Tag: noindex, nofollow" header in middleware.ts,
# not by this file.

# OpenAI: ChatGPT search index, user-initiated fetches, model training
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
Allow: /
Disallow: /404
Disallow: /p404
Disallow: /api/

# Anthropic: Claude user-initiated fetches, search, training
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: ClaudeBot
Allow: /
Disallow: /404
Disallow: /p404
Disallow: /api/

# Perplexity: answer-engine index and user-initiated fetches
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /
Disallow: /404
Disallow: /p404
Disallow: /api/

# Microsoft Bing (also feeds Copilot answers)
User-agent: Bingbot
Allow: /
Disallow: /404
Disallow: /p404
Disallow: /api/

# Google-Extended (Gemini / AI use) and Applebot-Extended (Apple Intelligence) are
# control tokens, not separate crawlers. Googlebot and Applebot do the fetching.
# Naming them with Allow is an explicit opt-in to AI use of already-crawled pages.
User-agent: Google-Extended
User-agent: Applebot-Extended
Allow: /
Disallow: /404
Disallow: /p404
Disallow: /api/

# Common Crawl. Its open dataset is widely used to train LLMs.
# CLIENT DECISION (default here: allow). To opt out, replace the two lines under
# CCBot with "Disallow: /". Opting out does not affect search or answer-engine
# visibility above.
User-agent: CCBot
Allow: /
Disallow: /404
Disallow: /p404
Disallow: /api/

# Everyone else, including Googlebot and Applebot.
User-agent: *
Disallow: /404
Disallow: /p404
Disallow: /api/

Sitemap: https://perfectpuppy4u.com/sitemap.xml