# Fyndify robots.txt — https://fyndify.com
# Public content is fully crawlable. Admin + internal APIs are disallowed.

User-agent: *
Allow: /
Allow: /cars/
Allow: /cities/
Allow: /finance/
Allow: /compare/
Disallow: /admin/
Disallow: /admin
Disallow: /finance/api/
Disallow: /search?

# Be polite to all crawlers — rate-limit crawl frequency.
Crawl-delay: 1

# Explicit sitemap + feed references for search engines.
Sitemap: https://fyndify.com/sitemap.xml
Sitemap: https://fyndify.com/feed.xml

# Block AI training crawlers that ignore robots.txt anyway — explicit refusal.
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /