# Robots.txt for ScanMalware.com
# Allow all crawlers including AI systems
#
# /api/ is disallowed: scan pages fire ~15 JSON XHRs each and crawlers were
# spending 84% of their crawl budget re-fetching those endpoints instead of
# pages. The JSON is useless for indexing; page content is server-rendered.
# /api/v1/screenshot/ stays allowed so og:image / social cards / image search
# can still fetch scan screenshots.
#
# /search-domains?q= runs a live certificate-log query for every new search, so
# crawlers are kept off the result pages; the empty /search-domains form stays allowed.

User-agent: *
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?

# Crawl-delay for respectful crawling
Crawl-delay: 1

# Special rules for major search engines
User-agent: Googlebot
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?
Crawl-delay: 0

User-agent: Bingbot
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?
Crawl-delay: 0

User-agent: Slurp
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?
Crawl-delay: 1

# AI Crawlers - Explicitly allowed for AI SEO
User-agent: GPTBot
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?

User-agent: OAI-SearchBot
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?

User-agent: ChatGPT-User
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?

User-agent: ClaudeBot
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?

User-agent: Claude-Web
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?

User-agent: anthropic-ai
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?

User-agent: PerplexityBot
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?

User-agent: GoogleOther
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?

User-agent: Google-Extended
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?

User-agent: Bytespider
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?

User-agent: cohere-ai
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?

User-agent: Applebot-Extended
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?

# SEO tool bots - allowed
User-agent: AhrefsBot
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?

User-agent: SemrushBot
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?

User-agent: DotBot
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?

User-agent: MJ12bot
Allow: /api/v1/screenshot/
Disallow: /api/
Disallow: /hash-search
Disallow: /search-domains?

Sitemap: https://scanmalware.com/sitemap.xml