User-agent: *
Allow: /
Disallow: /admin/
Disallow: /backoffice/
Disallow: /accounts/
Disallow: /myprofile/
Disallow: /privacy/
Disallow: /update_profile/
Disallow: /update_password/
Disallow: /signin/
Disallow: /login/
Disallow: /photonew/
Disallow: /photoedit/
Disallow: /photodelete/
Disallow: /photorotate/
Disallow: /photocrop/
Disallow: /photofullscreen/
Disallow: /dishphotonew/
Disallow: /dishnew/
Disallow: /dishedit/
Disallow: /list_delete/
Disallow: /lists_mine/
Disallow: /cuisineedit/
Disallow: /cuisinenew/

# Amazon crawlers
User-agent: Amazonbot
Disallow: /

User-agent: Amzn-SearchBot
Disallow: /

# AI training crawlers (no user-facing benefit, large request volume)
User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: ChatGPT-User
Crawl-delay: 10

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: DuckAssistBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: AzureAI-SearchBot
Disallow: /

# SEO / analytics crawlers (no user-facing benefit, heavy traffic)
User-agent: SemrushBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: peer39_crawler
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# Social media crawlers
# meta-externalagent and meta-webindexer were the largest crawler-amplified
# request source observed in production logs (issue #2). Block both.
User-agent: meta-externalagent
Disallow: /

User-agent: meta-webindexer
Disallow: /

# Misc crawlers (rate-limit, don't block)
User-agent: TikTokSpider
Crawl-delay: 10

# Pinterestbot: highest-volume single crawler in prod logs (~58% of 200s).
# Real traffic referrer for the site, so rate-limit instead of blocking.
User-agent: Pinterestbot
Crawl-delay: 10

# Baiduspider: site does not target the Chinese market; throttle aggressively.
User-agent: Baiduspider
Crawl-delay: 30

# Sogou: Chinese search engine, no user-facing benefit for this site.
User-agent: Sogou web spider
Disallow: /

User-agent: Sogou inst spider
Disallow: /

User-agent: Sogou spider2
Disallow: /

# Yandex: throttle (we don't target Russia either, but don't fully block).
User-agent: YandexBot
Crawl-delay: 30

User-agent: CravingExplorer
Disallow: /

# Misc heavy crawlers observed in prod logs
User-agent: ShapBot
Disallow: /

User-agent: DomainStatsBot
Disallow: /

# rednote-websearch-bot (Xiaohongshu/RED): highest-volume crawler observed in
# prod logs (Jul 2026), no proven referral traffic. May ignore robots.txt, so
# also enforce via UA block in middleware if it persists.
User-agent: rednote-websearch-bot
Disallow: /

# Sitemap location
Sitemap: https://worldfood.guide/sitemap.xml