User-agent: *
Crawl-delay: 4
Disallow: /user/

# LLM and AI systems
Allow: /llms.txt

# AI search crawlers — allow full access for content discovery
User-agent: GPTBot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

Sitemap: https://fungenerators.com/sitemap.xml

#  DataForSEOBot
User-agent: DataForSeoBot
Crawl-delay: 10
Disallow: /

# dotbot moz.com 
User-agent: dotbot
Crawl-delay: 10
Disallow: 

#  PetalBot
User-agent: PetalBot
Crawl-delay: 10
Disallow: 

#  MJ12bot
User-agent: MJ12bot
Crawl-delay: 10 
Disallow: 
    
# SeznamBot
User-agent: SeznamBot
Crawl-delay: 10 
Disallow: 

# SEMrushBot
User-agent: SEMrushBot
Crawl-delay: 10 
Disallow: 
    
# AhrefsBot
User-agent: AhrefsBot
Crawl-delay: 10 
Disallow: 
    
# AspiegelBot
User-agent: AspiegelBot
Crawl-delay: 10
Disallow: