User-agent: *

# Normal things to discourage bots from accessing
Disallow: /cgi-bin/

# Bots should never get here as raw data
Disallow: /caches/
Disallow: /exports/
Disallow: /imports/
Disallow: /temp/

# Bots should never get here as exposes framework files
Disallow: /data/ace/
Disallow: /data/ckeditor/
Disallow: /text/
Disallow: /text_custom/
Disallow: /themes/*/text
Disallow: /themes/*/xml

# Bots should never get here as exposes admin functionality (individual admin/utility scripts have noindex built-in)
Disallow: /adminzone/
#Disallow: /cms/

# Bots don't need to index these special scripts (performance)
Disallow: /data/external_url_proxy.php

# Set Sitemap
Sitemap: https://composr.app/data_custom/sitemaps/index.xml
Sitemap: https://composr.app/data_custom/sitemaps/news_sitemap.xml

# Discourage some spammy bots
User-agent: SurdotlyBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: ZoominfoBot
Disallow: /
User-agent: Buck
Disallow: /
User-agent: trendictionbot
Disallow: /
User-agent: omgili
Disallow: /
User-agent: bbot
Disallow: /
User-agent: brands-bot-logo
Disallow: /
User-agent: brands-bot
Disallow: /

# Busy bots not discouraged by default as some sites may want them
#User-agent: MJ12bot
#Disallow: /

# Set reasonable Crawl Delay to discourage bots from being over-enthusiastic
Crawl-Delay: 10