User-agent: *
Allow: /
Allow: */dynamic-pages-router/*
Allow: */cloud-data/*
Allow: /hybrid-courses
Allow: /cyber-range
Allow: /kb
Allow: /soc-analyst-jobs
Allow: /shop
Disallow: *?lightbox=
Disallow: /_api/*
Disallow: /admin/*
Disallow: /private/*
Disallow: /temp/*

# Optimization for Google Ads Bot
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google
Disallow: /_partials*
Disallow: /pro-gallery-webapp/v1/galleries/*

# Block PetalBot and other unwanted bots
User-agent: PetalBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: MJ12bot
Disallow: /

# Crawl delay for overly enthusiastic bots
User-agent: dotbot
Crawl-delay: 10
User-agent: AhrefsBot
Crawl-delay: 10
User-agent: BLEXBot
Crawl-delay: 10

# Allow important educational content for search engines
User-agent: Googlebot
Allow: /hybrid-courses
Allow: /cyber-range
Allow: /kb
Allow: /soc-analyst-jobs
Allow: /post/*
Crawl-delay: 1

User-agent: Bingbot
Allow: /hybrid-courses
Allow: /cyber-range
Allow: /kb
Allow: /soc-analyst-jobs
Allow: /post/*
Crawl-delay: 2

Sitemap: https://www.cybernoweducation.com/sitemap.xml

# Optimized for cybersecurity education content - Updated for better SEO performance