# =====================================================
# KG2UNI ROBOTS.TXT
# =====================================================

# OpenAI training crawler - block
User-agent: GPTBot
Disallow: /

# Claude Search - allow discovery but slow it down
User-agent: Claude-SearchBot
Crawl-delay: 5
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /dashboard/
Disallow: /profile/
Disallow: /student-registration/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /courses/*?*filtering=1*
Disallow: /courses/*?*instructor=*
Disallow: /courses/*?*level=*
Disallow: /courses/*?*price_type=*
Disallow: /courses/*?*duration=*
Disallow: /course-category/*?*tutor-course-filter-category=*
Disallow: /*/feed/
Disallow: /feed/

# Anthropic training crawler - block
User-agent: ClaudeBot
Disallow: /

# Other unwanted crawlers
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: GoogleOther
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SERankingBacklinksBot
Disallow: /

# Normal search engines, social previews and other legitimate crawlers
User-agent: *
Allow: /

Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /dashboard/
Disallow: /profile/
Disallow: /student-registration/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/

# Tutor LMS dynamic/filter URLs
Disallow: /courses/*?*filtering=1*
Disallow: /courses/*?*instructor=*
Disallow: /courses/*?*level=*
Disallow: /courses/*?*price_type=*
Disallow: /courses/*?*duration=*
Disallow: /course-category/*?*tutor-course-filter-category=*

# WordPress feeds/search
Disallow: /*/feed/
Disallow: /feed/
Disallow: /*?s=*

Sitemap: https://kg2uni.com/sitemap_index.xml