User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-json/
Disallow: /xmlrpc.php
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /?s=
Disallow: /search/
Disallow: /category/
Disallow: /tag/
Disallow: /author/
Disallow: /page/
Disallow: /date/
Disallow: /20*/  # Blocks content older than 2 years (e.g., /2021/, /2020/, etc.)
Disallow: /*?*  # Blocks URLs with query parameters
Disallow: /*.php$
Disallow: /*.js$
Disallow: /*.css$
Disallow: /*.xml$
Disallow: /*.json$
Disallow: /our-community/csr/
Disallow: /our-community/stories/
Disallow: /our-community/sports/


# Block all subdomains except the main site
Disallow: /*.talentibex.com/

# Allow only the main important directories
Allow: /jobs/
Allow: /programs/
Allow: /our-community/events/
Allow: /our-community/blogs/

# Allow major search engines & AI bots
User-agent: Googlebot
Allow: /
User-agent: Bingbot
Allow: /
User-agent: Applebot
Allow: /
User-agent: Yandex
Allow: /
User-agent: Baiduspider
Allow: /
User-agent: GPTBot
Allow: /
User-agent: ClaudeBot
Allow: /

# Block irrelevant bots & crawlers
User-agent: MJ12bot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: Sogou
Disallow: /

Sitemap: https://www.talentibex.com/sitemap.xml