# =============================== 
# ✅ FINAL ROBOTS.TXT (YOAST + ADSENSE + SECURITY + CRAWLOPTIMIZED)
# ===============================

# ── General Bot Rules ──
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# 🔍 Block duplicate & internal search result pages
Disallow: /?s=
Disallow: /page/*/?s=
Disallow: /search/
Disallow: /*/search/*

# 🔐 Block REST API endpoints (JSON)
Disallow: /wp-json/
Disallow: /?rest_route=

# 🔐 Block unnecessary or sensitive WordPress paths
Disallow: /cgi-bin/
Disallow: /trackback/
Disallow: /comments/
Disallow: /xmlrpc.php
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /readme.html
Disallow: /license.txt
Disallow: /.git
Disallow: /*?replytocom=

# ✅ Allow essential static assets for rendering
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/
Allow: /wp-content/astra-local-fonts/
Allow: /wp-includes/

# 🎯 Googlebot (SEO + Ads compatibility)
User-agent: Googlebot
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.woff$
Allow: /*.woff2$
Allow: /*.ttf$
Allow: /*.otf$

# 🖼️ Googlebot-Image
User-agent: Googlebot-Image
Allow: /wp-content/uploads/
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.gif$
Allow: /*.svg$

# 💰 Google Ad Bot
User-agent: Mediapartners-Google
Allow: /

# ✅ Structured Data Testing
User-agent: Google-Structured-Data-Testing-Tool
Allow: /

# 🤖 Social Media Bots (optional but safe to allow)
User-agent: FacebookExternalHit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

# 🚫 Block known aggressive bots
User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: DotBot
Disallow: /

# 🐌 Crawl-delay for high-impact bots
User-agent: Bingbot
Crawl-delay: 10

User-agent: Yandex
Crawl-delay: 10

User-agent: AhrefsBot
Crawl-delay: 20

User-agent: SemrushBot
Crawl-delay: 20

# 🗺️ Yoast SEO Sitemap
Sitemap: https://careeraidhub.com/sitemap_index.xml

# ===============================
# 🚀 END OF ROBOTS.TXT
# ===============================