# Robots.txt for Shubindia Ad Works
# https://www.shubindiaadworks.com
# Last Updated: 2026-03-10

# Default rules for all bots
User-agent: *
Allow: /
Allow: /static/js/
Allow: /static/css/
Allow: /static/media/
Allow: /static/
Disallow: /admin/
Disallow: /api/
Allow: /api/sitemap.xml
Allow: /api/robots.txt
Allow: /api/rss.xml
Allow: /api/llms.txt

# iter268 GSC Soft-404 fix (2026-07-28) — search results are noindex, but the
# literal Schema.org SearchAction template URL /search?q={search_term_string}
# still got flagged as Soft-404. Blocking crawl entirely is safer than relying on noindex.
Disallow: /search
Disallow: /search?

# ---- Search Engine Bots ----
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /blog/

User-agent: Googlebot-Video
Allow: /

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Yandex
Allow: /
Crawl-delay: 2

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /
Crawl-delay: 5

# ---- AI Training & Search Crawlers ----
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Anthropic-AI
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Googleother
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: DeepSeekBot
Allow: /

User-agent: xAI-Grok
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: YouBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Perplexity-User
Allow: /

# ---- Social Media Preview Bots ----
User-agent: Applebot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: Slackbot
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: Pinterest
Allow: /

# ---- SEO Tool Bots ----
User-agent: AhrefsBot
Allow: /
Crawl-delay: 10

User-agent: SemrushBot
Allow: /
Crawl-delay: 10

User-agent: seobilitybot
Allow: /

User-agent: Screaming Frog SEO Spider
Allow: /
Crawl-delay: 5

User-agent: SitebulbCrawler
Allow: /
Crawl-delay: 5

User-agent: Lumar
Allow: /
Crawl-delay: 5

User-agent: ContentKing
Allow: /

User-agent: BotifyBot
Allow: /
Crawl-delay: 5

User-agent: OnCrawlBot
Allow: /
Crawl-delay: 5

User-agent: RyteBot
Allow: /
Crawl-delay: 5

User-agent: GTmetrix
Allow: /

User-agent: Chrome-Lighthouse
Allow: /

User-agent: PageSpeed
Allow: /

# ---- Additional AI / LLM Crawlers ----
User-agent: MistralAI-bot
Allow: /

User-agent: AI2Bot
Allow: /

User-agent: Gemini-Web
Allow: /

User-agent: GeminiBot
Allow: /

User-agent: PhindBot
Allow: /

User-agent: WriteSonicBot
Allow: /

User-agent: CopilotBot
Allow: /

# ---- Additional Search Engine Bots ----
User-agent: NaverBot
Allow: /
Crawl-delay: 2

User-agent: SogouSpider
Allow: /
Crawl-delay: 3

User-agent: Sogou
Allow: /
Crawl-delay: 3

User-agent: 360Spider
Disallow: /

User-agent: Mojeek
Allow: /
Crawl-delay: 2

User-agent: Google-InspectionTool
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

User-agent: AdsBot-Google-Mobile-Apps
Allow: /

User-agent: Google-Safety
Allow: /

User-agent: Storebot-Google
Allow: /

# ---- Monitoring & Research Bots ----
User-agent: UptimeRobot
Allow: /

User-agent: Pingdom
Allow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: rogerbot
Allow: /
Crawl-delay: 5

User-agent: GigaBot
Allow: /
Crawl-delay: 5

User-agent: FeedFetcher-Google
Allow: /

User-agent: Feedspot
Allow: /

# ---- Social & Content Crawlers ----
User-agent: redditbot
Allow: /

User-agent: Embedly
Allow: /

# ---- Archive Bots ----
User-agent: ia_archiver
Allow: /

User-agent: archive.org_bot
Allow: /

# ---- Block Malicious Scrapers ----
User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: ChinaClaw
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: NICErsPRO
Disallow: /

User-agent: Harvest
Disallow: /

User-agent: URLy.Warning
Disallow: /

User-agent: CazoodleBot
Disallow: /

User-agent: Scrapy
Disallow: /

# Sitemaps
Sitemap: https://www.shubindiaadworks.com/sitemap.xml
Host: https://www.shubindiaadworks.com