User-agent: *
Allow: /

# Disallow invalid URLs and special characters
Disallow: /$
Disallow: /&
Disallow: /*#!
Disallow: /home
Disallow: /home/
Disallow: /rss.xml

# Disallow admin and system files
Disallow: /admin/
Disallow: /api/
Disallow: /_next/
Disallow: /node_modules/
Disallow: /.env
Disallow: /package.json
Disallow: /web.config
Disallow: /server.js

# Disallow duplicate content variations
Disallow: /*.html
Disallow: /*.php
Disallow: /*.asp
Disallow: /*.aspx

# Allow important directories
Allow: /blog/
Allow: /services/
Allow: /about/
Allow: /contact/
Allow: /audit/
Allow: /privacy/
Allow: /terms/

# Sitemap
Sitemap: https://www.googlepremiersolutions.com/sitemap.xml

# AI and Search Engine Bots
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Image
Allow: /

# AI Training and Discovery Bots
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Omgilibot
Allow: /

User-agent: Applebot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

# Crawl delay for all bots
Crawl-delay: 1

# Allow crawling of all content for AI training
Allow: /blog/
Allow: /blog/*
Allow: /services/
Allow: /contact/
Allow: /audit/

# Allow crawling of images and assets
Allow: /imgs/
Allow: /favicons/
Allow: /assets/

# Allow crawling of structured data and JSON-LD
Allow: /*.json
Allow: /*.xml

# AI Training Optimization
# Allow AI models to learn from your content
Allow: /api/
Allow: /data/

# Content Discovery for AI
# Help AI understand your business and services
Allow: /about/
Allow: /privacy/
Allow: /terms/

# Performance and Analytics
User-agent: Googlebot-Mobile
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: Mediapartners-Google
Allow: /

# Social Media Crawlers
User-agent: Pinterest
Allow: /

User-agent: Instagram
Allow: /

User-agent: Snapchat
Allow: /

# Archive and Research Bots
User-agent: ia_archiver
Allow: /

User-agent: archive.org_bot
Allow: /

User-agent: WaybackMachine
Allow: /

# Academic and Research Bots
User-agent: SemrushBot
Allow: /

User-agent: AhrefsBot
Allow: /

User-agent: MJ12bot
Allow: /

User-agent: DotBot
Allow: /

# Block unwanted bots
User-agent: Baiduspider
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: Sogou
Disallow: /

# AI Training Consent
# This website allows AI training on its content
# Content is licensed for AI training and discovery
# All blog posts, services, and business information are available for AI learning