# robots.txt — lillytechsystems.com
# Policy: allow search engines & social preview bots; block SEO-data scrapers
# that burn bandwidth and provide us no value. (AWStats 2026-06: Barkrowler
# alone = 4.5k hits / 19 MB.) Impolite bots that ignore this file are blocked
# server-side in .htaccess.

# --- SEO-data / content scrapers: blocked ---
User-agent: Barkrowler
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: Bytespider
Disallow: /

# --- AI TRAINING bots: blocked (content absorbed into models, nothing back) ---
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# --- AI SEARCH / CITATION bots: ALLOWED (they cite + link us = referral traffic) ---
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# --- Everyone else (Googlebot, bingbot, LinkedInBot, facebookexternalhit, …) ---
User-agent: *
Allow: /

Sitemap: https://lillytechsystems.com/sitemap.xml

# Answer engines: /llms.txt is a short, curated map of what is here and where
# the machine-readable version lives. Most lesson pages are noindex, which is a
# ranking decision about templated pages, not a request not to cite them.
# https://lillytechsystems.com/llms.txt