# robots.txt for 1990 Web Archive
# https://1990webarchive.com/

# General crawler directives
User-agent: *
Allow: /archive/
Allow: /search/
Allow: /about/
Allow: /blog/
Allow: /assets/
Allow: /manifest.json

Disallow: /admin/
Disallow: /api/internal/
Disallow: /crawl-queue/
Disallow: /user-data/
Disallow: /export/
Disallow: /*?session=
Disallow: /*?token=
Disallow: /*?preview=

# Preservation & Archive Bots
User-agent: archive.org_bot
User-agent: ia_archiver
Allow: /
Crawl-delay: 2

# AI Training & Scraper Bots (Protect archived intellectual property)
User-agent: GPTBot
User-agent: CCBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Bytespider
Disallow: /

# Sitemap Locations
Sitemap: https://1990webarchive.com/sitemap.xml
Sitemap: https://1990webarchive.com/sitemap-archive-1990-1995.xml
Sitemap: https://1990webarchive.com/sitemap-archive-1996-2000.xml