User-agent: *
Allow: /
Crawl-delay: 1

# Block admin and private areas
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /node_modules/
Disallow: /_next/
Disallow: /auth/

# Block duplicate and utility paths
Disallow: /search?
Disallow: /*?q=
Disallow: /r/

# Block AI crawlers from consuming content
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Amazonbot
Disallow: /

# Sitemap index
Sitemap: https://discoverwebtools.com/sitemap.xml