# Aisles crawler policy. Generated from configuration; reviewed by the Aisles team.
# Search discovery and AI-training preferences are set separately (ALLOW_SEARCH_CRAWLERS,
# ALLOW_AI_SEARCH_CRAWLERS, ALLOW_TRAINING_CRAWLERS). These are preferences, not access controls.
# Content-Signal: search=yes, ai-input=yes, ai-train=yes

User-agent: *
Allow: /
Disallow: /go/
Disallow: /admin
Disallow: /api/v1/
Disallow: /search
Disallow: /ask
Disallow: /sandbox/
Disallow: /media/
Disallow: /partners/

# Bing: the HTML pages only (no Markdown/JSON copies) and no store-listing sitemaps; see /sitemap-priority.xml
User-agent: bingbot
User-agent: msnbot
Allow: /
Disallow: /go/
Disallow: /admin
Disallow: /api/v1/
Disallow: /search
Disallow: /ask
Disallow: /sandbox/
Disallow: /media/
Disallow: /partners/
Disallow: /api/md/
Disallow: /*.md$
Disallow: /*.json$
Disallow: /sitemap/listings-

# AI search crawlers that index pages to cite them: the HTML pages, not their Markdown/JSON copies
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Claude-SearchBot
User-agent: Amazonbot
Allow: /
Allow: /api/v1/docs
Disallow: /go/
Disallow: /admin
Disallow: /api/v1/
Disallow: /search
Disallow: /ask
Disallow: /sandbox/
Disallow: /media/
Disallow: /partners/
Disallow: /api/md/
Disallow: /*.md$
Disallow: /*.json$

# AI assistants fetching a page for a person's question: they may also use the search page and search API
User-agent: ChatGPT-User
User-agent: Perplexity-User
User-agent: Claude-User
Allow: /
Allow: /search
Allow: /api/v1/search
Allow: /api/v1/docs
Disallow: /go/
Disallow: /admin
Disallow: /api/v1/
Disallow: /ask
Disallow: /sandbox/
Disallow: /media/
Disallow: /partners/

# Model-training crawlers and opt-out tokens
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: cohere-ai
User-agent: Bytespider
User-agent: meta-externalagent
Allow: /
Disallow: /go/
Disallow: /admin
Disallow: /api/v1/
Disallow: /search
Disallow: /ask
Disallow: /sandbox/
Disallow: /media/
Disallow: /partners/

Sitemap: https://aislesinventory.com/sitemap/core.xml
Sitemap: https://aislesinventory.com/sitemap.xml
Sitemap: https://aislesinventory.com/sitemap-priority.xml
Schemamap: https://aislesinventory.com/sitemap.xml

# Guide for AI assistants: https://aislesinventory.com/llms.txt
# Includes in-store availability at shopping malls: Del Amo Fashion Center (Torrance, CA); King of Prussia (King of Prussia, PA); Mall of Georgia (Buford, GA); Roosevelt Field (Garden City, NY); South Shore Plaza (Braintree, MA); The Galleria (Houston, TX); University Park Mall (Mishawaka, IN); Woodfield Mall (Schaumburg, IL); Sawgrass Mills (Sunrise, FL)