Sitemap: https://spectruminfocus.com/sitemap.xml

User-agent: *

Crawl-delay: 1




# Allowed Paths
Allow: /$ #
Allow: /section/in-focus #
Allow: /section/in-focus/* #
Allow: /sitemap.xml #
Allow: /services/* #





# Excluded Paths
Disallow: /nys/* #
Disallow: /nyc/* #
Disallow: /tx/* #
Disallow: /fl/* #
Disallow: /nc/* #
Disallow: /wi/* #
Disallow: /oh/* #
Disallow: /ky/* #
Disallow: /ca/* #
Disallow: /ma/* #
Disallow: /me/* #
Disallow: /mo/* #
Disallow: /sc/* #
Disallow: /ny/* #
Disallow: /mi/* #
Disallow: /al/* #
Disallow: /*/*/partner-content/* #
Disallow: /content/news/stories/* #


# Additional Config
User-agent: GPTBot 
Disallow: / 

User-agent: anthropic-ai 
Disallow: / 

User-agent: PetalBot 
Disallow: / 

User-agent: Amazonbot 
Disallow: / 

User-agent: ChatGPT 
Disallow: / 

User-agent: Bytespider 
Disallow: / 

User-agent: ClaudeBot 
Disallow: / 

User-agent: PerplexityBot 
Disallow: / 

User-agent: meta-externalagent 
Disallow: / 

User-agent: chatgpt
Disallow: / 

User-agent: CCBot
Disallow: / 

User-agent: Yeti
Disallow: / 

User-agent: Diffbot
Disallow: / 

User-agent: Timpibot
Disallow: / 

User-agent: Ai2Bot
Disallow: / 

User-agent: MistralAI-User
Disallow: / 

User-agent: AnthropicBot
Disallow: / 

User-agent: claude-web
Disallow: / 

User-agent: cohere-ai
Disallow: / 

User-agent: omgili 
Disallow: / 

User-agent: Google-Extended
Disallow: / 

User-agent: Applebot-Extended
Disallow: /