# Allow legitimate search engine crawlers
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

# Block common scraper bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: GPTBot
Allow: /

User-agent: CCBot
Disallow: /


User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# ChatGPT-User fetches pages live when a user asks ChatGPT to look at them.
# Allow it so ChatGPT can read and recommend our jobs/companies/visas pages.
User-agent: ChatGPT-User
Allow: /

# OAI-SearchBot powers ChatGPT Search results — allow for visibility.
User-agent: OAI-SearchBot
Allow: /

# PerplexityBot fetches pages live for Perplexity answers — allow.
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Microsoft Copilot reads pages through Bingbot (allowed above).
# Cohere's web crawler — allow for AI search visibility.
User-agent: cohere-ai
Allow: /

# Apple Intelligence / Siri search.
User-agent: Applebot-Extended
Allow: /

# ---- Blocked scrapers / generic automation tools ----


User-agent: Scrapy
Disallow: /

User-agent: python-requests
Disallow: /

User-agent: curl
Disallow: /

User-agent: wget
Disallow: /

User-agent: Java
Disallow: /

User-agent: libwww-perl
Disallow: /

# Default for all other bots
User-agent: *
Allow: /

# Account, admin, authentication and internal tool routes are not listed here
# on purpose (listing them would publish their names). They are excluded from
# sitemap.xml and serve <meta name="robots" content="noindex, nofollow">.

# Filtered / search-parameter URLs duplicate the canonical directory pages.
Disallow: /*?*category=
Disallow: /*?*experience=
Disallow: /*?*search=
Disallow: /*?*page=

# Sitemap location (sitemap index pointing to live, dynamically generated sitemap)
Sitemap: https://nomadcareers.online/sitemap.xml