# Nakordoni.eu robots.txt
# Policy: allow all major search + AI + ads/preview crawlers responsibly.
# Only /cgi-bin/ is disallowed to everyone.

# --- Google (Search / Infra / Ads / AI) ---
User-agent: Googlebot
Disallow:
User-agent: GoogleOther
Disallow:
User-agent: Google-Extended
Disallow:
User-agent: APIs-Google
Disallow:
User-agent: Mediapartners-Google
Disallow:
User-agent: AdsBot-Google
Disallow:
User-agent: AdsBot-Google-Mobile
Disallow:
User-agent: AdsBot-Google-Mobile-Apps
Disallow:
User-agent: googlebot-image
Disallow:
User-agent: googlebot-video
Disallow:
User-agent: googlebot-news
Disallow:
User-agent: googlebot-mobile
Disallow:

# --- Microsoft / Bing (Search / Ads) ---
User-agent: Bingbot
Disallow:
User-agent: BingPreview
Disallow:
User-agent: adidxbot
Disallow:
User-agent: MicrosoftBot
Disallow:
User-agent: MSNBot
Disallow:

# --- OpenAI ---
User-agent: GPTBot
Disallow:
User-agent: ChatGPT-User
Disallow:
User-agent: OAI-SearchBot
Disallow:

# --- Anthropic ---
User-agent: ClaudeBot
Disallow:
User-agent: Claude-Web
Disallow:

# --- Perplexity ---
User-agent: PerplexityBot
Disallow:

# --- Meta (Facebook / Instagram) ---
User-agent: Meta-ExternalAgent
Disallow:
User-agent: facebookexternalhit
Disallow:
User-agent: Facebot
Disallow:

# --- xAI (Grok) ---
User-agent: Grok
Disallow:
User-agent: Grokbot
Disallow:
User-agent: xAIBot
Disallow:
User-agent: xai
Disallow:

# --- Apple ---
User-agent: Applebot
Disallow:
User-agent: Applebot-Extended
Disallow:

# --- ByteDance / TikTok ---
User-agent: Bytespider
Disallow:

# --- Common Crawl (research) ---
User-agent: CCBot
Disallow:

# --- Messaging / preview bots popular in UA ---
User-agent: TelegramBot
Disallow:
User-agent: WhatsApp
Disallow:
User-agent: SkypeUriPreview
Disallow:
User-agent: Viber
Disallow:
User-agent: Slackbot
Disallow:
User-agent: Discordbot
Disallow:

# --- Social / ads / discovery (global) ---
User-agent: Twitterbot
Disallow:
User-agent: LinkedInBot
Disallow:
User-agent: Pinterestbot
Disallow:
User-agent: CriteoBot
Disallow:
User-agent: Taboola
Disallow:
User-agent: Outbrain
Disallow:
User-agent: Snapchat-Proxy
Disallow:

# --- UA ad-market / local media bots (best-effort; verify in logs) ---
User-agent: MgidBot
Disallow:
User-agent: mgid
Disallow:
User-agent: admixer-crawler
Disallow:
User-agent: admixer-bot
Disallow:
User-agent: UkrNetBot
Disallow:
User-agent: bigmir-spider
Disallow:

# --- Popular SEO crawlers (allowed) ---
User-agent: AhrefsBot
Disallow:
User-agent: SemrushBot
Disallow:
User-agent: MJ12bot
Disallow:
User-agent: DotBot
Disallow:
User-agent: ia_archiver
Disallow:
User-agent: Slurp
Disallow:
User-agent: Teoma
Disallow:

# --- General rule for all others ---
User-agent: *
Disallow: /cgi-bin/

Sitemap: https://nakordoni.eu/cache_func/sitemap.xml