| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316 |
- User-agent: 2^32$
- User-agent: AddSearchBot
- User-agent: AdsBot-Google
- User-agent: Agentic
- User-agent: AhrefsBot
- User-agent: .ai
- User-agent: AI21 Labs
- User-agent: AI2Bot
- User-agent: Ai2Bot-Dolma
- User-agent: AI2Bot-Dolma
- User-agent: AI Article Writer
- User-agent: AIBot
- User-agent: AI Content Detector
- User-agent: AI Dungeon
- User-agent: aiHitBot
- User-agent: AIMatrix
- User-agent: AISearchBot
- User-agent: AI Search Engine
- User-agent: AI SEO Crawler
- User-agent: AI Training
- User-agent: AITraining
- User-agent: AI Writer
- User-agent: Alexa
- User-agent: Alpha AI
- User-agent: AlphaAI
- User-agent: a[mazing]{42}(robot)
- User-agent: Amazon Bedrock
- User-agent: Amazonbot
- User-agent: AmazonBot
- User-agent: Amazon Comprehend
- User-agent: Amazon-Kendra
- User-agent: Amazon Lex
- User-agent: Amazon Sagemaker
- User-agent: Amazon Silk
- User-agent: Amazon Textract
- User-agent: Amelia
- User-agent: AndersPinkBot
- User-agent: Andibot
- User-agent: Anthropic
- User-agent: anthropic-ai
- User-agent: AnyPicker
- User-agent: Anyword
- User-agent: Applebot
- User-agent: Applebot-Extended
- User-agent: Aria Browse
- User-agent: Articoolo
- User-agent: Automated Writer
- User-agent: Awario
- User-agent: AwarioBot
- User-agent: AwarioRssBot
- User-agent: AwarioSmartBot
- User-agent: Azure
- User-agent: BardBot
- User-agent: bedrockbot
- User-agent: bigsur.ai
- User-agent: BLEXBot
- User-agent: Brave Leo
- User-agent: Brightbot 1.0
- User-agent: ByteDance
- User-agent: Bytespider
- User-agent: CatBoost
- User-agent: CCBot
- User-agent: CC-Crawler
- User-agent: ChatGLM
- User-agent: ChatGPT Agent
- User-agent: ChatGPT-User
- User-agent: ChatGPT-User/2.0
- User-agent: Chinchilla
- User-agent: Claude
- User-agent: ClaudeBot
- User-agent: Claude-SearchBot
- User-agent: Claude-User
- User-agent: claude-web
- User-agent: Claude-Web
- User-agent: ClearScope
- User-agent: CloudVertexBot
- User-agent: Cohere
- User-agent: cohere-ai
- User-agent: cohere-training-data-crawler
- User-agent: Common Crawl
- User-agent: CommonCrawl
- User-agent: ContentAtScale
- User-agent: ContentBot
- User-agent: Contentedge
- User-agent: Content Harmony
- User-agent: Content King
- User-agent: Content Optimizer
- User-agent: Content Samurai
- User-agent: Conversion AI
- User-agent: Copilot
- User-agent: CopyAI
- User-agent: Copymatic
- User-agent: Copyscape
- User-agent: Cotoyogi
- User-agent: crawler.with.dots
- User-agent: CrawlQ AI
- User-agent: Crawlspace
- User-agent: Crew AI
- User-agent: CrewAI
- User-agent: curl|sudo bash
- User-agent: DALL-E
- User-agent: DataForSeoBot
- User-agent: DataProvider
- User-agent: Datenbank Crawler
- User-agent: DeepAI
- User-agent: DeepL
- User-agent: DeepMind
- User-agent: DeepSeek
- User-agent: Devin
- User-agent: diffbot
- User-agent: Diffbot
- User-agent: Doubao AI
- User-agent: DuckAssistBot
- User-agent: Echobot Bot
- User-agent: EchoboxBot
- User-agent: Facebookbot
- User-agent: FacebookBot
- User-agent: facebookexternalhit
- User-agent: FacebookExternalHit
- User-agent: Factset_spyderbot
- User-agent: Falcon
- User-agent: Firecrawl
- User-agent: FirecrawlAgent
- User-agent: Flyriver
- User-agent: Frase AI
- User-agent: FriendlyCrawler
- User-agent: Fuzz Faster U Fool
- User-agent: Fuzz Faster U Fool v2.0.0
- User-agent: Gemini
- User-agent: Gemini-Deep-Research
- User-agent: Gemma
- User-agent: GenAI
- User-agent: Genspark
- User-agent: Gigabot
- User-agent: GLM
- User-agent: GoogleAgent-Mariner
- User-agent: Google-CloudVertexBot
- User-agent: Google-Extended
- User-agent: Google-Firebase
- User-agent: GoogleOther
- User-agent: GoogleOther-Image
- User-agent: GoogleOther-Video
- User-agent: Goose
- User-agent: GPT
- User-agent: GPTBot
- User-agent: Grammarly
- User-agent: Grendizer
- User-agent: Grok
- User-agent: GT Bot
- User-agent: GTBot
- User-agent: Hemingway Editor
- User-agent: Hugging Face
- User-agent: Hypotenuse AI
- User-agent: iaskspider
- User-agent: iaskspider/2.0
- User-agent: ICC-Crawler
- User-agent: ImageGen
- User-agent: ImagesiftBot
- User-agent: img2dataset
- User-agent: imgproxy
- User-agent: Inferkit
- User-agent: INK Editor
- User-agent: INKforall
- User-agent: IntelliSeek
- User-agent: ISSCyberRiskCrawler
- User-agent: Is this a crawler?
- User-agent: JasperAI
- User-agent: Kafkai
- User-agent: Kangaroo
- User-agent: Kangaroo Bot
- User-agent: Keyword Density AI
- User-agent: Knowledge
- User-agent: KomoBot
- User-agent: LinerBot
- User-agent: LinkedInBot
- User-agent: LLaMA
- User-agent: LLMs
- User-agent: magpie-crawler
- User-agent: MarketMuse
- User-agent: Meltwater
- User-agent: Meta AI
- User-agent: Meta-AI
- User-agent: MetaAI
- User-agent: Meta-External
- User-agent: meta-externalagent
- User-agent: Meta-ExternalAgent
- User-agent: meta-externalfetcher
- User-agent: Meta-ExternalFetcher
- User-agent: MetaTagBot
- User-agent: meta-webindexer
- User-agent: Mistral
- User-agent: MistralAI-User
- User-agent: MistralAI-User/1.0
- User-agent: MJ12bot
- User-agent: MyCentralAIScraperBot
- User-agent: Narrative
- User-agent: NeevaBot
- User-agent: netEstate Imprint Crawler
- User-agent: NeuralSEO
- User-agent: Neural Text
- User-agent: Nova Act
- User-agent: NovaAct
- User-agent: Nutch
- User-agent: OAI-SearchBot
- User-agent: omgili
- User-agent: Omgili
- User-agent: omgilibot
- User-agent: Omgilibot
- User-agent: OmniExplorer_Bot
- User-agent: Open AI
- User-agent: OpenAI
- User-agent: OpenBot
- User-agent: OpenText AI
- User-agent: Operator
- User-agent: Outwrite
- User-agent: Page Analyzer AI
- User-agent: PanguBot
- User-agent: Panscient
- User-agent: panscient.com
- User-agent: Paperlibot
- User-agent: Paraphraser.io
- User-agent: peer39_crawler
- User-agent: Perplexity
- User-agent: PerplexityBot
- User-agent: Perplexity-User
- User-agent: Petalbot
- User-agent: PetalBot
- User-agent: Phindbot
- User-agent: PhindBot
- User-agent: PiplBot
- User-agent: Poseidon Research Crawler
- User-agent: prefetch-proxy
- User-agent: ProWritingAid
- User-agent: psbot
- User-agent: python-requests
- User-agent: QualifiedBot
- User-agent: QuillBot
- User-agent: quillbot.com
- User-agent: RobotSpider
- User-agent: Robozilla
- User-agent: Rytr
- User-agent: SaplingAI
- User-agent: SBIntuitionsBot
- User-agent: Scalenut
- User-agent: Scraper
- User-agent: Scrapy
- User-agent: ScriptBook
- User-agent: Seekr
- User-agent: SemrushBot-OCOB
- User-agent: SemrushBot-SWA
- User-agent: sentibot
- User-agent: Sentibot
- User-agent: SentiBot
- User-agent: SEO Content Machine
- User-agent: SEO Robot
- User-agent: ShapBot
- User-agent: Sidetrade
- User-agent: Sidetrade indexer bot
- User-agent: Simplified AI
- User-agent: Sitefinity
- User-agent: Skydancer
- User-agent: SlickWrite
- User-agent: Sonic
- User-agent: Spinbot
- User-agent: Spin Rewriter
- User-agent: Stability
- User-agent: StableDiffusionBot
- User-agent: star***crawler
- User-agent: Sudowrite
- User-agent: SummalyBot
- User-agent: Super Agent
- User-agent: Surfer AI
- User-agent: Teoma
- User-agent: TerraCotta
- User-agent: Text Blaze
- User-agent: TextCortex
- User-agent: The Knowledge AI
- User-agent: Thinkbot
- User-agent: ThinkChaos
- User-agent: TikTokSpider
- User-agent: Timpibot
- User-agent: TimpiBot
- User-agent: TurnitinBot
- User-agent: VelenPublicWebCrawler
- User-agent: Vidnami AI
- User-agent: WARDBot
- User-agent: Webzio
- User-agent: webzio-extended
- User-agent: Webzio-Extended
- User-agent: Whisper
- User-agent: WordAI
- User-agent: Wordtune
- User-agent: WormsGTP
- User-agent: wpbot
- User-agent: WPBot
- User-agent: Writecream
- User-agent: WriterZen
- User-agent: Writescope
- User-agent: Writesonic
- User-agent: xAI
- User-agent: xBot
- User-agent: YaK
- User-agent: YandexAdditional
- User-agent: YandexAdditionalBot
- User-agent: Youbot
- User-agent: YouBot
- User-agent: Zerochat
- User-agent: Zero GTP
- User-agent: Zhipu
- User-agent: Zimm
- Disallow: /
- Disallow: *
- DisallowAITraining: /
- Content-Usage: ai=n
|