robots.txt 7.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316
  1. User-agent: 2^32$
  2. User-agent: AddSearchBot
  3. User-agent: AdsBot-Google
  4. User-agent: Agentic
  5. User-agent: AhrefsBot
  6. User-agent: .ai
  7. User-agent: AI21 Labs
  8. User-agent: AI2Bot
  9. User-agent: Ai2Bot-Dolma
  10. User-agent: AI2Bot-Dolma
  11. User-agent: AI Article Writer
  12. User-agent: AIBot
  13. User-agent: AI Content Detector
  14. User-agent: AI Dungeon
  15. User-agent: aiHitBot
  16. User-agent: AIMatrix
  17. User-agent: AISearchBot
  18. User-agent: AI Search Engine
  19. User-agent: AI SEO Crawler
  20. User-agent: AI Training
  21. User-agent: AITraining
  22. User-agent: AI Writer
  23. User-agent: Alexa
  24. User-agent: Alpha AI
  25. User-agent: AlphaAI
  26. User-agent: a[mazing]{42}(robot)
  27. User-agent: Amazon Bedrock
  28. User-agent: Amazonbot
  29. User-agent: AmazonBot
  30. User-agent: Amazon Comprehend
  31. User-agent: Amazon-Kendra
  32. User-agent: Amazon Lex
  33. User-agent: Amazon Sagemaker
  34. User-agent: Amazon Silk
  35. User-agent: Amazon Textract
  36. User-agent: Amelia
  37. User-agent: AndersPinkBot
  38. User-agent: Andibot
  39. User-agent: Anthropic
  40. User-agent: anthropic-ai
  41. User-agent: AnyPicker
  42. User-agent: Anyword
  43. User-agent: Applebot
  44. User-agent: Applebot-Extended
  45. User-agent: Aria Browse
  46. User-agent: Articoolo
  47. User-agent: Automated Writer
  48. User-agent: Awario
  49. User-agent: AwarioBot
  50. User-agent: AwarioRssBot
  51. User-agent: AwarioSmartBot
  52. User-agent: Azure
  53. User-agent: BardBot
  54. User-agent: bedrockbot
  55. User-agent: bigsur.ai
  56. User-agent: BLEXBot
  57. User-agent: Brave Leo
  58. User-agent: Brightbot 1.0
  59. User-agent: ByteDance
  60. User-agent: Bytespider
  61. User-agent: CatBoost
  62. User-agent: CCBot
  63. User-agent: CC-Crawler
  64. User-agent: ChatGLM
  65. User-agent: ChatGPT Agent
  66. User-agent: ChatGPT-User
  67. User-agent: ChatGPT-User/2.0
  68. User-agent: Chinchilla
  69. User-agent: Claude
  70. User-agent: ClaudeBot
  71. User-agent: Claude-SearchBot
  72. User-agent: Claude-User
  73. User-agent: claude-web
  74. User-agent: Claude-Web
  75. User-agent: ClearScope
  76. User-agent: CloudVertexBot
  77. User-agent: Cohere
  78. User-agent: cohere-ai
  79. User-agent: cohere-training-data-crawler
  80. User-agent: Common Crawl
  81. User-agent: CommonCrawl
  82. User-agent: ContentAtScale
  83. User-agent: ContentBot
  84. User-agent: Contentedge
  85. User-agent: Content Harmony
  86. User-agent: Content King
  87. User-agent: Content Optimizer
  88. User-agent: Content Samurai
  89. User-agent: Conversion AI
  90. User-agent: Copilot
  91. User-agent: CopyAI
  92. User-agent: Copymatic
  93. User-agent: Copyscape
  94. User-agent: Cotoyogi
  95. User-agent: crawler.with.dots
  96. User-agent: CrawlQ AI
  97. User-agent: Crawlspace
  98. User-agent: Crew AI
  99. User-agent: CrewAI
  100. User-agent: curl|sudo bash
  101. User-agent: DALL-E
  102. User-agent: DataForSeoBot
  103. User-agent: DataProvider
  104. User-agent: Datenbank Crawler
  105. User-agent: DeepAI
  106. User-agent: DeepL
  107. User-agent: DeepMind
  108. User-agent: DeepSeek
  109. User-agent: Devin
  110. User-agent: diffbot
  111. User-agent: Diffbot
  112. User-agent: Doubao AI
  113. User-agent: DuckAssistBot
  114. User-agent: Echobot Bot
  115. User-agent: EchoboxBot
  116. User-agent: Facebookbot
  117. User-agent: FacebookBot
  118. User-agent: facebookexternalhit
  119. User-agent: FacebookExternalHit
  120. User-agent: Factset_spyderbot
  121. User-agent: Falcon
  122. User-agent: Firecrawl
  123. User-agent: FirecrawlAgent
  124. User-agent: Flyriver
  125. User-agent: Frase AI
  126. User-agent: FriendlyCrawler
  127. User-agent: Fuzz Faster U Fool
  128. User-agent: Fuzz Faster U Fool v2.0.0
  129. User-agent: Gemini
  130. User-agent: Gemini-Deep-Research
  131. User-agent: Gemma
  132. User-agent: GenAI
  133. User-agent: Genspark
  134. User-agent: Gigabot
  135. User-agent: GLM
  136. User-agent: GoogleAgent-Mariner
  137. User-agent: Google-CloudVertexBot
  138. User-agent: Google-Extended
  139. User-agent: Google-Firebase
  140. User-agent: GoogleOther
  141. User-agent: GoogleOther-Image
  142. User-agent: GoogleOther-Video
  143. User-agent: Goose
  144. User-agent: GPT
  145. User-agent: GPTBot
  146. User-agent: Grammarly
  147. User-agent: Grendizer
  148. User-agent: Grok
  149. User-agent: GT Bot
  150. User-agent: GTBot
  151. User-agent: Hemingway Editor
  152. User-agent: Hugging Face
  153. User-agent: Hypotenuse AI
  154. User-agent: iaskspider
  155. User-agent: iaskspider/2.0
  156. User-agent: ICC-Crawler
  157. User-agent: ImageGen
  158. User-agent: ImagesiftBot
  159. User-agent: img2dataset
  160. User-agent: imgproxy
  161. User-agent: Inferkit
  162. User-agent: INK Editor
  163. User-agent: INKforall
  164. User-agent: IntelliSeek
  165. User-agent: ISSCyberRiskCrawler
  166. User-agent: Is this a crawler?
  167. User-agent: JasperAI
  168. User-agent: Kafkai
  169. User-agent: Kangaroo
  170. User-agent: Kangaroo Bot
  171. User-agent: Keyword Density AI
  172. User-agent: Knowledge
  173. User-agent: KomoBot
  174. User-agent: LinerBot
  175. User-agent: LinkedInBot
  176. User-agent: LLaMA
  177. User-agent: LLMs
  178. User-agent: magpie-crawler
  179. User-agent: MarketMuse
  180. User-agent: Meltwater
  181. User-agent: Meta AI
  182. User-agent: Meta-AI
  183. User-agent: MetaAI
  184. User-agent: Meta-External
  185. User-agent: meta-externalagent
  186. User-agent: Meta-ExternalAgent
  187. User-agent: meta-externalfetcher
  188. User-agent: Meta-ExternalFetcher
  189. User-agent: MetaTagBot
  190. User-agent: meta-webindexer
  191. User-agent: Mistral
  192. User-agent: MistralAI-User
  193. User-agent: MistralAI-User/1.0
  194. User-agent: MJ12bot
  195. User-agent: MyCentralAIScraperBot
  196. User-agent: Narrative
  197. User-agent: NeevaBot
  198. User-agent: netEstate Imprint Crawler
  199. User-agent: NeuralSEO
  200. User-agent: Neural Text
  201. User-agent: Nova Act
  202. User-agent: NovaAct
  203. User-agent: Nutch
  204. User-agent: OAI-SearchBot
  205. User-agent: omgili
  206. User-agent: Omgili
  207. User-agent: omgilibot
  208. User-agent: Omgilibot
  209. User-agent: OmniExplorer_Bot
  210. User-agent: Open AI
  211. User-agent: OpenAI
  212. User-agent: OpenBot
  213. User-agent: OpenText AI
  214. User-agent: Operator
  215. User-agent: Outwrite
  216. User-agent: Page Analyzer AI
  217. User-agent: PanguBot
  218. User-agent: Panscient
  219. User-agent: panscient.com
  220. User-agent: Paperlibot
  221. User-agent: Paraphraser.io
  222. User-agent: peer39_crawler
  223. User-agent: Perplexity
  224. User-agent: PerplexityBot
  225. User-agent: Perplexity-User
  226. User-agent: Petalbot
  227. User-agent: PetalBot
  228. User-agent: Phindbot
  229. User-agent: PhindBot
  230. User-agent: PiplBot
  231. User-agent: Poseidon Research Crawler
  232. User-agent: prefetch-proxy
  233. User-agent: ProWritingAid
  234. User-agent: psbot
  235. User-agent: python-requests
  236. User-agent: QualifiedBot
  237. User-agent: QuillBot
  238. User-agent: quillbot.com
  239. User-agent: RobotSpider
  240. User-agent: Robozilla
  241. User-agent: Rytr
  242. User-agent: SaplingAI
  243. User-agent: SBIntuitionsBot
  244. User-agent: Scalenut
  245. User-agent: Scraper
  246. User-agent: Scrapy
  247. User-agent: ScriptBook
  248. User-agent: Seekr
  249. User-agent: SemrushBot-OCOB
  250. User-agent: SemrushBot-SWA
  251. User-agent: sentibot
  252. User-agent: Sentibot
  253. User-agent: SentiBot
  254. User-agent: SEO Content Machine
  255. User-agent: SEO Robot
  256. User-agent: ShapBot
  257. User-agent: Sidetrade
  258. User-agent: Sidetrade indexer bot
  259. User-agent: Simplified AI
  260. User-agent: Sitefinity
  261. User-agent: Skydancer
  262. User-agent: SlickWrite
  263. User-agent: Sonic
  264. User-agent: Spinbot
  265. User-agent: Spin Rewriter
  266. User-agent: Stability
  267. User-agent: StableDiffusionBot
  268. User-agent: star***crawler
  269. User-agent: Sudowrite
  270. User-agent: SummalyBot
  271. User-agent: Super Agent
  272. User-agent: Surfer AI
  273. User-agent: Teoma
  274. User-agent: TerraCotta
  275. User-agent: Text Blaze
  276. User-agent: TextCortex
  277. User-agent: The Knowledge AI
  278. User-agent: Thinkbot
  279. User-agent: ThinkChaos
  280. User-agent: TikTokSpider
  281. User-agent: Timpibot
  282. User-agent: TimpiBot
  283. User-agent: TurnitinBot
  284. User-agent: VelenPublicWebCrawler
  285. User-agent: Vidnami AI
  286. User-agent: WARDBot
  287. User-agent: Webzio
  288. User-agent: webzio-extended
  289. User-agent: Webzio-Extended
  290. User-agent: Whisper
  291. User-agent: WordAI
  292. User-agent: Wordtune
  293. User-agent: WormsGTP
  294. User-agent: wpbot
  295. User-agent: WPBot
  296. User-agent: Writecream
  297. User-agent: WriterZen
  298. User-agent: Writescope
  299. User-agent: Writesonic
  300. User-agent: xAI
  301. User-agent: xBot
  302. User-agent: YaK
  303. User-agent: YandexAdditional
  304. User-agent: YandexAdditionalBot
  305. User-agent: Youbot
  306. User-agent: YouBot
  307. User-agent: Zerochat
  308. User-agent: Zero GTP
  309. User-agent: Zhipu
  310. User-agent: Zimm
  311. Disallow: /
  312. Disallow: *
  313. DisallowAITraining: /
  314. Content-Usage: ai=n