{"schemaVersion":"ready-scan.resource-sources.v1","locale":"en","path":"/sources/source-d871053673d1c958","canonicalUrl":"https://readyscan.ai/sources/source-d871053673d1c958","machineUrl":"https://readyscan.ai/discovery/sources/source-d871053673d1c958.json","page":1,"pageSize":48,"totalCount":1,"totalPages":1,"previous":null,"next":null,"kind":"source","source":{"id":"source-d871053673d1c958","url":"https://www.rfc-editor.org/rfc/rfc9309.html","title":"RFC 9309: Robots Exclusion Protocol","publisher":"IETF","publicationCount":1,"modifiedOn":"2026-10-10","path":"/sources/source-d871053673d1c958","canonicalUrl":"https://readyscan.ai/sources/source-d871053673d1c958","machineUrl":"https://readyscan.ai/discovery/sources/source-d871053673d1c958.json"},"items":[{"key":"insight-ai-crawler-search-training-controls","citations":[{"id":"robots-standard","title":"RFC 9309: Robots Exclusion Protocol","publisher":"IETF","url":"https://www.rfc-editor.org/rfc/rfc9309.html","accessedOn":"2026-10-10"}],"editions":[{"language":"en","title":"AI crawler access is not one switch","description":"Separate controls for search discovery, possible model-training use and visits triggered by a user. The available controls and their effects differ by provider.","path":"/insights/ai-crawler-search-training-controls","canonicalUrl":"https://readyscan.ai/insights/ai-crawler-search-training-controls","machineUrl":"https://readyscan.ai/discovery/resources/ai-crawler-search-training-controls.json","modifiedOn":"2026-10-10"},{"language":"zh-Hant","title":"AI 爬蟲存取並非單一開關","description":"分開處理搜尋發現、可能用於模型訓練的內容，以及由用戶觸發的造訪。各供應商提供的控制及其效果各有不同。","path":"/zh/insights/ai-crawler-search-training-controls","canonicalUrl":"https://readyscan.ai/zh/insights/ai-crawler-search-training-controls","machineUrl":"https://readyscan.ai/zh/discovery/resources/ai-crawler-search-training-controls.json","modifiedOn":"2026-10-10"},{"language":"zh-Hans","title":"AI 爬虫访问并非单一开关","description":"分别处理搜索发现、可能用于模型训练的内容，以及由用户触发的访问。各供应商提供的控制及其效果各不相同。","path":"/zh-Hans/insights/ai-crawler-search-training-controls","canonicalUrl":"https://readyscan.ai/zh-Hans/insights/ai-crawler-search-training-controls","machineUrl":"https://readyscan.ai/zh-Hans/discovery/resources/ai-crawler-search-training-controls.json","modifiedOn":"2026-10-10"}],"selected":{"language":"en","title":"AI crawler access is not one switch","description":"Separate controls for search discovery, possible model-training use and visits triggered by a user. The available controls and their effects differ by provider.","path":"/insights/ai-crawler-search-training-controls","canonicalUrl":"https://readyscan.ai/insights/ai-crawler-search-training-controls","machineUrl":"https://readyscan.ai/discovery/resources/ai-crawler-search-training-controls.json","modifiedOn":"2026-10-10"}}],"versionId":"fnv1a64:dcca1844bd915ba4"}