{"adoption": {"forks": 11, "observed_at": "2026-08-28T04:07:49.679774+00:00", "stars": 3218}, "canonical_url": "https://ross.abutalabs.com/products/ai-crawler-py", "card": {"archived": false, "artifact_type": "library", "description": "Crawl a website starting from a URL, find relevant pages, and extract data – all guided by your natural language prompt.", "domain": ["crawlers", "artificial-intelligence"], "enriched": true, "function": ["web-scraping", "nlp", "etl", "data-generation"], "health_score": 79, "homepage": "https://aistudio.oxylabs.io/apps/crawl", "language": null, "license": null, "license_family": "other", "maturity": "active", "member_repos": ["oxylabs/ai-crawler-py"], "name": "oxylabs/ai-crawler-py", "platform": ["python", "cloud"], "pushed_at": "2026-08-21T11:42:16+00:00", "repo": "oxylabs/ai-crawler-py", "stars": 3218, "tags": ["ai-crawler", "web-crawler", "ai-scraping", "oxylabs", "api-client", "low-code", "commercial-service", "prompt-driven", "data-engineering", "automation", "web-server"], "topics": ["ai", "ai-agents", "ai-crawler", "ai-studio", "web-crawler", "ai-web-crawler", "crawl-agent", "ai-scraping"], "urls": [], "use_cases": ["crawl a website and extract data using a natural language prompt", "find relevant pages on a domain without writing custom scrapers", "extract structured JSON from websites using a schema", "convert website content to markdown for LLM pipelines", "feed crawled web data into AI agents or automation workflows"], "what_it_is": "A Python client library for Oxylabs AI Studio's AI-Crawler, a commercial service that crawls websites from a starting URL, uses natural language prompts to find relevant pages, and extracts structured JSON or Markdown data. It wraps a paid API (with a free trial of 1,000 credits) rather than performing crawling locally.", "when_to_avoid": ["you need a fully free, self-hosted crawler with no API costs", "you want full control over crawl logic, politeness, or proxies", "your project cannot depend on a third-party commercial service"], "when_to_choose": ["you want prompt-driven crawling without building or maintaining scrapers", "you need schema-based structured extraction from websites", "you already use Oxylabs AI Studio or want managed crawling infrastructure"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/ai-crawler-py", "repo": "oxylabs/ai-crawler-py", "role": "main", "score": 61}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:07:49.679774+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-29T18:44:54.736931+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "7c2c2a5e4c10305e81dac7a59d7ec65bd20d61c3a07851ed2f56d7f57c8bdd4f", "fetched_at": "2026-08-28T04:07:49.679774+00:00", "kind": "readme", "missing": false, "url": "https://github.com/oxylabs/ai-crawler-py"}, {"content_hash": "816b67fd1a585840dc4c5ebf99c50bb1724dc51c67a0257ef401d3ecf9aed5b5", "fetched_at": "2026-08-29T09:38:18.944914+00:00", "kind": "homepage", "missing": false, "url": "https://aistudio.oxylabs.io/apps/crawl"}, {"content_hash": "9226846bdd8d0ee2eab52adb401f4ee5c7058a2a3c46f8813d0a0a24ad623d23", "fetched_at": "2026-08-29T09:38:18.953741+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/pricing"}, {"content_hash": "2807647bbec53f2eddc1c38fe7b615226487cb760126ac5c8bc07dfca25bc019", "fetched_at": "2026-08-29T09:38:18.955702+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/releases"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:07:49.679774+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-29T18:44:54.736931+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "7c2c2a5e4c10305e81dac7a59d7ec65bd20d61c3a07851ed2f56d7f57c8bdd4f", "fetched_at": "2026-08-28T04:07:49.679774+00:00", "kind": "readme", "missing": false, "url": "https://github.com/oxylabs/ai-crawler-py"}, {"content_hash": "816b67fd1a585840dc4c5ebf99c50bb1724dc51c67a0257ef401d3ecf9aed5b5", "fetched_at": "2026-08-29T09:38:18.944914+00:00", "kind": "homepage", "missing": false, "url": "https://aistudio.oxylabs.io/apps/crawl"}, {"content_hash": "9226846bdd8d0ee2eab52adb401f4ee5c7058a2a3c46f8813d0a0a24ad623d23", "fetched_at": "2026-08-29T09:38:18.953741+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/pricing"}, {"content_hash": "2807647bbec53f2eddc1c38fe7b615226487cb760126ac5c8bc07dfca25bc019", "fetched_at": "2026-08-29T09:38:18.955702+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/releases"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-29T18:44:54.736931+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "7c2c2a5e4c10305e81dac7a59d7ec65bd20d61c3a07851ed2f56d7f57c8bdd4f", "fetched_at": "2026-08-28T04:07:49.679774+00:00", "kind": "readme", "missing": false, "url": "https://github.com/oxylabs/ai-crawler-py"}, {"content_hash": "816b67fd1a585840dc4c5ebf99c50bb1724dc51c67a0257ef401d3ecf9aed5b5", "fetched_at": "2026-08-29T09:38:18.944914+00:00", "kind": "homepage", "missing": false, "url": "https://aistudio.oxylabs.io/apps/crawl"}, {"content_hash": "9226846bdd8d0ee2eab52adb401f4ee5c7058a2a3c46f8813d0a0a24ad623d23", "fetched_at": "2026-08-29T09:38:18.953741+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/pricing"}, {"content_hash": "2807647bbec53f2eddc1c38fe7b615226487cb760126ac5c8bc07dfca25bc019", "fetched_at": "2026-08-29T09:38:18.955702+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/releases"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:07:49.679774+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:07:49.679774+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:07:49.679774+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-29T18:44:54.736931+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "7c2c2a5e4c10305e81dac7a59d7ec65bd20d61c3a07851ed2f56d7f57c8bdd4f", "fetched_at": "2026-08-28T04:07:49.679774+00:00", "kind": "readme", "missing": false, "url": "https://github.com/oxylabs/ai-crawler-py"}, {"content_hash": "816b67fd1a585840dc4c5ebf99c50bb1724dc51c67a0257ef401d3ecf9aed5b5", "fetched_at": "2026-08-29T09:38:18.944914+00:00", "kind": "homepage", "missing": false, "url": "https://aistudio.oxylabs.io/apps/crawl"}, {"content_hash": "9226846bdd8d0ee2eab52adb401f4ee5c7058a2a3c46f8813d0a0a24ad623d23", "fetched_at": "2026-08-29T09:38:18.953741+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/pricing"}, {"content_hash": "2807647bbec53f2eddc1c38fe7b615226487cb760126ac5c8bc07dfca25bc019", "fetched_at": "2026-08-29T09:38:18.955702+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/releases"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:07:49.679774+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:07:49.679774+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-29T18:44:54.736931+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "7c2c2a5e4c10305e81dac7a59d7ec65bd20d61c3a07851ed2f56d7f57c8bdd4f", "fetched_at": "2026-08-28T04:07:49.679774+00:00", "kind": "readme", "missing": false, "url": "https://github.com/oxylabs/ai-crawler-py"}, {"content_hash": "816b67fd1a585840dc4c5ebf99c50bb1724dc51c67a0257ef401d3ecf9aed5b5", "fetched_at": "2026-08-29T09:38:18.944914+00:00", "kind": "homepage", "missing": false, "url": "https://aistudio.oxylabs.io/apps/crawl"}, {"content_hash": "9226846bdd8d0ee2eab52adb401f4ee5c7058a2a3c46f8813d0a0a24ad623d23", "fetched_at": "2026-08-29T09:38:18.953741+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/pricing"}, {"content_hash": "2807647bbec53f2eddc1c38fe7b615226487cb760126ac5c8bc07dfca25bc019", "fetched_at": "2026-08-29T09:38:18.955702+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/releases"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:07:49.679774+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:07:49.679774+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:07:49.679774+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-29T18:44:54.736931+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "7c2c2a5e4c10305e81dac7a59d7ec65bd20d61c3a07851ed2f56d7f57c8bdd4f", "fetched_at": "2026-08-28T04:07:49.679774+00:00", "kind": "readme", "missing": false, "url": "https://github.com/oxylabs/ai-crawler-py"}, {"content_hash": "816b67fd1a585840dc4c5ebf99c50bb1724dc51c67a0257ef401d3ecf9aed5b5", "fetched_at": "2026-08-29T09:38:18.944914+00:00", "kind": "homepage", "missing": false, "url": "https://aistudio.oxylabs.io/apps/crawl"}, {"content_hash": "9226846bdd8d0ee2eab52adb401f4ee5c7058a2a3c46f8813d0a0a24ad623d23", "fetched_at": "2026-08-29T09:38:18.953741+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/pricing"}, {"content_hash": "2807647bbec53f2eddc1c38fe7b615226487cb760126ac5c8bc07dfca25bc019", "fetched_at": "2026-08-29T09:38:18.955702+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/releases"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:07:49.679774+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:07:49.679774+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-29T18:44:54.736931+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "7c2c2a5e4c10305e81dac7a59d7ec65bd20d61c3a07851ed2f56d7f57c8bdd4f", "fetched_at": "2026-08-28T04:07:49.679774+00:00", "kind": "readme", "missing": false, "url": "https://github.com/oxylabs/ai-crawler-py"}, {"content_hash": "816b67fd1a585840dc4c5ebf99c50bb1724dc51c67a0257ef401d3ecf9aed5b5", "fetched_at": "2026-08-29T09:38:18.944914+00:00", "kind": "homepage", "missing": false, "url": "https://aistudio.oxylabs.io/apps/crawl"}, {"content_hash": "9226846bdd8d0ee2eab52adb401f4ee5c7058a2a3c46f8813d0a0a24ad623d23", "fetched_at": "2026-08-29T09:38:18.953741+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/pricing"}, {"content_hash": "2807647bbec53f2eddc1c38fe7b615226487cb760126ac5c8bc07dfca25bc019", "fetched_at": "2026-08-29T09:38:18.955702+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/releases"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-29T18:44:54.736931+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "7c2c2a5e4c10305e81dac7a59d7ec65bd20d61c3a07851ed2f56d7f57c8bdd4f", "fetched_at": "2026-08-28T04:07:49.679774+00:00", "kind": "readme", "missing": false, "url": "https://github.com/oxylabs/ai-crawler-py"}, {"content_hash": "816b67fd1a585840dc4c5ebf99c50bb1724dc51c67a0257ef401d3ecf9aed5b5", "fetched_at": "2026-08-29T09:38:18.944914+00:00", "kind": "homepage", "missing": false, "url": "https://aistudio.oxylabs.io/apps/crawl"}, {"content_hash": "9226846bdd8d0ee2eab52adb401f4ee5c7058a2a3c46f8813d0a0a24ad623d23", "fetched_at": "2026-08-29T09:38:18.953741+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/pricing"}, {"content_hash": "2807647bbec53f2eddc1c38fe7b615226487cb760126ac5c8bc07dfca25bc019", "fetched_at": "2026-08-29T09:38:18.955702+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/releases"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-29T18:44:54.736931+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "7c2c2a5e4c10305e81dac7a59d7ec65bd20d61c3a07851ed2f56d7f57c8bdd4f", "fetched_at": "2026-08-28T04:07:49.679774+00:00", "kind": "readme", "missing": false, "url": "https://github.com/oxylabs/ai-crawler-py"}, {"content_hash": "816b67fd1a585840dc4c5ebf99c50bb1724dc51c67a0257ef401d3ecf9aed5b5", "fetched_at": "2026-08-29T09:38:18.944914+00:00", "kind": "homepage", "missing": false, "url": "https://aistudio.oxylabs.io/apps/crawl"}, {"content_hash": "9226846bdd8d0ee2eab52adb401f4ee5c7058a2a3c46f8813d0a0a24ad623d23", "fetched_at": "2026-08-29T09:38:18.953741+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/pricing"}, {"content_hash": "2807647bbec53f2eddc1c38fe7b615226487cb760126ac5c8bc07dfca25bc019", "fetched_at": "2026-08-29T09:38:18.955702+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/releases"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-29T18:44:54.736931+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "7c2c2a5e4c10305e81dac7a59d7ec65bd20d61c3a07851ed2f56d7f57c8bdd4f", "fetched_at": "2026-08-28T04:07:49.679774+00:00", "kind": "readme", "missing": false, "url": "https://github.com/oxylabs/ai-crawler-py"}, {"content_hash": "816b67fd1a585840dc4c5ebf99c50bb1724dc51c67a0257ef401d3ecf9aed5b5", "fetched_at": "2026-08-29T09:38:18.944914+00:00", "kind": "homepage", "missing": false, "url": "https://aistudio.oxylabs.io/apps/crawl"}, {"content_hash": "9226846bdd8d0ee2eab52adb401f4ee5c7058a2a3c46f8813d0a0a24ad623d23", "fetched_at": "2026-08-29T09:38:18.953741+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/pricing"}, {"content_hash": "2807647bbec53f2eddc1c38fe7b615226487cb760126ac5c8bc07dfca25bc019", "fetched_at": "2026-08-29T09:38:18.955702+00:00", "kind": "site_page", "missing": false, "url": "https://aistudio.oxylabs.io/releases"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 98, "longevity": 25, "rhythm": 35}, "computed_at": "2026-09-02T17:46:02.011165+00:00", "flags": ["no_releases", "no_license"], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 350, "days_push": 12, "days_rel": null, "gap_med": null, "n_releases_24m": 0}, "score": 61, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}