{"adoption": {"forks": 363, "observed_at": "2026-08-28T04:08:03.685893+00:00", "stars": 3415}, "canonical_url": "https://ross.abutalabs.com/products/anycrawl", "card": {"archived": false, "artifact_type": "service", "description": "AnyCrawl 🚀: A Node.js/TypeScript crawler that turns websites into LLM-ready data and extracts structured SERP results from Google/Bing/Baidu/etc. Native multi-threading for bulk processing.", "domain": ["artificial-intelligence", "large-language-models", "crawlers", "web-development", "developer-tools", "self-hosted"], "enriched": true, "function": ["web-scraping", "http-server", "rag", "search-engine", "mcp", "json", "markdown", "scheduling", "webhook", "caching"], "health_score": 98, "homepage": "https://anycrawl.dev", "language": "TypeScript", "license": "MIT", "license_family": "permissive", "maturity": "active", "member_repos": ["any4ai/AnyCrawl"], "name": "any4ai/AnyCrawl", "platform": ["self-hosted", "cli"], "pushed_at": "2026-08-11T18:32:25+00:00", "repo": "any4ai/AnyCrawl", "stars": 3415, "tags": ["serp-extraction", "html-to-markdown", "playwright", "multi-threading", "openapi", "batch-processing", "proxy-support", "llm-ready-data", "scraping-api", "firecrawl-alternative", "retrieval-augmented-generation", "search", "data-engineering", "nodejs", "docker", "web-server"], "topics": ["aitools", "crawl", "scrape", "serp", "webscraper", "ai-scraping", "data", "html-to-markdown", "rag", "scraping"], "urls": [], "use_cases": ["convert websites into clean markdown for LLM training or RAG pipelines", "scrape Google, Bing, and Baidu search results as structured JSON", "bulk crawl large sites with multi-threaded processing", "extract content from JavaScript-heavy SPAs and dynamic pages", "self-host a scraping API compatible with OpenAPI tooling", "schedule recurring crawl jobs with webhook notifications", "feed fresh web data to AI agents via MCP server"], "what_it_is": "AnyCrawl is a Node.js/TypeScript web crawler and scraping service that converts websites into LLM-ready markdown/JSON data and extracts structured SERP results from search engines like Google, Bing, and Baidu. It offers a self-hostable OpenAPI-compatible HTTP API with multi-threaded bulk processing, Playwright-based rendering for JavaScript-heavy sites, scheduled tasks, webhooks, and an MCP server.", "when_to_avoid": ["you only need a lightweight in-process scraping library without an HTTP service", "you require a fully managed hosted scraping API with no infrastructure", "your use case is simple single-page fetching that a plain HTTP client covers", "you need non-Node.js runtime integration as the primary dependency"], "when_to_choose": ["you need LLM-ready structured data from websites at scale", "you want SERP extraction from multiple search engines in one tool", "you prefer self-hosting a scraping service via Docker with zero configuration", "you need to handle JavaScript-rendered or dynamic content", "you want scheduled tasks, webhooks, and batch processing built in"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/anycrawl", "repo": "any4ai/AnyCrawl", "role": "main", "score": 85}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:08:03.685893+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-29T18:38:28.101408+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4bb9b807e186c93915d61c4494003fd6814bde8cdfcd114dee4be994fa3ef676", "fetched_at": "2026-08-28T04:08:03.685893+00:00", "kind": "readme", "missing": false, "url": "https://github.com/any4ai/AnyCrawl"}, {"content_hash": "ca7d89c3a621409ebde60c0b9802b791a0a2ff3f6469e74cf1344527f8391e0c", "fetched_at": "2026-08-29T09:32:25.444486+00:00", "kind": "homepage", "missing": false, "url": "https://anycrawl.dev"}, {"content_hash": "34bb2f1d12ca129e9b5803a0b73aeac72830d1455bbc2aa8212d1ada6109f435", "fetched_at": "2026-08-29T09:32:25.450324+00:00", "kind": "site_page", "missing": false, "url": "https://docs.anycrawl.dev"}, {"content_hash": "4b94807942c800e7022b10429cf80cf6c87dd32c7938bfbbe57828985064a0af", "fetched_at": "2026-08-29T09:32:25.448320+00:00", "kind": "site_page", "missing": false, "url": "https://anycrawl.dev/price"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:08:03.685893+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-29T18:38:28.101408+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4bb9b807e186c93915d61c4494003fd6814bde8cdfcd114dee4be994fa3ef676", "fetched_at": "2026-08-28T04:08:03.685893+00:00", "kind": "readme", "missing": false, "url": "https://github.com/any4ai/AnyCrawl"}, {"content_hash": "ca7d89c3a621409ebde60c0b9802b791a0a2ff3f6469e74cf1344527f8391e0c", "fetched_at": "2026-08-29T09:32:25.444486+00:00", "kind": "homepage", "missing": false, "url": "https://anycrawl.dev"}, {"content_hash": "34bb2f1d12ca129e9b5803a0b73aeac72830d1455bbc2aa8212d1ada6109f435", "fetched_at": "2026-08-29T09:32:25.450324+00:00", "kind": "site_page", "missing": false, "url": "https://docs.anycrawl.dev"}, {"content_hash": "4b94807942c800e7022b10429cf80cf6c87dd32c7938bfbbe57828985064a0af", "fetched_at": "2026-08-29T09:32:25.448320+00:00", "kind": "site_page", "missing": false, "url": "https://anycrawl.dev/price"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-29T18:38:28.101408+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4bb9b807e186c93915d61c4494003fd6814bde8cdfcd114dee4be994fa3ef676", "fetched_at": "2026-08-28T04:08:03.685893+00:00", "kind": "readme", "missing": false, "url": "https://github.com/any4ai/AnyCrawl"}, {"content_hash": "ca7d89c3a621409ebde60c0b9802b791a0a2ff3f6469e74cf1344527f8391e0c", "fetched_at": "2026-08-29T09:32:25.444486+00:00", "kind": "homepage", "missing": false, "url": "https://anycrawl.dev"}, {"content_hash": "34bb2f1d12ca129e9b5803a0b73aeac72830d1455bbc2aa8212d1ada6109f435", "fetched_at": "2026-08-29T09:32:25.450324+00:00", "kind": "site_page", "missing": false, "url": "https://docs.anycrawl.dev"}, {"content_hash": "4b94807942c800e7022b10429cf80cf6c87dd32c7938bfbbe57828985064a0af", "fetched_at": "2026-08-29T09:32:25.448320+00:00", "kind": "site_page", "missing": false, "url": "https://anycrawl.dev/price"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:08:03.685893+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:08:03.685893+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:08:03.685893+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-29T18:38:28.101408+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4bb9b807e186c93915d61c4494003fd6814bde8cdfcd114dee4be994fa3ef676", "fetched_at": "2026-08-28T04:08:03.685893+00:00", "kind": "readme", "missing": false, "url": "https://github.com/any4ai/AnyCrawl"}, {"content_hash": "ca7d89c3a621409ebde60c0b9802b791a0a2ff3f6469e74cf1344527f8391e0c", "fetched_at": "2026-08-29T09:32:25.444486+00:00", "kind": "homepage", "missing": false, "url": "https://anycrawl.dev"}, {"content_hash": "34bb2f1d12ca129e9b5803a0b73aeac72830d1455bbc2aa8212d1ada6109f435", "fetched_at": "2026-08-29T09:32:25.450324+00:00", "kind": "site_page", "missing": false, "url": "https://docs.anycrawl.dev"}, {"content_hash": "4b94807942c800e7022b10429cf80cf6c87dd32c7938bfbbe57828985064a0af", "fetched_at": "2026-08-29T09:32:25.448320+00:00", "kind": "site_page", "missing": false, "url": "https://anycrawl.dev/price"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:08:03.685893+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:08:03.685893+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-29T18:38:28.101408+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4bb9b807e186c93915d61c4494003fd6814bde8cdfcd114dee4be994fa3ef676", "fetched_at": "2026-08-28T04:08:03.685893+00:00", "kind": "readme", "missing": false, "url": "https://github.com/any4ai/AnyCrawl"}, {"content_hash": "ca7d89c3a621409ebde60c0b9802b791a0a2ff3f6469e74cf1344527f8391e0c", "fetched_at": "2026-08-29T09:32:25.444486+00:00", "kind": "homepage", "missing": false, "url": "https://anycrawl.dev"}, {"content_hash": "34bb2f1d12ca129e9b5803a0b73aeac72830d1455bbc2aa8212d1ada6109f435", "fetched_at": "2026-08-29T09:32:25.450324+00:00", "kind": "site_page", "missing": false, "url": "https://docs.anycrawl.dev"}, {"content_hash": "4b94807942c800e7022b10429cf80cf6c87dd32c7938bfbbe57828985064a0af", "fetched_at": "2026-08-29T09:32:25.448320+00:00", "kind": "site_page", "missing": false, "url": "https://anycrawl.dev/price"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:08:03.685893+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:08:03.685893+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:08:03.685893+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-29T18:38:28.101408+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4bb9b807e186c93915d61c4494003fd6814bde8cdfcd114dee4be994fa3ef676", "fetched_at": "2026-08-28T04:08:03.685893+00:00", "kind": "readme", "missing": false, "url": "https://github.com/any4ai/AnyCrawl"}, {"content_hash": "ca7d89c3a621409ebde60c0b9802b791a0a2ff3f6469e74cf1344527f8391e0c", "fetched_at": "2026-08-29T09:32:25.444486+00:00", "kind": "homepage", "missing": false, "url": "https://anycrawl.dev"}, {"content_hash": "34bb2f1d12ca129e9b5803a0b73aeac72830d1455bbc2aa8212d1ada6109f435", "fetched_at": "2026-08-29T09:32:25.450324+00:00", "kind": "site_page", "missing": false, "url": "https://docs.anycrawl.dev"}, {"content_hash": "4b94807942c800e7022b10429cf80cf6c87dd32c7938bfbbe57828985064a0af", "fetched_at": "2026-08-29T09:32:25.448320+00:00", "kind": "site_page", "missing": false, "url": "https://anycrawl.dev/price"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:08:03.685893+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:08:03.685893+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-29T18:38:28.101408+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4bb9b807e186c93915d61c4494003fd6814bde8cdfcd114dee4be994fa3ef676", "fetched_at": "2026-08-28T04:08:03.685893+00:00", "kind": "readme", "missing": false, "url": "https://github.com/any4ai/AnyCrawl"}, {"content_hash": "ca7d89c3a621409ebde60c0b9802b791a0a2ff3f6469e74cf1344527f8391e0c", "fetched_at": "2026-08-29T09:32:25.444486+00:00", "kind": "homepage", "missing": false, "url": "https://anycrawl.dev"}, {"content_hash": "34bb2f1d12ca129e9b5803a0b73aeac72830d1455bbc2aa8212d1ada6109f435", "fetched_at": "2026-08-29T09:32:25.450324+00:00", "kind": "site_page", "missing": false, "url": "https://docs.anycrawl.dev"}, {"content_hash": "4b94807942c800e7022b10429cf80cf6c87dd32c7938bfbbe57828985064a0af", "fetched_at": "2026-08-29T09:32:25.448320+00:00", "kind": "site_page", "missing": false, "url": "https://anycrawl.dev/price"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-29T18:38:28.101408+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4bb9b807e186c93915d61c4494003fd6814bde8cdfcd114dee4be994fa3ef676", "fetched_at": "2026-08-28T04:08:03.685893+00:00", "kind": "readme", "missing": false, "url": "https://github.com/any4ai/AnyCrawl"}, {"content_hash": "ca7d89c3a621409ebde60c0b9802b791a0a2ff3f6469e74cf1344527f8391e0c", "fetched_at": "2026-08-29T09:32:25.444486+00:00", "kind": "homepage", "missing": false, "url": "https://anycrawl.dev"}, {"content_hash": "34bb2f1d12ca129e9b5803a0b73aeac72830d1455bbc2aa8212d1ada6109f435", "fetched_at": "2026-08-29T09:32:25.450324+00:00", "kind": "site_page", "missing": false, "url": "https://docs.anycrawl.dev"}, {"content_hash": "4b94807942c800e7022b10429cf80cf6c87dd32c7938bfbbe57828985064a0af", "fetched_at": "2026-08-29T09:32:25.448320+00:00", "kind": "site_page", "missing": false, "url": "https://anycrawl.dev/price"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-29T18:38:28.101408+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4bb9b807e186c93915d61c4494003fd6814bde8cdfcd114dee4be994fa3ef676", "fetched_at": "2026-08-28T04:08:03.685893+00:00", "kind": "readme", "missing": false, "url": "https://github.com/any4ai/AnyCrawl"}, {"content_hash": "ca7d89c3a621409ebde60c0b9802b791a0a2ff3f6469e74cf1344527f8391e0c", "fetched_at": "2026-08-29T09:32:25.444486+00:00", "kind": "homepage", "missing": false, "url": "https://anycrawl.dev"}, {"content_hash": "34bb2f1d12ca129e9b5803a0b73aeac72830d1455bbc2aa8212d1ada6109f435", "fetched_at": "2026-08-29T09:32:25.450324+00:00", "kind": "site_page", "missing": false, "url": "https://docs.anycrawl.dev"}, {"content_hash": "4b94807942c800e7022b10429cf80cf6c87dd32c7938bfbbe57828985064a0af", "fetched_at": "2026-08-29T09:32:25.448320+00:00", "kind": "site_page", "missing": false, "url": "https://anycrawl.dev/price"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-29T18:38:28.101408+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4bb9b807e186c93915d61c4494003fd6814bde8cdfcd114dee4be994fa3ef676", "fetched_at": "2026-08-28T04:08:03.685893+00:00", "kind": "readme", "missing": false, "url": "https://github.com/any4ai/AnyCrawl"}, {"content_hash": "ca7d89c3a621409ebde60c0b9802b791a0a2ff3f6469e74cf1344527f8391e0c", "fetched_at": "2026-08-29T09:32:25.444486+00:00", "kind": "homepage", "missing": false, "url": "https://anycrawl.dev"}, {"content_hash": "34bb2f1d12ca129e9b5803a0b73aeac72830d1455bbc2aa8212d1ada6109f435", "fetched_at": "2026-08-29T09:32:25.450324+00:00", "kind": "site_page", "missing": false, "url": "https://docs.anycrawl.dev"}, {"content_hash": "4b94807942c800e7022b10429cf80cf6c87dd32c7938bfbbe57828985064a0af", "fetched_at": "2026-08-29T09:32:25.448320+00:00", "kind": "site_page", "missing": false, "url": "https://anycrawl.dev/price"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 97, "longevity": 37, "rhythm": 96}, "computed_at": "2026-09-03T02:20:16.233290+00:00", "flags": [], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 520, "days_push": 22, "days_rel": 28, "gap_med": 11, "n_releases_24m": 18}, "score": 85, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}