{"adoption": {"forks": 1888, "observed_at": "2026-08-28T04:10:52.181481+00:00", "stars": 12262}, "canonical_url": "https://ross.abutalabs.com/products/crawlab", "card": {"archived": false, "artifact_type": "application", "description": "Distributed web crawler admin platform for spiders management regardless of languages and frameworks. 分布式爬虫管理平台，支持任何语言和框架", "domain": ["crawlers", "web-development", "self-hosted"], "enriched": true, "function": ["web-scraping", "scheduling", "monitoring", "logging", "self-hosted", "gui", "orm", "database"], "health_score": 67, "homepage": "https://www.crawlab.cn", "language": "Go", "license": "BSD-3-Clause", "license_family": "permissive", "maturity": "active", "member_repos": ["crawlab-team/crawlab"], "name": "crawlab-team/crawlab", "platform": ["self-hosted", "cross-platform", "go"], "pushed_at": "2026-02-10T00:31:17+00:00", "repo": "crawlab-team/crawlab", "stars": 12262, "tags": ["web-crawler", "spider-management", "distributed-crawling", "scrapy", "scrapyd", "task-scheduling", "grpc", "monaco-editor", "data-extraction", "data-engineering", "automation", "docker", "web-server", "linux"], "topics": ["webcrawler", "scrapy", "crawlab", "spiders-management", "go", "scrapyd-ui", "spider", "crawler", "webspider", "web-crawler", "docker", "platform", "crawling-tasks"], "urls": [], "use_cases": ["manage and schedule web crawlers from a central dashboard", "run distributed scraping across multiple worker nodes", "monitor spider task logs and results in real time", "store scraped data into MySQL, PostgreSQL, or MongoDB automatically", "edit and deploy spider scripts in the browser", "replace scrapyd with a multi-language crawler platform"], "what_it_is": "Crawlab is a Go-based distributed web crawler management platform with a web UI for managing, scheduling, and monitoring spiders written in any language or framework (Scrapy, Puppeteer, Selenium, etc.). It supports multi-node clusters over gRPC, a built-in Monaco code editor, task logs, and automatic storage of scraped data into MongoDB or SQL databases via native ORM.", "when_to_avoid": ["you only need a single simple scraper script without orchestration", "you want a scraping framework/library to write crawler code rather than a management platform", "you cannot run Docker or MongoDB in your environment"], "when_to_choose": ["you run many spiders in different languages or frameworks and need unified management", "you need distributed crawling with master/worker nodes and scheduling", "you want a self-hosted admin UI with logs, results, and database integration"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/crawlab", "repo": "crawlab-team/crawlab", "role": "main", "score": 52}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:10:52.181481+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-29T17:14:12.921749+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "87f15e160d04f64eef5780b2d7db03dad33350660912c1e4a25aa28f5c728bef", "fetched_at": "2026-08-28T04:10:52.181481+00:00", "kind": "readme", "missing": false, "url": "https://github.com/crawlab-team/crawlab"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.167518+00:00", "kind": "homepage", "missing": false, "url": "https://www.crawlab.cn"}, {"content_hash": "df0bdf97e2651f64e9f91f07ead57d3ecd0a4962986576ff7254c273e0969c46", "fetched_at": "2026-08-29T08:11:41.173834+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io"}, {"content_hash": "68f37fba588845fac2c5fbd65a31591c6935fee822b3fbc3ac73c32374c78c1b", "fetched_at": "2026-08-29T08:11:41.175520+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io/docs/whats-new"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.171707+00:00", "kind": "site_page", "missing": false, "url": "https://www.crawlab.io/"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:10:52.181481+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-29T17:14:12.921749+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "87f15e160d04f64eef5780b2d7db03dad33350660912c1e4a25aa28f5c728bef", "fetched_at": "2026-08-28T04:10:52.181481+00:00", "kind": "readme", "missing": false, "url": "https://github.com/crawlab-team/crawlab"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.167518+00:00", "kind": "homepage", "missing": false, "url": "https://www.crawlab.cn"}, {"content_hash": "df0bdf97e2651f64e9f91f07ead57d3ecd0a4962986576ff7254c273e0969c46", "fetched_at": "2026-08-29T08:11:41.173834+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io"}, {"content_hash": "68f37fba588845fac2c5fbd65a31591c6935fee822b3fbc3ac73c32374c78c1b", "fetched_at": "2026-08-29T08:11:41.175520+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io/docs/whats-new"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.171707+00:00", "kind": "site_page", "missing": false, "url": "https://www.crawlab.io/"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-29T17:14:12.921749+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "87f15e160d04f64eef5780b2d7db03dad33350660912c1e4a25aa28f5c728bef", "fetched_at": "2026-08-28T04:10:52.181481+00:00", "kind": "readme", "missing": false, "url": "https://github.com/crawlab-team/crawlab"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.167518+00:00", "kind": "homepage", "missing": false, "url": "https://www.crawlab.cn"}, {"content_hash": "df0bdf97e2651f64e9f91f07ead57d3ecd0a4962986576ff7254c273e0969c46", "fetched_at": "2026-08-29T08:11:41.173834+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io"}, {"content_hash": "68f37fba588845fac2c5fbd65a31591c6935fee822b3fbc3ac73c32374c78c1b", "fetched_at": "2026-08-29T08:11:41.175520+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io/docs/whats-new"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.171707+00:00", "kind": "site_page", "missing": false, "url": "https://www.crawlab.io/"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:10:52.181481+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:10:52.181481+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:10:52.181481+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-29T17:14:12.921749+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "87f15e160d04f64eef5780b2d7db03dad33350660912c1e4a25aa28f5c728bef", "fetched_at": "2026-08-28T04:10:52.181481+00:00", "kind": "readme", "missing": false, "url": "https://github.com/crawlab-team/crawlab"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.167518+00:00", "kind": "homepage", "missing": false, "url": "https://www.crawlab.cn"}, {"content_hash": "df0bdf97e2651f64e9f91f07ead57d3ecd0a4962986576ff7254c273e0969c46", "fetched_at": "2026-08-29T08:11:41.173834+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io"}, {"content_hash": "68f37fba588845fac2c5fbd65a31591c6935fee822b3fbc3ac73c32374c78c1b", "fetched_at": "2026-08-29T08:11:41.175520+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io/docs/whats-new"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.171707+00:00", "kind": "site_page", "missing": false, "url": "https://www.crawlab.io/"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:10:52.181481+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:10:52.181481+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-29T17:14:12.921749+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "87f15e160d04f64eef5780b2d7db03dad33350660912c1e4a25aa28f5c728bef", "fetched_at": "2026-08-28T04:10:52.181481+00:00", "kind": "readme", "missing": false, "url": "https://github.com/crawlab-team/crawlab"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.167518+00:00", "kind": "homepage", "missing": false, "url": "https://www.crawlab.cn"}, {"content_hash": "df0bdf97e2651f64e9f91f07ead57d3ecd0a4962986576ff7254c273e0969c46", "fetched_at": "2026-08-29T08:11:41.173834+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io"}, {"content_hash": "68f37fba588845fac2c5fbd65a31591c6935fee822b3fbc3ac73c32374c78c1b", "fetched_at": "2026-08-29T08:11:41.175520+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io/docs/whats-new"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.171707+00:00", "kind": "site_page", "missing": false, "url": "https://www.crawlab.io/"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:10:52.181481+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:10:52.181481+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:10:52.181481+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-29T17:14:12.921749+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "87f15e160d04f64eef5780b2d7db03dad33350660912c1e4a25aa28f5c728bef", "fetched_at": "2026-08-28T04:10:52.181481+00:00", "kind": "readme", "missing": false, "url": "https://github.com/crawlab-team/crawlab"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.167518+00:00", "kind": "homepage", "missing": false, "url": "https://www.crawlab.cn"}, {"content_hash": "df0bdf97e2651f64e9f91f07ead57d3ecd0a4962986576ff7254c273e0969c46", "fetched_at": "2026-08-29T08:11:41.173834+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io"}, {"content_hash": "68f37fba588845fac2c5fbd65a31591c6935fee822b3fbc3ac73c32374c78c1b", "fetched_at": "2026-08-29T08:11:41.175520+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io/docs/whats-new"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.171707+00:00", "kind": "site_page", "missing": false, "url": "https://www.crawlab.io/"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:10:52.181481+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:10:52.181481+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-29T17:14:12.921749+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "87f15e160d04f64eef5780b2d7db03dad33350660912c1e4a25aa28f5c728bef", "fetched_at": "2026-08-28T04:10:52.181481+00:00", "kind": "readme", "missing": false, "url": "https://github.com/crawlab-team/crawlab"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.167518+00:00", "kind": "homepage", "missing": false, "url": "https://www.crawlab.cn"}, {"content_hash": "df0bdf97e2651f64e9f91f07ead57d3ecd0a4962986576ff7254c273e0969c46", "fetched_at": "2026-08-29T08:11:41.173834+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io"}, {"content_hash": "68f37fba588845fac2c5fbd65a31591c6935fee822b3fbc3ac73c32374c78c1b", "fetched_at": "2026-08-29T08:11:41.175520+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io/docs/whats-new"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.171707+00:00", "kind": "site_page", "missing": false, "url": "https://www.crawlab.io/"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-29T17:14:12.921749+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "87f15e160d04f64eef5780b2d7db03dad33350660912c1e4a25aa28f5c728bef", "fetched_at": "2026-08-28T04:10:52.181481+00:00", "kind": "readme", "missing": false, "url": "https://github.com/crawlab-team/crawlab"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.167518+00:00", "kind": "homepage", "missing": false, "url": "https://www.crawlab.cn"}, {"content_hash": "df0bdf97e2651f64e9f91f07ead57d3ecd0a4962986576ff7254c273e0969c46", "fetched_at": "2026-08-29T08:11:41.173834+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io"}, {"content_hash": "68f37fba588845fac2c5fbd65a31591c6935fee822b3fbc3ac73c32374c78c1b", "fetched_at": "2026-08-29T08:11:41.175520+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io/docs/whats-new"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.171707+00:00", "kind": "site_page", "missing": false, "url": "https://www.crawlab.io/"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-29T17:14:12.921749+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "87f15e160d04f64eef5780b2d7db03dad33350660912c1e4a25aa28f5c728bef", "fetched_at": "2026-08-28T04:10:52.181481+00:00", "kind": "readme", "missing": false, "url": "https://github.com/crawlab-team/crawlab"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.167518+00:00", "kind": "homepage", "missing": false, "url": "https://www.crawlab.cn"}, {"content_hash": "df0bdf97e2651f64e9f91f07ead57d3ecd0a4962986576ff7254c273e0969c46", "fetched_at": "2026-08-29T08:11:41.173834+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io"}, {"content_hash": "68f37fba588845fac2c5fbd65a31591c6935fee822b3fbc3ac73c32374c78c1b", "fetched_at": "2026-08-29T08:11:41.175520+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io/docs/whats-new"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.171707+00:00", "kind": "site_page", "missing": false, "url": "https://www.crawlab.io/"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-29T17:14:12.921749+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "87f15e160d04f64eef5780b2d7db03dad33350660912c1e4a25aa28f5c728bef", "fetched_at": "2026-08-28T04:10:52.181481+00:00", "kind": "readme", "missing": false, "url": "https://github.com/crawlab-team/crawlab"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.167518+00:00", "kind": "homepage", "missing": false, "url": "https://www.crawlab.cn"}, {"content_hash": "df0bdf97e2651f64e9f91f07ead57d3ecd0a4962986576ff7254c273e0969c46", "fetched_at": "2026-08-29T08:11:41.173834+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io"}, {"content_hash": "68f37fba588845fac2c5fbd65a31591c6935fee822b3fbc3ac73c32374c78c1b", "fetched_at": "2026-08-29T08:11:41.175520+00:00", "kind": "site_page", "missing": false, "url": "https://docs.crawlab.io/docs/whats-new"}, {"content_hash": "c5b65147607f44aa40ee8c2860ceea353dabe9d9c82f79e58d54ed06d1df3896", "fetched_at": "2026-08-29T08:11:41.171707+00:00", "kind": "site_page", "missing": false, "url": "https://www.crawlab.io/"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 66, "longevity": 100, "rhythm": 8}, "computed_at": "2026-09-03T02:20:16.233290+00:00", "flags": [], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 2761, "days_push": 205, "days_rel": null, "gap_med": null, "n_releases_24m": 0}, "score": 52, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}