{"adoption": {"forks": 123, "observed_at": "2026-08-28T04:03:38.147888+00:00", "stars": 1114}, "canonical_url": "https://ross.abutalabs.com/products/crawly", "card": {"archived": false, "artifact_type": "framework", "description": "Crawly, a high-level web crawling & scraping framework for Elixir. ", "domain": ["crawlers", "web-development", "data-science"], "enriched": true, "function": ["web-scraping", "http-client", "parser", "etl"], "health_score": 39, "homepage": "https://hexdocs.pm/crawly", "language": "Elixir", "license": "Apache-2.0", "license_family": "permissive", "maturity": "active", "member_repos": ["elixir-crawly/crawly"], "name": "elixir-crawly/crawly", "platform": ["elixir", "windows", "bsd", "cross-platform"], "pushed_at": "2025-07-16T13:44:19+00:00", "repo": "elixir-crawly/crawly", "stars": 1114, "tags": ["crawler", "spider", "data-extraction", "html-parsing", "scrapy-inspired", "erlang", "beam-concurrency", "data-engineering", "linux", "macos"], "topics": ["elixir", "erlang", "scraper", "scraping", "scraping-websites", "extract-data", "spider", "crawler", "crawling"], "urls": [], "use_cases": ["scrape product prices and titles from e-commerce sites", "crawl a website and extract structured data into a database", "build a scalable web crawler in Elixir", "follow pagination links automatically while collecting items", "collect news articles or listings for data mining", "archive website content periodically", "extract data from HTML pages for downstream ETL pipelines"], "what_it_is": "Crawly is a high-level web crawling and scraping framework for Elixir, modeled after Scrapy, where developers define spiders that fetch pages and extract structured data. It ships with built-in support for concurrent requests, retries, request throttling, automatic link following, and pluggable item-processing pipelines.", "when_to_avoid": ["You only need a one-off simple HTTP fetch, where a plain HTTP client like Req or HTTPoison is simpler", "You need to scrape heavily JavaScript-rendered pages without setting up browser rendering yourself", "Your project is not on the Elixir/Erlang platform", "You need a turnkey scraping service rather than a framework you write spider code in"], "when_to_choose": ["You are building a medium-to-large scale crawler in Elixir and want BEAM concurrency for parallel fetches", "You want a Scrapy-like workflow with spiders, item pipelines, and configurable middleware", "You need out-of-the-box features like retries, rate limiting, robots.txt handling, and automatic URL deduplication", "You want structured data extraction integrated with parsers like Floki"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/crawly", "repo": "elixir-crawly/crawly", "role": "main", "score": 37}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:03:38.147888+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-30T06:42:30.576814+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4add3e6494261c9d878a4e569a36ec70b705effabd95abaa660cee419ec6f6a0", "fetched_at": "2026-08-28T04:03:38.147888+00:00", "kind": "readme", "missing": false, "url": "https://github.com/elixir-crawly/crawly"}, {"content_hash": "84fd8c9cc750805341624223b1687abb3c8f93a04b3b17292597cfc85cecb8a2", "fetched_at": "2026-08-29T12:46:06.236974+00:00", "kind": "homepage", "missing": false, "url": "https://hexdocs.pm/crawly"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:03:38.147888+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-30T06:42:30.576814+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4add3e6494261c9d878a4e569a36ec70b705effabd95abaa660cee419ec6f6a0", "fetched_at": "2026-08-28T04:03:38.147888+00:00", "kind": "readme", "missing": false, "url": "https://github.com/elixir-crawly/crawly"}, {"content_hash": "84fd8c9cc750805341624223b1687abb3c8f93a04b3b17292597cfc85cecb8a2", "fetched_at": "2026-08-29T12:46:06.236974+00:00", "kind": "homepage", "missing": false, "url": "https://hexdocs.pm/crawly"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-30T06:42:30.576814+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4add3e6494261c9d878a4e569a36ec70b705effabd95abaa660cee419ec6f6a0", "fetched_at": "2026-08-28T04:03:38.147888+00:00", "kind": "readme", "missing": false, "url": "https://github.com/elixir-crawly/crawly"}, {"content_hash": "84fd8c9cc750805341624223b1687abb3c8f93a04b3b17292597cfc85cecb8a2", "fetched_at": "2026-08-29T12:46:06.236974+00:00", "kind": "homepage", "missing": false, "url": "https://hexdocs.pm/crawly"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:03:38.147888+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:03:38.147888+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:03:38.147888+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-30T06:42:30.576814+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4add3e6494261c9d878a4e569a36ec70b705effabd95abaa660cee419ec6f6a0", "fetched_at": "2026-08-28T04:03:38.147888+00:00", "kind": "readme", "missing": false, "url": "https://github.com/elixir-crawly/crawly"}, {"content_hash": "84fd8c9cc750805341624223b1687abb3c8f93a04b3b17292597cfc85cecb8a2", "fetched_at": "2026-08-29T12:46:06.236974+00:00", "kind": "homepage", "missing": false, "url": "https://hexdocs.pm/crawly"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:03:38.147888+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:03:38.147888+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-30T06:42:30.576814+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4add3e6494261c9d878a4e569a36ec70b705effabd95abaa660cee419ec6f6a0", "fetched_at": "2026-08-28T04:03:38.147888+00:00", "kind": "readme", "missing": false, "url": "https://github.com/elixir-crawly/crawly"}, {"content_hash": "84fd8c9cc750805341624223b1687abb3c8f93a04b3b17292597cfc85cecb8a2", "fetched_at": "2026-08-29T12:46:06.236974+00:00", "kind": "homepage", "missing": false, "url": "https://hexdocs.pm/crawly"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:03:38.147888+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:03:38.147888+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:03:38.147888+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-30T06:42:30.576814+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4add3e6494261c9d878a4e569a36ec70b705effabd95abaa660cee419ec6f6a0", "fetched_at": "2026-08-28T04:03:38.147888+00:00", "kind": "readme", "missing": false, "url": "https://github.com/elixir-crawly/crawly"}, {"content_hash": "84fd8c9cc750805341624223b1687abb3c8f93a04b3b17292597cfc85cecb8a2", "fetched_at": "2026-08-29T12:46:06.236974+00:00", "kind": "homepage", "missing": false, "url": "https://hexdocs.pm/crawly"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:03:38.147888+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:03:38.147888+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-30T06:42:30.576814+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4add3e6494261c9d878a4e569a36ec70b705effabd95abaa660cee419ec6f6a0", "fetched_at": "2026-08-28T04:03:38.147888+00:00", "kind": "readme", "missing": false, "url": "https://github.com/elixir-crawly/crawly"}, {"content_hash": "84fd8c9cc750805341624223b1687abb3c8f93a04b3b17292597cfc85cecb8a2", "fetched_at": "2026-08-29T12:46:06.236974+00:00", "kind": "homepage", "missing": false, "url": "https://hexdocs.pm/crawly"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-30T06:42:30.576814+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4add3e6494261c9d878a4e569a36ec70b705effabd95abaa660cee419ec6f6a0", "fetched_at": "2026-08-28T04:03:38.147888+00:00", "kind": "readme", "missing": false, "url": "https://github.com/elixir-crawly/crawly"}, {"content_hash": "84fd8c9cc750805341624223b1687abb3c8f93a04b3b17292597cfc85cecb8a2", "fetched_at": "2026-08-29T12:46:06.236974+00:00", "kind": "homepage", "missing": false, "url": "https://hexdocs.pm/crawly"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-30T06:42:30.576814+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4add3e6494261c9d878a4e569a36ec70b705effabd95abaa660cee419ec6f6a0", "fetched_at": "2026-08-28T04:03:38.147888+00:00", "kind": "readme", "missing": false, "url": "https://github.com/elixir-crawly/crawly"}, {"content_hash": "84fd8c9cc750805341624223b1687abb3c8f93a04b3b17292597cfc85cecb8a2", "fetched_at": "2026-08-29T12:46:06.236974+00:00", "kind": "homepage", "missing": false, "url": "https://hexdocs.pm/crawly"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-30T06:42:30.576814+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "4add3e6494261c9d878a4e569a36ec70b705effabd95abaa660cee419ec6f6a0", "fetched_at": "2026-08-28T04:03:38.147888+00:00", "kind": "readme", "missing": false, "url": "https://github.com/elixir-crawly/crawly"}, {"content_hash": "84fd8c9cc750805341624223b1687abb3c8f93a04b3b17292597cfc85cecb8a2", "fetched_at": "2026-08-29T12:46:06.236974+00:00", "kind": "homepage", "missing": false, "url": "https://hexdocs.pm/crawly"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 32, "longevity": 100, "rhythm": 8}, "computed_at": "2026-09-02T17:46:02.011165+00:00", "flags": [], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 2734, "days_push": 413, "days_rel": null, "gap_med": null, "n_releases_24m": 0}, "score": 37, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}