{"adoption": {"forks": 792, "observed_at": "2026-08-28T04:07:55.816075+00:00", "stars": 3305}, "canonical_url": "https://ross.abutalabs.com/products/heritrix3", "card": {"archived": false, "artifact_type": "application", "description": "Heritrix is the Internet Archive's open-source, extensible, web-scale, archival-quality web crawler project.  ", "domain": ["crawlers", "web-development"], "enriched": true, "function": ["web-scraping", "http-client", "configuration-management"], "health_score": 100, "homepage": "https://heritrix.readthedocs.io/", "language": "Java", "license": "NOASSERTION", "license_family": "other", "maturity": "active", "member_repos": ["internetarchive/heritrix3"], "name": "internetarchive/heritrix3", "platform": ["jvm"], "pushed_at": "2026-08-25T02:06:42+00:00", "repo": "internetarchive/heritrix3", "stars": 3305, "tags": ["web-crawler", "warc", "web-archiving", "digital-preservation", "internet-archive", "robots-txt", "archival-quality", "archives", "search", "linux", "docker", "web-server"], "topics": ["java", "webcrawling", "warc", "heritrix"], "urls": [], "use_cases": ["crawl entire websites for archival purposes", "build web archive collections in WARC format", "mirror large portions of the web at scale", "respectfully crawl sites honoring robots.txt and politeness settings", "run scheduled recurring crawls of web resources", "extend crawler behavior with custom Java modules"], "what_it_is": "Heritrix is the Internet Archive's open-source, extensible, web-scale, archival-quality web crawler written in Java. It crawls websites and stores content in WARC archives while respecting robots.txt and politeness policies.", "when_to_avoid": ["you only need lightweight scraping of a few pages", "you want a simple scriptable crawler without a web UI or job configuration", "you need JavaScript-heavy rendering out of the box without extra setup"], "when_to_choose": ["you need archival-quality, standards-compliant web crawling with WARC output", "you want a mature, battle-tested crawler used by the Internet Archive", "you need fine-grained politeness and robots.txt compliance", "you want an extensible crawler with a REST API and bean-based configuration"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/heritrix3", "repo": "internetarchive/heritrix3", "role": "main", "score": 95}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:07:55.816075+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-29T18:42:39.290122+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d99049684fac404d6b29d6119d25893aaf3dbdf1075f2339e230670de2eca318", "fetched_at": "2026-08-28T04:07:55.816075+00:00", "kind": "readme", "missing": false, "url": "https://github.com/internetarchive/heritrix3"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:07:55.816075+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-29T18:42:39.290122+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d99049684fac404d6b29d6119d25893aaf3dbdf1075f2339e230670de2eca318", "fetched_at": "2026-08-28T04:07:55.816075+00:00", "kind": "readme", "missing": false, "url": "https://github.com/internetarchive/heritrix3"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-29T18:42:39.290122+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d99049684fac404d6b29d6119d25893aaf3dbdf1075f2339e230670de2eca318", "fetched_at": "2026-08-28T04:07:55.816075+00:00", "kind": "readme", "missing": false, "url": "https://github.com/internetarchive/heritrix3"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:07:55.816075+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:07:55.816075+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:07:55.816075+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-29T18:42:39.290122+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d99049684fac404d6b29d6119d25893aaf3dbdf1075f2339e230670de2eca318", "fetched_at": "2026-08-28T04:07:55.816075+00:00", "kind": "readme", "missing": false, "url": "https://github.com/internetarchive/heritrix3"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:07:55.816075+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:07:55.816075+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-29T18:42:39.290122+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d99049684fac404d6b29d6119d25893aaf3dbdf1075f2339e230670de2eca318", "fetched_at": "2026-08-28T04:07:55.816075+00:00", "kind": "readme", "missing": false, "url": "https://github.com/internetarchive/heritrix3"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:07:55.816075+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:07:55.816075+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:07:55.816075+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-29T18:42:39.290122+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d99049684fac404d6b29d6119d25893aaf3dbdf1075f2339e230670de2eca318", "fetched_at": "2026-08-28T04:07:55.816075+00:00", "kind": "readme", "missing": false, "url": "https://github.com/internetarchive/heritrix3"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:07:55.816075+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:07:55.816075+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-29T18:42:39.290122+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d99049684fac404d6b29d6119d25893aaf3dbdf1075f2339e230670de2eca318", "fetched_at": "2026-08-28T04:07:55.816075+00:00", "kind": "readme", "missing": false, "url": "https://github.com/internetarchive/heritrix3"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-29T18:42:39.290122+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d99049684fac404d6b29d6119d25893aaf3dbdf1075f2339e230670de2eca318", "fetched_at": "2026-08-28T04:07:55.816075+00:00", "kind": "readme", "missing": false, "url": "https://github.com/internetarchive/heritrix3"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-29T18:42:39.290122+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d99049684fac404d6b29d6119d25893aaf3dbdf1075f2339e230670de2eca318", "fetched_at": "2026-08-28T04:07:55.816075+00:00", "kind": "readme", "missing": false, "url": "https://github.com/internetarchive/heritrix3"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-29T18:42:39.290122+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d99049684fac404d6b29d6119d25893aaf3dbdf1075f2339e230670de2eca318", "fetched_at": "2026-08-28T04:07:55.816075+00:00", "kind": "readme", "missing": false, "url": "https://github.com/internetarchive/heritrix3"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 99, "longevity": 100, "rhythm": 87}, "computed_at": "2026-09-03T02:20:16.233290+00:00", "flags": ["no_license"], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 5430, "days_push": 9, "days_rel": 9, "gap_med": 41.5, "n_releases_24m": 17}, "score": 95, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}