{"adoption": {"forks": 150, "observed_at": "2026-08-28T04:05:20.322563+00:00", "stars": 1673}, "canonical_url": "https://ross.abutalabs.com/products/the-pile", "card": {"archived": false, "artifact_type": "dataset", "description": null, "domain": ["large-language-models", "machine-learning"], "enriched": true, "function": ["etl", "data-generation", "machine-learning", "llm-training"], "health_score": 20, "homepage": null, "language": "Python", "license": "MIT", "license_family": "permissive", "maturity": "maintenance", "member_repos": ["EleutherAI/the-pile"], "name": "EleutherAI/the-pile", "platform": ["python"], "pushed_at": "2023-04-27T21:00:48+00:00", "repo": "EleutherAI/the-pile", "stars": 1673, "tags": ["language-modeling", "text-corpus", "dataset-replication", "eleutherai", "open-source-data", "natural-language-processing", "data-engineering", "linux", "macos"], "topics": [], "urls": [], "use_cases": ["build a large text corpus for training language models", "replicate the Pile dataset from its component sources", "create a custom variant of the Pile with different data mixtures", "download diverse text data for pretraining research", "study the composition and weighting of an LLM pretraining dataset"], "what_it_is": "The Pile is a large, diverse, open-source language modeling dataset composed of many smaller text sources combined together. This repository contains the code for replicating the Pile or building variants of it, rather than the dataset itself.", "when_to_avoid": ["you just want to download and use the Pile dataset itself (get it from the official download link instead)", "you need a maintained, actively updated corpus pipeline", "you need a small curated dataset rather than hundreds of gigabytes of text"], "when_to_choose": ["you need to reproduce or modify the Pile's data pipeline", "you want to understand how a large pretraining corpus is assembled from many sources", "you are doing research on dataset composition for language modeling"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/the-pile", "repo": "EleutherAI/the-pile", "role": "main", "score": 32}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:05:20.322563+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-30T03:42:34.343493+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c278c93424d7d978edfec98f526dee7f8ff5714b17354833169d58370558367e", "fetched_at": "2026-08-28T04:05:20.322563+00:00", "kind": "readme", "missing": false, "url": "https://github.com/EleutherAI/the-pile"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:05:20.322563+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-30T03:42:34.343493+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c278c93424d7d978edfec98f526dee7f8ff5714b17354833169d58370558367e", "fetched_at": "2026-08-28T04:05:20.322563+00:00", "kind": "readme", "missing": false, "url": "https://github.com/EleutherAI/the-pile"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-30T03:42:34.343493+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c278c93424d7d978edfec98f526dee7f8ff5714b17354833169d58370558367e", "fetched_at": "2026-08-28T04:05:20.322563+00:00", "kind": "readme", "missing": false, "url": "https://github.com/EleutherAI/the-pile"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:05:20.322563+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:05:20.322563+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:05:20.322563+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-30T03:42:34.343493+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c278c93424d7d978edfec98f526dee7f8ff5714b17354833169d58370558367e", "fetched_at": "2026-08-28T04:05:20.322563+00:00", "kind": "readme", "missing": false, "url": "https://github.com/EleutherAI/the-pile"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:05:20.322563+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:05:20.322563+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-30T03:42:34.343493+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c278c93424d7d978edfec98f526dee7f8ff5714b17354833169d58370558367e", "fetched_at": "2026-08-28T04:05:20.322563+00:00", "kind": "readme", "missing": false, "url": "https://github.com/EleutherAI/the-pile"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:05:20.322563+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:05:20.322563+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:05:20.322563+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-30T03:42:34.343493+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c278c93424d7d978edfec98f526dee7f8ff5714b17354833169d58370558367e", "fetched_at": "2026-08-28T04:05:20.322563+00:00", "kind": "readme", "missing": false, "url": "https://github.com/EleutherAI/the-pile"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:05:20.322563+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:05:20.322563+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-30T03:42:34.343493+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c278c93424d7d978edfec98f526dee7f8ff5714b17354833169d58370558367e", "fetched_at": "2026-08-28T04:05:20.322563+00:00", "kind": "readme", "missing": false, "url": "https://github.com/EleutherAI/the-pile"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-30T03:42:34.343493+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c278c93424d7d978edfec98f526dee7f8ff5714b17354833169d58370558367e", "fetched_at": "2026-08-28T04:05:20.322563+00:00", "kind": "readme", "missing": false, "url": "https://github.com/EleutherAI/the-pile"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-30T03:42:34.343493+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c278c93424d7d978edfec98f526dee7f8ff5714b17354833169d58370558367e", "fetched_at": "2026-08-28T04:05:20.322563+00:00", "kind": "readme", "missing": false, "url": "https://github.com/EleutherAI/the-pile"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-30T03:42:34.343493+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c278c93424d7d978edfec98f526dee7f8ff5714b17354833169d58370558367e", "fetched_at": "2026-08-28T04:05:20.322563+00:00", "kind": "readme", "missing": false, "url": "https://github.com/EleutherAI/the-pile"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 0, "longevity": 100, "rhythm": 35}, "computed_at": "2026-09-03T02:20:16.233290+00:00", "flags": ["no_releases"], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 2198, "days_push": 1224, "days_rel": null, "gap_med": null, "n_releases_24m": 0}, "score": 32, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}