{"adoption": {"forks": 236, "observed_at": "2026-08-28T04:07:47.428669+00:00", "stars": 3169}, "canonical_url": "https://ross.abutalabs.com/products/webdataset", "card": {"archived": false, "artifact_type": "library", "description": "A high-performance Python-based I/O system for large (and small) deep learning problems, with strong support for PyTorch.", "domain": ["deep-learning", "machine-learning", "files"], "enriched": true, "function": ["etl", "streaming", "file-system", "machine-learning", "data-science"], "health_score": 69, "homepage": null, "language": "Python", "license": "BSD-3-Clause", "license_family": "permissive", "maturity": "stable", "member_repos": ["webdataset/webdataset"], "name": "webdataset/webdataset", "platform": ["python", "cross-platform"], "pushed_at": "2026-02-09T14:48:08+00:00", "repo": "webdataset/webdataset", "stars": 3169, "tags": ["pytorch", "tar-shards", "data-loading", "object-storage", "data-pipeline", "data-engineering", "gpu"], "topics": ["webdataset", "webdataset-format", "data-augmentation", "pytorch", "deep-learning"], "urls": [], "use_cases": ["stream large image datasets from S3 or GCS into PyTorch training", "load sharded training data sequentially for high-throughput I/O", "create tar-based dataset shards for deep learning", "train models on datasets too large to fit in memory", "augment and preprocess samples in a streaming pipeline", "read MsgPack or CBOR concatenated data sources"], "what_it_is": "A Python library providing a high-performance sequential I/O system based on tar-shard files for large-scale deep learning training, with strong PyTorch integration. It reads data from local disk or cloud object stores via pipes, enabling efficient streaming pipelines for images, audio, video, and other training samples.", "when_to_avoid": ["you need random access to individual samples without sharding overhead", "your project is not Python/PyTorch based (though other language ports exist)", "you need a full dataset management platform with versioning and metadata"], "when_to_choose": ["you train PyTorch models on large datasets stored locally or in cloud object stores", "you need sequential I/O performance instead of random-access file reads", "you want a simple, tool-friendly format (plain tar archives) for training data"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/webdataset", "repo": "webdataset/webdataset", "role": "main", "score": 52}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:07:47.428669+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-29T18:45:30.946153+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "b8b195ecbc91dbdb4a6b7a80e62e5a42e74cb75d931073425b6e053f4092eaf9", "fetched_at": "2026-08-28T04:07:47.428669+00:00", "kind": "readme", "missing": false, "url": "https://github.com/webdataset/webdataset"}, {"content_hash": "85462eb8f19837af01ec4052a74b069d1aca6dc759f88fde3f2edfe108bc8efa", "fetched_at": "2026-08-29T09:39:40.672355+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/webdataset/json"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:07:47.428669+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-29T18:45:30.946153+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "b8b195ecbc91dbdb4a6b7a80e62e5a42e74cb75d931073425b6e053f4092eaf9", "fetched_at": "2026-08-28T04:07:47.428669+00:00", "kind": "readme", "missing": false, "url": "https://github.com/webdataset/webdataset"}, {"content_hash": "85462eb8f19837af01ec4052a74b069d1aca6dc759f88fde3f2edfe108bc8efa", "fetched_at": "2026-08-29T09:39:40.672355+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/webdataset/json"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-29T18:45:30.946153+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "b8b195ecbc91dbdb4a6b7a80e62e5a42e74cb75d931073425b6e053f4092eaf9", "fetched_at": "2026-08-28T04:07:47.428669+00:00", "kind": "readme", "missing": false, "url": "https://github.com/webdataset/webdataset"}, {"content_hash": "85462eb8f19837af01ec4052a74b069d1aca6dc759f88fde3f2edfe108bc8efa", "fetched_at": "2026-08-29T09:39:40.672355+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/webdataset/json"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:07:47.428669+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:07:47.428669+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:07:47.428669+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-29T18:45:30.946153+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "b8b195ecbc91dbdb4a6b7a80e62e5a42e74cb75d931073425b6e053f4092eaf9", "fetched_at": "2026-08-28T04:07:47.428669+00:00", "kind": "readme", "missing": false, "url": "https://github.com/webdataset/webdataset"}, {"content_hash": "85462eb8f19837af01ec4052a74b069d1aca6dc759f88fde3f2edfe108bc8efa", "fetched_at": "2026-08-29T09:39:40.672355+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/webdataset/json"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:07:47.428669+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:07:47.428669+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-29T18:45:30.946153+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "b8b195ecbc91dbdb4a6b7a80e62e5a42e74cb75d931073425b6e053f4092eaf9", "fetched_at": "2026-08-28T04:07:47.428669+00:00", "kind": "readme", "missing": false, "url": "https://github.com/webdataset/webdataset"}, {"content_hash": "85462eb8f19837af01ec4052a74b069d1aca6dc759f88fde3f2edfe108bc8efa", "fetched_at": "2026-08-29T09:39:40.672355+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/webdataset/json"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:07:47.428669+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:07:47.428669+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:07:47.428669+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-29T18:45:30.946153+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "b8b195ecbc91dbdb4a6b7a80e62e5a42e74cb75d931073425b6e053f4092eaf9", "fetched_at": "2026-08-28T04:07:47.428669+00:00", "kind": "readme", "missing": false, "url": "https://github.com/webdataset/webdataset"}, {"content_hash": "85462eb8f19837af01ec4052a74b069d1aca6dc759f88fde3f2edfe108bc8efa", "fetched_at": "2026-08-29T09:39:40.672355+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/webdataset/json"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:07:47.428669+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:07:47.428669+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-29T18:45:30.946153+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "b8b195ecbc91dbdb4a6b7a80e62e5a42e74cb75d931073425b6e053f4092eaf9", "fetched_at": "2026-08-28T04:07:47.428669+00:00", "kind": "readme", "missing": false, "url": "https://github.com/webdataset/webdataset"}, {"content_hash": "85462eb8f19837af01ec4052a74b069d1aca6dc759f88fde3f2edfe108bc8efa", "fetched_at": "2026-08-29T09:39:40.672355+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/webdataset/json"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-29T18:45:30.946153+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "b8b195ecbc91dbdb4a6b7a80e62e5a42e74cb75d931073425b6e053f4092eaf9", "fetched_at": "2026-08-28T04:07:47.428669+00:00", "kind": "readme", "missing": false, "url": "https://github.com/webdataset/webdataset"}, {"content_hash": "85462eb8f19837af01ec4052a74b069d1aca6dc759f88fde3f2edfe108bc8efa", "fetched_at": "2026-08-29T09:39:40.672355+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/webdataset/json"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-29T18:45:30.946153+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "b8b195ecbc91dbdb4a6b7a80e62e5a42e74cb75d931073425b6e053f4092eaf9", "fetched_at": "2026-08-28T04:07:47.428669+00:00", "kind": "readme", "missing": false, "url": "https://github.com/webdataset/webdataset"}, {"content_hash": "85462eb8f19837af01ec4052a74b069d1aca6dc759f88fde3f2edfe108bc8efa", "fetched_at": "2026-08-29T09:39:40.672355+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/webdataset/json"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-29T18:45:30.946153+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "b8b195ecbc91dbdb4a6b7a80e62e5a42e74cb75d931073425b6e053f4092eaf9", "fetched_at": "2026-08-28T04:07:47.428669+00:00", "kind": "readme", "missing": false, "url": "https://github.com/webdataset/webdataset"}, {"content_hash": "85462eb8f19837af01ec4052a74b069d1aca6dc759f88fde3f2edfe108bc8efa", "fetched_at": "2026-08-29T09:39:40.672355+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/webdataset/json"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 66, "longevity": 100, "rhythm": 8}, "computed_at": "2026-09-02T17:46:02.011165+00:00", "flags": [], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 2583, "days_push": 205, "days_rel": 705, "gap_med": null, "n_releases_24m": 1}, "score": 52, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}