{"adoption": {"forks": 285, "observed_at": "2026-08-28T04:05:49.621321+00:00", "stars": 1891}, "canonical_url": "https://ross.abutalabs.com/products/petastorm", "card": {"archived": false, "artifact_type": "library", "description": "Petastorm library enables single machine or distributed training and evaluation of deep learning models from datasets in Apache Parquet format. It supports ML frameworks such as Tensorflow, Pytorch, and PySpark and can be used from pure Python code.", "domain": ["machine-learning", "deep-learning", "big-data"], "enriched": true, "function": ["machine-learning", "etl", "data-science", "serialization"], "health_score": 72, "homepage": null, "language": "Python", "license": "Apache-2.0", "license_family": "permissive", "maturity": "maintenance", "member_repos": ["uber/petastorm"], "name": "uber/petastorm", "platform": ["python", "cross-platform"], "pushed_at": "2026-01-02T23:08:02+00:00", "repo": "uber/petastorm", "stars": 1891, "tags": ["parquet", "tensorflow", "pytorch", "pyspark", "pyarrow", "data-loading", "distributed-training", "data-engineering"], "topics": ["tensorflow", "pytorch", "deep-learning", "machine-learning", "sysml", "pyspark", "pyarrow", "parquet", "parquet-files"], "urls": [], "use_cases": ["train deep learning models from parquet datasets", "load parquet data into pytorch or tensorflow", "generate ml datasets with pyspark", "distributed training data loading from spark clusters", "read multidimensional arrays stored in parquet", "stream large datasets to gpu training pipelines"], "what_it_is": "Petastorm is a Python data access library from Uber that enables single-machine or distributed training and evaluation of deep learning models directly from Apache Parquet datasets. It integrates with TensorFlow, PyTorch, and PySpark, and supports multidimensional arrays and extensible codecs on top of Parquet schemas.", "when_to_avoid": ["your data is in CSV, JSON, or other non-Parquet formats", "you need a general-purpose ETL tool rather than ML data loading", "you require a project with frequent updates and active development"], "when_to_choose": ["your training data lives in Apache Parquet and you use TensorFlow, PyTorch, or PySpark", "you need distributed data loading for deep learning from Spark-generated datasets", "you want native support for multidimensional arrays and image codecs in Parquet"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/petastorm", "repo": "uber/petastorm", "role": "main", "score": 58}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:05:49.621321+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-30T03:13:00.882647+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "2652d2a600df21db06c54ad79874bb798e60c02f82ce35fad89d96fcef131a69", "fetched_at": "2026-08-28T04:05:49.621321+00:00", "kind": "readme", "missing": false, "url": "https://github.com/uber/petastorm"}, {"content_hash": "a26b018237576cdedd6aabc465a7063c2acd2e9280298348614b021f6a82d799", "fetched_at": "2026-08-29T10:52:09.656980+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/petastorm/json"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:05:49.621321+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-30T03:13:00.882647+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "2652d2a600df21db06c54ad79874bb798e60c02f82ce35fad89d96fcef131a69", "fetched_at": "2026-08-28T04:05:49.621321+00:00", "kind": "readme", "missing": false, "url": "https://github.com/uber/petastorm"}, {"content_hash": "a26b018237576cdedd6aabc465a7063c2acd2e9280298348614b021f6a82d799", "fetched_at": "2026-08-29T10:52:09.656980+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/petastorm/json"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-30T03:13:00.882647+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "2652d2a600df21db06c54ad79874bb798e60c02f82ce35fad89d96fcef131a69", "fetched_at": "2026-08-28T04:05:49.621321+00:00", "kind": "readme", "missing": false, "url": "https://github.com/uber/petastorm"}, {"content_hash": "a26b018237576cdedd6aabc465a7063c2acd2e9280298348614b021f6a82d799", "fetched_at": "2026-08-29T10:52:09.656980+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/petastorm/json"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:05:49.621321+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:05:49.621321+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:05:49.621321+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-30T03:13:00.882647+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "2652d2a600df21db06c54ad79874bb798e60c02f82ce35fad89d96fcef131a69", "fetched_at": "2026-08-28T04:05:49.621321+00:00", "kind": "readme", "missing": false, "url": "https://github.com/uber/petastorm"}, {"content_hash": "a26b018237576cdedd6aabc465a7063c2acd2e9280298348614b021f6a82d799", "fetched_at": "2026-08-29T10:52:09.656980+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/petastorm/json"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:05:49.621321+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:05:49.621321+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-30T03:13:00.882647+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "2652d2a600df21db06c54ad79874bb798e60c02f82ce35fad89d96fcef131a69", "fetched_at": "2026-08-28T04:05:49.621321+00:00", "kind": "readme", "missing": false, "url": "https://github.com/uber/petastorm"}, {"content_hash": "a26b018237576cdedd6aabc465a7063c2acd2e9280298348614b021f6a82d799", "fetched_at": "2026-08-29T10:52:09.656980+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/petastorm/json"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:05:49.621321+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:05:49.621321+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:05:49.621321+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-30T03:13:00.882647+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "2652d2a600df21db06c54ad79874bb798e60c02f82ce35fad89d96fcef131a69", "fetched_at": "2026-08-28T04:05:49.621321+00:00", "kind": "readme", "missing": false, "url": "https://github.com/uber/petastorm"}, {"content_hash": "a26b018237576cdedd6aabc465a7063c2acd2e9280298348614b021f6a82d799", "fetched_at": "2026-08-29T10:52:09.656980+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/petastorm/json"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:05:49.621321+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:05:49.621321+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-30T03:13:00.882647+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "2652d2a600df21db06c54ad79874bb798e60c02f82ce35fad89d96fcef131a69", "fetched_at": "2026-08-28T04:05:49.621321+00:00", "kind": "readme", "missing": false, "url": "https://github.com/uber/petastorm"}, {"content_hash": "a26b018237576cdedd6aabc465a7063c2acd2e9280298348614b021f6a82d799", "fetched_at": "2026-08-29T10:52:09.656980+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/petastorm/json"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-30T03:13:00.882647+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "2652d2a600df21db06c54ad79874bb798e60c02f82ce35fad89d96fcef131a69", "fetched_at": "2026-08-28T04:05:49.621321+00:00", "kind": "readme", "missing": false, "url": "https://github.com/uber/petastorm"}, {"content_hash": "a26b018237576cdedd6aabc465a7063c2acd2e9280298348614b021f6a82d799", "fetched_at": "2026-08-29T10:52:09.656980+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/petastorm/json"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-30T03:13:00.882647+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "2652d2a600df21db06c54ad79874bb798e60c02f82ce35fad89d96fcef131a69", "fetched_at": "2026-08-28T04:05:49.621321+00:00", "kind": "readme", "missing": false, "url": "https://github.com/uber/petastorm"}, {"content_hash": "a26b018237576cdedd6aabc465a7063c2acd2e9280298348614b021f6a82d799", "fetched_at": "2026-08-29T10:52:09.656980+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/petastorm/json"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-30T03:13:00.882647+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "2652d2a600df21db06c54ad79874bb798e60c02f82ce35fad89d96fcef131a69", "fetched_at": "2026-08-28T04:05:49.621321+00:00", "kind": "readme", "missing": false, "url": "https://github.com/uber/petastorm"}, {"content_hash": "a26b018237576cdedd6aabc465a7063c2acd2e9280298348614b021f6a82d799", "fetched_at": "2026-08-29T10:52:09.656980+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/petastorm/json"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 60, "longevity": 100, "rhythm": 32}, "computed_at": "2026-09-03T02:20:16.233290+00:00", "flags": [], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 3001, "days_push": 243, "days_rel": 243, "gap_med": null, "n_releases_24m": 1}, "score": 58, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}