{"adoption": {"forks": 319, "observed_at": "2026-08-28T04:05:29.778187+00:00", "stars": 1736}, "canonical_url": "https://ross.abutalabs.com/products/nvidia-nemo-curator", "card": {"archived": false, "artifact_type": "library", "description": "Scalable data pre processing and curation toolkit for LLMs", "domain": ["machine-learning", "large-language-models", "developer-tools"], "enriched": true, "function": ["etl", "data-science", "machine-learning", "llm-training", "nlp", "image-processing", "audio-processing", "video-processing", "streaming"], "health_score": 100, "homepage": null, "language": "Python", "license": "Apache-2.0", "license_family": "permissive", "maturity": "active", "member_repos": ["NVIDIA-NeMo/Curator"], "name": "NVIDIA-NeMo/Curator", "platform": ["python", "cloud"], "pushed_at": "2026-08-26T19:15:40+00:00", "repo": "NVIDIA-NeMo/Curator", "stars": 1736, "tags": ["data-curation", "deduplication", "gpu-accelerated", "ray", "synthetic-data-generation", "fine-tuning", "data-quality", "multi-modal", "data-engineering", "natural-language-processing", "linux", "docker", "gpu"], "topics": ["data-curation", "llm", "data", "data-prep", "data-preparation", "data-processing", "data-processing-pipelines", "data-quality", "datacuration", "datarecipes", "deduplication", "fast-data-processing", "fine-tuning", "large-language-models", "large-scale-data-processing", "llmapps", "python", "llm-data-quality", "semantic-deduplication"], "urls": [], "use_cases": ["deduplicate large web-scale text corpora for LLM training", "filter and quality-score training data for fine-tuning", "build speech and audio datasets with speaker diarization", "curate image datasets with aesthetic and NSFW filtering", "run data preparation pipelines on multi-node Ray or Slurm clusters", "generate synthetic training data with an in-pipeline LLM inference server", "detect languages and classify documents at scale"], "what_it_is": "NVIDIA NeMo Curator is a scalable, GPU-accelerated toolkit for preprocessing and curating large text, image, video, and audio datasets for AI model training. It provides repeatable pipelines for loading, filtering, deduplicating, and transforming data that can run on a laptop or across multi-node Ray clusters.", "when_to_avoid": ["you only need small-scale data cleaning on a CPU laptop", "you want a no-code GUI tool rather than a Python library", "your pipeline is simple enough for pandas or plain scripts"], "when_to_choose": ["you need GPU-accelerated processing of very large multi-modal datasets", "you want repeatable, production-grade data curation pipelines for LLM training", "you need semantic or exact deduplication at scale", "you run on HPC clusters (Slurm) or multi-node Ray setups"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/nvidia-nemo-curator", "repo": "NVIDIA-NeMo/Curator", "role": "main", "score": 86}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:05:29.778187+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-30T03:30:39.997942+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c65fba70b6ae28a25e03e0bc2f8e92db77ee815704ff36d34983c297b5aa8ca8", "fetched_at": "2026-08-28T04:05:29.778187+00:00", "kind": "readme", "missing": false, "url": "https://github.com/NVIDIA-NeMo/Curator"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:05:29.778187+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-30T03:30:39.997942+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c65fba70b6ae28a25e03e0bc2f8e92db77ee815704ff36d34983c297b5aa8ca8", "fetched_at": "2026-08-28T04:05:29.778187+00:00", "kind": "readme", "missing": false, "url": "https://github.com/NVIDIA-NeMo/Curator"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-30T03:30:39.997942+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c65fba70b6ae28a25e03e0bc2f8e92db77ee815704ff36d34983c297b5aa8ca8", "fetched_at": "2026-08-28T04:05:29.778187+00:00", "kind": "readme", "missing": false, "url": "https://github.com/NVIDIA-NeMo/Curator"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:05:29.778187+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:05:29.778187+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:05:29.778187+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-30T03:30:39.997942+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c65fba70b6ae28a25e03e0bc2f8e92db77ee815704ff36d34983c297b5aa8ca8", "fetched_at": "2026-08-28T04:05:29.778187+00:00", "kind": "readme", "missing": false, "url": "https://github.com/NVIDIA-NeMo/Curator"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:05:29.778187+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:05:29.778187+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-30T03:30:39.997942+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c65fba70b6ae28a25e03e0bc2f8e92db77ee815704ff36d34983c297b5aa8ca8", "fetched_at": "2026-08-28T04:05:29.778187+00:00", "kind": "readme", "missing": false, "url": "https://github.com/NVIDIA-NeMo/Curator"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:05:29.778187+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:05:29.778187+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:05:29.778187+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-30T03:30:39.997942+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c65fba70b6ae28a25e03e0bc2f8e92db77ee815704ff36d34983c297b5aa8ca8", "fetched_at": "2026-08-28T04:05:29.778187+00:00", "kind": "readme", "missing": false, "url": "https://github.com/NVIDIA-NeMo/Curator"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:05:29.778187+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:05:29.778187+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-30T03:30:39.997942+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c65fba70b6ae28a25e03e0bc2f8e92db77ee815704ff36d34983c297b5aa8ca8", "fetched_at": "2026-08-28T04:05:29.778187+00:00", "kind": "readme", "missing": false, "url": "https://github.com/NVIDIA-NeMo/Curator"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-30T03:30:39.997942+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c65fba70b6ae28a25e03e0bc2f8e92db77ee815704ff36d34983c297b5aa8ca8", "fetched_at": "2026-08-28T04:05:29.778187+00:00", "kind": "readme", "missing": false, "url": "https://github.com/NVIDIA-NeMo/Curator"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-30T03:30:39.997942+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c65fba70b6ae28a25e03e0bc2f8e92db77ee815704ff36d34983c297b5aa8ca8", "fetched_at": "2026-08-28T04:05:29.778187+00:00", "kind": "readme", "missing": false, "url": "https://github.com/NVIDIA-NeMo/Curator"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-30T03:30:39.997942+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "c65fba70b6ae28a25e03e0bc2f8e92db77ee815704ff36d34983c297b5aa8ca8", "fetched_at": "2026-08-28T04:05:29.778187+00:00", "kind": "readme", "missing": false, "url": "https://github.com/NVIDIA-NeMo/Curator"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 99, "longevity": 64, "rhythm": 83}, "computed_at": "2026-09-03T02:20:16.233290+00:00", "flags": [], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 902, "days_push": 7, "days_rel": 37, "gap_med": 64, "n_releases_24m": 12}, "score": 86, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}