{"adoption": {"forks": 584, "observed_at": "2026-08-28T04:08:12.883792+00:00", "stars": 3643}, "canonical_url": "https://ross.abutalabs.com/products/deequ", "card": {"archived": false, "artifact_type": "library", "description": "Deequ is a library built on top of Apache Spark for defining \"unit tests for data\", which measure data quality in large datasets.", "domain": ["data-science", "big-data", "analytics"], "enriched": true, "function": ["testing", "data-science", "etl", "monitoring"], "health_score": 95, "homepage": null, "language": "Scala", "license": "Apache-2.0", "license_family": "permissive", "maturity": "active", "member_repos": ["awslabs/deequ"], "name": "awslabs/deequ", "platform": ["jvm", "python"], "pushed_at": "2026-07-21T17:55:47+00:00", "repo": "awslabs/deequ", "stars": 3643, "tags": ["data-quality", "apache-spark", "unit-tests-for-data", "data-validation", "pydeequ", "data-engineering", "spark"], "topics": ["dataquality", "spark", "unit-testing", "scala"], "urls": [], "use_cases": ["validate data quality in large spark dataframes", "define unit tests for datasets", "detect data errors before loading into ml pipelines", "compute data quality metrics on big data", "check constraints like completeness and uniqueness on tables", "profile large datasets for quality issues"], "what_it_is": "Deequ is a Scala library built on Apache Spark for defining 'unit tests for data' that measure data quality in large datasets. It computes data quality metrics and verifies constraints on tabular data before it feeds downstream systems or machine learning pipelines.", "when_to_avoid": ["your data is small and fits in pandas or a single machine", "you don't use Spark and can't justify its overhead", "you need streaming or non-tabular (e.g., unstructured text) data validation"], "when_to_choose": ["you already run Apache Spark and need scalable data quality checks", "you want declarative constraint verification on large tabular datasets", "you need to catch data errors before feeding consuming systems or ML models"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/deequ", "repo": "awslabs/deequ", "role": "main", "score": 93}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:08:12.883792+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-29T18:31:46.404538+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "ceddc246f62919c3a39457aa3cbed7b4a12c7ba72e70065fcf8583922b07da1b", "fetched_at": "2026-08-28T04:08:12.883792+00:00", "kind": "readme", "missing": false, "url": "https://github.com/awslabs/deequ"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:08:12.883792+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-29T18:31:46.404538+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "ceddc246f62919c3a39457aa3cbed7b4a12c7ba72e70065fcf8583922b07da1b", "fetched_at": "2026-08-28T04:08:12.883792+00:00", "kind": "readme", "missing": false, "url": "https://github.com/awslabs/deequ"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-29T18:31:46.404538+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "ceddc246f62919c3a39457aa3cbed7b4a12c7ba72e70065fcf8583922b07da1b", "fetched_at": "2026-08-28T04:08:12.883792+00:00", "kind": "readme", "missing": false, "url": "https://github.com/awslabs/deequ"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:08:12.883792+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:08:12.883792+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:08:12.883792+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-29T18:31:46.404538+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "ceddc246f62919c3a39457aa3cbed7b4a12c7ba72e70065fcf8583922b07da1b", "fetched_at": "2026-08-28T04:08:12.883792+00:00", "kind": "readme", "missing": false, "url": "https://github.com/awslabs/deequ"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:08:12.883792+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:08:12.883792+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-29T18:31:46.404538+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "ceddc246f62919c3a39457aa3cbed7b4a12c7ba72e70065fcf8583922b07da1b", "fetched_at": "2026-08-28T04:08:12.883792+00:00", "kind": "readme", "missing": false, "url": "https://github.com/awslabs/deequ"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:08:12.883792+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:08:12.883792+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:08:12.883792+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-29T18:31:46.404538+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "ceddc246f62919c3a39457aa3cbed7b4a12c7ba72e70065fcf8583922b07da1b", "fetched_at": "2026-08-28T04:08:12.883792+00:00", "kind": "readme", "missing": false, "url": "https://github.com/awslabs/deequ"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:08:12.883792+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:08:12.883792+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-29T18:31:46.404538+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "ceddc246f62919c3a39457aa3cbed7b4a12c7ba72e70065fcf8583922b07da1b", "fetched_at": "2026-08-28T04:08:12.883792+00:00", "kind": "readme", "missing": false, "url": "https://github.com/awslabs/deequ"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-29T18:31:46.404538+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "ceddc246f62919c3a39457aa3cbed7b4a12c7ba72e70065fcf8583922b07da1b", "fetched_at": "2026-08-28T04:08:12.883792+00:00", "kind": "readme", "missing": false, "url": "https://github.com/awslabs/deequ"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-29T18:31:46.404538+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "ceddc246f62919c3a39457aa3cbed7b4a12c7ba72e70065fcf8583922b07da1b", "fetched_at": "2026-08-28T04:08:12.883792+00:00", "kind": "readme", "missing": false, "url": "https://github.com/awslabs/deequ"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-29T18:31:46.404538+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "ceddc246f62919c3a39457aa3cbed7b4a12c7ba72e70065fcf8583922b07da1b", "fetched_at": "2026-08-28T04:08:12.883792+00:00", "kind": "readme", "missing": false, "url": "https://github.com/awslabs/deequ"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 93, "longevity": 100, "rhythm": 89}, "computed_at": "2026-09-03T02:20:16.233290+00:00", "flags": [], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 2948, "days_push": 43, "days_rel": 76, "gap_med": 5.0, "n_releases_24m": 13}, "score": 93, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}