{"adoption": {"forks": 364, "observed_at": "2026-09-03T02:15:09.172587+00:00", "stars": 1262}, "canonical_url": "https://ross.abutalabs.com/products/datafusion-comet", "card": {"archived": false, "artifact_type": "library", "description": "Apache DataFusion Comet Spark Accelerator", "domain": ["big-data", "analytics", "databases"], "enriched": true, "function": ["data-science", "etl", "streaming"], "health_score": 100, "homepage": "https://datafusion.apache.org/comet", "language": "Scala", "license": "Apache-2.0", "license_family": "permissive", "maturity": "active", "member_repos": ["apache/datafusion-comet"], "name": "apache/datafusion-comet", "platform": ["jvm", "rust"], "pushed_at": "2026-09-02T20:46:17+00:00", "repo": "apache/datafusion-comet", "stars": 1262, "tags": ["apache-spark", "apache-arrow", "datafusion", "query-acceleration", "columnar", "parquet", "iceberg", "native-execution", "query-engine", "data-engineering", "linux", "spark"], "topics": ["arrow", "datafusion", "rust", "spark"], "urls": [], "use_cases": ["speed up existing Spark SQL jobs without code changes", "reduce cloud costs for large Spark workloads", "accelerate Parquet scans for Apache Iceberg tables", "run TPC-DS style analytics faster on commodity hardware", "replace row-based Spark execution with Arrow columnar processing", "offload Spark operators and shuffles to a native Rust engine"], "what_it_is": "Apache DataFusion Comet is a high-performance accelerator plugin for Apache Spark that keeps queries Arrow-native end-to-end, executing operators and expressions as Rust code via the Apache DataFusion engine. It requires no code changes to existing Spark SQL, DataFrame, or PySpark workloads and delivers roughly 2x speedup on TPC-DS benchmarks.", "when_to_avoid": ["you run Spark on macOS in production (published jars bundle Linux native libraries only)", "you rely on proprietary cloud-provider Spark forks", "you need Spark 3.4 or JDK 11 (deprecated)", "your queries use features Comet does not support and frequent fallback negates gains"], "when_to_choose": ["you have existing Spark workloads and want faster queries with no code changes", "you want to cut compute costs by running the same jobs on fewer resources", "your data is in Parquet or Iceberg format on Linux clusters", "you prefer an Apache Software Foundation project with active releases"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/datafusion-comet", "repo": "apache/datafusion-comet", "role": "main", "score": 81}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-09-03T02:15:09.172587+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-30T05:05:34.231132+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "41154ac1324f8b4bb99242149354f1ff2ae69b2740734a8f9438abc75e6529cf", "fetched_at": "2026-09-03T02:15:09.172587+00:00", "kind": "readme", "missing": false, "url": "https://github.com/apache/datafusion-comet"}, {"content_hash": "42f8c9b44a17e71e300f1b0c8da511bf4ddf26f6f6c562e1b60c1fda796f0df9", "fetched_at": "2026-08-29T12:17:10.356415+00:00", "kind": "homepage", "missing": false, "url": "https://datafusion.apache.org/comet"}, {"content_hash": "f79bba2bc6c3f3caf6c5bf9349646d4d73acb2c0fb52cac8f3f37bb10fa5514e", "fetched_at": "2026-08-29T12:17:10.361049+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/about/index.html"}, {"content_hash": "8dadd8ba0a3308d521ccdb5e32de0479a466033fd2dbbe85eb0b3193d3983f3d", "fetched_at": "2026-08-29T12:17:10.362692+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/user-guide/latest/installation.html"}, {"content_hash": "cd9e11aa04e6e4e1e3aaff966399b9a7edeb6cd8c1ba207268e3653cc9ebecb5", "fetched_at": "2026-08-29T12:17:10.359219+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/changelog/index.html"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-09-03T02:15:09.172587+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-30T05:05:34.231132+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "41154ac1324f8b4bb99242149354f1ff2ae69b2740734a8f9438abc75e6529cf", "fetched_at": "2026-09-03T02:15:09.172587+00:00", "kind": "readme", "missing": false, "url": "https://github.com/apache/datafusion-comet"}, {"content_hash": "42f8c9b44a17e71e300f1b0c8da511bf4ddf26f6f6c562e1b60c1fda796f0df9", "fetched_at": "2026-08-29T12:17:10.356415+00:00", "kind": "homepage", "missing": false, "url": "https://datafusion.apache.org/comet"}, {"content_hash": "f79bba2bc6c3f3caf6c5bf9349646d4d73acb2c0fb52cac8f3f37bb10fa5514e", "fetched_at": "2026-08-29T12:17:10.361049+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/about/index.html"}, {"content_hash": "8dadd8ba0a3308d521ccdb5e32de0479a466033fd2dbbe85eb0b3193d3983f3d", "fetched_at": "2026-08-29T12:17:10.362692+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/user-guide/latest/installation.html"}, {"content_hash": "cd9e11aa04e6e4e1e3aaff966399b9a7edeb6cd8c1ba207268e3653cc9ebecb5", "fetched_at": "2026-08-29T12:17:10.359219+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/changelog/index.html"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-30T05:05:34.231132+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "41154ac1324f8b4bb99242149354f1ff2ae69b2740734a8f9438abc75e6529cf", "fetched_at": "2026-09-03T02:15:09.172587+00:00", "kind": "readme", "missing": false, "url": "https://github.com/apache/datafusion-comet"}, {"content_hash": "42f8c9b44a17e71e300f1b0c8da511bf4ddf26f6f6c562e1b60c1fda796f0df9", "fetched_at": "2026-08-29T12:17:10.356415+00:00", "kind": "homepage", "missing": false, "url": "https://datafusion.apache.org/comet"}, {"content_hash": "f79bba2bc6c3f3caf6c5bf9349646d4d73acb2c0fb52cac8f3f37bb10fa5514e", "fetched_at": "2026-08-29T12:17:10.361049+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/about/index.html"}, {"content_hash": "8dadd8ba0a3308d521ccdb5e32de0479a466033fd2dbbe85eb0b3193d3983f3d", "fetched_at": "2026-08-29T12:17:10.362692+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/user-guide/latest/installation.html"}, {"content_hash": "cd9e11aa04e6e4e1e3aaff966399b9a7edeb6cd8c1ba207268e3653cc9ebecb5", "fetched_at": "2026-08-29T12:17:10.359219+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/changelog/index.html"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-09-03T02:15:09.172587+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-09-03T02:15:09.172587+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-09-03T02:15:09.172587+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-30T05:05:34.231132+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "41154ac1324f8b4bb99242149354f1ff2ae69b2740734a8f9438abc75e6529cf", "fetched_at": "2026-09-03T02:15:09.172587+00:00", "kind": "readme", "missing": false, "url": "https://github.com/apache/datafusion-comet"}, {"content_hash": "42f8c9b44a17e71e300f1b0c8da511bf4ddf26f6f6c562e1b60c1fda796f0df9", "fetched_at": "2026-08-29T12:17:10.356415+00:00", "kind": "homepage", "missing": false, "url": "https://datafusion.apache.org/comet"}, {"content_hash": "f79bba2bc6c3f3caf6c5bf9349646d4d73acb2c0fb52cac8f3f37bb10fa5514e", "fetched_at": "2026-08-29T12:17:10.361049+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/about/index.html"}, {"content_hash": "8dadd8ba0a3308d521ccdb5e32de0479a466033fd2dbbe85eb0b3193d3983f3d", "fetched_at": "2026-08-29T12:17:10.362692+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/user-guide/latest/installation.html"}, {"content_hash": "cd9e11aa04e6e4e1e3aaff966399b9a7edeb6cd8c1ba207268e3653cc9ebecb5", "fetched_at": "2026-08-29T12:17:10.359219+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/changelog/index.html"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-09-03T02:15:09.172587+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-09-03T02:15:09.172587+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-30T05:05:34.231132+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "41154ac1324f8b4bb99242149354f1ff2ae69b2740734a8f9438abc75e6529cf", "fetched_at": "2026-09-03T02:15:09.172587+00:00", "kind": "readme", "missing": false, "url": "https://github.com/apache/datafusion-comet"}, {"content_hash": "42f8c9b44a17e71e300f1b0c8da511bf4ddf26f6f6c562e1b60c1fda796f0df9", "fetched_at": "2026-08-29T12:17:10.356415+00:00", "kind": "homepage", "missing": false, "url": "https://datafusion.apache.org/comet"}, {"content_hash": "f79bba2bc6c3f3caf6c5bf9349646d4d73acb2c0fb52cac8f3f37bb10fa5514e", "fetched_at": "2026-08-29T12:17:10.361049+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/about/index.html"}, {"content_hash": "8dadd8ba0a3308d521ccdb5e32de0479a466033fd2dbbe85eb0b3193d3983f3d", "fetched_at": "2026-08-29T12:17:10.362692+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/user-guide/latest/installation.html"}, {"content_hash": "cd9e11aa04e6e4e1e3aaff966399b9a7edeb6cd8c1ba207268e3653cc9ebecb5", "fetched_at": "2026-08-29T12:17:10.359219+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/changelog/index.html"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-09-03T02:15:09.172587+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-09-03T02:15:09.172587+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-09-03T02:15:09.172587+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-30T05:05:34.231132+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "41154ac1324f8b4bb99242149354f1ff2ae69b2740734a8f9438abc75e6529cf", "fetched_at": "2026-09-03T02:15:09.172587+00:00", "kind": "readme", "missing": false, "url": "https://github.com/apache/datafusion-comet"}, {"content_hash": "42f8c9b44a17e71e300f1b0c8da511bf4ddf26f6f6c562e1b60c1fda796f0df9", "fetched_at": "2026-08-29T12:17:10.356415+00:00", "kind": "homepage", "missing": false, "url": "https://datafusion.apache.org/comet"}, {"content_hash": "f79bba2bc6c3f3caf6c5bf9349646d4d73acb2c0fb52cac8f3f37bb10fa5514e", "fetched_at": "2026-08-29T12:17:10.361049+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/about/index.html"}, {"content_hash": "8dadd8ba0a3308d521ccdb5e32de0479a466033fd2dbbe85eb0b3193d3983f3d", "fetched_at": "2026-08-29T12:17:10.362692+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/user-guide/latest/installation.html"}, {"content_hash": "cd9e11aa04e6e4e1e3aaff966399b9a7edeb6cd8c1ba207268e3653cc9ebecb5", "fetched_at": "2026-08-29T12:17:10.359219+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/changelog/index.html"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-09-03T02:15:09.172587+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-09-03T02:15:09.172587+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-30T05:05:34.231132+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "41154ac1324f8b4bb99242149354f1ff2ae69b2740734a8f9438abc75e6529cf", "fetched_at": "2026-09-03T02:15:09.172587+00:00", "kind": "readme", "missing": false, "url": "https://github.com/apache/datafusion-comet"}, {"content_hash": "42f8c9b44a17e71e300f1b0c8da511bf4ddf26f6f6c562e1b60c1fda796f0df9", "fetched_at": "2026-08-29T12:17:10.356415+00:00", "kind": "homepage", "missing": false, "url": "https://datafusion.apache.org/comet"}, {"content_hash": "f79bba2bc6c3f3caf6c5bf9349646d4d73acb2c0fb52cac8f3f37bb10fa5514e", "fetched_at": "2026-08-29T12:17:10.361049+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/about/index.html"}, {"content_hash": "8dadd8ba0a3308d521ccdb5e32de0479a466033fd2dbbe85eb0b3193d3983f3d", "fetched_at": "2026-08-29T12:17:10.362692+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/user-guide/latest/installation.html"}, {"content_hash": "cd9e11aa04e6e4e1e3aaff966399b9a7edeb6cd8c1ba207268e3653cc9ebecb5", "fetched_at": "2026-08-29T12:17:10.359219+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/changelog/index.html"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-30T05:05:34.231132+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "41154ac1324f8b4bb99242149354f1ff2ae69b2740734a8f9438abc75e6529cf", "fetched_at": "2026-09-03T02:15:09.172587+00:00", "kind": "readme", "missing": false, "url": "https://github.com/apache/datafusion-comet"}, {"content_hash": "42f8c9b44a17e71e300f1b0c8da511bf4ddf26f6f6c562e1b60c1fda796f0df9", "fetched_at": "2026-08-29T12:17:10.356415+00:00", "kind": "homepage", "missing": false, "url": "https://datafusion.apache.org/comet"}, {"content_hash": "f79bba2bc6c3f3caf6c5bf9349646d4d73acb2c0fb52cac8f3f37bb10fa5514e", "fetched_at": "2026-08-29T12:17:10.361049+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/about/index.html"}, {"content_hash": "8dadd8ba0a3308d521ccdb5e32de0479a466033fd2dbbe85eb0b3193d3983f3d", "fetched_at": "2026-08-29T12:17:10.362692+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/user-guide/latest/installation.html"}, {"content_hash": "cd9e11aa04e6e4e1e3aaff966399b9a7edeb6cd8c1ba207268e3653cc9ebecb5", "fetched_at": "2026-08-29T12:17:10.359219+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/changelog/index.html"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-30T05:05:34.231132+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "41154ac1324f8b4bb99242149354f1ff2ae69b2740734a8f9438abc75e6529cf", "fetched_at": "2026-09-03T02:15:09.172587+00:00", "kind": "readme", "missing": false, "url": "https://github.com/apache/datafusion-comet"}, {"content_hash": "42f8c9b44a17e71e300f1b0c8da511bf4ddf26f6f6c562e1b60c1fda796f0df9", "fetched_at": "2026-08-29T12:17:10.356415+00:00", "kind": "homepage", "missing": false, "url": "https://datafusion.apache.org/comet"}, {"content_hash": "f79bba2bc6c3f3caf6c5bf9349646d4d73acb2c0fb52cac8f3f37bb10fa5514e", "fetched_at": "2026-08-29T12:17:10.361049+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/about/index.html"}, {"content_hash": "8dadd8ba0a3308d521ccdb5e32de0479a466033fd2dbbe85eb0b3193d3983f3d", "fetched_at": "2026-08-29T12:17:10.362692+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/user-guide/latest/installation.html"}, {"content_hash": "cd9e11aa04e6e4e1e3aaff966399b9a7edeb6cd8c1ba207268e3653cc9ebecb5", "fetched_at": "2026-08-29T12:17:10.359219+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/changelog/index.html"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-30T05:05:34.231132+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "41154ac1324f8b4bb99242149354f1ff2ae69b2740734a8f9438abc75e6529cf", "fetched_at": "2026-09-03T02:15:09.172587+00:00", "kind": "readme", "missing": false, "url": "https://github.com/apache/datafusion-comet"}, {"content_hash": "42f8c9b44a17e71e300f1b0c8da511bf4ddf26f6f6c562e1b60c1fda796f0df9", "fetched_at": "2026-08-29T12:17:10.356415+00:00", "kind": "homepage", "missing": false, "url": "https://datafusion.apache.org/comet"}, {"content_hash": "f79bba2bc6c3f3caf6c5bf9349646d4d73acb2c0fb52cac8f3f37bb10fa5514e", "fetched_at": "2026-08-29T12:17:10.361049+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/about/index.html"}, {"content_hash": "8dadd8ba0a3308d521ccdb5e32de0479a466033fd2dbbe85eb0b3193d3983f3d", "fetched_at": "2026-08-29T12:17:10.362692+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/user-guide/latest/installation.html"}, {"content_hash": "cd9e11aa04e6e4e1e3aaff966399b9a7edeb6cd8c1ba207268e3653cc9ebecb5", "fetched_at": "2026-08-29T12:17:10.359219+00:00", "kind": "site_page", "missing": false, "url": "https://datafusion.apache.org/comet/changelog/index.html"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 100, "longevity": 68, "rhythm": 64}, "computed_at": "2026-09-03T02:20:16.233290+00:00", "flags": [], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 961, "days_push": 0, "days_rel": 26, "gap_med": null, "n_releases_24m": 1}, "score": 81, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 3, "stale_scrape": false}}