{"adoption": {"forks": 443, "observed_at": "2026-08-28T04:04:31.329279+00:00", "stars": 1366}, "canonical_url": "https://ross.abutalabs.com/products/sparkmagic", "card": {"archived": false, "artifact_type": "library", "description": "Jupyter magics and kernels for working with remote Spark clusters", "domain": ["data-science", "big-data", "developer-tools", "analytics"], "enriched": true, "function": ["sdk", "data-science", "developer-tools"], "health_score": 58, "homepage": null, "language": "Python", "license": "NOASSERTION", "license_family": "other", "maturity": "active", "member_repos": ["jupyter-incubator/sparkmagic"], "name": "jupyter-incubator/sparkmagic", "platform": ["python", "cross-platform"], "pushed_at": "2025-09-09T14:08:11+00:00", "repo": "jupyter-incubator/sparkmagic", "stars": 1366, "tags": ["spark", "livy", "jupyter", "pyspark", "ipython-magics", "remote-spark-cluster", "sql-magic", "kerberos", "pandas"], "topics": ["spark", "kernel", "cluster", "livy", "magic", "sql-query", "pandas-dataframe", "jupyter", "pyspark", "kerberos", "notebook", "jupyter-notebook"], "urls": [], "use_cases": ["run Spark code against a remote Spark cluster from a Jupyter notebook", "execute SparkSQL queries with the %%sql magic and visualize results", "capture SQL query output as pandas dataframes for use with Python libraries", "connect Jupyter to a Livy server with Basic or Kerberos authentication", "send local files or dataframes to a remote Spark cluster", "manage Spark sessions and view application info and logs from a notebook"], "what_it_is": "Sparkmagic is a set of Jupyter magics and kernels for interactively working with remote Spark clusters through a REST server such as Livy, Lighter, or Ilum. It lets users run Spark code in multiple languages, execute SparkSQL queries, visualize results, and exchange data with remote clusters from Jupyter notebooks.", "when_to_avoid": ["you run Spark locally and prefer pyspark directly without a REST intermediary", "you need a non-notebook programmatic Spark client", "your cluster has no Livy-compatible REST server"], "when_to_choose": ["you need to work with a remote Spark cluster from Jupyter without a local Spark installation", "you want interactive SparkSQL querying with automatic visualization in notebooks", "your cluster exposes sessions via Livy, Lighter, or Ilum"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/sparkmagic", "repo": "jupyter-incubator/sparkmagic", "role": "main", "score": 41}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:04:31.329279+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-30T04:41:13.101173+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d068a024ed48998ddb2e640e9b701960b7ae89e96ca5b84a761159459b086464", "fetched_at": "2026-08-28T04:04:31.329279+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jupyter-incubator/sparkmagic"}, {"content_hash": "92bd7f8b220a066f72f38fa9e33da307947e373c0a30a1c8ea56e0f3ea98f434", "fetched_at": "2026-08-29T11:58:22.493494+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/sparkmagic/json"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:04:31.329279+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-30T04:41:13.101173+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d068a024ed48998ddb2e640e9b701960b7ae89e96ca5b84a761159459b086464", "fetched_at": "2026-08-28T04:04:31.329279+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jupyter-incubator/sparkmagic"}, {"content_hash": "92bd7f8b220a066f72f38fa9e33da307947e373c0a30a1c8ea56e0f3ea98f434", "fetched_at": "2026-08-29T11:58:22.493494+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/sparkmagic/json"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-30T04:41:13.101173+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d068a024ed48998ddb2e640e9b701960b7ae89e96ca5b84a761159459b086464", "fetched_at": "2026-08-28T04:04:31.329279+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jupyter-incubator/sparkmagic"}, {"content_hash": "92bd7f8b220a066f72f38fa9e33da307947e373c0a30a1c8ea56e0f3ea98f434", "fetched_at": "2026-08-29T11:58:22.493494+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/sparkmagic/json"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:04:31.329279+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:04:31.329279+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:04:31.329279+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-30T04:41:13.101173+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d068a024ed48998ddb2e640e9b701960b7ae89e96ca5b84a761159459b086464", "fetched_at": "2026-08-28T04:04:31.329279+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jupyter-incubator/sparkmagic"}, {"content_hash": "92bd7f8b220a066f72f38fa9e33da307947e373c0a30a1c8ea56e0f3ea98f434", "fetched_at": "2026-08-29T11:58:22.493494+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/sparkmagic/json"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:04:31.329279+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:04:31.329279+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-30T04:41:13.101173+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d068a024ed48998ddb2e640e9b701960b7ae89e96ca5b84a761159459b086464", "fetched_at": "2026-08-28T04:04:31.329279+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jupyter-incubator/sparkmagic"}, {"content_hash": "92bd7f8b220a066f72f38fa9e33da307947e373c0a30a1c8ea56e0f3ea98f434", "fetched_at": "2026-08-29T11:58:22.493494+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/sparkmagic/json"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:04:31.329279+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:04:31.329279+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:04:31.329279+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-30T04:41:13.101173+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d068a024ed48998ddb2e640e9b701960b7ae89e96ca5b84a761159459b086464", "fetched_at": "2026-08-28T04:04:31.329279+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jupyter-incubator/sparkmagic"}, {"content_hash": "92bd7f8b220a066f72f38fa9e33da307947e373c0a30a1c8ea56e0f3ea98f434", "fetched_at": "2026-08-29T11:58:22.493494+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/sparkmagic/json"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:04:31.329279+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:04:31.329279+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-30T04:41:13.101173+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d068a024ed48998ddb2e640e9b701960b7ae89e96ca5b84a761159459b086464", "fetched_at": "2026-08-28T04:04:31.329279+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jupyter-incubator/sparkmagic"}, {"content_hash": "92bd7f8b220a066f72f38fa9e33da307947e373c0a30a1c8ea56e0f3ea98f434", "fetched_at": "2026-08-29T11:58:22.493494+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/sparkmagic/json"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-30T04:41:13.101173+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d068a024ed48998ddb2e640e9b701960b7ae89e96ca5b84a761159459b086464", "fetched_at": "2026-08-28T04:04:31.329279+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jupyter-incubator/sparkmagic"}, {"content_hash": "92bd7f8b220a066f72f38fa9e33da307947e373c0a30a1c8ea56e0f3ea98f434", "fetched_at": "2026-08-29T11:58:22.493494+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/sparkmagic/json"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-30T04:41:13.101173+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d068a024ed48998ddb2e640e9b701960b7ae89e96ca5b84a761159459b086464", "fetched_at": "2026-08-28T04:04:31.329279+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jupyter-incubator/sparkmagic"}, {"content_hash": "92bd7f8b220a066f72f38fa9e33da307947e373c0a30a1c8ea56e0f3ea98f434", "fetched_at": "2026-08-29T11:58:22.493494+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/sparkmagic/json"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-30T04:41:13.101173+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d068a024ed48998ddb2e640e9b701960b7ae89e96ca5b84a761159459b086464", "fetched_at": "2026-08-28T04:04:31.329279+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jupyter-incubator/sparkmagic"}, {"content_hash": "92bd7f8b220a066f72f38fa9e33da307947e373c0a30a1c8ea56e0f3ea98f434", "fetched_at": "2026-08-29T11:58:22.493494+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/sparkmagic/json"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 41, "longevity": 100, "rhythm": 8}, "computed_at": "2026-09-03T02:20:16.233290+00:00", "flags": ["no_license"], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 3999, "days_push": 358, "days_rel": 422, "gap_med": 236, "n_releases_24m": 2}, "score": 41, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}