{"adoption": {"forks": 808, "observed_at": "2026-08-28T04:06:16.458916+00:00", "stars": 2119}, "canonical_url": "https://ross.abutalabs.com/products/pyspark-example-project", "card": {"archived": true, "artifact_type": "learning-resource", "description": "Implementing best practices for PySpark ETL jobs and applications.", "domain": ["big-data", "tutorials"], "enriched": true, "function": ["etl", "testing", "developer-tools"], "health_score": 10, "homepage": null, "language": "Python", "license": null, "license_family": "other", "maturity": "maintenance", "member_repos": ["AlexIoannides/pyspark-example-project"], "name": "AlexIoannides/pyspark-example-project", "platform": ["python", "jvm", "cross-platform"], "pushed_at": "2023-01-01T22:08:44+00:00", "repo": "AlexIoannides/pyspark-example-project", "stars": 2119, "tags": ["pyspark", "apache-spark", "etl-pipeline", "best-practices", "example-project", "data-science", "data-engineering"], "topics": ["pyspark", "etl-job", "python", "data-engineering", "spark", "data-science", "etl", "etl-pipeline"], "urls": [], "use_cases": ["structure a pyspark etl job for testability", "learn best practices for spark etl projects", "unit test spark transformations with test data", "pass  config parameters to a pyspark job", "package dependencies for spark-submit", "template for starting a pyspark data engineering project"], "what_it_is": "A reference example project demonstrating best practices for structuring PySpark ETL jobs and applications. It shows how to organize code for testability, pass configuration, handle dependencies, and write meaningful unit tests for Spark ETL pipelines.", "when_to_avoid": ["you need a production ETL framework or orchestrator rather than a reference example", "you use Scala/Java Spark instead of Python", "you need streaming or complex pipeline orchestration features"], "when_to_choose": ["starting a new PySpark ETL project and wanting a proven structure", "learning how to test Spark transformations in isolation", "looking for conventions for configs, dependencies, and packaging for Spark clusters"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/pyspark-example-project", "repo": "AlexIoannides/pyspark-example-project", "role": "main", "score": 10}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:06:16.458916+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-30T02:53:01.494000+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "384dde179c1eb83a4bb0f9c9b8ff65aa7b19d42751b1be0765f995a73befef81", "fetched_at": "2026-08-28T04:06:16.458916+00:00", "kind": "readme", "missing": false, "url": "https://github.com/AlexIoannides/pyspark-example-project"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:06:16.458916+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-30T02:53:01.494000+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "384dde179c1eb83a4bb0f9c9b8ff65aa7b19d42751b1be0765f995a73befef81", "fetched_at": "2026-08-28T04:06:16.458916+00:00", "kind": "readme", "missing": false, "url": "https://github.com/AlexIoannides/pyspark-example-project"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-30T02:53:01.494000+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "384dde179c1eb83a4bb0f9c9b8ff65aa7b19d42751b1be0765f995a73befef81", "fetched_at": "2026-08-28T04:06:16.458916+00:00", "kind": "readme", "missing": false, "url": "https://github.com/AlexIoannides/pyspark-example-project"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:06:16.458916+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:06:16.458916+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:06:16.458916+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-30T02:53:01.494000+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "384dde179c1eb83a4bb0f9c9b8ff65aa7b19d42751b1be0765f995a73befef81", "fetched_at": "2026-08-28T04:06:16.458916+00:00", "kind": "readme", "missing": false, "url": "https://github.com/AlexIoannides/pyspark-example-project"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:06:16.458916+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:06:16.458916+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-30T02:53:01.494000+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "384dde179c1eb83a4bb0f9c9b8ff65aa7b19d42751b1be0765f995a73befef81", "fetched_at": "2026-08-28T04:06:16.458916+00:00", "kind": "readme", "missing": false, "url": "https://github.com/AlexIoannides/pyspark-example-project"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:06:16.458916+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:06:16.458916+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:06:16.458916+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-30T02:53:01.494000+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "384dde179c1eb83a4bb0f9c9b8ff65aa7b19d42751b1be0765f995a73befef81", "fetched_at": "2026-08-28T04:06:16.458916+00:00", "kind": "readme", "missing": false, "url": "https://github.com/AlexIoannides/pyspark-example-project"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:06:16.458916+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:06:16.458916+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-30T02:53:01.494000+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "384dde179c1eb83a4bb0f9c9b8ff65aa7b19d42751b1be0765f995a73befef81", "fetched_at": "2026-08-28T04:06:16.458916+00:00", "kind": "readme", "missing": false, "url": "https://github.com/AlexIoannides/pyspark-example-project"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-30T02:53:01.494000+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "384dde179c1eb83a4bb0f9c9b8ff65aa7b19d42751b1be0765f995a73befef81", "fetched_at": "2026-08-28T04:06:16.458916+00:00", "kind": "readme", "missing": false, "url": "https://github.com/AlexIoannides/pyspark-example-project"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-30T02:53:01.494000+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "384dde179c1eb83a4bb0f9c9b8ff65aa7b19d42751b1be0765f995a73befef81", "fetched_at": "2026-08-28T04:06:16.458916+00:00", "kind": "readme", "missing": false, "url": "https://github.com/AlexIoannides/pyspark-example-project"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-30T02:53:01.494000+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "384dde179c1eb83a4bb0f9c9b8ff65aa7b19d42751b1be0765f995a73befef81", "fetched_at": "2026-08-28T04:06:16.458916+00:00", "kind": "readme", "missing": false, "url": "https://github.com/AlexIoannides/pyspark-example-project"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 0, "longevity": 100, "rhythm": 35}, "computed_at": "2026-09-03T02:20:16.233290+00:00", "flags": ["no_releases", "archived", "no_license"], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 3170, "days_push": 1340, "days_rel": null, "gap_med": null, "n_releases_24m": 0}, "score": 10, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}