{"adoption": {"forks": 278, "observed_at": "2026-08-28T04:04:42.157602+00:00", "stars": 1429}, "canonical_url": "https://ross.abutalabs.com/products/dataengineeringproject", "card": {"archived": false, "artifact_type": "application", "description": "Example end to end data engineering project.", "domain": ["big-data", "crawlers", "apis"], "enriched": true, "function": ["web-scraping", "streaming", "etl", "search-engine", "api-framework", "scheduling", "caching"], "health_score": 20, "homepage": null, "language": "Python", "license": "MIT", "license_family": "permissive", "maturity": "maintenance", "member_repos": ["damklis/DataEngineeringProject"], "name": "damklis/DataEngineeringProject", "platform": ["python", "self-hosted"], "pushed_at": "2022-12-08T11:28:55+00:00", "repo": "damklis/DataEngineeringProject", "stars": 1429, "tags": ["kafka", "airflow", "debezium", "elasticsearch", "mongodb", "minio", "redis", "cqrs", "rss-feeds", "data-pipeline", "example-project", "data-engineering", "search", "automation", "docker", "web-server"], "topics": ["big-data", "scraping", "mongodb", "elasticsearch", "data-engineering", "kafka", "kafka-connect", "debezium", "django-rest-framework", "redis", "airflow", "minio", "s3", "python", "data-pipeline", "hacktoberfest"], "urls": [], "use_cases": ["build an end-to-end data pipeline with kafka and airflow", "scrape news from rss feeds into a searchable store", "learn how debezium cdc syncs mongodb to elasticsearch", "example of cqrs with mongodb and elasticsearch", "set up kafka connect sinks for mongo, elasticsearch, and s3", "serve scraped data through a rest api", "reference architecture for a data engineering portfolio project"], "what_it_is": "An end-to-end data engineering project that scrapes news from RSS feeds via Airflow-scheduled Python scrapers and streams them through Kafka, MongoDB, Debezium, and Elasticsearch with MinIO for persistence. It exposes the collected news through a Django REST API, demonstrating a CQRS-style pipeline built entirely from open-source components.", "when_to_avoid": ["you need a production-grade, actively maintained product rather than a demo", "your stack doesn't include kafka, mongodb, or elasticsearch", "you need a lightweight pipeline without heavy infrastructure"], "when_to_choose": ["you want a working reference implementation of a streaming data pipeline", "you're learning kafka, kafka connect, debezium, and airflow together", "you need a template for scraping plus serving data via an API"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/dataengineeringproject", "repo": "damklis/DataEngineeringProject", "role": "main", "score": 32}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:04:42.157602+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-30T04:37:14.953770+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "19dc6b79268895eec986df6e765c589d96d7bcd626bd3b0f2285aa9e552fc302", "fetched_at": "2026-08-28T04:04:42.157602+00:00", "kind": "readme", "missing": false, "url": "https://github.com/damklis/DataEngineeringProject"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:04:42.157602+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-30T04:37:14.953770+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "19dc6b79268895eec986df6e765c589d96d7bcd626bd3b0f2285aa9e552fc302", "fetched_at": "2026-08-28T04:04:42.157602+00:00", "kind": "readme", "missing": false, "url": "https://github.com/damklis/DataEngineeringProject"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-30T04:37:14.953770+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "19dc6b79268895eec986df6e765c589d96d7bcd626bd3b0f2285aa9e552fc302", "fetched_at": "2026-08-28T04:04:42.157602+00:00", "kind": "readme", "missing": false, "url": "https://github.com/damklis/DataEngineeringProject"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:04:42.157602+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:04:42.157602+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:04:42.157602+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-30T04:37:14.953770+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "19dc6b79268895eec986df6e765c589d96d7bcd626bd3b0f2285aa9e552fc302", "fetched_at": "2026-08-28T04:04:42.157602+00:00", "kind": "readme", "missing": false, "url": "https://github.com/damklis/DataEngineeringProject"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:04:42.157602+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:04:42.157602+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-30T04:37:14.953770+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "19dc6b79268895eec986df6e765c589d96d7bcd626bd3b0f2285aa9e552fc302", "fetched_at": "2026-08-28T04:04:42.157602+00:00", "kind": "readme", "missing": false, "url": "https://github.com/damklis/DataEngineeringProject"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:04:42.157602+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:04:42.157602+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:04:42.157602+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-30T04:37:14.953770+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "19dc6b79268895eec986df6e765c589d96d7bcd626bd3b0f2285aa9e552fc302", "fetched_at": "2026-08-28T04:04:42.157602+00:00", "kind": "readme", "missing": false, "url": "https://github.com/damklis/DataEngineeringProject"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:04:42.157602+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:04:42.157602+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-30T04:37:14.953770+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "19dc6b79268895eec986df6e765c589d96d7bcd626bd3b0f2285aa9e552fc302", "fetched_at": "2026-08-28T04:04:42.157602+00:00", "kind": "readme", "missing": false, "url": "https://github.com/damklis/DataEngineeringProject"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-30T04:37:14.953770+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "19dc6b79268895eec986df6e765c589d96d7bcd626bd3b0f2285aa9e552fc302", "fetched_at": "2026-08-28T04:04:42.157602+00:00", "kind": "readme", "missing": false, "url": "https://github.com/damklis/DataEngineeringProject"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-30T04:37:14.953770+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "19dc6b79268895eec986df6e765c589d96d7bcd626bd3b0f2285aa9e552fc302", "fetched_at": "2026-08-28T04:04:42.157602+00:00", "kind": "readme", "missing": false, "url": "https://github.com/damklis/DataEngineeringProject"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-30T04:37:14.953770+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "19dc6b79268895eec986df6e765c589d96d7bcd626bd3b0f2285aa9e552fc302", "fetched_at": "2026-08-28T04:04:42.157602+00:00", "kind": "readme", "missing": false, "url": "https://github.com/damklis/DataEngineeringProject"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 0, "longevity": 100, "rhythm": 35}, "computed_at": "2026-09-02T17:46:02.011165+00:00", "flags": ["no_releases"], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 2255, "days_push": 1364, "days_rel": null, "gap_med": null, "n_releases_24m": 0}, "score": 32, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}