{"adoption": {"forks": 159, "observed_at": "2026-08-28T04:03:42.527140+00:00", "stars": 1131}, "canonical_url": "https://ross.abutalabs.com/products/bigflow", "card": {"archived": false, "artifact_type": "framework", "description": "Baidu Bigflow is an interface that allows for writing distributed computing programs and provides lots of simple, flexible, powerful APIs. Using Bigflow, you can easily handle data of any scale. Bigflow processes 4P+ data inside Baidu and runs about 10k jobs every day.", "domain": ["big-data", "microservices", "data-science"], "enriched": true, "function": ["etl", "streaming", "data-science", "workflow-automation"], "health_score": 54, "homepage": "http://baidu.github.io/bigflow", "language": "C++", "license": "Apache-2.0", "license_family": "permissive", "maturity": "active", "member_repos": ["baidu/bigflow"], "name": "baidu/bigflow", "platform": ["python", "cpp"], "pushed_at": "2025-12-11T20:38:47+00:00", "repo": "baidu/bigflow", "stars": 1131, "tags": ["distributed-computing", "flumejava", "batch-processing", "pyspark-alternative", "data-pipeline", "data-engineering", "spark", "linux", "docker"], "topics": [], "urls": [], "use_cases": ["write distributed data processing jobs in python", "run batch etl pipelines on spark", "replace pyspark with a faster python-first framework", "process large-scale datasets without managing partitioners", "port single-machine python logic to distributed execution"], "what_it_is": "Baidu Bigflow is a distributed computing framework offering simple, flexible Python APIs for writing data processing programs that can run on different execution engines. It is inspired by FlumeJava, Cloud Dataflow, and Spark, with an open-source release targeting Spark as the backend.", "when_to_avoid": ["you need streaming engines other than spark, since the open-source version only supports bigflow on spark", "you need a large community and ecosystem, as bigflow is baidu-centric with limited external adoption", "you work primarily in scala or java rather than python"], "when_to_choose": ["you want a python-first distributed computing api with better performance than pyspark", "you want code portable across execution engines", "you prefer spark-like apis with simpler semantics like nested distributed datasets"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/bigflow", "repo": "baidu/bigflow", "role": "main", "score": 48}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:03:42.527140+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-30T06:38:01.092682+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cc49c8940282e0626026d698083deab1aa7cfb895d2166f397ebc57fcf66f477", "fetched_at": "2026-08-28T04:03:42.527140+00:00", "kind": "readme", "missing": false, "url": "https://github.com/baidu/bigflow"}, {"content_hash": "82c2723a3b55a81b92f1e6fa3eb09fabdd9f14236c23498727d9002a4ac62fde", "fetched_at": "2026-08-29T12:42:24.389024+00:00", "kind": "homepage", "missing": false, "url": "http://baidu.github.io/bigflow"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:03:42.527140+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-30T06:38:01.092682+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cc49c8940282e0626026d698083deab1aa7cfb895d2166f397ebc57fcf66f477", "fetched_at": "2026-08-28T04:03:42.527140+00:00", "kind": "readme", "missing": false, "url": "https://github.com/baidu/bigflow"}, {"content_hash": "82c2723a3b55a81b92f1e6fa3eb09fabdd9f14236c23498727d9002a4ac62fde", "fetched_at": "2026-08-29T12:42:24.389024+00:00", "kind": "homepage", "missing": false, "url": "http://baidu.github.io/bigflow"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-30T06:38:01.092682+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cc49c8940282e0626026d698083deab1aa7cfb895d2166f397ebc57fcf66f477", "fetched_at": "2026-08-28T04:03:42.527140+00:00", "kind": "readme", "missing": false, "url": "https://github.com/baidu/bigflow"}, {"content_hash": "82c2723a3b55a81b92f1e6fa3eb09fabdd9f14236c23498727d9002a4ac62fde", "fetched_at": "2026-08-29T12:42:24.389024+00:00", "kind": "homepage", "missing": false, "url": "http://baidu.github.io/bigflow"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:03:42.527140+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:03:42.527140+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:03:42.527140+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-30T06:38:01.092682+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cc49c8940282e0626026d698083deab1aa7cfb895d2166f397ebc57fcf66f477", "fetched_at": "2026-08-28T04:03:42.527140+00:00", "kind": "readme", "missing": false, "url": "https://github.com/baidu/bigflow"}, {"content_hash": "82c2723a3b55a81b92f1e6fa3eb09fabdd9f14236c23498727d9002a4ac62fde", "fetched_at": "2026-08-29T12:42:24.389024+00:00", "kind": "homepage", "missing": false, "url": "http://baidu.github.io/bigflow"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:03:42.527140+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:03:42.527140+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-30T06:38:01.092682+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cc49c8940282e0626026d698083deab1aa7cfb895d2166f397ebc57fcf66f477", "fetched_at": "2026-08-28T04:03:42.527140+00:00", "kind": "readme", "missing": false, "url": "https://github.com/baidu/bigflow"}, {"content_hash": "82c2723a3b55a81b92f1e6fa3eb09fabdd9f14236c23498727d9002a4ac62fde", "fetched_at": "2026-08-29T12:42:24.389024+00:00", "kind": "homepage", "missing": false, "url": "http://baidu.github.io/bigflow"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:03:42.527140+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:03:42.527140+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:03:42.527140+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-30T06:38:01.092682+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cc49c8940282e0626026d698083deab1aa7cfb895d2166f397ebc57fcf66f477", "fetched_at": "2026-08-28T04:03:42.527140+00:00", "kind": "readme", "missing": false, "url": "https://github.com/baidu/bigflow"}, {"content_hash": "82c2723a3b55a81b92f1e6fa3eb09fabdd9f14236c23498727d9002a4ac62fde", "fetched_at": "2026-08-29T12:42:24.389024+00:00", "kind": "homepage", "missing": false, "url": "http://baidu.github.io/bigflow"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:03:42.527140+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:03:42.527140+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-30T06:38:01.092682+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cc49c8940282e0626026d698083deab1aa7cfb895d2166f397ebc57fcf66f477", "fetched_at": "2026-08-28T04:03:42.527140+00:00", "kind": "readme", "missing": false, "url": "https://github.com/baidu/bigflow"}, {"content_hash": "82c2723a3b55a81b92f1e6fa3eb09fabdd9f14236c23498727d9002a4ac62fde", "fetched_at": "2026-08-29T12:42:24.389024+00:00", "kind": "homepage", "missing": false, "url": "http://baidu.github.io/bigflow"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-30T06:38:01.092682+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cc49c8940282e0626026d698083deab1aa7cfb895d2166f397ebc57fcf66f477", "fetched_at": "2026-08-28T04:03:42.527140+00:00", "kind": "readme", "missing": false, "url": "https://github.com/baidu/bigflow"}, {"content_hash": "82c2723a3b55a81b92f1e6fa3eb09fabdd9f14236c23498727d9002a4ac62fde", "fetched_at": "2026-08-29T12:42:24.389024+00:00", "kind": "homepage", "missing": false, "url": "http://baidu.github.io/bigflow"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-30T06:38:01.092682+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cc49c8940282e0626026d698083deab1aa7cfb895d2166f397ebc57fcf66f477", "fetched_at": "2026-08-28T04:03:42.527140+00:00", "kind": "readme", "missing": false, "url": "https://github.com/baidu/bigflow"}, {"content_hash": "82c2723a3b55a81b92f1e6fa3eb09fabdd9f14236c23498727d9002a4ac62fde", "fetched_at": "2026-08-29T12:42:24.389024+00:00", "kind": "homepage", "missing": false, "url": "http://baidu.github.io/bigflow"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-30T06:38:01.092682+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cc49c8940282e0626026d698083deab1aa7cfb895d2166f397ebc57fcf66f477", "fetched_at": "2026-08-28T04:03:42.527140+00:00", "kind": "readme", "missing": false, "url": "https://github.com/baidu/bigflow"}, {"content_hash": "82c2723a3b55a81b92f1e6fa3eb09fabdd9f14236c23498727d9002a4ac62fde", "fetched_at": "2026-08-29T12:42:24.389024+00:00", "kind": "homepage", "missing": false, "url": "http://baidu.github.io/bigflow"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 56, "longevity": 100, "rhythm": 8}, "computed_at": "2026-09-03T02:20:16.233290+00:00", "flags": [], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 3214, "days_push": 265, "days_rel": null, "gap_med": null, "n_releases_24m": 0}, "score": 48, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}