{"adoption": {"forks": 122, "observed_at": "2026-08-28T04:04:17.553329+00:00", "stars": 1300}, "canonical_url": "https://ross.abutalabs.com/products/data_engineering_book", "card": {"archived": false, "artifact_type": "learning-resource", "description": "大模型数据工程：架构、算法及项目实战", "domain": ["large-language-models", "machine-learning", "data-science", "tutorials", "privacy"], "enriched": true, "function": ["etl", "data-science", "rag", "llm-training", "nlp", "machine-learning", "workflow-automation", "security"], "health_score": 98, "homepage": "https://datascale-ai.github.io/data_engineering_book/", "language": "Python", "license": "MIT", "license_family": "permissive", "maturity": "active", "member_repos": ["datascale-ai/data_engineering_book"], "name": "datascale-ai/data_engineering_book", "platform": ["python", "cross-platform"], "pushed_at": "2026-08-25T15:04:46+00:00", "repo": "datascale-ai/data_engineering_book", "stars": 1300, "tags": ["book", "data-engineering", "llm-data-pipelines", "dataops", "synthetic-data", "multimodal-data", "rlhf", "pretraining-data", "chinese", "springer", "retrieval-augmented-generation", "ai-agents", "natural-language-processing"], "topics": [], "urls": [], "use_cases": ["learn how to clean and deduplicate pretraining corpora from Common Crawl", "build SFT instruction datasets and RLHF preference data", "construct a RAG document parsing and semantic chunking pipeline", "generate synthetic training data with quality control", "set up DataOps with data versioning and observability", "engineer multimodal image-text and video datasets", "apply privacy-preserving techniques like federated learning and differential privacy", "build agent tool-use and chain-of-thought training data"], "what_it_is": "An open-source book (with GitHub Pages site and runnable Python code) on data engineering for large language models, covering pretraining data cleaning, multimodal alignment, RAG pipelines, synthetic data, DataOps, and privacy compliance. It includes 48 chapters, 15 end-to-end project case studies, and 8 appendices, published as a 2026 Springer title.", "when_to_avoid": ["you need production-ready tooling rather than educational material", "you want a general data engineering book unrelated to LLMs", "you need the English or Japanese edition fully finalized (translations are still in progress)"], "when_to_choose": ["you want a systematic, end-to-end curriculum on LLM data engineering with runnable code", "your team needs practical recipes for pretraining data cleaning, dedup, and quality evaluation", "you are building RAG, SFT, RLHF, or synthetic data pipelines and want worked project examples", "you need guidance on DataOps, data versioning, and data governance for ML"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/data_engineering_book", "repo": "datascale-ai/data_engineering_book", "role": "main", "score": 65}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:04:17.553329+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-30T04:52:18.348985+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "641bdf9652fdf668c3f4bddf0e8427825d344a16bb90572fcad80295d9b99f40", "fetched_at": "2026-08-28T04:04:17.553329+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datascale-ai/data_engineering_book"}, {"content_hash": "87b4c28429d87c58818a48623023f1288026897c3754d7c2847032951a558a3b", "fetched_at": "2026-08-29T12:09:51.115872+00:00", "kind": "homepage", "missing": false, "url": "https://datascale-ai.github.io/data_engineering_book/"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:04:17.553329+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-30T04:52:18.348985+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "641bdf9652fdf668c3f4bddf0e8427825d344a16bb90572fcad80295d9b99f40", "fetched_at": "2026-08-28T04:04:17.553329+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datascale-ai/data_engineering_book"}, {"content_hash": "87b4c28429d87c58818a48623023f1288026897c3754d7c2847032951a558a3b", "fetched_at": "2026-08-29T12:09:51.115872+00:00", "kind": "homepage", "missing": false, "url": "https://datascale-ai.github.io/data_engineering_book/"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-30T04:52:18.348985+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "641bdf9652fdf668c3f4bddf0e8427825d344a16bb90572fcad80295d9b99f40", "fetched_at": "2026-08-28T04:04:17.553329+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datascale-ai/data_engineering_book"}, {"content_hash": "87b4c28429d87c58818a48623023f1288026897c3754d7c2847032951a558a3b", "fetched_at": "2026-08-29T12:09:51.115872+00:00", "kind": "homepage", "missing": false, "url": "https://datascale-ai.github.io/data_engineering_book/"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:04:17.553329+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:04:17.553329+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:04:17.553329+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-30T04:52:18.348985+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "641bdf9652fdf668c3f4bddf0e8427825d344a16bb90572fcad80295d9b99f40", "fetched_at": "2026-08-28T04:04:17.553329+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datascale-ai/data_engineering_book"}, {"content_hash": "87b4c28429d87c58818a48623023f1288026897c3754d7c2847032951a558a3b", "fetched_at": "2026-08-29T12:09:51.115872+00:00", "kind": "homepage", "missing": false, "url": "https://datascale-ai.github.io/data_engineering_book/"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:04:17.553329+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:04:17.553329+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-30T04:52:18.348985+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "641bdf9652fdf668c3f4bddf0e8427825d344a16bb90572fcad80295d9b99f40", "fetched_at": "2026-08-28T04:04:17.553329+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datascale-ai/data_engineering_book"}, {"content_hash": "87b4c28429d87c58818a48623023f1288026897c3754d7c2847032951a558a3b", "fetched_at": "2026-08-29T12:09:51.115872+00:00", "kind": "homepage", "missing": false, "url": "https://datascale-ai.github.io/data_engineering_book/"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:04:17.553329+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:04:17.553329+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:04:17.553329+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-30T04:52:18.348985+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "641bdf9652fdf668c3f4bddf0e8427825d344a16bb90572fcad80295d9b99f40", "fetched_at": "2026-08-28T04:04:17.553329+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datascale-ai/data_engineering_book"}, {"content_hash": "87b4c28429d87c58818a48623023f1288026897c3754d7c2847032951a558a3b", "fetched_at": "2026-08-29T12:09:51.115872+00:00", "kind": "homepage", "missing": false, "url": "https://datascale-ai.github.io/data_engineering_book/"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:04:17.553329+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:04:17.553329+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-30T04:52:18.348985+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "641bdf9652fdf668c3f4bddf0e8427825d344a16bb90572fcad80295d9b99f40", "fetched_at": "2026-08-28T04:04:17.553329+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datascale-ai/data_engineering_book"}, {"content_hash": "87b4c28429d87c58818a48623023f1288026897c3754d7c2847032951a558a3b", "fetched_at": "2026-08-29T12:09:51.115872+00:00", "kind": "homepage", "missing": false, "url": "https://datascale-ai.github.io/data_engineering_book/"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-30T04:52:18.348985+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "641bdf9652fdf668c3f4bddf0e8427825d344a16bb90572fcad80295d9b99f40", "fetched_at": "2026-08-28T04:04:17.553329+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datascale-ai/data_engineering_book"}, {"content_hash": "87b4c28429d87c58818a48623023f1288026897c3754d7c2847032951a558a3b", "fetched_at": "2026-08-29T12:09:51.115872+00:00", "kind": "homepage", "missing": false, "url": "https://datascale-ai.github.io/data_engineering_book/"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-30T04:52:18.348985+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "641bdf9652fdf668c3f4bddf0e8427825d344a16bb90572fcad80295d9b99f40", "fetched_at": "2026-08-28T04:04:17.553329+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datascale-ai/data_engineering_book"}, {"content_hash": "87b4c28429d87c58818a48623023f1288026897c3754d7c2847032951a558a3b", "fetched_at": "2026-08-29T12:09:51.115872+00:00", "kind": "homepage", "missing": false, "url": "https://datascale-ai.github.io/data_engineering_book/"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-30T04:52:18.348985+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "641bdf9652fdf668c3f4bddf0e8427825d344a16bb90572fcad80295d9b99f40", "fetched_at": "2026-08-28T04:04:17.553329+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datascale-ai/data_engineering_book"}, {"content_hash": "87b4c28429d87c58818a48623023f1288026897c3754d7c2847032951a558a3b", "fetched_at": "2026-08-29T12:09:51.115872+00:00", "kind": "homepage", "missing": false, "url": "https://datascale-ai.github.io/data_engineering_book/"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 99, "longevity": 15, "rhythm": 49}, "computed_at": "2026-09-03T02:20:16.233290+00:00", "flags": [], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 214, "days_push": 8, "days_rel": 129, "gap_med": null, "n_releases_24m": 1}, "score": 65, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}