{"adoption": {"forks": 156, "observed_at": "2026-08-28T04:07:23.670050+00:00", "stars": 2811}, "canonical_url": "https://ross.abutalabs.com/products/datachain", "card": {"archived": false, "artifact_type": "library", "description": "The Context Layer for unstructured data: typed, versioned datasets over S3, GCS, Azure", "domain": ["machine-learning", "artificial-intelligence", "developer-tools", "cloud-computing"], "enriched": true, "function": ["etl", "data-science", "machine-learning", "rag", "mcp", "search-engine", "file-system", "cli"], "health_score": 100, "homepage": "https://docs.datachain.ai", "language": "Python", "license": "Apache-2.0", "license_family": "permissive", "maturity": "active", "member_repos": ["datachain-ai/datachain"], "name": "datachain-ai/datachain", "platform": ["python", "cli", "cloud", "cross-platform"], "pushed_at": "2026-08-26T14:34:25+00:00", "repo": "datachain-ai/datachain", "stars": 2811, "tags": ["unstructured-data", "data-versioning", "object-storage", "pydantic", "mlops", "multimodal", "agent-harness", "knowledge-base", "data-curation", "s3", "gcs", "azure", "data-engineering", "ai-agents", "docker"], "topics": ["claude-code", "codex", "data-processing", "harness-engineering", "mlops", "unstructured-data", "data-context-layer", "ai-agents", "knowledge-base", "multimodal", "pydantic"], "urls": [], "use_cases": ["process and curate unstructured data in S3, GCS, or Azure at scale", "build versioned, typed datasets from files in cloud object storage", "run batch inference with ML models over images or text in cloud storage", "give coding agents like Claude Code persistent knowledge of my datasets", "find images similar to a reference image in a large cloud bucket", "filter and join files with JSON metadata annotations", "avoid recomputing embeddings and filters across agent sessions"], "what_it_is": "DataChain is a Python library that turns unstructured files in S3, GCS, and Azure into typed, versioned datasets queryable at warehouse speed, with a distributed compute engine for parallel processing over files. It also provides an optional Knowledge Base and Agent Harness (skill + MCP server) that plug into Claude Code, Cursor, Codex, and other coding agents so they can understand and reuse your data.", "when_to_avoid": ["your data is already structured and lives in a relational database or data warehouse with existing tooling", "you need a full visual data-pipeline orchestration platform rather than a Python library", "your datasets are small enough that pandas or plain scripts suffice", "you cannot use cloud object storage and need only local single-machine processing"], "when_to_choose": ["you work with large volumes of unstructured files (images, text, video) in S3, GCS, or Azure and need typed, versioned, queryable datasets", "you want distributed, checkpointed Python processing over cloud files without moving bytes out of storage", "you use AI coding agents and want them to reuse prior data-processing results instead of starting from raw files each session", "you need warehouse-speed filtering, joining, or similarity search over hundreds of millions of file records"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/datachain", "repo": "datachain-ai/datachain", "role": "main", "score": 86}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:07:23.670050+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-30T08:13:32.166940+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "fd6a056021c72698f1ac0c181587946c75b289e6de9bdefdbae624c03a938132", "fetched_at": "2026-08-28T04:07:23.670050+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datachain-ai/datachain"}, {"content_hash": "99e362052d8000a04f9c9008331402035af1cb214017f09394e5f0335627d1fe", "fetched_at": "2026-08-29T09:54:07.625978+00:00", "kind": "homepage", "missing": false, "url": "https://docs.datachain.ai"}, {"content_hash": "cbbc871baacbab0a20873da46e66ceba305c44f85757c3db4cd67ca92ff2b949", "fetched_at": "2026-08-29T09:54:07.635083+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/python"}, {"content_hash": "28f67aeaa66c901979638e5402dec6d92f121c66fa91b8c7b2fe5cce78772a32", "fetched_at": "2026-08-29T09:54:07.637223+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/agents"}, {"content_hash": "a74a64729a96e0e36c4b095d7c6d452f1ff0f0940ba85301e0688eaad6f596d7", "fetched_at": "2026-08-29T09:54:07.638902+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/datachain/json"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:07:23.670050+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-30T08:13:32.166940+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "fd6a056021c72698f1ac0c181587946c75b289e6de9bdefdbae624c03a938132", "fetched_at": "2026-08-28T04:07:23.670050+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datachain-ai/datachain"}, {"content_hash": "99e362052d8000a04f9c9008331402035af1cb214017f09394e5f0335627d1fe", "fetched_at": "2026-08-29T09:54:07.625978+00:00", "kind": "homepage", "missing": false, "url": "https://docs.datachain.ai"}, {"content_hash": "cbbc871baacbab0a20873da46e66ceba305c44f85757c3db4cd67ca92ff2b949", "fetched_at": "2026-08-29T09:54:07.635083+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/python"}, {"content_hash": "28f67aeaa66c901979638e5402dec6d92f121c66fa91b8c7b2fe5cce78772a32", "fetched_at": "2026-08-29T09:54:07.637223+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/agents"}, {"content_hash": "a74a64729a96e0e36c4b095d7c6d452f1ff0f0940ba85301e0688eaad6f596d7", "fetched_at": "2026-08-29T09:54:07.638902+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/datachain/json"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-30T08:13:32.166940+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "fd6a056021c72698f1ac0c181587946c75b289e6de9bdefdbae624c03a938132", "fetched_at": "2026-08-28T04:07:23.670050+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datachain-ai/datachain"}, {"content_hash": "99e362052d8000a04f9c9008331402035af1cb214017f09394e5f0335627d1fe", "fetched_at": "2026-08-29T09:54:07.625978+00:00", "kind": "homepage", "missing": false, "url": "https://docs.datachain.ai"}, {"content_hash": "cbbc871baacbab0a20873da46e66ceba305c44f85757c3db4cd67ca92ff2b949", "fetched_at": "2026-08-29T09:54:07.635083+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/python"}, {"content_hash": "28f67aeaa66c901979638e5402dec6d92f121c66fa91b8c7b2fe5cce78772a32", "fetched_at": "2026-08-29T09:54:07.637223+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/agents"}, {"content_hash": "a74a64729a96e0e36c4b095d7c6d452f1ff0f0940ba85301e0688eaad6f596d7", "fetched_at": "2026-08-29T09:54:07.638902+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/datachain/json"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:07:23.670050+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:07:23.670050+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:07:23.670050+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-30T08:13:32.166940+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "fd6a056021c72698f1ac0c181587946c75b289e6de9bdefdbae624c03a938132", "fetched_at": "2026-08-28T04:07:23.670050+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datachain-ai/datachain"}, {"content_hash": "99e362052d8000a04f9c9008331402035af1cb214017f09394e5f0335627d1fe", "fetched_at": "2026-08-29T09:54:07.625978+00:00", "kind": "homepage", "missing": false, "url": "https://docs.datachain.ai"}, {"content_hash": "cbbc871baacbab0a20873da46e66ceba305c44f85757c3db4cd67ca92ff2b949", "fetched_at": "2026-08-29T09:54:07.635083+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/python"}, {"content_hash": "28f67aeaa66c901979638e5402dec6d92f121c66fa91b8c7b2fe5cce78772a32", "fetched_at": "2026-08-29T09:54:07.637223+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/agents"}, {"content_hash": "a74a64729a96e0e36c4b095d7c6d452f1ff0f0940ba85301e0688eaad6f596d7", "fetched_at": "2026-08-29T09:54:07.638902+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/datachain/json"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:07:23.670050+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:07:23.670050+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-30T08:13:32.166940+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "fd6a056021c72698f1ac0c181587946c75b289e6de9bdefdbae624c03a938132", "fetched_at": "2026-08-28T04:07:23.670050+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datachain-ai/datachain"}, {"content_hash": "99e362052d8000a04f9c9008331402035af1cb214017f09394e5f0335627d1fe", "fetched_at": "2026-08-29T09:54:07.625978+00:00", "kind": "homepage", "missing": false, "url": "https://docs.datachain.ai"}, {"content_hash": "cbbc871baacbab0a20873da46e66ceba305c44f85757c3db4cd67ca92ff2b949", "fetched_at": "2026-08-29T09:54:07.635083+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/python"}, {"content_hash": "28f67aeaa66c901979638e5402dec6d92f121c66fa91b8c7b2fe5cce78772a32", "fetched_at": "2026-08-29T09:54:07.637223+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/agents"}, {"content_hash": "a74a64729a96e0e36c4b095d7c6d452f1ff0f0940ba85301e0688eaad6f596d7", "fetched_at": "2026-08-29T09:54:07.638902+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/datachain/json"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:07:23.670050+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:07:23.670050+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:07:23.670050+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-30T08:13:32.166940+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "fd6a056021c72698f1ac0c181587946c75b289e6de9bdefdbae624c03a938132", "fetched_at": "2026-08-28T04:07:23.670050+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datachain-ai/datachain"}, {"content_hash": "99e362052d8000a04f9c9008331402035af1cb214017f09394e5f0335627d1fe", "fetched_at": "2026-08-29T09:54:07.625978+00:00", "kind": "homepage", "missing": false, "url": "https://docs.datachain.ai"}, {"content_hash": "cbbc871baacbab0a20873da46e66ceba305c44f85757c3db4cd67ca92ff2b949", "fetched_at": "2026-08-29T09:54:07.635083+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/python"}, {"content_hash": "28f67aeaa66c901979638e5402dec6d92f121c66fa91b8c7b2fe5cce78772a32", "fetched_at": "2026-08-29T09:54:07.637223+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/agents"}, {"content_hash": "a74a64729a96e0e36c4b095d7c6d452f1ff0f0940ba85301e0688eaad6f596d7", "fetched_at": "2026-08-29T09:54:07.638902+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/datachain/json"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:07:23.670050+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:07:23.670050+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-30T08:13:32.166940+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "fd6a056021c72698f1ac0c181587946c75b289e6de9bdefdbae624c03a938132", "fetched_at": "2026-08-28T04:07:23.670050+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datachain-ai/datachain"}, {"content_hash": "99e362052d8000a04f9c9008331402035af1cb214017f09394e5f0335627d1fe", "fetched_at": "2026-08-29T09:54:07.625978+00:00", "kind": "homepage", "missing": false, "url": "https://docs.datachain.ai"}, {"content_hash": "cbbc871baacbab0a20873da46e66ceba305c44f85757c3db4cd67ca92ff2b949", "fetched_at": "2026-08-29T09:54:07.635083+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/python"}, {"content_hash": "28f67aeaa66c901979638e5402dec6d92f121c66fa91b8c7b2fe5cce78772a32", "fetched_at": "2026-08-29T09:54:07.637223+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/agents"}, {"content_hash": "a74a64729a96e0e36c4b095d7c6d452f1ff0f0940ba85301e0688eaad6f596d7", "fetched_at": "2026-08-29T09:54:07.638902+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/datachain/json"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-30T08:13:32.166940+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "fd6a056021c72698f1ac0c181587946c75b289e6de9bdefdbae624c03a938132", "fetched_at": "2026-08-28T04:07:23.670050+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datachain-ai/datachain"}, {"content_hash": "99e362052d8000a04f9c9008331402035af1cb214017f09394e5f0335627d1fe", "fetched_at": "2026-08-29T09:54:07.625978+00:00", "kind": "homepage", "missing": false, "url": "https://docs.datachain.ai"}, {"content_hash": "cbbc871baacbab0a20873da46e66ceba305c44f85757c3db4cd67ca92ff2b949", "fetched_at": "2026-08-29T09:54:07.635083+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/python"}, {"content_hash": "28f67aeaa66c901979638e5402dec6d92f121c66fa91b8c7b2fe5cce78772a32", "fetched_at": "2026-08-29T09:54:07.637223+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/agents"}, {"content_hash": "a74a64729a96e0e36c4b095d7c6d452f1ff0f0940ba85301e0688eaad6f596d7", "fetched_at": "2026-08-29T09:54:07.638902+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/datachain/json"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-30T08:13:32.166940+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "fd6a056021c72698f1ac0c181587946c75b289e6de9bdefdbae624c03a938132", "fetched_at": "2026-08-28T04:07:23.670050+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datachain-ai/datachain"}, {"content_hash": "99e362052d8000a04f9c9008331402035af1cb214017f09394e5f0335627d1fe", "fetched_at": "2026-08-29T09:54:07.625978+00:00", "kind": "homepage", "missing": false, "url": "https://docs.datachain.ai"}, {"content_hash": "cbbc871baacbab0a20873da46e66ceba305c44f85757c3db4cd67ca92ff2b949", "fetched_at": "2026-08-29T09:54:07.635083+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/python"}, {"content_hash": "28f67aeaa66c901979638e5402dec6d92f121c66fa91b8c7b2fe5cce78772a32", "fetched_at": "2026-08-29T09:54:07.637223+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/agents"}, {"content_hash": "a74a64729a96e0e36c4b095d7c6d452f1ff0f0940ba85301e0688eaad6f596d7", "fetched_at": "2026-08-29T09:54:07.638902+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/datachain/json"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-30T08:13:32.166940+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "fd6a056021c72698f1ac0c181587946c75b289e6de9bdefdbae624c03a938132", "fetched_at": "2026-08-28T04:07:23.670050+00:00", "kind": "readme", "missing": false, "url": "https://github.com/datachain-ai/datachain"}, {"content_hash": "99e362052d8000a04f9c9008331402035af1cb214017f09394e5f0335627d1fe", "fetched_at": "2026-08-29T09:54:07.625978+00:00", "kind": "homepage", "missing": false, "url": "https://docs.datachain.ai"}, {"content_hash": "cbbc871baacbab0a20873da46e66ceba305c44f85757c3db4cd67ca92ff2b949", "fetched_at": "2026-08-29T09:54:07.635083+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/python"}, {"content_hash": "28f67aeaa66c901979638e5402dec6d92f121c66fa91b8c7b2fe5cce78772a32", "fetched_at": "2026-08-29T09:54:07.637223+00:00", "kind": "site_page", "missing": false, "url": "https://docs.datachain.ai/getting-started/agents"}, {"content_hash": "a74a64729a96e0e36c4b095d7c6d452f1ff0f0940ba85301e0688eaad6f596d7", "fetched_at": "2026-08-29T09:54:07.638902+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/datachain/json"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 99, "longevity": 57, "rhythm": 87}, "computed_at": "2026-09-03T02:39:23.370411+00:00", "flags": [], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 799, "days_push": 7, "days_rel": 9, "gap_med": 1, "n_releases_24m": 250}, "score": 86, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}