{"adoption": {"forks": 1092, "observed_at": "2026-08-28T04:10:43.784508+00:00", "stars": 10691}, "canonical_url": "https://ross.abutalabs.com/products/minbpe", "card": {"archived": false, "artifact_type": "library", "description": "Minimal, clean code for the Byte Pair Encoding (BPE) algorithm commonly used in LLM tokenization.", "domain": ["large-language-models", "machine-learning", "education", "developer-tools"], "enriched": true, "function": ["nlp", "parser", "machine-learning", "llm-training", "developer-tools"], "health_score": 20, "homepage": null, "language": "Python", "license": "MIT", "license_family": "permissive", "maturity": "stable", "member_repos": ["karpathy/minbpe"], "name": "karpathy/minbpe", "platform": ["python", "cross-platform"], "pushed_at": "2024-07-01T14:20:22+00:00", "repo": "karpathy/minbpe", "stars": 10691, "tags": ["tokenizer", "byte-pair-encoding", "bpe", "llm", "educational", "minimal-implementation", "natural-language-processing"], "topics": [], "urls": [], "use_cases": ["train a BPE tokenizer on custom text", "understand how LLM tokenization works", "reproduce GPT-4 tokenization in Python", "encode and decode text to tokens and back", "learn the byte pair encoding algorithm from clean code", "build a custom tokenizer for an LLM project"], "what_it_is": "A minimal, clean Python implementation of the byte-level Byte Pair Encoding (BPE) algorithm used for tokenization in modern LLMs like GPT-4 and Llama. It provides train, encode, and decode functionality with a GPT-4-compatible tokenizer that reproduces tiktoken's output.", "when_to_avoid": ["you need a fast production tokenizer for large-scale workloads (use tiktoken or HuggingFace tokenizers)", "you need tokenizer features like parallel encoding or vocabularies beyond BPE"], "when_to_choose": ["you want readable, educational BPE code to study or modify", "you need to train a small custom tokenizer without heavy dependencies", "you want to exactly reproduce GPT-4 tokenization behavior"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/minbpe", "repo": "karpathy/minbpe", "role": "main", "score": 25}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:10:43.784508+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-29T17:17:58.659744+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "3257561ae23077ecde6625e35f8fd01250901051c4501b894d9a7618a34aa223", "fetched_at": "2026-08-28T04:10:43.784508+00:00", "kind": "readme", "missing": false, "url": "https://github.com/karpathy/minbpe"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:10:43.784508+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-29T17:17:58.659744+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "3257561ae23077ecde6625e35f8fd01250901051c4501b894d9a7618a34aa223", "fetched_at": "2026-08-28T04:10:43.784508+00:00", "kind": "readme", "missing": false, "url": "https://github.com/karpathy/minbpe"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-29T17:17:58.659744+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "3257561ae23077ecde6625e35f8fd01250901051c4501b894d9a7618a34aa223", "fetched_at": "2026-08-28T04:10:43.784508+00:00", "kind": "readme", "missing": false, "url": "https://github.com/karpathy/minbpe"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:10:43.784508+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:10:43.784508+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:10:43.784508+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-29T17:17:58.659744+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "3257561ae23077ecde6625e35f8fd01250901051c4501b894d9a7618a34aa223", "fetched_at": "2026-08-28T04:10:43.784508+00:00", "kind": "readme", "missing": false, "url": "https://github.com/karpathy/minbpe"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:10:43.784508+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:10:43.784508+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-29T17:17:58.659744+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "3257561ae23077ecde6625e35f8fd01250901051c4501b894d9a7618a34aa223", "fetched_at": "2026-08-28T04:10:43.784508+00:00", "kind": "readme", "missing": false, "url": "https://github.com/karpathy/minbpe"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:10:43.784508+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:10:43.784508+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:10:43.784508+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-29T17:17:58.659744+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "3257561ae23077ecde6625e35f8fd01250901051c4501b894d9a7618a34aa223", "fetched_at": "2026-08-28T04:10:43.784508+00:00", "kind": "readme", "missing": false, "url": "https://github.com/karpathy/minbpe"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:10:43.784508+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:10:43.784508+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-29T17:17:58.659744+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "3257561ae23077ecde6625e35f8fd01250901051c4501b894d9a7618a34aa223", "fetched_at": "2026-08-28T04:10:43.784508+00:00", "kind": "readme", "missing": false, "url": "https://github.com/karpathy/minbpe"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-29T17:17:58.659744+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "3257561ae23077ecde6625e35f8fd01250901051c4501b894d9a7618a34aa223", "fetched_at": "2026-08-28T04:10:43.784508+00:00", "kind": "readme", "missing": false, "url": "https://github.com/karpathy/minbpe"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-29T17:17:58.659744+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "3257561ae23077ecde6625e35f8fd01250901051c4501b894d9a7618a34aa223", "fetched_at": "2026-08-28T04:10:43.784508+00:00", "kind": "readme", "missing": false, "url": "https://github.com/karpathy/minbpe"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-29T17:17:58.659744+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "3257561ae23077ecde6625e35f8fd01250901051c4501b894d9a7618a34aa223", "fetched_at": "2026-08-28T04:10:43.784508+00:00", "kind": "readme", "missing": false, "url": "https://github.com/karpathy/minbpe"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 0, "longevity": 66, "rhythm": 35}, "computed_at": "2026-09-02T17:46:02.011165+00:00", "flags": ["no_releases"], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 929, "days_push": 793, "days_rel": null, "gap_med": null, "n_releases_24m": 0}, "score": 25, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}