{"adoption": {"forks": 217, "observed_at": "2026-08-28T04:08:34.018313+00:00", "stars": 4061}, "canonical_url": "https://ross.abutalabs.com/products/gigatoken", "card": {"archived": false, "artifact_type": "library", "description": "Language model tokenization at GB/s", "domain": ["large-language-models", "machine-learning", "performance"], "enriched": true, "function": ["nlp", "llm-training", "parser", "benchmarking"], "health_score": 78, "homepage": null, "language": "Rust", "license": "MIT", "license_family": "permissive", "maturity": "active", "member_repos": ["marcelroed/gigatoken"], "name": "marcelroed/gigatoken", "platform": ["python", "rust", "windows", "cross-platform"], "pushed_at": "2026-08-06T00:11:22+00:00", "repo": "marcelroed/gigatoken", "stars": 4061, "tags": ["tokenizer", "llm-pretraining", "huggingface-compatibility", "tiktoken-compatibility", "high-throughput", "data-pipeline", "natural-language-processing", "linux", "macos"], "topics": ["llm", "nlp", "tokenization", "tokenizer"], "urls": [], "use_cases": ["tokenize large text corpora for LLM pretraining", "speed up tokenization bottleneck in training data pipelines", "drop-in replacement for HuggingFace tokenizers", "replace tiktoken with a faster tokenizer", "batch encode large text files at GB/s", "tokenize text for models like GPT-2, Llama 3, or Qwen"], "what_it_is": "Gigatoken is a Rust-based tokenizer library for language modeling that tokenizes text at GB/s throughput, claiming ~1000x speedups over HuggingFace tokenizers. It offers a native fast API plus drop-in compatibility modes with HuggingFace Tokenizers and tiktoken, installable via pip.", "when_to_avoid": ["you need tokenizers for exotic models not among the supported set", "you only tokenize small amounts of text where speed is irrelevant", "you depend on HF tokenizers features like training new tokenizers or advanced preprocessing"], "when_to_choose": ["you need to tokenize gigabytes of training data quickly", "tokenization is the bottleneck in your LLM data pipeline", "you want exact-output compatibility with HF tokenizers or tiktoken but faster", "you have many CPU cores available for parallel tokenization"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/gigatoken", "repo": "marcelroed/gigatoken", "role": "main", "score": 60}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:08:34.018313+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-29T18:23:35.560539+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "a35859b9c398815e6429a4bf6f62179b1ff1f490b541ff542c9b3995a6325f10", "fetched_at": "2026-08-28T04:08:34.018313+00:00", "kind": "readme", "missing": false, "url": "https://github.com/marcelroed/gigatoken"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:08:34.018313+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-29T18:23:35.560539+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "a35859b9c398815e6429a4bf6f62179b1ff1f490b541ff542c9b3995a6325f10", "fetched_at": "2026-08-28T04:08:34.018313+00:00", "kind": "readme", "missing": false, "url": "https://github.com/marcelroed/gigatoken"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-29T18:23:35.560539+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "a35859b9c398815e6429a4bf6f62179b1ff1f490b541ff542c9b3995a6325f10", "fetched_at": "2026-08-28T04:08:34.018313+00:00", "kind": "readme", "missing": false, "url": "https://github.com/marcelroed/gigatoken"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:08:34.018313+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:08:34.018313+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:08:34.018313+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-29T18:23:35.560539+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "a35859b9c398815e6429a4bf6f62179b1ff1f490b541ff542c9b3995a6325f10", "fetched_at": "2026-08-28T04:08:34.018313+00:00", "kind": "readme", "missing": false, "url": "https://github.com/marcelroed/gigatoken"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:08:34.018313+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:08:34.018313+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-29T18:23:35.560539+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "a35859b9c398815e6429a4bf6f62179b1ff1f490b541ff542c9b3995a6325f10", "fetched_at": "2026-08-28T04:08:34.018313+00:00", "kind": "readme", "missing": false, "url": "https://github.com/marcelroed/gigatoken"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:08:34.018313+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:08:34.018313+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:08:34.018313+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-29T18:23:35.560539+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "a35859b9c398815e6429a4bf6f62179b1ff1f490b541ff542c9b3995a6325f10", "fetched_at": "2026-08-28T04:08:34.018313+00:00", "kind": "readme", "missing": false, "url": "https://github.com/marcelroed/gigatoken"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:08:34.018313+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:08:34.018313+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-29T18:23:35.560539+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "a35859b9c398815e6429a4bf6f62179b1ff1f490b541ff542c9b3995a6325f10", "fetched_at": "2026-08-28T04:08:34.018313+00:00", "kind": "readme", "missing": false, "url": "https://github.com/marcelroed/gigatoken"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-29T18:23:35.560539+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "a35859b9c398815e6429a4bf6f62179b1ff1f490b541ff542c9b3995a6325f10", "fetched_at": "2026-08-28T04:08:34.018313+00:00", "kind": "readme", "missing": false, "url": "https://github.com/marcelroed/gigatoken"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-29T18:23:35.560539+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "a35859b9c398815e6429a4bf6f62179b1ff1f490b541ff542c9b3995a6325f10", "fetched_at": "2026-08-28T04:08:34.018313+00:00", "kind": "readme", "missing": false, "url": "https://github.com/marcelroed/gigatoken"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-29T18:23:35.560539+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "a35859b9c398815e6429a4bf6f62179b1ff1f490b541ff542c9b3995a6325f10", "fetched_at": "2026-08-28T04:08:34.018313+00:00", "kind": "readme", "missing": false, "url": "https://github.com/marcelroed/gigatoken"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 96, "longevity": 21, "rhythm": 35}, "computed_at": "2026-09-03T02:20:16.233290+00:00", "flags": ["no_releases"], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 296, "days_push": 28, "days_rel": null, "gap_med": null, "n_releases_24m": 0}, "score": 60, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}