{"adoption": {"forks": 226, "observed_at": "2026-08-28T04:04:35.940318+00:00", "stars": 1391}, "canonical_url": "https://ross.abutalabs.com/products/nlg-eval", "card": {"archived": false, "artifact_type": "library", "description": "Evaluation code for various unsupervised automated metrics for Natural Language Generation.", "domain": ["machine-learning", "data-science"], "enriched": true, "function": ["nlp", "machine-learning", "benchmarking", "cli"], "health_score": 23, "homepage": "http://arxiv.org/abs/1706.09799", "language": "Python", "license": "NOASSERTION", "license_family": "other", "maturity": "maintenance", "member_repos": ["Maluuba/nlg-eval"], "name": "Maluuba/nlg-eval", "platform": ["python", "cli", "cross-platform"], "pushed_at": "2024-08-20T16:26:23+00:00", "repo": "Maluuba/nlg-eval", "stars": 1391, "tags": ["nlg-evaluation", "bleu", "meteor", "rouge", "cider", "spice", "skip-thoughts", "text-generation-metrics", "dialogue-evaluation", "natural-language-processing"], "topics": ["natural-language-generation", "natural-language-processing", "nlg", "nlp", "evaluation", "bleu", "bleu-score", "meteor", "cider", "rouge", "rouge-l", "task-oriented-dialogue", "machine-translation", "dialog", "dialogue", "skip-thought-vectors", "skip-thoughts"], "urls": [], "use_cases": ["evaluate generated dialogue responses against references", "compute BLEU and ROUGE scores for machine translation output", "score image captions with CIDEr and SPICE", "compare NLG model outputs with embedding-based similarity metrics", "benchmark text generation models with standard NLG metrics", "evaluate task-oriented dialogue response generation"], "what_it_is": "A Python library and CLI tool that computes unsupervised automated metrics (BLEU, METEOR, ROUGE, CIDEr, SPICE, SkipThought, and embedding-based similarities) for evaluating Natural Language Generation output. It takes hypothesis and reference files as input and outputs metric scores, commonly used for dialogue and captioning evaluation.", "when_to_avoid": ["you need human evaluation or learned metrics like BERTScore", "you cannot install Java 1.8+ (required for METEOR/SPICE)", "you need actively maintained tooling with modern Python support", "evaluating LLM outputs where GPT-based judges are preferred"], "when_to_choose": ["you need multiple standard NLG metrics from a single tool", "evaluating dialogue systems or captioning models in Python", "you want reproducible metric scores for research papers"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/nlg-eval", "repo": "Maluuba/nlg-eval", "role": "main", "score": 23}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:04:35.940318+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-30T04:39:35.121200+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "927eb7b36463ff31f8869a03234519b75655975b69509cc4b4f8e213a0e2ec09", "fetched_at": "2026-08-28T04:04:35.940318+00:00", "kind": "readme", "missing": false, "url": "https://github.com/Maluuba/nlg-eval"}, {"content_hash": "4aaab4c1334bb5614f40a5d585a12e2346dbc4637f6550487bb1cdf267428fb3", "fetched_at": "2026-08-29T11:54:40.830899+00:00", "kind": "homepage", "missing": false, "url": "http://arxiv.org/abs/1706.09799"}, {"content_hash": "cca9c3a11c56b1212864e03fb8a5511a3bdb6df53398ba4dd2ed778f931607dd", "fetched_at": "2026-08-29T11:54:40.839984+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/donate.html"}, {"content_hash": "47cbc55ff1deaca3f36b463373de863a7053d91cacf9f95725618e11610209ae", "fetched_at": "2026-08-29T11:54:40.843323+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/ourmembers.html"}, {"content_hash": "a1f16f915a9ad89ee57c3e929a1b6577d481f52c05f8015fc720b16ee8014f19", "fetched_at": "2026-08-29T11:54:40.845030+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about"}, {"content_hash": "b14a8d05a0ec337409ab63bffd91306b5317b16fac358aa6d1fe6afbaf3c8628", "fetched_at": "2026-08-29T11:54:40.841757+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/labs/index.html"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:04:35.940318+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-30T04:39:35.121200+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "927eb7b36463ff31f8869a03234519b75655975b69509cc4b4f8e213a0e2ec09", "fetched_at": "2026-08-28T04:04:35.940318+00:00", "kind": "readme", "missing": false, "url": "https://github.com/Maluuba/nlg-eval"}, {"content_hash": "4aaab4c1334bb5614f40a5d585a12e2346dbc4637f6550487bb1cdf267428fb3", "fetched_at": "2026-08-29T11:54:40.830899+00:00", "kind": "homepage", "missing": false, "url": "http://arxiv.org/abs/1706.09799"}, {"content_hash": "cca9c3a11c56b1212864e03fb8a5511a3bdb6df53398ba4dd2ed778f931607dd", "fetched_at": "2026-08-29T11:54:40.839984+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/donate.html"}, {"content_hash": "47cbc55ff1deaca3f36b463373de863a7053d91cacf9f95725618e11610209ae", "fetched_at": "2026-08-29T11:54:40.843323+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/ourmembers.html"}, {"content_hash": "a1f16f915a9ad89ee57c3e929a1b6577d481f52c05f8015fc720b16ee8014f19", "fetched_at": "2026-08-29T11:54:40.845030+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about"}, {"content_hash": "b14a8d05a0ec337409ab63bffd91306b5317b16fac358aa6d1fe6afbaf3c8628", "fetched_at": "2026-08-29T11:54:40.841757+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/labs/index.html"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-30T04:39:35.121200+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "927eb7b36463ff31f8869a03234519b75655975b69509cc4b4f8e213a0e2ec09", "fetched_at": "2026-08-28T04:04:35.940318+00:00", "kind": "readme", "missing": false, "url": "https://github.com/Maluuba/nlg-eval"}, {"content_hash": "4aaab4c1334bb5614f40a5d585a12e2346dbc4637f6550487bb1cdf267428fb3", "fetched_at": "2026-08-29T11:54:40.830899+00:00", "kind": "homepage", "missing": false, "url": "http://arxiv.org/abs/1706.09799"}, {"content_hash": "cca9c3a11c56b1212864e03fb8a5511a3bdb6df53398ba4dd2ed778f931607dd", "fetched_at": "2026-08-29T11:54:40.839984+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/donate.html"}, {"content_hash": "47cbc55ff1deaca3f36b463373de863a7053d91cacf9f95725618e11610209ae", "fetched_at": "2026-08-29T11:54:40.843323+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/ourmembers.html"}, {"content_hash": "a1f16f915a9ad89ee57c3e929a1b6577d481f52c05f8015fc720b16ee8014f19", "fetched_at": "2026-08-29T11:54:40.845030+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about"}, {"content_hash": "b14a8d05a0ec337409ab63bffd91306b5317b16fac358aa6d1fe6afbaf3c8628", "fetched_at": "2026-08-29T11:54:40.841757+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/labs/index.html"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:04:35.940318+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:04:35.940318+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:04:35.940318+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-30T04:39:35.121200+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "927eb7b36463ff31f8869a03234519b75655975b69509cc4b4f8e213a0e2ec09", "fetched_at": "2026-08-28T04:04:35.940318+00:00", "kind": "readme", "missing": false, "url": "https://github.com/Maluuba/nlg-eval"}, {"content_hash": "4aaab4c1334bb5614f40a5d585a12e2346dbc4637f6550487bb1cdf267428fb3", "fetched_at": "2026-08-29T11:54:40.830899+00:00", "kind": "homepage", "missing": false, "url": "http://arxiv.org/abs/1706.09799"}, {"content_hash": "cca9c3a11c56b1212864e03fb8a5511a3bdb6df53398ba4dd2ed778f931607dd", "fetched_at": "2026-08-29T11:54:40.839984+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/donate.html"}, {"content_hash": "47cbc55ff1deaca3f36b463373de863a7053d91cacf9f95725618e11610209ae", "fetched_at": "2026-08-29T11:54:40.843323+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/ourmembers.html"}, {"content_hash": "a1f16f915a9ad89ee57c3e929a1b6577d481f52c05f8015fc720b16ee8014f19", "fetched_at": "2026-08-29T11:54:40.845030+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about"}, {"content_hash": "b14a8d05a0ec337409ab63bffd91306b5317b16fac358aa6d1fe6afbaf3c8628", "fetched_at": "2026-08-29T11:54:40.841757+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/labs/index.html"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:04:35.940318+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:04:35.940318+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-30T04:39:35.121200+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "927eb7b36463ff31f8869a03234519b75655975b69509cc4b4f8e213a0e2ec09", "fetched_at": "2026-08-28T04:04:35.940318+00:00", "kind": "readme", "missing": false, "url": "https://github.com/Maluuba/nlg-eval"}, {"content_hash": "4aaab4c1334bb5614f40a5d585a12e2346dbc4637f6550487bb1cdf267428fb3", "fetched_at": "2026-08-29T11:54:40.830899+00:00", "kind": "homepage", "missing": false, "url": "http://arxiv.org/abs/1706.09799"}, {"content_hash": "cca9c3a11c56b1212864e03fb8a5511a3bdb6df53398ba4dd2ed778f931607dd", "fetched_at": "2026-08-29T11:54:40.839984+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/donate.html"}, {"content_hash": "47cbc55ff1deaca3f36b463373de863a7053d91cacf9f95725618e11610209ae", "fetched_at": "2026-08-29T11:54:40.843323+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/ourmembers.html"}, {"content_hash": "a1f16f915a9ad89ee57c3e929a1b6577d481f52c05f8015fc720b16ee8014f19", "fetched_at": "2026-08-29T11:54:40.845030+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about"}, {"content_hash": "b14a8d05a0ec337409ab63bffd91306b5317b16fac358aa6d1fe6afbaf3c8628", "fetched_at": "2026-08-29T11:54:40.841757+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/labs/index.html"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:04:35.940318+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:04:35.940318+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:04:35.940318+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-30T04:39:35.121200+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "927eb7b36463ff31f8869a03234519b75655975b69509cc4b4f8e213a0e2ec09", "fetched_at": "2026-08-28T04:04:35.940318+00:00", "kind": "readme", "missing": false, "url": "https://github.com/Maluuba/nlg-eval"}, {"content_hash": "4aaab4c1334bb5614f40a5d585a12e2346dbc4637f6550487bb1cdf267428fb3", "fetched_at": "2026-08-29T11:54:40.830899+00:00", "kind": "homepage", "missing": false, "url": "http://arxiv.org/abs/1706.09799"}, {"content_hash": "cca9c3a11c56b1212864e03fb8a5511a3bdb6df53398ba4dd2ed778f931607dd", "fetched_at": "2026-08-29T11:54:40.839984+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/donate.html"}, {"content_hash": "47cbc55ff1deaca3f36b463373de863a7053d91cacf9f95725618e11610209ae", "fetched_at": "2026-08-29T11:54:40.843323+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/ourmembers.html"}, {"content_hash": "a1f16f915a9ad89ee57c3e929a1b6577d481f52c05f8015fc720b16ee8014f19", "fetched_at": "2026-08-29T11:54:40.845030+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about"}, {"content_hash": "b14a8d05a0ec337409ab63bffd91306b5317b16fac358aa6d1fe6afbaf3c8628", "fetched_at": "2026-08-29T11:54:40.841757+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/labs/index.html"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:04:35.940318+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:04:35.940318+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-30T04:39:35.121200+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "927eb7b36463ff31f8869a03234519b75655975b69509cc4b4f8e213a0e2ec09", "fetched_at": "2026-08-28T04:04:35.940318+00:00", "kind": "readme", "missing": false, "url": "https://github.com/Maluuba/nlg-eval"}, {"content_hash": "4aaab4c1334bb5614f40a5d585a12e2346dbc4637f6550487bb1cdf267428fb3", "fetched_at": "2026-08-29T11:54:40.830899+00:00", "kind": "homepage", "missing": false, "url": "http://arxiv.org/abs/1706.09799"}, {"content_hash": "cca9c3a11c56b1212864e03fb8a5511a3bdb6df53398ba4dd2ed778f931607dd", "fetched_at": "2026-08-29T11:54:40.839984+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/donate.html"}, {"content_hash": "47cbc55ff1deaca3f36b463373de863a7053d91cacf9f95725618e11610209ae", "fetched_at": "2026-08-29T11:54:40.843323+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/ourmembers.html"}, {"content_hash": "a1f16f915a9ad89ee57c3e929a1b6577d481f52c05f8015fc720b16ee8014f19", "fetched_at": "2026-08-29T11:54:40.845030+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about"}, {"content_hash": "b14a8d05a0ec337409ab63bffd91306b5317b16fac358aa6d1fe6afbaf3c8628", "fetched_at": "2026-08-29T11:54:40.841757+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/labs/index.html"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-30T04:39:35.121200+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "927eb7b36463ff31f8869a03234519b75655975b69509cc4b4f8e213a0e2ec09", "fetched_at": "2026-08-28T04:04:35.940318+00:00", "kind": "readme", "missing": false, "url": "https://github.com/Maluuba/nlg-eval"}, {"content_hash": "4aaab4c1334bb5614f40a5d585a12e2346dbc4637f6550487bb1cdf267428fb3", "fetched_at": "2026-08-29T11:54:40.830899+00:00", "kind": "homepage", "missing": false, "url": "http://arxiv.org/abs/1706.09799"}, {"content_hash": "cca9c3a11c56b1212864e03fb8a5511a3bdb6df53398ba4dd2ed778f931607dd", "fetched_at": "2026-08-29T11:54:40.839984+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/donate.html"}, {"content_hash": "47cbc55ff1deaca3f36b463373de863a7053d91cacf9f95725618e11610209ae", "fetched_at": "2026-08-29T11:54:40.843323+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/ourmembers.html"}, {"content_hash": "a1f16f915a9ad89ee57c3e929a1b6577d481f52c05f8015fc720b16ee8014f19", "fetched_at": "2026-08-29T11:54:40.845030+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about"}, {"content_hash": "b14a8d05a0ec337409ab63bffd91306b5317b16fac358aa6d1fe6afbaf3c8628", "fetched_at": "2026-08-29T11:54:40.841757+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/labs/index.html"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-30T04:39:35.121200+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "927eb7b36463ff31f8869a03234519b75655975b69509cc4b4f8e213a0e2ec09", "fetched_at": "2026-08-28T04:04:35.940318+00:00", "kind": "readme", "missing": false, "url": "https://github.com/Maluuba/nlg-eval"}, {"content_hash": "4aaab4c1334bb5614f40a5d585a12e2346dbc4637f6550487bb1cdf267428fb3", "fetched_at": "2026-08-29T11:54:40.830899+00:00", "kind": "homepage", "missing": false, "url": "http://arxiv.org/abs/1706.09799"}, {"content_hash": "cca9c3a11c56b1212864e03fb8a5511a3bdb6df53398ba4dd2ed778f931607dd", "fetched_at": "2026-08-29T11:54:40.839984+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/donate.html"}, {"content_hash": "47cbc55ff1deaca3f36b463373de863a7053d91cacf9f95725618e11610209ae", "fetched_at": "2026-08-29T11:54:40.843323+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/ourmembers.html"}, {"content_hash": "a1f16f915a9ad89ee57c3e929a1b6577d481f52c05f8015fc720b16ee8014f19", "fetched_at": "2026-08-29T11:54:40.845030+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about"}, {"content_hash": "b14a8d05a0ec337409ab63bffd91306b5317b16fac358aa6d1fe6afbaf3c8628", "fetched_at": "2026-08-29T11:54:40.841757+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/labs/index.html"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-30T04:39:35.121200+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "927eb7b36463ff31f8869a03234519b75655975b69509cc4b4f8e213a0e2ec09", "fetched_at": "2026-08-28T04:04:35.940318+00:00", "kind": "readme", "missing": false, "url": "https://github.com/Maluuba/nlg-eval"}, {"content_hash": "4aaab4c1334bb5614f40a5d585a12e2346dbc4637f6550487bb1cdf267428fb3", "fetched_at": "2026-08-29T11:54:40.830899+00:00", "kind": "homepage", "missing": false, "url": "http://arxiv.org/abs/1706.09799"}, {"content_hash": "cca9c3a11c56b1212864e03fb8a5511a3bdb6df53398ba4dd2ed778f931607dd", "fetched_at": "2026-08-29T11:54:40.839984+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/donate.html"}, {"content_hash": "47cbc55ff1deaca3f36b463373de863a7053d91cacf9f95725618e11610209ae", "fetched_at": "2026-08-29T11:54:40.843323+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about/ourmembers.html"}, {"content_hash": "a1f16f915a9ad89ee57c3e929a1b6577d481f52c05f8015fc720b16ee8014f19", "fetched_at": "2026-08-29T11:54:40.845030+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/about"}, {"content_hash": "b14a8d05a0ec337409ab63bffd91306b5317b16fac358aa6d1fe6afbaf3c8628", "fetched_at": "2026-08-29T11:54:40.841757+00:00", "kind": "site_page", "missing": false, "url": "https://info.arxiv.org/labs/index.html"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 0, "longevity": 100, "rhythm": 8}, "computed_at": "2026-09-03T02:20:16.233290+00:00", "flags": ["no_license"], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 3354, "days_push": 743, "days_rel": null, "gap_med": null, "n_releases_24m": 0}, "score": 23, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}