{"adoption": {"forks": 92, "observed_at": "2026-08-28T04:03:37.486894+00:00", "stars": 1110}, "canonical_url": "https://ross.abutalabs.com/products/ragchecker", "card": {"archived": false, "artifact_type": "framework", "description": "RAGChecker: A Fine-grained Framework For Diagnosing RAG", "domain": ["large-language-models", "machine-learning", "developer-tools"], "enriched": true, "function": ["rag", "benchmarking", "testing", "nlp", "llm-inference", "cli"], "health_score": 29, "homepage": null, "language": "Python", "license": "Apache-2.0", "license_family": "permissive", "maturity": "active", "member_repos": ["amazon-science/RAGChecker"], "name": "amazon-science/RAGChecker", "platform": ["python", "cli", "cross-platform"], "pushed_at": "2024-12-13T13:24:41+00:00", "repo": "amazon-science/RAGChecker", "stars": 1110, "tags": ["rag-evaluation", "diagnostic-metrics", "claim-level-entailment", "evaluation-framework", "retrieval-evaluation", "generator-evaluation", "retrieval-augmented-generation", "natural-language-processing"], "topics": [], "urls": [], "use_cases": ["evaluate my rag pipeline quality", "diagnose whether retrieval or generation is failing in my rag system", "benchmark rag systems across domains", "measure faithfulness and relevance of rag answers", "compare rag checker metrics with human judgments", "run claim-level entailment evaluation on rag outputs"], "what_it_is": "RAGChecker is an automatic evaluation framework for diagnosing Retrieval-Augmented Generation (RAG) systems. It provides holistic and diagnostic metrics for retrieval and generation components, using claim-level entailment for fine-grained analysis, plus a benchmark dataset.", "when_to_avoid": ["you need to build or serve a RAG system rather than evaluate one", "you need a lightweight evaluation without LLM-based entailment costs", "you need non-English evaluation support"], "when_to_choose": ["you need fine-grained, component-level diagnosis of a RAG pipeline", "you want automated metrics correlated with human judgments", "you need a benchmark dataset for RAG evaluation research"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/ragchecker", "repo": "amazon-science/RAGChecker", "role": "main", "score": 25}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:03:37.486894+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-30T06:43:26.315280+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d2a5e03773b47c8a8b75b77edc8db7658ffb4b36ddc61d3ee7a521318e94ff1f", "fetched_at": "2026-08-28T04:03:37.486894+00:00", "kind": "readme", "missing": false, "url": "https://github.com/amazon-science/RAGChecker"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:03:37.486894+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-30T06:43:26.315280+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d2a5e03773b47c8a8b75b77edc8db7658ffb4b36ddc61d3ee7a521318e94ff1f", "fetched_at": "2026-08-28T04:03:37.486894+00:00", "kind": "readme", "missing": false, "url": "https://github.com/amazon-science/RAGChecker"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-30T06:43:26.315280+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d2a5e03773b47c8a8b75b77edc8db7658ffb4b36ddc61d3ee7a521318e94ff1f", "fetched_at": "2026-08-28T04:03:37.486894+00:00", "kind": "readme", "missing": false, "url": "https://github.com/amazon-science/RAGChecker"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:03:37.486894+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:03:37.486894+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:03:37.486894+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-30T06:43:26.315280+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d2a5e03773b47c8a8b75b77edc8db7658ffb4b36ddc61d3ee7a521318e94ff1f", "fetched_at": "2026-08-28T04:03:37.486894+00:00", "kind": "readme", "missing": false, "url": "https://github.com/amazon-science/RAGChecker"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:03:37.486894+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:03:37.486894+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-30T06:43:26.315280+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d2a5e03773b47c8a8b75b77edc8db7658ffb4b36ddc61d3ee7a521318e94ff1f", "fetched_at": "2026-08-28T04:03:37.486894+00:00", "kind": "readme", "missing": false, "url": "https://github.com/amazon-science/RAGChecker"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:03:37.486894+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:03:37.486894+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:03:37.486894+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-30T06:43:26.315280+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d2a5e03773b47c8a8b75b77edc8db7658ffb4b36ddc61d3ee7a521318e94ff1f", "fetched_at": "2026-08-28T04:03:37.486894+00:00", "kind": "readme", "missing": false, "url": "https://github.com/amazon-science/RAGChecker"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:03:37.486894+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:03:37.486894+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-30T06:43:26.315280+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d2a5e03773b47c8a8b75b77edc8db7658ffb4b36ddc61d3ee7a521318e94ff1f", "fetched_at": "2026-08-28T04:03:37.486894+00:00", "kind": "readme", "missing": false, "url": "https://github.com/amazon-science/RAGChecker"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-30T06:43:26.315280+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d2a5e03773b47c8a8b75b77edc8db7658ffb4b36ddc61d3ee7a521318e94ff1f", "fetched_at": "2026-08-28T04:03:37.486894+00:00", "kind": "readme", "missing": false, "url": "https://github.com/amazon-science/RAGChecker"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-30T06:43:26.315280+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d2a5e03773b47c8a8b75b77edc8db7658ffb4b36ddc61d3ee7a521318e94ff1f", "fetched_at": "2026-08-28T04:03:37.486894+00:00", "kind": "readme", "missing": false, "url": "https://github.com/amazon-science/RAGChecker"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-30T06:43:26.315280+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "d2a5e03773b47c8a8b75b77edc8db7658ffb4b36ddc61d3ee7a521318e94ff1f", "fetched_at": "2026-08-28T04:03:37.486894+00:00", "kind": "readme", "missing": false, "url": "https://github.com/amazon-science/RAGChecker"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 0, "longevity": 57, "rhythm": 40}, "computed_at": "2026-09-02T17:46:02.011165+00:00", "flags": [], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 800, "days_push": 628, "days_rel": 707, "gap_med": 0, "n_releases_24m": 6}, "score": 25, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}