{"adoption": {"forks": 458, "observed_at": "2026-08-28T04:06:55.406856+00:00", "stars": 2482}, "canonical_url": "https://ross.abutalabs.com/products/news-please", "card": {"archived": false, "artifact_type": "library", "description": "news-please - an integrated web crawler and information extractor for news that just works", "domain": ["crawlers"], "enriched": true, "function": ["web-scraping", "parser", "nlp"], "health_score": 66, "homepage": null, "language": "Python", "license": "Apache-2.0", "license_family": "permissive", "maturity": "active", "member_repos": ["fhamborg/news-please"], "name": "fhamborg/news-please", "platform": ["python", "cli"], "pushed_at": "2026-04-14T21:17:42+00:00", "repo": "fhamborg/news-please", "stars": 2482, "tags": ["news-crawler", "news-extraction", "article-extraction", "commoncrawl", "rss", "scrapy", "information-extraction", "news-archive", "media-monitoring", "elasticsearch", "natural-language-processing", "data-engineering"], "topics": ["news-crawler", "news-extractor", "crawler", "extractor", "news", "news-websites", "elasticsearch", "json", "python", "nlp", "data-gathering", "news-archive", "news-articles", "commoncrawl", "extract-articles", "extract-information", "news-scraper", "ccnews", "cc-news", "roberta"], "urls": [], "use_cases": ["scrape and extract full text from news article URLs", "crawl an entire news website from its root URL or RSS feed", "build a labeled news dataset for NLP or media research", "download and filter articles from the Common Crawl news archive", "extract metadata like author names and publication dates from articles", "store crawled news articles in Elasticsearch, PostgreSQL, or JSON files", "track how a news article changes over time via repeated crawl revisions"], "what_it_is": "news-please is an open-source Python news crawler and information extractor that pulls structured article data (headline, lead, main text, authors, date, language, images) from almost any news website using RSS feeds, recursive link crawling, or the Common Crawl news archive. It works out of the box via a CLI or as an embeddable library, storing results as JSON or in PostgreSQL, Elasticsearch, or Redis.", "when_to_avoid": ["you need general-purpose web scraping beyond news articles (use Scrapy directly)", "target sites rely heavily on client-side JavaScript rendering", "you need a real-time, high-throughput streaming crawl system", "you only need RSS feed aggregation without full-text extraction"], "when_to_choose": ["you need turnkey news crawling and extraction that works with just a root URL", "you want to mine the Common Crawl news archive with publisher/date filters", "you need structured article attributes (headline, text, authors, dates) rather than raw HTML", "you want a Python library you can embed in your own data pipeline", "you need pluggable storage backends like JSON, PostgreSQL, Elasticsearch, or Redis"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/news-please", "repo": "fhamborg/news-please", "role": "main", "score": 67}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:06:55.406856+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-30T02:28:12.849341+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "462cc1753e666d6c7732765b3d0e5cdcb1e0a96dfeacb980f63bb04e676ef5c4", "fetched_at": "2026-08-28T04:06:55.406856+00:00", "kind": "readme", "missing": false, "url": "https://github.com/fhamborg/news-please"}, {"content_hash": "3e9dc62197f1ca43ab74bbcb944b70afac5d0d58f9d7e2bd385fa61d83b10f45", "fetched_at": "2026-08-29T10:10:18.120930+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/news-please/json"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:06:55.406856+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-30T02:28:12.849341+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "462cc1753e666d6c7732765b3d0e5cdcb1e0a96dfeacb980f63bb04e676ef5c4", "fetched_at": "2026-08-28T04:06:55.406856+00:00", "kind": "readme", "missing": false, "url": "https://github.com/fhamborg/news-please"}, {"content_hash": "3e9dc62197f1ca43ab74bbcb944b70afac5d0d58f9d7e2bd385fa61d83b10f45", "fetched_at": "2026-08-29T10:10:18.120930+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/news-please/json"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-30T02:28:12.849341+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "462cc1753e666d6c7732765b3d0e5cdcb1e0a96dfeacb980f63bb04e676ef5c4", "fetched_at": "2026-08-28T04:06:55.406856+00:00", "kind": "readme", "missing": false, "url": "https://github.com/fhamborg/news-please"}, {"content_hash": "3e9dc62197f1ca43ab74bbcb944b70afac5d0d58f9d7e2bd385fa61d83b10f45", "fetched_at": "2026-08-29T10:10:18.120930+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/news-please/json"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:06:55.406856+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:06:55.406856+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:06:55.406856+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-30T02:28:12.849341+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "462cc1753e666d6c7732765b3d0e5cdcb1e0a96dfeacb980f63bb04e676ef5c4", "fetched_at": "2026-08-28T04:06:55.406856+00:00", "kind": "readme", "missing": false, "url": "https://github.com/fhamborg/news-please"}, {"content_hash": "3e9dc62197f1ca43ab74bbcb944b70afac5d0d58f9d7e2bd385fa61d83b10f45", "fetched_at": "2026-08-29T10:10:18.120930+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/news-please/json"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:06:55.406856+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:06:55.406856+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-30T02:28:12.849341+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "462cc1753e666d6c7732765b3d0e5cdcb1e0a96dfeacb980f63bb04e676ef5c4", "fetched_at": "2026-08-28T04:06:55.406856+00:00", "kind": "readme", "missing": false, "url": "https://github.com/fhamborg/news-please"}, {"content_hash": "3e9dc62197f1ca43ab74bbcb944b70afac5d0d58f9d7e2bd385fa61d83b10f45", "fetched_at": "2026-08-29T10:10:18.120930+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/news-please/json"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:06:55.406856+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:06:55.406856+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:06:55.406856+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-30T02:28:12.849341+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "462cc1753e666d6c7732765b3d0e5cdcb1e0a96dfeacb980f63bb04e676ef5c4", "fetched_at": "2026-08-28T04:06:55.406856+00:00", "kind": "readme", "missing": false, "url": "https://github.com/fhamborg/news-please"}, {"content_hash": "3e9dc62197f1ca43ab74bbcb944b70afac5d0d58f9d7e2bd385fa61d83b10f45", "fetched_at": "2026-08-29T10:10:18.120930+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/news-please/json"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:06:55.406856+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:06:55.406856+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-30T02:28:12.849341+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "462cc1753e666d6c7732765b3d0e5cdcb1e0a96dfeacb980f63bb04e676ef5c4", "fetched_at": "2026-08-28T04:06:55.406856+00:00", "kind": "readme", "missing": false, "url": "https://github.com/fhamborg/news-please"}, {"content_hash": "3e9dc62197f1ca43ab74bbcb944b70afac5d0d58f9d7e2bd385fa61d83b10f45", "fetched_at": "2026-08-29T10:10:18.120930+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/news-please/json"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-30T02:28:12.849341+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "462cc1753e666d6c7732765b3d0e5cdcb1e0a96dfeacb980f63bb04e676ef5c4", "fetched_at": "2026-08-28T04:06:55.406856+00:00", "kind": "readme", "missing": false, "url": "https://github.com/fhamborg/news-please"}, {"content_hash": "3e9dc62197f1ca43ab74bbcb944b70afac5d0d58f9d7e2bd385fa61d83b10f45", "fetched_at": "2026-08-29T10:10:18.120930+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/news-please/json"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-30T02:28:12.849341+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "462cc1753e666d6c7732765b3d0e5cdcb1e0a96dfeacb980f63bb04e676ef5c4", "fetched_at": "2026-08-28T04:06:55.406856+00:00", "kind": "readme", "missing": false, "url": "https://github.com/fhamborg/news-please"}, {"content_hash": "3e9dc62197f1ca43ab74bbcb944b70afac5d0d58f9d7e2bd385fa61d83b10f45", "fetched_at": "2026-08-29T10:10:18.120930+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/news-please/json"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-30T02:28:12.849341+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "462cc1753e666d6c7732765b3d0e5cdcb1e0a96dfeacb980f63bb04e676ef5c4", "fetched_at": "2026-08-28T04:06:55.406856+00:00", "kind": "readme", "missing": false, "url": "https://github.com/fhamborg/news-please"}, {"content_hash": "3e9dc62197f1ca43ab74bbcb944b70afac5d0d58f9d7e2bd385fa61d83b10f45", "fetched_at": "2026-08-29T10:10:18.120930+00:00", "kind": "registry_pypi", "missing": false, "url": "https://pypi.org/pypi/news-please/json"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 77, "longevity": 100, "rhythm": 35}, "computed_at": "2026-09-03T02:20:16.233290+00:00", "flags": ["no_releases"], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 3545, "days_push": 141, "days_rel": null, "gap_med": null, "n_releases_24m": 0}, "score": 67, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}