{"adoption": {"forks": 333, "observed_at": "2026-08-28T04:07:36.175007+00:00", "stars": 2993}, "canonical_url": "https://ross.abutalabs.com/products/gospider", "card": {"archived": false, "artifact_type": "cli-tool", "description": "Gospider - Fast web spider written in Go", "domain": ["security", "crawlers", "developer-tools"], "enriched": true, "function": ["web-scraping", "cli", "parser"], "health_score": 20, "homepage": null, "language": "Go", "license": "MIT", "license_family": "permissive", "maturity": "maintenance", "member_repos": ["jaeles-project/gospider"], "name": "jaeles-project/gospider", "platform": ["cli", "windows", "go"], "pushed_at": "2024-04-21T08:12:13+00:00", "repo": "jaeles-project/gospider", "stars": 2993, "tags": ["bug-bounty", "recon", "osint", "sitemap", "wayback-machine", "link-finder", "subdomain-discovery", "command-line", "linux", "macos", "docker"], "topics": ["spider", "crawler", "go", "gospider", "bugbounty"], "urls": [], "use_cases": ["crawl a website and extract all URLs", "find subdomains and AWS S3 buckets in page sources", "pull archived URLs from the Wayback Machine and Common Crawl", "discover links hidden in JavaScript files", "enumerate sitemap.xml and robots.txt entries", "run recon for bug bounty targets", "crawl multiple sites in parallel with custom headers and proxies"], "what_it_is": "GoSpider is a fast web spider/crawler written in Go that crawls sites in parallel and extracts URLs from sitemaps, robots.txt, JavaScript files, and third-party archives like the Wayback Machine and Common Crawl. It is designed for reconnaissance workflows, especially in bug bounty hunting, with grep-friendly output.", "when_to_avoid": ["you need a full-featured scraping framework with rendering or data extraction pipelines", "you want a polite, rate-limited crawler for production site indexing", "you need a maintained library API rather than a standalone CLI"], "when_to_choose": ["you need a fast, parallel CLI crawler for reconnaissance", "you want URL discovery from JS files and third-party archives in one tool", "you need grep-friendly output to pipe into other recon tools"]}, "data_as_of": "2026-08-30T08:39:29.467469+00:00", "members": [{"path": "/products/gospider", "repo": "jaeles-project/gospider", "role": "main", "score": 23}], "provenance": {"archived": {"kind": "observed", "observed_at": "2026-08-28T04:07:36.175007+00:00", "source": "github"}, "artifact_type": {"confidence": null, "enriched_at": "2026-08-30T07:30:51.522466+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cb7e315ef496c3ba2cb247f7a5f4dd15648557d597831c68ff9e33a5a8735b25", "fetched_at": "2026-08-28T04:07:36.175007+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jaeles-project/gospider"}], "taxonomy_version": 1}, "description": {"kind": "observed", "observed_at": "2026-08-28T04:07:36.175007+00:00", "source": "github"}, "domain": {"confidence": null, "enriched_at": "2026-08-30T07:30:51.522466+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cb7e315ef496c3ba2cb247f7a5f4dd15648557d597831c68ff9e33a5a8735b25", "fetched_at": "2026-08-28T04:07:36.175007+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jaeles-project/gospider"}], "taxonomy_version": 1}, "enriched": {"inputs": [], "kind": "computed", "method": "enrichment_status"}, "function": {"confidence": null, "enriched_at": "2026-08-30T07:30:51.522466+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cb7e315ef496c3ba2cb247f7a5f4dd15648557d597831c68ff9e33a5a8735b25", "fetched_at": "2026-08-28T04:07:36.175007+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jaeles-project/gospider"}], "taxonomy_version": 1}, "health_score": {"inputs": ["days_since_push", "days_since_release", "archived"], "kind": "computed", "method": "health_v1"}, "homepage": {"kind": "observed", "observed_at": "2026-08-28T04:07:36.175007+00:00", "source": "github"}, "language": {"kind": "observed", "observed_at": "2026-08-28T04:07:36.175007+00:00", "source": "github"}, "license": {"kind": "observed", "observed_at": "2026-08-28T04:07:36.175007+00:00", "source": "github"}, "license_family": {"inputs": ["license"], "kind": "computed", "method": "license_family"}, "maturity": {"confidence": null, "enriched_at": "2026-08-30T07:30:51.522466+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cb7e315ef496c3ba2cb247f7a5f4dd15648557d597831c68ff9e33a5a8735b25", "fetched_at": "2026-08-28T04:07:36.175007+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jaeles-project/gospider"}], "taxonomy_version": 1}, "member_repos": {"kind": "observed", "observed_at": "2026-08-28T04:07:36.175007+00:00", "source": "github"}, "name": {"kind": "observed", "observed_at": "2026-08-28T04:07:36.175007+00:00", "source": "github"}, "platform": {"confidence": null, "enriched_at": "2026-08-30T07:30:51.522466+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cb7e315ef496c3ba2cb247f7a5f4dd15648557d597831c68ff9e33a5a8735b25", "fetched_at": "2026-08-28T04:07:36.175007+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jaeles-project/gospider"}], "taxonomy_version": 1}, "pushed_at": {"kind": "observed", "observed_at": "2026-08-28T04:07:36.175007+00:00", "source": "github"}, "repo": {"kind": "observed", "observed_at": "2026-08-28T04:07:36.175007+00:00", "source": "github"}, "stars": {"kind": "observed", "observed_at": "2026-08-28T04:07:36.175007+00:00", "source": "github"}, "tags": {"confidence": null, "enriched_at": "2026-08-30T07:30:51.522466+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cb7e315ef496c3ba2cb247f7a5f4dd15648557d597831c68ff9e33a5a8735b25", "fetched_at": "2026-08-28T04:07:36.175007+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jaeles-project/gospider"}], "taxonomy_version": 1}, "topics": {"kind": "observed", "observed_at": "2026-08-28T04:07:36.175007+00:00", "source": "github"}, "urls": {"kind": "observed", "observed_at": "2026-08-28T04:07:36.175007+00:00", "source": "github"}, "use_cases": {"confidence": null, "enriched_at": "2026-08-30T07:30:51.522466+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cb7e315ef496c3ba2cb247f7a5f4dd15648557d597831c68ff9e33a5a8735b25", "fetched_at": "2026-08-28T04:07:36.175007+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jaeles-project/gospider"}], "taxonomy_version": 1}, "what_it_is": {"confidence": null, "enriched_at": "2026-08-30T07:30:51.522466+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cb7e315ef496c3ba2cb247f7a5f4dd15648557d597831c68ff9e33a5a8735b25", "fetched_at": "2026-08-28T04:07:36.175007+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jaeles-project/gospider"}], "taxonomy_version": 1}, "when_to_avoid": {"confidence": null, "enriched_at": "2026-08-30T07:30:51.522466+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cb7e315ef496c3ba2cb247f7a5f4dd15648557d597831c68ff9e33a5a8735b25", "fetched_at": "2026-08-28T04:07:36.175007+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jaeles-project/gospider"}], "taxonomy_version": 1}, "when_to_choose": {"confidence": null, "enriched_at": "2026-08-30T07:30:51.522466+00:00", "kind": "inferred", "prompt_version": 1, "sources": [{"content_hash": "cb7e315ef496c3ba2cb247f7a5f4dd15648557d597831c68ff9e33a5a8735b25", "fetched_at": "2026-08-28T04:07:36.175007+00:00", "kind": "readme", "missing": false, "url": "https://github.com/jaeles-project/gospider"}], "taxonomy_version": 1}}, "score": {"components": {"activity": 0, "longevity": 100, "rhythm": 8}, "computed_at": "2026-09-02T17:46:02.011165+00:00", "flags": [], "formula": "round(0.45*activity + 0.35*rhythm + 0.20*longevity); archived -> min(score, 10)", "inputs": {"age_days": 2415, "days_push": 864, "days_rel": null, "gap_med": null, "n_releases_24m": 0}, "score": 23, "version": 2}, "staleness": {"enrichment_outdated": false, "low_confidence": false, "scrape_days": 9, "stale_scrape": false}}