反爬虫与机器人治理（频率/指纹/动态挑战）最佳实践

背景与价值爬虫与机器人会带来资源消耗与数据风险。统一频率、指纹与挑战策略能降低风险。统一规范频率控制：按IP与指纹进行速率限制。指纹聚合：指纹库聚合异常行为并提升风险评分。动态挑战：高风险请求触发挑战（验证码或前端挑战流）。核心实现速率与指纹class Rate { tokens = new Map<string, { left: number; resetAt: number }>(); now(): number { return Date.now() } refill(k: string, capacity: number, windowMs: number) { const r = this.tokens.get(k); const n = this.now(); if (!r || n >= r.resetAt) this.tokens.set(k, { left: capacity, resetAt: n + windowMs }) } take(k: string): boolean { const r = this.tokens.get(k); if (!r) return false; if (r.left <= 0) return false; r.left--; return true } } class Fingerprints { risky = new Set<string>(); mark(k: string) { this.risky.add(k) } isRisky(k: string): boolean { return this.risky.has(k) } } 挑战门禁type Req = { ip: string; fingerprint: string } function gate(req: Req, rate: Rate, fps: Fingerprints): { ok: boolean; challenge?: 'captcha' | 'front_challenge' } { const key = req.ip + '|' + req.fingerprint rate.refill(key, 60, 60000) if (!rate.take(key)) return { ok: false, challenge: 'captcha' } if (fps.isRisky(req.fingerprint)) return { ok: false, challenge: 'front_challenge' } return { ok: true } } 落地建议按IP与设备指纹联合限速并记录审计，针对异常行为提升风险。高风险流量统一触发动态挑战并收敛接口能力，保护核心资源。验证清单限速是否按联合维度生效；挑战是否在高风险判定时触发。