Continuous Profiling 工程实践：从 eBPF 采样到生产级性能治理体系

%3Ch1%3EContinuous%20Profiling%20%E5%B7%A5%E7%A8%8B%E5%AE%9E%E8%B7%B5%EF%BC%9A%E4%BB%8E%20eBPF%20%E9%87%87%E6%A0%B7%E5%88%B0%E7%94%9F%E4%BA%A7%E7%BA%A7%E6%80%A7%E8%83%BD%E6%B2%BB%E7%90%86%E4%BD%93%E7%B3%BB%3C/h1%3E%0A%3Ch2%3E%E4%B8%80%E3%80%81%E4%B8%BA%E4%BB%80%E4%B9%88%E9%9C%80%E8%A6%81%E6%8C%81%E7%BB%AD%E5%89%96%E6%9E%90%EF%BC%9F%3C/h2%3E%0A%3Cp%3E%E4%BC%A0%E7%BB%9F%E6%80%A7%E8%83%BD%E6%8E%92%E6%9F%A5%E7%9A%84%E5%9B%B0%E5%A2%83%E5%9C%A8%E4%BA%8E%EF%BC%9A%E9%97%AE%E9%A2%98%E5%8F%91%E7%94%9F%E6%97%B6%E4%BD%A0%E4%B8%8D%E5%9C%A8%E5%9C%BA%EF%BC%8C%E7%AD%89%E4%BD%A0%E5%9B%9E%E6%9D%A5%E6%97%B6%E7%8E%B0%E5%9C%BA%E5%B7%B2%E7%A0%B4%E5%9D%8F%E3%80%82%E5%BC%80%E5%8F%91%E8%80%85%E9%80%9A%E5%B8%B8%E4%BE%9D%E8%B5%96%20pprof%E3%80%81perf%20record%20%E7%AD%89%22%E6%A1%88%E5%8F%91%E5%90%8E%22%E9%87%87%E6%A0%B7%E5%B7%A5%E5%85%B7%EF%BC%8C%E4%BD%86%E8%BF%99%E4%BA%9B%E5%B7%A5%E5%85%B7%E5%AD%98%E5%9C%A8%E4%B8%89%E4%B8%AA%E6%A0%B9%E6%9C%AC%E6%80%A7%E7%BC%BA%E9%99%B7%EF%BC%9A%3C/p%3E%0A%3Cp%3E%3Cstrong%3E%E7%AC%AC%E4%B8%80%EF%BC%8C%E9%87%87%E6%A0%B7%E7%AA%97%E5%8F%A3%E6%9C%89%E9%99%90%E3%80%82%3C/strong%3E%20%3Ccode%3Eperf%20record%20-F%2099%3C/code%3E%20%E7%9F%AD%E6%9A%82%E5%BF%AB%E7%85%A7%E9%9A%BE%E4%BB%A5%E6%8D%95%E8%8E%B7%E5%81%B6%E5%8F%91%E5%B0%96%E5%B3%B0%E3%80%82%E4%B8%80%E4%B8%AA%E6%AF%8F%2030%20%E5%88%86%E9%92%9F%E6%89%8D%E5%87%BA%E7%8E%B0%E4%B8%80%E6%AC%A1%E7%9A%84%20goroutine%20%E6%B3%84%E6%BC%8F%EF%BC%8C%E5%9C%A8%205%20%E5%88%86%E9%92%9F%E7%9A%84%E9%87%87%E6%A0%B7%E7%AA%97%E5%8F%A3%E5%86%85%E6%A6%82%E7%8E%87%E4%B8%8A%E5%87%A0%E4%B9%8E%E4%B8%8D%E5%8F%AF%E8%83%BD%E8%A2%AB%E5%91%BD%E4%B8%AD%E3%80%82%3C/p%3E%0A%3Cp%3E%3Cstrong%3E%E7%AC%AC%E4%BA%8C%EF%BC%8C%E4%BE%B5%E5%85%A5%E6%80%A7%E8%BF%87%E9%AB%98%E3%80%82%3C/strong%3E%20%E4%BC%A0%E7%BB%9F%20profiler%EF%BC%88%E5%A6%82%20gperftools%EF%BC%89%E9%9C%80%E8%A6%81%E9%A2%84%E7%BC%96%E8%AF%91%E9%93%BE%E6%8E%A5%E3%80%81%E9%87%8D%E5%90%AF%E8%BF%9B%E7%A8%8B%EF%BC%8C%E5%9C%A8%20K8s%20%E7%8E%AF%E5%A2%83%E4%B8%AD%E6%84%8F%E5%91%B3%E7%9D%80%E8%A6%81%E9%87%8D%E5%BB%BA%20Pod%EF%BC%8C%E8%80%8C%E9%87%8D%E5%BB%BA%20Pod%20%E5%90%8E%E8%B4%9F%E8%BD%BD%E6%A8%A1%E5%BC%8F%E6%94%B9%E5%8F%98%EF%BC%8C%E9%97%AE%E9%A2%98%E5%8F%AF%E8%83%BD%22%E6%B6%88%E5%A4%B1%22%E3%80%82%3C/p%3E%0A%3Cp%3E%3Cstrong%3E%E7%AC%AC%E4%B8%89%EF%BC%8C%E7%BC%BA%E4%B9%8F%E5%8E%86%E5%8F%B2%E5%9F%BA%E7%BA%BF%E3%80%82%3C/strong%3E%20%E5%8D%95%E6%AC%A1%E9%87%87%E6%A0%B7%E5%8F%AA%E8%83%BD%E5%9B%9E%E7%AD%94%22%E7%8E%B0%E5%9C%A8%E6%98%AF%E4%BB%80%E4%B9%88%E7%8A%B6%E6%80%81%22%EF%BC%8C%E6%97%A0%E6%B3%95%E5%9B%9E%E7%AD%94%22%E7%9B%B8%E6%AF%94%E4%B8%8A%E5%91%A8%EF%BC%8C%E5%93%AA%E6%AE%B5%E4%BB%A3%E7%A0%81%E5%8F%98%E6%85%A2%E4%BA%86%22%E3%80%82%3C/p%3E%0A%3Cp%3EContinuous%20Profiling%EF%BC%88%E6%8C%81%E7%BB%AD%E5%89%96%E6%9E%90%EF%BC%89%E8%A7%A3%E5%86%B3%E8%BF%99%E4%BA%9B%E9%97%AE%E9%A2%98%E7%9A%84%E6%96%B9%E5%BC%8F%E6%98%AF%EF%BC%9A%E4%BB%A5%E6%9E%81%E4%BD%8E%E5%BC%80%E9%94%80%EF%BC%88%3C1%25%20CPU%EF%BC%89%E6%8C%81%E7%BB%AD%E9%87%87%E9%9B%86%E7%94%9F%E4%BA%A7%E7%8E%AF%E5%A2%83%E7%9A%84%E6%80%A7%E8%83%BD%E6%95%B0%E6%8D%AE%EF%BC%8C%E9%95%BF%E6%9C%9F%E5%AD%98%E5%82%A8%E5%B9%B6%E5%BB%BA%E7%AB%8B%E6%97%B6%E5%BA%8F%E5%9F%BA%E7%BA%BF%E3%80%82%E6%AD%A3%E5%A6%82%20Datadog%E3%80%81Grafana%20Labs%20%E7%AD%89%E5%9C%A8%E8%BF%87%E5%8E%BB%E4%B8%A4%E5%B9%B4%E5%B0%86%20Continuous%20Profiling%20%E5%88%97%E4%B8%BA%E4%B8%80%E7%BA%A7%E5%8F%AF%E8%A7%82%E6%B5%8B%E6%80%A7%E6%94%AF%E6%9F%B1%E6%89%80%E5%B1%95%E7%A4%BA%E7%9A%84%E8%B6%8B%E5%8A%BF%E2%80%94%E2%80%94%E5%AE%83%E7%BB%88%E4%BA%8E%E4%BB%8E%22%E9%AB%98%E7%BA%A7%E6%8A%80%E5%B7%A7%22%E5%8F%98%E6%88%90%E4%BA%86%22%E5%BF%85%E5%A4%87%E5%9F%BA%E7%A1%80%E8%AE%BE%E6%96%BD%22%E3%80%82%3C/p%3E%0A%3Ch2%3E%E4%BA%8C%E3%80%81%E6%A0%B8%E5%BF%83%E5%89%96%E6%9E%90%E7%B1%BB%E5%9E%8B%E4%B8%8E%E5%BA%95%E5%B1%82%E5%8E%9F%E7%90%86%3C/h2%3E%0A%3Ch3%3E2.1%20On-CPU%20Profiling%3C/h3%3E%0A%3Cp%3EOn-CPU%20%E5%89%96%E6%9E%90%E6%B5%8B%E9%87%8F%E7%9A%84%E6%98%AF%E7%BA%BF%E7%A8%8B%E5%AE%9E%E9%99%85%E5%8D%A0%E7%94%A8%20CPU%20%E6%89%A7%E8%A1%8C%E6%8C%87%E4%BB%A4%E7%9A%84%E6%97%B6%E9%97%B4%E3%80%82%E6%9C%80%E7%BB%8F%E5%85%B8%E7%9A%84%E5%AE%9E%E7%8E%B0%E6%98%AF%20Linux%20perf_event%20%E5%AD%90%E7%B3%BB%E7%BB%9F%E4%BB%A5%E5%9B%BA%E5%AE%9A%E9%A2%91%E7%8E%87%EF%BC%88%E9%80%9A%E5%B8%B8%2099Hz%20%E4%BB%A5%E9%81%BF%E5%85%8D%E4%B8%8E%E5%86%85%E6%A0%B8%E5%AE%9A%E6%97%B6%E5%99%A8%E5%90%8C%E6%AD%A5%EF%BC%89%E8%A7%A6%E5%8F%91%E4%B8%AD%E6%96%AD%EF%BC%8C%E8%AE%B0%E5%BD%95%E5%BD%93%E5%89%8D%E6%8C%87%E4%BB%A4%E6%8C%87%E9%92%88%EF%BC%88IP%EF%BC%89%E5%92%8C%E8%B0%83%E7%94%A8%E6%A0%88%E3%80%82%3C/p%3E%0A%3Cp%3EeBPF%20%E5%AF%B9%20On-CPU%20%E7%9A%84%E5%A2%9E%E5%BC%BA%E5%9C%A8%E4%BA%8E%EF%BC%9A%E9%80%9A%E8%BF%87%20%3Ccode%3EPERF_EVENT%3C/code%3E%20%E7%B1%BB%E5%9E%8B%E7%9A%84%20BPF%20%E7%A8%8B%E5%BA%8F%EF%BC%8C%E5%8F%AF%E4%BB%A5%E5%9C%A8%E4%B8%AD%E6%96%AD%E5%A4%84%E7%90%86%E4%B8%AD%E6%89%A7%E8%A1%8C%E8%87%AA%E5%AE%9A%E4%B9%89%E9%80%BB%E8%BE%91%E3%80%82%E4%BE%8B%E5%A6%82%E8%BF%87%E6%BB%A4%E5%8F%AA%E9%87%87%E6%A0%B7%E7%89%B9%E5%AE%9A%20cgroup%20%E7%9A%84%E8%BF%9B%E7%A8%8B%EF%BC%8C%E6%88%96%E8%AE%B0%E5%BD%95%E9%A2%9D%E5%A4%96%E7%9A%84%E4%B8%8A%E4%B8%8B%E6%96%87%EF%BC%88%E5%A6%82%E5%BD%93%E5%89%8D%E7%B3%BB%E7%BB%9F%E8%B0%83%E7%94%A8%E5%8F%B7%E3%80%81%E6%8C%81%E6%9C%89%E7%9A%84%E9%94%81%EF%BC%89%E3%80%82%3C/p%3E%0A%3Ch3%3E2.2%20Off-CPU%20Profiling%3C/h3%3E%0A%3Cp%3EOff-CPU%20%E6%97%B6%E9%97%B4%E6%8C%87%E7%BA%BF%E7%A8%8B%E8%A2%AB%E9%98%BB%E5%A1%9E%E7%AD%89%E5%BE%85%20I/O%E3%80%81%E9%94%81%E3%80%81%E8%B0%83%E5%BA%A6%E7%9A%84%E6%97%B6%E9%97%B4%E3%80%82%E8%BF%99%E9%83%A8%E5%88%86%E5%9C%A8%E4%BC%A0%E7%BB%9F%20On-CPU%20%E5%89%96%E6%9E%90%E4%B8%AD%E6%98%AF%E4%B8%8D%E5%8F%AF%E8%A7%81%E7%9A%84%E2%80%94%E2%80%94%E8%80%8C%E5%AE%9E%E9%99%85%E7%94%9F%E4%BA%A7%E7%8E%AF%E5%A2%83%E4%B8%AD%EF%BC%8COff-CPU%20%E6%97%B6%E9%97%B4%E6%98%AF%E5%BB%B6%E8%BF%9F%E7%9A%84%E4%B8%BB%E8%A6%81%E6%9D%A5%E6%BA%90%E3%80%82%3C/p%3E%0A%3Cp%3EOff-CPU%20%E5%89%96%E6%9E%90%E7%9A%84%E6%A0%B8%E5%BF%83%E6%96%B9%E6%B3%95%E6%98%AF%EF%BC%9A%E8%B7%9F%E8%B8%AA%20%3Ccode%3Esched_switch%3C/code%3E%20%E4%BA%8B%E4%BB%B6%EF%BC%8C%E8%AE%B0%E5%BD%95%E7%BA%BF%E7%A8%8B%E4%BB%8E%20RUNNING%20%E5%88%87%E6%8D%A2%E5%88%B0%20NOT_RUNNABLE%20%E7%9A%84%E6%97%B6%E9%97%B4%E6%88%B3%E5%92%8C%E8%B0%83%E7%94%A8%E6%A0%88%EF%BC%8C%E5%86%8D%E5%9C%A8%E4%B8%8B%E4%B8%80%20RUNNING%20%E5%88%87%E6%8D%A2%E6%97%B6%E8%AE%A1%E7%AE%97%E5%B7%AE%E5%80%BC%E3%80%82%E8%BF%99%E7%A7%8D%E6%96%B9%E5%BC%8F%E6%9C%89%E4%B8%A4%E4%B8%AA%E9%9A%BE%E7%82%B9%EF%BC%9A%3C/p%3E%0A%3Col%3E%3Cli%3E%3Cstrong%3E%E8%B0%83%E7%94%A8%E6%A0%88%E6%8D%95%E8%8E%B7%E6%97%B6%E6%9C%BA%3C/strong%3E%EF%BC%9A%E5%9C%A8%20%3Ccode%3Esched_switch%3C/code%3E%20%E6%97%B6%EF%BC%8C%E8%A2%AB%E5%88%87%E6%8D%A2%E5%87%BA%E7%9A%84%E7%BA%BF%E7%A8%8B%E5%8F%AF%E8%83%BD%E6%AD%A3%E5%A4%84%E4%BA%8E%E5%86%85%E6%A0%B8%E6%80%81%EF%BC%88%E5%A6%82%E6%89%A7%E8%A1%8C%20%3Ccode%3Efutex_wait%3C/code%3E%EF%BC%89%EF%BC%8C%E6%AD%A4%E6%97%B6%E7%94%A8%E6%88%B7%E6%80%81%E8%B0%83%E7%94%A8%E6%A0%88%E6%98%AF%E7%A8%B3%E5%AE%9A%E7%9A%84%E3%80%82%E4%BD%86%E5%A6%82%E6%9E%9C%E4%BD%BF%E7%94%A8%20%3Ccode%3E%24rsp%3C/code%3E%20%E6%89%8B%E5%8A%A8%E8%A7%A3%E6%A0%88%EF%BC%8C%E9%9C%80%E8%A6%81%E5%8C%BA%E5%88%86%E7%94%A8%E6%88%B7%E6%80%81/%E5%86%85%E6%A0%B8%E6%80%81%E6%A0%88%E8%BE%B9%E7%95%8C%E3%80%82%3C/li%3E%0A%3C/ol%3E%0A%3Col%3E%3Cli%3E%3Cstrong%3E%E5%86%85%E5%AD%98%E5%BC%80%E9%94%80%3C/strong%3E%EF%BC%9A%E6%AF%8F%E4%B8%AA%E7%BA%BF%E7%A8%8B%E7%9A%84%20Off-CPU%20%E7%AD%89%E5%BE%85%E4%B8%8A%E4%B8%8B%E6%96%87%E9%9C%80%E8%A6%81%E6%9A%82%E5%AD%98%EF%BC%8C%E9%AB%98%E5%B9%B6%E5%8F%91%E5%9C%BA%E6%99%AF%E4%B8%8B%E5%8F%AF%E8%83%BD%E8%BE%BE%E5%88%B0%E6%95%B0%E4%B8%87%E6%9D%A1%E7%9B%AE%E3%80%82%3C/li%3E%0A%3C/ol%3E%0A%3Ch3%3E2.3%20%E5%86%85%E5%AD%98%E5%89%96%E6%9E%90%3C/h3%3E%0A%3Cp%3E%E5%86%85%E5%AD%98%E5%89%96%E6%9E%90%E6%9C%89%E4%B8%A4%E7%A7%8D%E4%BA%92%E8%A1%A5%E7%9A%84%E6%96%B9%E6%B3%95%EF%BC%9A%3C/p%3E%0A%3Cul%3E%3Cli%3E%3Cstrong%3E%E5%88%86%E9%85%8D%E9%87%87%E6%A0%B7%3C/strong%3E%EF%BC%9A%E6%AF%8F%E9%9A%94%20N%20%E5%AD%97%E8%8A%82%EF%BC%88%E5%A6%82%20512KB%EF%BC%89%E9%87%87%E6%A0%B7%E4%B8%80%E6%AC%A1%20malloc%20%E8%B0%83%E7%94%A8%EF%BC%8C%E7%94%A8%E5%8A%A0%E6%9D%83%E7%BB%9F%E8%AE%A1%E6%8E%A8%E6%96%AD%E5%86%85%E5%AD%98%E5%88%86%E9%85%8D%E7%83%AD%E7%82%B9%E3%80%82%E8%BF%99%E6%98%AF%20TCMalloc%E3%80%81Jemalloc%20%E7%9A%84%E6%A0%87%E5%87%86%E6%96%B9%E6%B3%95%E3%80%82%3C/li%3E%0A%3Cli%3E%3Cstrong%3E%E7%BC%BA%E9%A1%B5%E8%B7%9F%E8%B8%AA%3C/strong%3E%EF%BC%9A%E9%80%9A%E8%BF%87%20%3Ccode%3Emmap%3C/code%3E%20%E8%B7%9F%E8%B8%AA%E5%92%8C%20page%20fault%20%E4%BA%8B%E4%BB%B6%EF%BC%8C%E5%AE%9A%E4%BD%8D%E5%AE%9E%E9%99%85%20RSS%20%E6%B6%88%E8%80%97%E7%9A%84%E4%BB%A3%E7%A0%81%E8%B7%AF%E5%BE%84%E3%80%82%3C/li%3E%0A%3C/ul%3E%0A%3Cp%3EeBPF%20%E5%9C%A8%E5%86%85%E5%AD%98%E5%89%96%E6%9E%90%E4%B8%AD%E7%9A%84%E7%8B%AC%E7%89%B9%E4%BC%98%E5%8A%BF%E6%98%AF%EF%BC%9A%E5%8F%AF%E4%BB%A5%E8%B7%9F%E8%B8%AA%E6%9C%AA%E5%AF%BC%E5%87%BA%E7%AC%A6%E5%8F%B7%E7%9A%84%E5%86%85%E6%A0%B8%E5%88%86%E9%85%8D%E8%B7%AF%E5%BE%84%EF%BC%88%E5%A6%82%20%3Ccode%3Ekmem_cache_alloc%3C/code%3E%EF%BC%89%EF%BC%8C%E8%BF%99%E5%9C%A8%E6%8E%92%E6%9F%A5%E5%86%85%E6%A0%B8%E6%80%81%E5%86%85%E5%AD%98%E6%B3%84%E6%BC%8F%E6%97%B6%E4%B8%8D%E5%8F%AF%E6%88%96%E7%BC%BA%E3%80%82%3C/p%3E%0A%3Ch3%3E2.4%20%E9%94%81%E5%89%96%E6%9E%90%3C/h3%3E%0A%3Cp%3E%E9%94%81%E7%AB%9E%E4%BA%89%E5%89%96%E6%9E%90%E5%9C%A8%20Go%20%E7%AD%89%E9%AB%98%E5%BA%A6%E5%B9%B6%E5%8F%91%E7%9A%84%E8%BF%90%E8%A1%8C%E6%97%B6%E4%B8%AD%E5%B0%A4%E4%B8%BA%E9%87%8D%E8%A6%81%E3%80%82eBPF%20%E5%8F%AF%E4%BB%A5%E8%B7%9F%E8%B8%AA%20%3Ccode%3Efutex%3C/code%3E%20%E7%B3%BB%E7%BB%9F%E8%B0%83%E7%94%A8%E7%9A%84%20WAIT%20%E5%92%8C%20WAKE%20%E4%BA%8B%E4%BB%B6%EF%BC%8C%E7%BB%93%E5%90%88%E7%94%A8%E6%88%B7%E6%80%81%E8%B0%83%E7%94%A8%E6%A0%88%E7%B2%BE%E7%A1%AE%E5%AE%9A%E4%BD%8D%E7%AB%9E%E4%BA%89%E6%9C%80%E6%BF%80%E7%83%88%E7%9A%84%E9%94%81%E3%80%82%3C/p%3E%0A%3Ch2%3E%E4%B8%89%E3%80%81eBPF%20%E5%AE%9E%E6%88%98%EF%BC%9AOff-CPU%20%E5%88%86%E6%9E%90%E5%99%A8%E4%BB%8E%E9%9B%B6%E5%86%99%E8%B5%B7%3C/h2%3E%0A%3Cp%3E%E4%B8%8B%E9%9D%A2%E4%BB%A5%20BCC%20%E5%92%8C%20libbpf%20%E4%B8%BA%E4%BE%8B%EF%BC%8C%E5%B1%95%E7%A4%BA%E4%B8%80%E4%B8%AA%E6%9C%80%E5%B0%8F%E5%8F%AF%E5%B7%A5%E4%BD%9C%E7%9A%84%20Off-CPU%20%E5%88%86%E6%9E%90%E5%99%A8%E3%80%82%3C/p%3E%0A%3Ch3%3E3.1%20BCC%20%E5%BF%AB%E9%80%9F%E5%8E%9F%E5%9E%8B%3C/h3%3E%0A%3Cpre%3E%3Ccode%20class%3D%22language-python%22%3E%0A%23%21/usr/bin/env%20python3%0Afrom%20bcc%20import%20BPF%0Aimport%20ctypes%20as%20ct%0A%0A%23%20BPF%20%E5%86%85%E6%A0%B8%E6%80%81%E7%A8%8B%E5%BA%8F%0Abpf_text%20%3D%20%22%22%22%0A%23include%20%26lt%3Buapi/linux/ptrace.h%26gt%3B%0A%23include%20%26lt%3Blinux/sched.h%26gt%3B%0A%0A//%20%E5%AD%98%E5%82%A8%E6%AF%8F%E4%B8%AA%E7%BA%BF%E7%A8%8B%E7%9A%84%20Off-CPU%20%E8%B5%B7%E5%A7%8B%E6%97%B6%E9%97%B4%0ABPF_HASH%28start%2C%20u32%2C%20u64%29%3B%0A//%20%E8%AE%B0%E5%BD%95%20Off-CPU%20%E6%97%B6%E9%95%BF%E5%88%86%E5%B8%83%EF%BC%8Ckey%20%3D%20%28pid%2C%20kernel_stack_id%2C%20user_stack_id%29%0ABPF_HASH%28dist%2C%20struct%20key_t%2C%20u64%29%3B%0A%0Astruct%20key_t%20%7B%0A%20%20%20%20u32%20pid%3B%0A%20%20%20%20int%20kernel_stack_id%3B%0A%20%20%20%20int%20user_stack_id%3B%0A%7D%3B%0A%0ABPF_STACK_TRACE%28stack_traces%2C%2016384%29%3B%0A%0A//%20%E7%BA%BF%E7%A8%8B%E8%A2%AB%E5%88%87%E6%8D%A2%E5%87%BA%20CPU%20%E6%97%B6%E8%B0%83%E7%94%A8%0Aint%20trace_sched_switch%28struct%20pt_regs%20%2Aargs%2C%20bool%20preempt%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20struct%20task_struct%20%2Aprev%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20struct%20task_struct%20%2Anext%29%20%7B%0A%20%20%20%20u32%20pid%20%3D%20prev-%26gt%3Bpid%3B%0A%20%20%20%20u64%20ts%20%3D%20bpf_ktime_get_ns%28%29%3B%0A%20%20%20%20%0A%20%20%20%20//%20%E5%8F%AA%E8%BF%BD%E8%B8%AA%20RUNNING%20-%26gt%3B%20NOT_RUNNING%20%E7%9A%84%E8%BD%AC%E6%8D%A2%EF%BC%88%E5%8D%B3%20prev%20%E7%8A%B6%E6%80%81%20%21%3D%20TASK_RUNNING%EF%BC%89%0A%20%20%20%20if%20%28prev-%26gt%3Bstate%20%3D%3D%20TASK_RUNNING%29%20%7B%0A%20%20%20%20%20%20%20%20//%20%E5%AE%9E%E9%99%85%E4%B8%8A%20state%20%E5%9C%A8%E8%B0%83%E7%94%A8%E6%97%B6%E5%B7%B2%E6%9B%B4%E6%96%B0%EF%BC%8C%E9%9C%80%E4%BD%BF%E7%94%A8%E6%9B%B4%E7%B2%BE%E7%A1%AE%E7%9A%84%E5%88%A4%E6%96%AD%E6%96%B9%E5%BC%8F%0A%20%20%20%20%20%20%20%20return%200%3B%0A%20%20%20%20%7D%0A%20%20%20%20%0A%20%20%20%20//%20%E8%AE%B0%E5%BD%95%E5%88%87%E6%8D%A2%E5%87%BA%E6%97%B6%E9%97%B4%0A%20%20%20%20start.update%28%26pid%2C%20%26ts%29%3B%0A%20%20%20%20return%200%3B%0A%7D%0A%0A//%20%E7%BA%BF%E7%A8%8B%E8%A2%AB%E5%88%87%E6%8D%A2%E5%9B%9E%20CPU%20%E6%97%B6%E8%AE%A1%E7%AE%97%E5%B7%AE%E5%80%BC%0Aint%20trace_sched_wakeup%28struct%20pt_regs%20%2Aargs%2C%20struct%20task_struct%20%2Ap%29%20%7B%0A%20%20%20%20u32%20pid%20%3D%20p-%26gt%3Bpid%3B%0A%20%20%20%20u64%20%2Atsp%20%3D%20start.lookup%28%26pid%29%3B%0A%20%20%20%20if%20%28tsp%20%3D%3D%200%29%0A%20%20%20%20%20%20%20%20return%200%3B%0A%20%20%20%20%0A%20%20%20%20u64%20delta%20%3D%20bpf_ktime_get_ns%28%29%20-%20%2Atsp%3B%0A%20%20%20%20if%20%28delta%20%26lt%3B%201000%29%20%20//%20%E8%BF%87%E6%BB%A4%20%26lt%3B1us%20%E7%9A%84%E5%99%AA%E5%A3%B0%0A%20%20%20%20%20%20%20%20return%200%3B%0A%20%20%20%20%0A%20%20%20%20//%20%E6%9E%84%E5%BB%BA%E8%81%9A%E5%90%88%20key%0A%20%20%20%20struct%20key_t%20key%20%3D%20%7B%7D%3B%0A%20%20%20%20key.pid%20%3D%20pid%3B%0A%20%20%20%20key.kernel_stack_id%20%3D%20stack_traces.get_stackid%28args%2C%20BPF_F_USER_STACK%29%3B%0A%20%20%20%20key.user_stack_id%20%3D%20stack_traces.get_stackid%28args%2C%20BPF_F_USER_STACK%29%3B%0A%20%20%20%20%0A%20%20%20%20//%20%E7%B4%AF%E5%8A%A0%0A%20%20%20%20u64%20%2Aexisting%20%3D%20dist.lookup%28%26key%29%3B%0A%20%20%20%20if%20%28existing%29%20%7B%0A%20%20%20%20%20%20%20%20%28%2Aexisting%29%20%2B%3D%20delta%3B%0A%20%20%20%20%7D%20else%20%7B%0A%20%20%20%20%20%20%20%20dist.update%28%26key%2C%20%26delta%29%3B%0A%20%20%20%20%7D%0A%20%20%20%20%0A%20%20%20%20start.delete%28%26pid%29%3B%0A%20%20%20%20return%200%3B%0A%7D%0A%22%22%22%0A%0Ab%20%3D%20BPF%28text%3Dbpf_text%29%0Ab.attach_tracepoint%28name%3D%22sched%3Asched_switch%22%2C%20fn_name%3D%22trace_sched_switch%22%29%0Ab.attach_tracepoint%28name%3D%22sched%3Asched_wakeup%22%2C%20fn_name%3D%22trace_sched_wakeup%22%29%0A%0Aprint%28%22Tracing%20Off-CPU%20time...%20Ctrl-C%20to%20end.%22%29%0A%0Atry%3A%0A%20%20%20%20sleep%2830%29%0Aexcept%20KeyboardInterrupt%3A%0A%20%20%20%20pass%0A%0A%23%20%E8%BE%93%E5%87%BA%E7%BB%93%E6%9E%9C%0Adist%20%3D%20b.get_table%28%22dist%22%29%0Afor%20k%2C%20v%20in%20sorted%28dist.items%28%29%2C%20key%3Dlambda%20x%3A%20-x.value%29%3A%0A%20%20%20%20comm%20%3D%20b.get_sysname%28k.pid%29%20if%20k.pid%20%3D%3D%200%20else%20%5C%0A%20%20%20%20%20%20%20%20%20%20%20%28b%22unknown%22%29.decode%28%27utf-8%27%2C%20errors%3D%27replace%27%29%0A%20%20%20%20if%20k.pid%20%26gt%3B%200%3A%0A%20%20%20%20%20%20%20%20try%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20with%20open%28f%22/proc/%7Bk.pid%7D/comm%22%2C%20%22r%22%29%20as%20f%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20comm%20%3D%20f.read%28%29.strip%28%29%0A%20%20%20%20%20%20%20%20except%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20comm%20%3D%20%22%3F%3F%3F%22%0A%20%20%20%20print%28f%22%7Bcomm%3A%26lt%3B16%7D%20PID%3D%7Bk.pid%3A%26lt%3B6%7D%20avg_offcpu%3D%7Bv.value//1000%7D%20us%22%29%0A%60%60%60%3C/code%3E%3C/pre%3E%0A%3Ch3%3E3.2%20libbpf%20%2B%20BPF%20CO-RE%20%E7%94%9F%E4%BA%A7%E7%BA%A7%E5%AE%9E%E7%8E%B0%3C/h3%3E%0A%3Cp%3E%E4%B8%8A%E8%BF%B0%20BCC%20%E5%8E%9F%E5%9E%8B%E9%80%82%E5%90%88%E5%BC%80%E5%8F%91%E8%B0%83%E8%AF%95%EF%BC%8C%E4%BD%86%E7%94%9F%E4%BA%A7%E7%8E%AF%E5%A2%83%E9%9C%80%E8%A6%81%20BPF%20CO-RE%EF%BC%88Compile%20Once%2C%20Run%20Everywhere%EF%BC%89%E4%BF%9D%E8%AF%81%E8%B7%A8%E5%86%85%E6%A0%B8%E7%89%88%E6%9C%AC%E5%85%BC%E5%AE%B9%E3%80%82libbpf%20%E9%85%8D%E5%90%88%20BTF%20%E4%BF%A1%E6%81%AF%E5%8F%AF%E4%BB%A5%E7%BC%96%E8%AF%91%E4%B8%80%E6%AC%A1%20ELF%20%E6%96%87%E4%BB%B6%EF%BC%8C%E5%9C%A8%E4%B8%8D%E5%90%8C%E5%86%85%E6%A0%B8%E7%89%88%E6%9C%AC%EF%BC%88%E9%9C%80%20%3E%3D%205.4%20%E4%B8%94%E5%90%AF%E7%94%A8%20CONFIG_DEBUG_INFO_BTF%EF%BC%89%E4%B8%8A%E5%8A%A0%E8%BD%BD%E3%80%82%3C/p%3E%0A%3Cp%3E%E5%85%B3%E9%94%AE%E5%B7%AE%E5%BC%82%E4%BD%93%E7%8E%B0%E5%9C%A8%E8%B0%83%E7%94%A8%E6%A0%88%E8%8E%B7%E5%8F%96%EF%BC%9A%3C/p%3E%0A%3Cpre%3E%3Ccode%20class%3D%22language-c%22%3E%0A//%20%E4%BD%BF%E7%94%A8%20BPF_CORE_READ%20%E5%AE%8F%E8%87%AA%E5%8A%A8%E9%80%82%E9%85%8D%E5%86%85%E6%A0%B8%20task_struct%20%E5%8F%98%E5%8C%96%0Astruct%20task_struct%20%2Aprev%20%3D%20%28struct%20task_struct%20%2A%29PT_REGS_PARM3%28ctx%29%3B%0Au32%20pid%20%3D%20BPF_CORE_READ%28prev%2C%20pid%29%3B%0Au64%20ts%20%3D%20bpf_ktime_get_ns%28%29%3B%0A%0A//%20%E8%8E%B7%E5%8F%96%E7%94%A8%E6%88%B7%E6%80%81%E8%B0%83%E7%94%A8%E6%A0%88%0Astruct%20key_t%20key%20%3D%20%7B%7D%3B%0Akey.pid%20%3D%20bpf_get_current_pid_tgid%28%29%20%26gt%3B%26gt%3B%2032%3B%0Akey.user_stack_id%20%3D%20bpf_get_stackid%28ctx%2C%20%26stack_traces%2C%20BPF_F_USER_STACK%29%3B%0A%60%60%60%3C/code%3E%3C/pre%3E%0A%3Cp%3E%3Ccode%3EBPF_F_USER_STACK%3C/code%3E%20%E6%A0%87%E5%BF%97%E4%BC%9A%E5%B0%9D%E8%AF%95%E8%B5%B0%20ORC%EF%BC%88ORC%20Unwinder%EF%BC%89%E6%88%96%E4%BC%A0%E7%BB%9F%20FP%EF%BC%88Frame%20Pointer%EF%BC%89%E4%B8%A4%E7%A7%8D%E6%A0%88%E5%9B%9E%E6%BA%AF%E6%96%B9%E5%BC%8F%E3%80%82%E7%8E%B0%E4%BB%A3%E5%8F%91%E8%A1%8C%E7%89%88%EF%BC%88Ubuntu%2022.04%2B%E3%80%81Debian%2012%2B%EF%BC%89%E9%BB%98%E8%AE%A4%E5%90%AF%E7%94%A8%20ORC%EF%BC%8C%E6%80%A7%E8%83%BD%E4%BC%98%E4%BA%8E%20FP%20%E4%B8%94%E4%B8%8D%E9%9C%80%E8%A6%81%20%3Ccode%3E-fno-omit-frame-pointer%3C/code%3E%20%E7%BC%96%E8%AF%91%E6%A0%87%E5%BF%97%E3%80%82%3C/p%3E%0A%3Ch2%3E%E5%9B%9B%E3%80%81%E7%81%AB%E7%84%B0%E5%9B%BE%E6%B7%B1%E5%BA%A6%E5%88%86%E6%9E%90%3C/h2%3E%0A%3Cp%3EBrendan%20Gregg%20%E5%8F%91%E6%98%8E%E7%9A%84%20Flame%20Graph%EF%BC%88%E7%81%AB%E7%84%B0%E5%9B%BE%EF%BC%89%E6%98%AF%20Continuous%20Profiling%20%E7%9A%84%E6%A0%87%E5%87%86%E5%8F%AF%E8%A7%86%E5%8C%96%E3%80%82%E6%8E%8C%E6%8F%A1%E4%B8%89%E7%A7%8D%E8%BF%9B%E9%98%B6%E7%81%AB%E7%84%B0%E5%9B%BE%E6%98%AF%E9%AB%98%E7%BA%A7%E5%B7%A5%E7%A8%8B%E5%B8%88%E7%9A%84%E5%BF%85%E5%A4%87%E6%8A%80%E8%83%BD%EF%BC%9A%3C/p%3E%0A%3Ch3%3E4.1%20Differential%20Flame%20Graph%EF%BC%88%E5%B7%AE%E5%BC%82%E7%81%AB%E7%84%B0%E5%9B%BE%EF%BC%89%3C/h3%3E%0A%3Cp%3E%E5%BD%93%E4%BD%A0%E9%87%8D%E6%9E%84%E4%BA%86%E4%BB%A3%E7%A0%81%EF%BC%8C%E6%83%B3%E7%A1%AE%E8%AE%A4%E4%BC%98%E5%8C%96%E6%98%AF%E5%90%A6%E7%94%9F%E6%95%88%E6%97%B6%EF%BC%8C%E5%AF%B9%E6%AF%94%E4%B8%A4%E6%AC%A1%E9%87%87%E6%A0%B7%E7%9A%84%20Diff%20Flame%20Graph%20%E6%9C%80%E6%9C%89%E4%BB%B7%E5%80%BC%E3%80%82%E7%94%9F%E6%88%90%E6%96%B9%E5%BC%8F%EF%BC%9A%3C/p%3E%0A%3Cpre%3E%3Ccode%20class%3D%22language-bash%22%3E%0A%23%20%E5%85%88%E5%88%86%E5%88%AB%E9%87%87%E9%9B%86%0Aperf%20record%20-F%2099%20-p%20%24PID%20-g%20--%20sleep%2030%20-o%20perf_before.data%0Aperf%20inject%20-i%20perf_before.data%20-o%20perf_before.jit.data%0Astackcollapse-perf.pl%20perf_before.jit.data%20%26gt%3B%20perf_before.folded%0A%0A%23%20%E5%BA%94%E7%94%A8%E6%96%B0%E4%BB%A3%E7%A0%81%E5%90%8E%E5%86%8D%E6%AC%A1%E9%87%87%E9%9B%86%EF%BC%8C%E7%94%9F%E6%88%90%20folded%20%E6%96%87%E4%BB%B6%0A%23%20...%0A%23%20%E7%94%9F%E6%88%90%E5%B7%AE%E5%BC%82%E7%81%AB%E7%84%B0%E5%9B%BE%0Adifffolded.pl%20perf_before.folded%20perf_after.folded%20%7C%20flamegraph.pl%20%26gt%3B%20diff.svg%0A%60%60%60%3C/code%3E%3C/pre%3E%0A%3Cp%3E%E9%A2%9C%E8%89%B2%E8%AF%AD%E4%B9%89%EF%BC%9A%E7%BA%A2%E8%89%B2%E8%A1%A8%E7%A4%BA%E6%96%B0%E5%A2%9E%E7%9A%84%E7%83%AD%E7%82%B9%EF%BC%8C%E8%93%9D%E8%89%B2%E8%A1%A8%E7%A4%BA%E5%B7%B2%E6%B6%88%E9%99%A4%E7%9A%84%E7%83%AD%E7%82%B9%EF%BC%8C%E6%97%A0%E8%89%B2%E8%A1%A8%E7%A4%BA%E6%97%A0%E5%8F%98%E5%8C%96%E3%80%82%E9%81%BF%E5%85%8D%E8%A2%AB%22%E6%80%BB%E9%AB%98%E5%BA%A6%E5%B7%AE%E5%BC%82%22%E8%AF%AF%E5%AF%BC%E2%80%94%E2%80%94%E5%85%B3%E6%B3%A8%E7%9A%84%E5%BA%94%E6%98%AF%E5%85%B7%E4%BD%93%E9%A2%9C%E8%89%B2%E5%9D%97%E7%9A%84%E9%9D%A2%E7%A7%AF%E5%8F%98%E5%8C%96%E3%80%82%3C/p%3E%0A%3Ch3%3E4.2%20Off-CPU%20Flame%20Graph%3C/h3%3E%0A%3Cp%3EOff-CPU%20%E7%81%AB%E7%84%B0%E5%9B%BE%E7%9A%84%20x%20%E8%BD%B4%E4%B8%8D%E6%98%AF%E6%A0%B7%E6%9C%AC%E6%95%B0%EF%BC%8C%3Cstrong%3E%E8%80%8C%E6%98%AF%E9%98%BB%E5%A1%9E%E6%97%B6%E9%97%B4%E7%9A%84%E5%8A%A0%E6%9D%83%E5%92%8C%3C/strong%3E%E3%80%82%E8%BF%99%E6%84%8F%E5%91%B3%E7%9D%80%E4%B8%80%E4%B8%AA%E5%8F%AA%E5%87%BA%E7%8E%B0%201%20%E6%AC%A1%E4%BD%86%E9%98%BB%E5%A1%9E%2010%20%E7%A7%92%E7%9A%84%E8%B0%83%E7%94%A8%E8%B7%AF%E5%BE%84%E4%BC%9A%E6%98%8E%E6%98%BE%E5%AE%BD%E4%BA%8E%E5%87%BA%E7%8E%B0%201000%20%E6%AC%A1%E4%BD%86%E6%AF%8F%E6%AC%A1%E9%98%BB%E5%A1%9E%201%20%E6%AF%AB%E7%A7%92%E7%9A%84%E8%B7%AF%E5%BE%84%E3%80%82%E9%98%85%E8%AF%BB%20Off-CPU%20%E7%81%AB%E7%84%B0%E5%9B%BE%E6%97%B6%EF%BC%8C%E5%BA%94%E4%BC%98%E5%85%88%E5%85%B3%E6%B3%A8%E5%AE%BD%E5%9D%97%E8%80%8C%E9%9D%9E%E9%AB%98%E9%A2%91%E5%9D%97%E2%80%94%E2%80%94%E8%BF%99%E4%B8%8E%20On-CPU%20%E7%9A%84%E5%88%86%E6%9E%90%E7%AD%96%E7%95%A5%E6%81%B0%E5%A5%BD%E7%9B%B8%E5%8F%8D%E3%80%82%3C/p%3E%0A%3Ch3%3E4.3%20Memory%20Flame%20Graph%3C/h3%3E%0A%3Cp%3E%E5%B1%95%E7%A4%BA%E5%88%86%E9%85%8D%E8%B0%83%E7%94%A8%E8%B7%AF%E5%BE%84%E8%80%8C%E9%9D%9E%E5%AE%9E%E9%99%85%E4%BD%BF%E7%94%A8%E8%B7%AF%E5%BE%84%E3%80%82%20memory%20%E7%81%AB%E7%84%B0%E5%9B%BE%E7%9A%84%E9%99%B7%E9%98%B1%E6%98%AF%EF%BC%9A%3Cstrong%3E%E9%A2%91%E7%B9%81%E5%88%86%E9%85%8D%E5%B0%8F%E5%AF%B9%E8%B1%A1%3C/strong%3E%E7%9A%84%E8%B7%AF%E5%BE%84%E5%8F%AF%E8%83%BD%E6%AF%94%3Cstrong%3E%E5%81%B6%E5%B0%94%E5%88%86%E9%85%8D%E5%A4%A7%E5%AF%B9%E8%B1%A1%3C/strong%3E%E7%9A%84%E8%B7%AF%E5%BE%84%E6%9B%B4%E5%AE%BD%EF%BC%8C%E4%BD%86%E4%BB%8E%20GC%20%E5%8E%8B%E5%8A%9B%E8%A7%92%E5%BA%A6%E5%89%8D%E8%80%85%E5%BD%B1%E5%93%8D%E6%9B%B4%E5%A4%A7%E3%80%82%E5%9B%A0%E6%AD%A4%E9%9C%80%E7%BB%93%E5%90%88%E5%88%86%E9%85%8D%E9%A2%91%E6%AC%A1%EF%BC%88allocation%20rate%EF%BC%89%E8%80%8C%E9%9D%9E%E4%BB%85%E7%9C%8B%E6%80%BB%E5%A4%A7%E5%B0%8F%E3%80%82%3C/p%3E%0A%3Ch2%3E%E4%BA%94%E3%80%81%E7%94%9F%E4%BA%A7%E7%BA%A7%E6%9E%B6%E6%9E%84%E8%AE%BE%E8%AE%A1%3C/h2%3E%0A%3Ch3%3E5.1%20%E6%A0%B8%E5%BF%83%E7%BB%84%E4%BB%B6%E6%8B%93%E6%89%91%3C/h3%3E%0A%3Cpre%3E%3Ccode%3E%0A%E2%94%8C%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%90%20%20%20%20%20%E2%94%8C%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%90%20%20%20%20%20%E2%94%8C%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%90%0A%E2%94%82%20eBPF%20Agent%20%20%E2%94%82%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%96%B6%E2%94%82%20%20Collector%20%20%E2%94%82%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%96%B6%E2%94%82%20%20%20Storage%20%20%20%E2%94%82%0A%E2%94%82%20%28DaemonSet%29%20%E2%94%82%20%20%20%20%20%E2%94%82%20%20%28%E5%8F%AF%E9%80%89%29%20%20%20%20%20%E2%94%82%20%20%20%20%20%E2%94%82%20%28Parca/PSPS%29%E2%94%82%0A%E2%94%94%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%98%20%20%20%20%20%E2%94%94%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%98%20%20%20%20%20%E2%94%94%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%98%0A%20%20%20%20%20%20%20%E2%94%82%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%94%82%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%94%82%0A%20%20%20%20%20%20%20%E2%96%BC%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%96%BC%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%96%BC%0A%20%20node-level%20%20%20%20%20%20%20%20%20%20%20label-based%20%20%20%20%20%20%20%20pprof%20format%0A%20%20croup%20filter%20%20%20%20%20%20%20%20%20remote%20write%20%20%20%20%20%20%20%E5%8E%8B%E7%BC%A9%E5%AD%98%E5%82%A8%0A%60%60%60%3C/code%3E%3C/pre%3E%0A%3Ch3%3E5.2%20%E8%B5%84%E6%BA%90%E5%BC%80%E9%94%80%E6%8E%A7%E5%88%B6%3C/h3%3E%0A%3Cp%3E%E7%94%9F%E4%BA%A7%E7%8E%AF%E5%A2%83%20Continuous%20Profiling%20%E5%BF%85%E9%A1%BB%E4%B8%A5%E6%A0%BC%E6%8E%A7%E5%88%B6%E5%BC%80%E9%94%80%EF%BC%8C%E5%90%A6%E5%88%99%E4%BC%9A%E5%BC%95%E5%8F%91%22%E8%A7%82%E6%B5%8B%E6%80%A7%E5%AF%BC%E8%87%B4%E7%9A%84%E6%80%A7%E8%83%BD%E6%95%85%E9%9A%9C%22%EF%BC%9A%3C/p%3E%0A%3Cp%3E%7C%20%E7%BB%B4%E5%BA%A6%20%7C%20%E5%9F%BA%E7%BA%BF%E5%BC%80%E9%94%80%20%7C%20%E8%B0%83%E4%BC%98%E5%8F%82%E6%95%B0%20%7C%3C/p%3E%0A%3Cp%3E%7C------%7C---------%7C---------%7C%3C/p%3E%0A%3Cp%3E%7C%20CPU%20%7C%200.3-0.8%25%20%7C%20%E9%87%87%E6%A0%B7%E9%A2%91%E7%8E%87%E4%BB%8E%2099Hz%20%E9%99%8D%E8%87%B3%2049Hz%20%7C%3C/p%3E%0A%3Cp%3E%7C%20%E5%86%85%E5%AD%98%20%7C%2010-30MB%20%7C%20%E7%BC%A9%E5%B0%8F%20%3Ccode%3Estack_traces%3C/code%3E%20map%20%E5%AE%B9%E9%87%8F%20%7C%3C/p%3E%0A%3Cp%3E%7C%20%E7%A3%81%E7%9B%98%20%7C%205-20%20MB/hour%20%7C%20%E4%BD%BF%E7%94%A8%20pprof-gzip%20%E5%8E%8B%E7%BC%A9%20%7C%3C/p%3E%0A%3Cp%3E%7C%20%E7%BD%91%E7%BB%9C%20%7C%205-20%20MB/hour%20%7C%20%E4%BB%85%E4%B8%8A%E4%BC%A0%20diff%20%E8%80%8C%E9%9D%9E%E5%85%A8%E9%87%8F%20%7C%3C/p%3E%0A%3Cp%3E%E5%85%B3%E9%94%AE%E4%BC%98%E5%8C%96%E7%82%B9%EF%BC%9A%E4%BD%BF%E7%94%A8%20cgroup%20eBPF%20filter%EF%BC%88%3Ccode%3Ebpf_cgroup_acquire%3C/code%3E%EF%BC%89%E5%B0%86%E9%87%87%E6%A0%B7%E9%99%90%E5%88%B6%E5%9C%A8%E5%90%8C%E4%B8%80%E8%8A%82%E7%82%B9%20Namespace%20%E5%86%85%E7%9A%84%E7%89%B9%E5%AE%9A%20Pod%20%E7%BB%84%EF%BC%8C%E9%81%BF%E5%85%8D%E5%9B%A0%E5%85%A8%E5%B1%80%E9%87%87%E6%A0%B7%E5%AF%BC%E8%87%B4%E7%9A%84%20Kubelet%20%E7%AD%89%E7%B3%BB%E7%BB%9F%E7%BB%84%E4%BB%B6%E6%80%A7%E8%83%BD%E6%8A%96%E5%8A%A8%E3%80%82%3C/p%3E%0A%3Ch3%3E5.3%20%E5%AD%98%E5%82%A8%E9%80%89%E5%9E%8B%E5%AF%B9%E6%AF%94%3C/h3%3E%0A%3Cp%3E%3Cstrong%3EParca%3C/strong%3E%EF%BC%88Grafana%20%E7%94%9F%E6%80%81%EF%BC%89%EF%BC%9A%E5%8E%9F%E7%94%9F%E6%94%AF%E6%8C%81%20pprof%20%E6%A0%BC%E5%BC%8F%EF%BC%8C%E4%BD%BF%E7%94%A8%20Apache%20Arrow%20%2B%20Object%20Store%EF%BC%88S3/GCS%EF%BC%89%E5%90%8E%E7%AB%AF%EF%BC%8C%E6%94%AF%E6%8C%81%E6%8C%89%E6%A0%87%E7%AD%BE%E7%BB%B4%E5%BA%A6%E6%9F%A5%E8%AF%A2%E3%80%82%E9%83%A8%E7%BD%B2%20manifests%20%E7%BA%A6%20500%20%E8%A1%8C%20YAML%EF%BC%8C%E4%B8%AD%E7%AD%89%E8%BF%90%E7%BB%B4%E5%A4%8D%E6%9D%82%E5%BA%A6%E3%80%82%3C/p%3E%0A%3Cp%3E%3Cstrong%3EPyroscope%3C/strong%3E%EF%BC%9A%E5%9F%BA%E4%BA%8E%20go%EF%BC%8C%E5%8D%95%E4%BA%8C%E8%BF%9B%E5%88%B6%E9%83%A8%E7%BD%B2%E3%80%82%E4%BD%BF%E7%94%A8%20BadgerDB%20%2B%20Phlare%20%E6%9E%B6%E6%9E%84%E7%9A%84%E6%A0%91%E7%8A%B6%E7%B4%A2%E5%BC%95%EF%BC%8C%E6%9F%A5%E8%AF%A2%E5%BF%AB%E4%BD%86%E6%89%A9%E5%B1%95%E6%80%A7%E5%8F%97%E9%99%90%E3%80%82%E9%80%82%E5%90%88%E4%B8%AD%E5%B0%8F%E8%A7%84%E6%A8%A1%E9%9B%86%E7%BE%A4%EF%BC%88%3C500%20node%EF%BC%89%E3%80%82%3C/p%3E%0A%3Cp%3E%3Cstrong%3EGrafana%20Cloud%20Profiles%3C/strong%3E%EF%BC%9A%E6%89%98%E7%AE%A1%E6%9C%8D%E5%8A%A1%EF%BC%8C%E6%8C%89%20ingested%20samples%20%E8%AE%A1%E8%B4%B9%E3%80%82%E9%80%82%E5%90%88%E4%B8%8D%E6%83%B3%E8%87%AA%E8%BF%90%E7%BB%B4%E7%9A%84%E5%9B%A2%E9%98%9F%EF%BC%8C%E4%BD%86%E6%88%90%E6%9C%AC%E9%9A%8F%E9%87%87%E6%A0%B7%E9%A2%91%E7%8E%87%E7%BA%BF%E6%80%A7%E5%A2%9E%E9%95%BF%E3%80%82%3C/p%3E%0A%3Ch3%3E5.4%20%E4%B8%8E%20K8s%20%E9%9B%86%E6%88%90%E7%9A%84%E5%85%B3%E9%94%AE%E6%A8%A1%E5%BC%8F%3C/h3%3E%0A%3Cpre%3E%3Ccode%20class%3D%22language-yaml%22%3E%0AapiVersion%3A%20apps/v1%0Akind%3A%20DaemonSet%0Ametadata%3A%0A%20%20name%3A%20parca-agent%0Aspec%3A%0A%20%20template%3A%0A%20%20%20%20spec%3A%0A%20%20%20%20%20%20hostPID%3A%20true%20%20%20%20%20%20%23%20%E5%BF%85%E9%A1%BB%EF%BC%9A%E5%85%81%E8%AE%B8%E8%AF%BB%E5%8F%96%E5%AE%BF%E4%B8%BB%E6%9C%BA%20/proc%0A%20%20%20%20%20%20hostNetwork%3A%20true%20%20%23%20%E5%8F%AF%E9%80%89%EF%BC%9A%E7%B2%BE%E7%A1%AE%E8%8E%B7%E5%8F%96%E6%9C%AC%E5%9C%B0%E8%8A%82%E7%82%B9%E5%90%8D%0A%20%20%20%20%20%20containers%3A%0A%20%20%20%20%20%20-%20name%3A%20parca-agent%0A%20%20%20%20%20%20%20%20image%3A%20ghcr.io/parca-dev/parca-agent%3Alatest%0A%20%20%20%20%20%20%20%20securityContext%3A%0A%20%20%20%20%20%20%20%20%20%20privileged%3A%20true%20%20%20%23%20eBPF%20%E9%80%9A%E5%B8%B8%E9%9C%80%E8%A6%81%20CAP_BPF%20%28Linux%205.8%2B%29%0A%20%20%20%20%20%20%20%20env%3A%0A%20%20%20%20%20%20%20%20-%20name%3A%20NODE_NAME%0A%20%20%20%20%20%20%20%20%20%20valueFrom%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20fieldRef%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20fieldPath%3A%20spec.nodeName%0A%20%20%20%20%20%20%20%20volumeMounts%3A%0A%20%20%20%20%20%20%20%20-%20name%3A%20debugfs%0A%20%20%20%20%20%20%20%20%20%20mountPath%3A%20/sys/kernel/debug%0A%20%20%20%20%20%20%20%20%20%20readOnly%3A%20false%20%20%20%20%23%20debugfs%20%E9%9C%80%E8%A6%81%E8%AF%BB%E5%86%99%E4%BB%A5%E5%8A%A0%E8%BD%BD%20kprobe%0A%20%20%20%20%20%20volumes%3A%0A%20%20%20%20%20%20-%20name%3A%20debugfs%0A%20%20%20%20%20%20%20%20hostPath%3A%0A%20%20%20%20%20%20%20%20%20%20path%3A%20/sys/kernel/debug%0A%60%60%60%3C/code%3E%3C/pre%3E%0A%3Cp%3ELinux%205.8%2B%20%E5%BC%95%E5%85%A5%E4%BA%86%20%3Ccode%3ECAP_BPF%3C/code%3E%20capability%EF%BC%8C%E9%85%8D%E5%90%88%20%3Ccode%3ECAP_PERFMON%3C/code%3E%20%E5%92%8C%20%3Ccode%3ECAP_SYS_ADMIN%3C/code%3E%EF%BC%88%E4%BB%85%E7%94%A8%E4%BA%8E%E5%8A%A0%E8%BD%BD%E6%9F%90%E4%BA%9B%E7%B1%BB%E5%9E%8B%E7%9A%84%20BPF%20program%EF%BC%89%E5%8F%AF%E4%BB%A5%E5%AE%9E%E7%8E%B0%E9%9D%9E%E7%89%B9%E6%9D%83%20BPF%20%E6%93%8D%E4%BD%9C%E3%80%82%E4%BD%86%E5%AE%9E%E9%99%85%E7%94%9F%E4%BA%A7%E4%B8%AD%EF%BC%8CeBPF%20%E5%AE%B9%E5%99%A8%E9%80%9A%E5%B8%B8%E4%BB%8D%E9%85%8D%E7%BD%AE%20%3Ccode%3Eprivileged%3A%20true%3C/code%3E%20%E6%88%96%E8%87%B3%E5%B0%91%20%3Ccode%3ECAP_SYS_ADMIN%3C/code%3E%EF%BC%8C%E8%BF%99%E9%9C%80%E8%A6%81%E5%9C%A8%E5%AE%89%E5%85%A8%E5%8A%A0%E5%9B%BA%E7%9A%84%E9%9B%86%E7%BE%A4%E4%B8%AD%E9%80%9A%E8%BF%87%20Pod%20Security%20Admission%20%E5%81%9A%E7%99%BD%E5%90%8D%E5%8D%95%E3%80%82%3C/p%3E%0A%3Ch2%3E%E5%85%AD%E3%80%81%E5%AE%9E%E6%88%98%E6%A1%88%E4%BE%8B%EF%BC%9A20%25%20CPU%20%E6%B5%AA%E8%B4%B9%E5%9C%A8%E8%87%AA%E6%97%8B%E9%94%81%3C/h2%3E%0A%3Cp%3E%E4%BB%A5%E4%B8%8B%E6%98%AF%E4%B8%80%E4%B8%AA%E7%9C%9F%E5%AE%9E%E7%94%9F%E4%BA%A7%E9%97%AE%E9%A2%98%E7%9A%84%E5%AE%8C%E6%95%B4%E6%8E%92%E6%9F%A5%E6%B5%81%E7%A8%8B%EF%BC%8C%E5%B1%95%E7%A4%BA%20Continuous%20Profiling%20%E7%9A%84%E4%BB%B7%E5%80%BC%E3%80%82%3C/p%3E%0A%3Ch3%3E6.1%20%E7%8E%B0%E8%B1%A1%3C/h3%3E%0A%3Cp%3E%E6%9F%90%E5%9B%A2%E9%98%9F%E5%9C%A8%E5%91%A8%E4%BA%94%E4%BE%8B%E8%A1%8C%E6%89%A9%E5%AE%B9%E5%90%8E%E5%8F%91%E7%8E%B0%EF%BC%8C%E6%96%B0%20Pod%20%E7%9A%84%20QPS%20%E4%BB%85%E4%B8%BA%E6%97%A7%20Pod%20%E7%9A%84%2080%25%EF%BC%8C%E4%BD%86%20CPU%20%E4%BD%BF%E7%94%A8%E7%8E%87%E5%8F%8D%E8%80%8C%E4%BB%8E%2060%25%20%E4%B8%8A%E5%8D%87%E5%88%B0%2075%25%E3%80%82%E4%BC%A0%E7%BB%9F%E7%9A%84%20%3Ccode%3Etop%3C/code%3E%20%E5%92%8C%20%3Ccode%3Evmstats%3C/code%3E%20%E4%BB%85%E6%98%BE%E7%A4%BA%3Ccode%3E%20sy%3C/code%3E%EF%BC%88%E5%86%85%E6%A0%B8%E6%80%81%EF%BC%89%E6%97%B6%E9%97%B4%E5%A2%9E%E5%8A%A0%E4%BA%86%2012%25%EF%BC%8C%E6%97%A0%E6%B3%95%E7%A1%AE%E5%AE%9A%E5%85%B7%E4%BD%93%E5%8E%9F%E5%9B%A0%E3%80%82%3C/p%3E%0A%3Ch3%3E6.2%20Continuous%20Profiling%20%E6%8C%87%E7%A4%BA%E7%9A%84%E5%BC%82%E5%B8%B8%3C/h3%3E%0A%3Cp%3E%E5%9C%A8%20Pyroscope%20%E9%9D%A2%E6%9D%BF%E4%B8%AD%EF%BC%8C%E6%96%B0%E5%A2%9E%20Pod%20%E7%9A%84%20On-CPU%20%E7%81%AB%E7%84%B0%E5%9B%BE%E6%98%BE%E7%A4%BA%EF%BC%9A%3C/p%3E%0A%3Cpre%3E%3Ccode%3E%0A_raw_spin_lock_irqsave%20%20%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%20%2018.7%25%0A%20%20_raw_spin_lock%20%20%20%20%20%20%20%20%20%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%20%2012.1%25%0A%20%20%20%20copy_page_to_iter%20%20%20%20%20%E2%96%88%E2%96%88%E2%96%88%20%204.2%25%0A%20%20%20%20tcp_sendmsg_locked%20%20%20%20%20%E2%96%88%E2%96%88%E2%96%88%20%203.8%25%0A%20%20%20%20kmem_cache_alloc_node%20%20%E2%96%88%E2%96%88%20%202.9%25%0A%60%60%60%3C/code%3E%3C/pre%3E%0A%3Cp%3E%E6%97%A7%20Pod%20%E4%B8%AD%20%3Ccode%3Espin_lock%3C/code%3E%20%E7%9B%B8%E5%85%B3%E8%B7%AF%E5%BE%84%E4%BB%85%E5%8D%A0%203%25%EF%BC%8C%E6%96%B0%20Pod%20%E6%9A%B4%E6%B6%A8%E8%87%B3%2018.7%25%E3%80%82%E7%BD%91%E7%BB%9C%E6%A0%88%E4%B8%AD%E7%9A%84%E9%94%81%E7%AB%9E%E4%BA%89%E6%98%BE%E8%91%97%E3%80%82%3C/p%3E%0A%3Ch3%3E6.3%20%E6%A0%B9%E5%9B%A0%E5%88%86%E6%9E%90%3C/h3%3E%0A%3Cp%3E%E8%BF%9B%E4%B8%80%E6%AD%A5%E6%9F%A5%E7%9C%8B%20Off-CPU%20%E7%81%AB%E7%84%B0%E5%9B%BE%EF%BC%8C%E5%8F%91%E7%8E%B0%E5%AE%BD%E7%9A%84%E5%9D%97%E5%88%86%E5%B8%83%E5%9C%A8%20%3Ccode%3Etcp_sendmsg_locked%3C/code%3E%20%E5%86%85%E7%9A%84%20%3Ccode%3Elock_sock%3C/code%3E%E3%80%82%E8%BF%9E%E7%BB%AD%E6%95%B0%E5%A4%A9%E7%9A%84%E6%95%B0%E6%8D%AE%E9%87%87%E9%9B%86%E6%98%BE%E7%A4%BA%EF%BC%8C%E5%B0%96%E5%B3%B0%E9%9B%86%E4%B8%AD%E5%9C%A8%E6%AF%8F%E5%A4%A9%2014%3A00-15%3A00%EF%BC%88%E6%89%B9%E5%A4%84%E7%90%86%E4%BB%BB%E5%8A%A1%E7%9A%84%E9%AB%98%E5%B3%B0%EF%BC%89%E3%80%82%3C/p%3E%0A%3Cp%3E%E6%A0%B9%E5%9B%A0%E6%98%AF%E6%89%B9%E5%A4%84%E7%90%86%E4%BB%BB%E5%8A%A1%E9%AA%A4%E5%A2%9E%E4%B8%8E%E6%B4%BB%E8%B7%83%E8%BF%9E%E6%8E%A5%E6%95%B0%E7%BA%BF%E6%80%A7%E7%9B%B8%E5%85%B3%EF%BC%88%E6%96%B0%E5%BB%BA%E8%BF%9E%E6%8E%A5%E7%9A%84%20TCP%20%E6%8F%A1%E6%89%8B%E8%A7%A6%E5%8F%91%20%3Ccode%3Etcp_md5_do_lookup%3C/code%3E%20%E4%B8%AD%E7%9A%84%20%3Ccode%3Ercu_read_lock%3C/code%3E%EF%BC%89%E3%80%82%E8%80%8C%E6%96%B0%E5%BB%BA%20Pod%20%E7%9A%84%20%3Ccode%3Enf_conntrack_max%3C/code%3E%20%E6%9C%AA%E9%9A%8F%20%3Ccode%3Enet.core.somaxconn%3C/code%3E%20%E5%90%8C%E6%AF%94%E4%BE%8B%E8%B0%83%E6%95%B4%EF%BC%8C%E5%AF%BC%E8%87%B4%20conntrack%20%E8%A1%A8%E6%BB%A1%E5%90%8E%E8%A7%A6%E5%8F%91%E5%A4%A7%E9%87%8F%20%3Ccode%3Etcp_do_syn_retries%3C/code%3E%E3%80%82%3C/p%3E%0A%3Cp%3E%3Ccode%3Etcp_do_syn_retries%3C/code%3E%20%E5%86%85%E9%83%A8%E4%BC%9A%E8%B0%83%E7%94%A8%20%3Ccode%3Eraw_spin_lock%3C/code%3E%20%E4%BF%9D%E6%8A%A4%20%3Ccode%3Esk-%3Esk_lock.slock%3C/code%3E%EF%BC%8C%E5%BD%93%20retry%20%E9%A3%8E%E6%9A%B4%E5%8F%91%E7%94%9F%E6%97%B6%EF%BC%8C%E5%A4%9A%E6%A0%B8%20CPU%20%E5%9C%A8%E9%94%81%E4%B8%8A%E7%96%AF%E7%8B%82%E8%87%AA%E6%97%8B%E3%80%82%3C/p%3E%0A%3Ch3%3E6.4%20%E4%BF%AE%E5%A4%8D%E4%B8%8E%E9%AA%8C%E8%AF%81%3C/h3%3E%0A%3Cp%3E%E8%B0%83%E6%95%B4%20%3Ccode%3Enf_conntrack_max%3C/code%3E%20%E5%90%8E%EF%BC%8CPyroscope%20diff%20flame%20graph%20%E6%98%BE%E7%A4%BA%E7%BA%A2%E8%89%B2%E5%8C%BA%E5%9F%9F%EF%BC%88spinlock%20%E5%BC%80%E9%94%80%EF%BC%89%E5%AE%8C%E5%85%A8%E6%B6%88%E5%A4%B1%E3%80%82%E5%85%B3%E9%94%AE%E6%8C%87%E6%A0%87%E6%81%A2%E5%A4%8D%EF%BC%9A%3C/p%3E%0A%3Cul%3E%3Cli%3E%E8%B0%83%E5%BA%A6%E5%BB%B6%E8%BF%9F%20P99%20%E4%BB%8E%2034ms%20%E9%99%8D%E8%87%B3%206ms%3C/li%3E%0A%3Cli%3ECPU%20steal%20%E4%BB%8E%200.8%25%20%E9%99%8D%E8%87%B3%200.1%25%3C/li%3E%0A%3Cli%3EQPS%20%E6%81%A2%E5%A4%8D%E6%AD%A3%E5%B8%B8%EF%BC%88%E5%9B%A0%20syn%20retry%20%E5%87%8F%E5%B0%91%EF%BC%89%3C/li%3E%0A%3C/ul%3E%0A%3Cp%3E%E8%BF%99%E4%B8%AA%E6%A1%88%E4%BE%8B%E7%9A%84%E6%95%99%E8%AE%AD%EF%BC%9A%3Cstrong%3EOn-CPU%20%E7%81%AB%E7%84%B0%E5%9B%BE%E6%98%AF%E5%85%A5%E5%8F%A3%EF%BC%88%E6%98%BE%E7%A4%BA%E5%93%AA%E6%AE%B5%E4%BB%A3%E7%A0%81%E5%8D%A0%E7%94%A8%20CPU%EF%BC%89%EF%BC%8COff-CPU%20%E7%81%AB%E7%84%B0%E5%9B%BE%E6%98%AF%E5%87%BA%E5%8F%A3%EF%BC%88%E6%98%BE%E7%A4%BA%E4%B8%BA%E4%BB%80%E4%B9%88%E9%9C%80%E8%A6%81%E7%AD%89%E9%82%A3%E4%B9%88%E4%B9%85%EF%BC%89%3C/strong%3E%E3%80%82%3C/p%3E%0A%3Ch2%3E%E4%B8%83%E3%80%81%E8%BF%9B%E9%98%B6%EF%BC%9A%E5%9F%BA%E4%BA%8E%20eBPF%20%E5%86%85%E5%AD%98%E6%B3%84%E6%BC%8F%E6%A3%80%E6%B5%8B%3C/h2%3E%0A%3Cp%3E%E5%86%85%E5%AD%98%E6%B3%84%E6%BC%8F%E6%98%AF%E6%A3%98%E6%89%8B%E7%9A%84%E7%94%9F%E4%BA%A7%E9%97%AE%E9%A2%98%EF%BC%8C%E5%9B%A0%E4%B8%BA%E6%B3%84%E6%BC%8F%E7%9A%84%E4%BF%A1%E5%8F%B7%EF%BC%88RSS%20%E6%8C%81%E7%BB%AD%E5%A2%9E%E9%95%BF%EF%BC%89%E5%9C%A8%E4%BB%A3%E7%A0%81%E5%B1%82%E9%9D%A2%E4%B8%8D%E5%8F%AF%E8%A7%81%E3%80%82eBPF%20%E5%8F%AF%E4%BB%A5%E9%80%9A%E8%BF%87%E8%B7%9F%E8%B8%AA%20%3Ccode%3Ekmem_cache_alloc%3C/code%3E/%3Ccode%3Ekmem_cache_free%3C/code%3E%20%E7%9A%84%E9%85%8D%E5%AF%B9%E5%85%B3%E7%B3%BB%E6%9D%A5%E5%AE%9A%E4%BD%8D%E6%B3%84%E6%BC%8F%E7%82%B9%E3%80%82%3C/p%3E%0A%3Cpre%3E%3Ccode%20class%3D%22language-c%22%3E%0A//%20%E5%85%B3%E9%94%AE%E6%8A%80%E6%9C%AF%E7%82%B9%EF%BC%9A%E8%AE%B0%E5%BD%95%E6%AF%8F%E6%AC%A1%20kmem%20%E5%88%86%E9%85%8D%E7%9A%84%E8%B0%83%E7%94%A8%E6%A0%88%E5%92%8C%E6%97%B6%E9%97%B4%0ABPF_HASH%28allocs%2C%20u64%2C%20struct%20alloc_info%29%3B%0A%0Aint%20trace_kmem_alloc%28struct%20pt_regs%20%2Actx%2C%20const%20void%20%2Aaddr%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20size_t%20bytes%29%20%7B%0A%20%20%20%20if%20%28%21addr%29%20return%200%3B%0A%20%20%20%20%0A%20%20%20%20struct%20alloc_info%20info%20%3D%20%7B%7D%3B%0A%20%20%20%20info.ts%20%3D%20bpf_ktime_get_ns%28%29%3B%0A%20%20%20%20info.bytes%20%3D%20bytes%3B%0A%20%20%20%20info.stack_id%20%3D%20bpf_get_stackid%28ctx%2C%20%26alloc_traces%2C%20BPF_F_USER_STACK%29%3B%0A%20%20%20%20%0A%20%20%20%20u64%20addr_key%20%3D%20%28u64%29addr%3B%0A%20%20%20%20allocs.update%28%26addr_key%2C%20%26info%29%3B%0A%20%20%20%20return%200%3B%0A%7D%0A%0Aint%20trace_kmem_free%28struct%20pt_regs%20%2Actx%2C%20const%20void%20%2Aaddr%29%20%7B%0A%20%20%20%20if%20%28%21addr%29%20return%200%3B%0A%20%20%20%20%0A%20%20%20%20u64%20addr_key%20%3D%20%28u64%29addr%3B%0A%20%20%20%20allocs.delete%28%26addr_key%29%3B%0A%20%20%20%20return%200%3B%0A%7D%0A%60%60%60%3C/code%3E%3C/pre%3E%0A%3Cp%3E%E8%BF%90%E8%A1%8C%2024%20%E5%B0%8F%E6%97%B6%E5%90%8E%EF%BC%8C%3Ccode%3Eallocs%3C/code%3E%20%E8%A1%A8%E4%B8%AD%E5%8E%9F%E9%85%8D%E6%9C%AA%E9%87%8A%E6%94%BE%E7%9A%84%E8%AE%B0%E5%BD%95%E5%8D%B3%E4%B8%BA%E6%BD%9C%E5%9C%A8%E6%B3%84%E6%BC%8F%E3%80%82%E9%80%9A%E8%BF%87%E5%AF%B9%E6%AF%94%E6%97%B6%E9%97%B4%E6%88%B3%EF%BC%8C%E5%8F%AF%E5%AE%9A%E4%BD%8D%22%E5%AD%98%E6%B4%BB%E6%97%B6%E9%97%B4%20%3E%20%E9%98%88%E5%80%BC%22%E7%9A%84%E5%BC%82%E5%B8%B8%E5%88%86%E9%85%8D%E8%B7%AF%E5%BE%84%E3%80%82%3C/p%3E%0A%3Ch2%3E%E5%85%AB%E3%80%81%E8%B8%A9%E5%9D%91%E4%B8%8E%E6%9C%80%E4%BD%B3%E5%AE%9E%E8%B7%B5%3C/h2%3E%0A%3Ch3%3E8.1%20%E9%87%87%E6%A0%B7%E9%A2%91%E7%8E%87%E9%80%89%E6%8B%A9%3C/h3%3E%0A%3Cp%3E99Hz%20%E4%B8%8D%E6%98%AF%E9%AD%94%E6%B3%95%E6%95%B0%E5%AD%97%E3%80%82%E5%AF%B9%E4%BA%8E%E7%9F%AD%E5%AF%BF%E5%91%BD%E4%BA%8B%E4%BB%B6%EF%BC%88%3C10ms%EF%BC%89%EF%BC%8C1000Hz%20%E5%8F%AF%E8%83%BD%E6%9B%B4%E6%9C%89%E6%95%88%EF%BC%8C%E4%BD%86%20CPU%20%E5%BC%80%E9%94%80%E9%9D%9E%E7%BA%BF%E6%80%A7%E5%A2%9E%E9%95%BF%E2%80%94%E2%80%94%E4%BB%8E%2099Hz%20%E6%8F%90%E5%8D%87%E5%88%B0%20999Hz%EF%BC%8CCPU%20%E5%BC%80%E9%94%80%E5%A4%A7%E7%BA%A6%E5%A2%9E%E5%8A%A0%205%20%E5%80%8D%E8%80%8C%E9%9D%9E%2010%20%E5%80%8D%EF%BC%8C%E5%9B%A0%E4%B8%BA%E5%A4%A7%E9%83%A8%E5%88%86%E6%88%90%E6%9C%AC%E6%9D%A5%E8%87%AA%E4%B8%AD%E6%96%AD%E5%A4%84%E7%90%86%E8%80%8C%E9%9D%9E%E6%9E%81%E7%AE%80%E9%87%87%E6%A0%B7%E3%80%82%3C/p%3E%0A%3Ch3%3E8.2%20%E7%AC%A6%E5%8F%B7%E5%8C%96%E9%99%B7%E9%98%B1%3C/h3%3E%0A%3Cp%3E%E7%94%9F%E4%BA%A7%E7%8E%AF%E5%A2%83%E4%B8%AD%E7%BB%8F%E5%B8%B8%E9%81%87%E5%88%B0%E6%B2%A1%E6%9C%89%E7%AC%A6%E5%8F%B7%E4%BF%A1%E6%81%AF%E7%9A%84%E4%BA%8C%E8%BF%9B%E5%88%B6%EF%BC%88strip%20%E5%90%8E%E7%9A%84%20Go%20%E4%BA%8C%E8%BF%9B%E5%88%B6%E3%80%81JITted%20%E4%BB%A3%E7%A0%81%EF%BC%89%E3%80%82%E8%A7%A3%E5%86%B3%E6%96%B9%E6%A1%88%EF%BC%9A%3C/p%3E%0A%3Col%3E%3Cli%3EGo%20%E7%A8%8B%E5%BA%8F%E4%BD%BF%E7%94%A8%20%3Ccode%3E-ldflags%3D%22-s%3Dfalse%22%3C/code%3E%20%E4%BF%9D%E7%95%99%E7%AC%A6%E5%8F%B7%E8%A1%A8%E3%80%82%3C/li%3E%0A%3Cli%3E%E5%AF%B9%20eBPF%20Off-CPU%EF%BC%8C%E4%BD%BF%E7%94%A8%20%3Ccode%3Eperf%20inject%20--jit%3C/code%3E%20%E5%B0%86%20JIT%20%E4%BA%8B%E4%BB%B6%E8%BD%AC%E6%8D%A2%E4%B8%BA%E8%A7%A3%E6%9E%90%E5%90%8E%E7%9A%84%E7%AC%A6%E5%8F%B7%E3%80%82%3C/li%3E%0A%3Cli%3E%E4%BD%BF%E7%94%A8%20debuginfod%EF%BC%88%E5%A6%82%20%3Ccode%3Edebuginfod-find%3C/code%3E%EF%BC%89%E6%8C%89%E9%9C%80%E4%B8%8B%E8%BD%BD%20ELF%20%E8%B0%83%E8%AF%95%E4%BF%A1%E6%81%AF%E3%80%82%3C/li%3E%0A%3C/ol%3E%0A%3Ch3%3E8.3%20BPF%20Map%20%E6%BA%A2%E5%87%BA%3C/h3%3E%0A%3Cp%3E%E9%AB%98%E5%B9%B6%E5%8F%91%E7%B3%BB%E7%BB%9F%E5%9C%A8%20%3Ccode%3Esched_switch%3C/code%3E%20%E8%BF%BD%E8%B8%AA%E4%B8%8B%E5%8F%AF%E8%83%BD%E6%AF%8F%E7%A7%92%E4%BA%A7%E7%94%9F%E6%95%B0%E5%8D%81%E4%B8%87%E4%BA%8B%E4%BB%B6%E3%80%82%E5%A6%82%E6%9E%9C%20BPF%20%E7%A8%8B%E5%BA%8F%E5%A4%84%E7%90%86%E9%80%9F%E5%BA%A6%E8%B7%9F%E4%B8%8D%E4%B8%8A%EF%BC%8C%3Ccode%3Estart%3C/code%3E%20hash%20map%20%E4%BC%9A%E6%BA%A2%E5%87%BA%EF%BC%8C%E5%AF%BC%E8%87%B4%20Off-CPU%20%E4%BA%8B%E4%BB%B6%E8%A2%AB%E4%B8%A2%E5%BC%83%E3%80%82%E8%B0%83%E6%95%B4%E7%AD%96%E7%95%A5%EF%BC%9A%3C/p%3E%0A%3Cul%3E%3Cli%3E%E4%BD%BF%E7%94%A8%20%3Ccode%3EBPF_MAP_TYPE_LRU_HASH%3C/code%3E%20%E6%9B%BF%E4%BB%A3%20%3Ccode%3EBPF_MAP_TYPE_HASH%3C/code%3E%E3%80%82%3C/li%3E%0A%3Cli%3E%E5%AF%B9%20map%20%E5%A4%A7%E5%B0%8F%E8%AE%BE%E7%BD%AE%E4%B8%8A%E9%99%90%EF%BC%88%E5%A6%82%20%3Ccode%3Emax_entries%3D65536%3C/code%3E%EF%BC%89%E3%80%82%3C/li%3E%0A%3Cli%3E%E5%9C%A8%20BPF%20%E7%A8%8B%E5%BA%8F%E4%B8%AD%E5%8A%A0%E5%BF%AB%E9%80%9F%E5%BA%A6%EF%BC%9A%E9%81%BF%E5%85%8D%E5%A4%8D%E6%9D%82%E7%9A%84%E5%BE%AA%E7%8E%AF%E5%92%8C%E5%88%86%E6%94%AF%EF%BC%8C%E5%B0%BD%E6%97%A9%20%3Ccode%3Ereturn%200%3C/code%3E%E3%80%82%3C/li%3E%0A%3C/ul%3E%0A%3Ch3%3E8.4%20%E5%AE%B9%E5%99%A8%20PID%20Namespace%20%E4%B8%AD%E7%9A%84%E8%BF%87%E6%BB%A4%3C/h3%3E%0A%3Cp%3E%E5%9C%A8%20K8s%20Pod%20%E4%B8%AD%E8%BF%90%E8%A1%8C%20eBPF%20agent%20%E6%97%B6%EF%BC%8C%E9%9C%80%E8%A6%81%E5%8C%BA%E5%88%86%E5%AE%BF%E4%B8%BB%E6%9C%BA%20PID%20%E4%B8%8E%E5%AE%B9%E5%99%A8%E5%86%85_PID_%EF%BC%9A%3C/p%3E%0A%3Cpre%3E%3Ccode%20class%3D%22language-c%22%3E%0A//%20%E4%BD%BF%E7%94%A8%20bpf_get_current_pid_tgid%28%29%20%E8%8E%B7%E5%8F%96%E7%9A%84%E6%98%AF%E5%AE%BF%E4%B8%BB%E6%9C%BA%20PID%0A//%20%E9%9C%80%E8%A6%81%E7%94%A8%20bpf_get_ns_current_pid_tgid%28%29%20%E8%8E%B7%E5%8F%96%E7%89%B9%E5%AE%9A%20PID%20Namespace%20%E4%B8%AD%E7%9A%84%20PID%0A%23endif%0A%60%60%60%3C/code%3E%3C/pre%3E%0A%3Ch2%3E%E4%B9%9D%E3%80%81%E5%B1%95%E6%9C%9B%3C/h2%3E%0A%3Cp%3EContinuous%20Profiling%20%E6%AD%A3%E5%9C%A8%E4%BB%8E%22%E9%9D%99%E6%80%81%E5%88%86%E6%9E%90%22%E8%B5%B0%E5%90%91%22%E5%8A%A8%E6%80%81%E9%97%AD%E7%8E%AF%22%E3%80%82%E4%B8%89%E4%B8%AA%E8%B6%8B%E5%8A%BF%E5%80%BC%E5%BE%97%E5%85%B3%E6%B3%A8%EF%BC%9A%3C/p%3E%0A%3Col%3E%3Cli%3E%3Cstrong%3EeBPF%20%2B%20LLM%20%E6%99%BA%E8%83%BD%E8%AF%8A%E6%96%AD%3C/strong%3E%EF%BC%9AGrafana%20Beyla%E3%80%81Parca%20%E6%96%B0%E9%A1%B9%E7%9B%AE%E5%B0%9D%E8%AF%95%E5%B0%86%E7%81%AB%E7%84%B0%E5%9B%BE%E5%BC%82%E5%B8%B8%EF%BC%88%E5%A6%82%E6%96%B0%E5%A2%9E%E7%9A%84%E8%B6%85%E5%AE%BD%E5%9D%97%EF%BC%89%E4%B8%8E%20LLM%20%E7%BB%93%E5%90%88%EF%BC%8C%E8%87%AA%E5%8A%A8%E7%94%9F%E6%88%90%E4%BF%AE%E5%A4%8D%E5%BB%BA%E8%AE%AE%E3%80%82%E4%BD%86%E9%9C%80%E6%B3%A8%E6%84%8F%E5%B9%BB%E8%A7%89%E9%A3%8E%E9%99%A9%E2%80%94%E2%80%94AI%20%E7%94%9F%E6%88%90%E7%9A%84%E4%BC%98%E5%8C%96%E4%BB%A3%E7%A0%81%E9%9C%80%E7%BB%8F%E4%BA%BA%E5%B7%A5%E5%AE%A1%E6%9F%A5%E3%80%82%3C/li%3E%0A%3C/ol%3E%0A%3Col%3E%3Cli%3E%3Cstrong%3EProfile-guided%20Optimization%EF%BC%88PGO%EF%BC%89%E5%8F%8D%E9%A6%88%E7%8E%AF%3C/strong%3E%EF%BC%9A%E5%B0%86%E7%94%9F%E4%BA%A7%E7%8E%AF%E5%A2%83%E7%9A%84%E7%9C%9F%E5%AE%9E%E5%89%96%E6%9E%90%E6%95%B0%E6%8D%AE%E5%8F%8D%E9%A6%88%E7%BB%99%E7%BC%96%E8%AF%91%E5%99%A8%EF%BC%8C%E6%8C%87%E5%AF%BC%E5%88%86%E6%94%AF%E9%A2%84%E6%B5%8B%E3%80%81%E5%86%85%E8%81%94%E5%86%B3%E7%AD%96%E3%80%82Google%20%E5%B7%B2%E5%9C%A8%20Chrome%20%E4%B8%AD%E5%AE%9E%E8%B7%B5%E5%A4%9A%E5%B9%B4%EF%BC%8C%E9%80%9A%E8%BF%87%20Continuous%20Profiling%20%E6%95%B0%E6%8D%AE%E9%A9%B1%E5%8A%A8%20PGO%20%E5%B7%B2%E6%88%90%E4%B8%BA%20Rust/C%2B%2B%20%E7%BC%96%E8%AF%91%E5%99%A8%E7%9A%84%E5%AE%9E%E9%AA%8C%E7%89%B9%E6%80%A7%E3%80%82%3C/li%3E%0A%3C/ol%3E%0A%3Col%3E%3Cli%3E%3Cstrong%3E%E8%83%BD%E8%80%97%E5%89%96%E6%9E%90%EF%BC%88Energy%20Profiling%EF%BC%89%3C/strong%3E%EF%BC%9AIntel%20RAPL%E3%80%81ARM%20Energy%20Aware%20Scheduling%20API%20%E9%80%9A%E8%BF%87%20perf_event%20%E6%9A%B4%E9%9C%B2%E8%83%BD%E8%80%97%E8%AE%A1%E6%95%B0%E5%99%A8%EF%BC%8C%E4%B8%BA%20Green%20Computing%20%E6%8F%90%E4%BE%9B%E6%95%B0%E6%8D%AE%E5%9F%BA%E7%A1%80%E3%80%82%E6%AC%A7%E6%B4%B2%E6%95%B0%E6%8D%AE%E4%B8%AD%E5%BF%83%E8%83%BD%E6%95%88%E6%B3%95%E8%A7%84%EF%BC%88%E5%A6%82%20EU%20CoC%EF%BC%89%E7%9A%84%E8%90%BD%E5%9C%B0%E5%B0%86%E6%8E%A8%E5%8A%A8%E8%BF%99%E4%B8%80%E6%96%B9%E5%90%91%E3%80%82%3C/li%3E%0A%3C/ol%3E%0A%3Chr/%3E%0A%3Cp%3EContinuous%20Profiling%20%E4%B8%8D%E5%8F%AA%E6%98%AF%E4%B8%80%E4%B8%AA%E5%B7%A5%E5%85%B7%EF%BC%8C%E5%AE%83%E4%BB%A3%E8%A1%A8%E7%9D%80%E5%8F%AF%E8%A7%82%E6%B5%8B%E6%80%A7%E9%A2%86%E5%9F%9F%E4%BB%8E%22%E8%A2%AB%E5%8A%A8%E5%93%8D%E5%BA%94%22%E5%88%B0%22%E4%B8%BB%E5%8A%A8%E9%A2%84%E9%98%B2%22%E7%9A%84%E8%8C%83%E5%BC%8F%E8%BD%AC%E5%8F%98%E3%80%82%E5%BD%93%E4%BD%A0%E7%9A%84%E5%91%8A%E8%AD%A6%E5%9C%A8%E7%81%AB%E7%84%B0%E5%9B%BE%E4%B8%8A%E6%8F%90%E5%89%8D%202%20%E5%B0%8F%E6%97%B6%E5%B0%B1%E8%83%BD%E5%AF%BB%E8%A7%85%E5%88%B0%E6%81%B6%E5%8C%96%E7%9A%84%E8%9B%9B%E4%B8%9D%E9%A9%AC%E8%BF%B9%EF%BC%8C%E5%BD%93%E4%BD%A0%E7%9A%84%E5%B7%A5%E7%A8%8B%E5%B8%88%E4%B8%8D%E5%86%8D%E9%9C%80%E8%A6%81%E8%84%91%E8%A1%A5%202%20%E5%91%A8%E5%89%8D%E7%9A%84%E6%80%A7%E8%83%BD%E5%9C%BA%E6%99%AF%E2%80%94%E2%80%94%E8%BF%99%E6%89%8D%E6%98%AF%E5%8F%AF%E8%A7%82%E6%B5%8B%E6%80%A7%E5%B7%A5%E7%A8%8B%E7%9A%84%E7%9C%9F%E6%AD%A3%E4%BB%B7%E5%80%BC%E3%80%82%3C/p%3E%0A%3Cblockquote%3E%3Cp%3E%3Cstrong%3E%E5%85%B3%E9%94%AE%E5%8F%82%E8%80%83%E8%B5%84%E6%96%99%EF%BC%9A%3C/strong%3E%3C/p%3E%0A%3Cp%3E-%20BPF%20%E6%80%A7%E8%83%BD%E5%B7%A5%E5%85%B7%EF%BC%88Brendan%20Gregg%EF%BC%89%EF%BC%9Attps%3A//www.brendangregg.com/bpf-performance-tools-book.html%3C/p%3E%0A%3Cp%3E-%20Parca%20%E6%96%87%E6%A1%A3%EF%BC%9Ahttps%3A//www.parca.dev/docs/overview%3C/p%3E%0A%3Cp%3E-%20Pyroscope%20%E6%9E%B6%E6%9E%84%E7%99%BD%E7%9A%AE%E4%B9%A6%EF%BC%9Ahttps%3A//pyroscope.io/docs/architecture/%3C/p%3E%0A%3C/blockquote%3E