LLM 推理优化深度实战（完整版）

%3Ch2%3ELLM%20%E6%8E%A8%E7%90%86%E4%BC%98%E5%8C%96%E6%B7%B1%E5%BA%A6%E5%AE%9E%E6%88%98%EF%BC%9A%E4%BB%8E%20PagedAttention%20%E5%88%B0%20Speculative%20Decoding%20%E7%9A%84%E5%B7%A5%E7%A8%8B%E6%9E%B6%E6%9E%84%3C/h2%3E%0A%3Cp%3E%E5%A4%A7%E8%AF%AD%E8%A8%80%E6%A8%A1%E5%9E%8B%EF%BC%88LLM%EF%BC%89%E6%8E%A8%E7%90%86%E6%98%AF%20AI%20%E8%90%BD%E5%9C%B0%E4%B8%AD%E6%9C%80%E6%98%82%E8%B4%B5%E7%9A%84%E7%8E%AF%E8%8A%82%E3%80%82%E4%B8%80%E4%B8%AA%2070B%20%E5%8F%82%E6%95%B0%E9%87%8F%E7%9A%84%E6%A8%A1%E5%9E%8B%EF%BC%8C%E5%8D%95%E6%AC%A1%E6%8E%A8%E7%90%86%E5%8F%AF%E8%83%BD%E6%B6%88%E8%80%97%E6%95%B0%E5%8D%81%20GB%20%E6%98%BE%E5%AD%98%EF%BC%8C%E8%80%8C%E5%9C%A8%E7%BA%BF%E6%9C%8D%E5%8A%A1%E8%A6%81%E6%B1%82%E7%9A%84%20P99%20%E5%BB%B6%E8%BF%9F%E5%BE%80%E5%BE%80%E9%9C%80%E8%A6%81%E6%8E%A7%E5%88%B6%E5%9C%A8%202%20%E7%A7%92%E4%BB%A5%E5%86%85%E3%80%82%E5%A6%82%E4%BD%95%E5%9C%A8%E6%9C%89%E9%99%90%E7%9A%84%20GPU%20%E8%B5%84%E6%BA%90%E4%B8%8A%E5%90%8C%E6%97%B6%E5%AE%9E%E7%8E%B0%E9%AB%98%E5%90%9E%E5%90%90%E5%92%8C%E4%BD%8E%E5%BB%B6%E8%BF%9F%EF%BC%8C%E6%98%AF%E8%BF%87%E5%8E%BB%E4%B8%A4%E5%B9%B4%E7%B3%BB%E7%BB%9F%E4%BC%98%E5%8C%96%E9%A2%86%E5%9F%9F%E6%9C%80%E6%BF%80%E7%83%88%E7%9A%84%E6%88%98%E5%9C%BA%E4%B9%8B%E4%B8%80%E3%80%82%E6%9C%AC%E6%96%87%E5%B0%86%E4%BB%8E%E6%98%BE%E5%AD%98%E7%AE%A1%E7%90%86%E3%80%81%E6%89%B9%E5%A4%84%E7%90%86%E7%AD%96%E7%95%A5%E5%92%8C%E8%A7%A3%E7%A0%81%E7%AE%97%E6%B3%95%E4%B8%89%E4%B8%AA%E7%BB%B4%E5%BA%A6%EF%BC%8C%E6%B7%B1%E5%85%A5%E6%8B%86%E8%A7%A3%E7%8E%B0%E4%BB%A3%20LLM%20%E6%8E%A8%E7%90%86%E5%BC%95%E6%93%8E%E7%9A%84%E6%A0%B8%E5%BF%83%E6%8A%80%E6%9C%AF%EF%BC%8C%E5%B9%B6%E7%BB%99%E5%87%BA%E5%8F%AF%E8%90%BD%E5%9C%B0%E7%9A%84%E5%B7%A5%E7%A8%8B%E5%AE%9E%E8%B7%B5%E6%96%B9%E6%A1%88%E3%80%82%3C/p%3E%0A%3Ch2%3E%E4%B8%80%E3%80%81%E6%98%BE%E5%AD%98%E7%93%B6%E9%A2%88%EF%BC%9AKV%20Cache%20%E7%AE%A1%E7%90%86%E7%9A%84%E5%B7%A5%E7%A8%8B%E5%9B%B0%E5%B1%80%3C/h2%3E%0A%3Cp%3ETransformer%20%E8%87%AA%E5%9B%9E%E5%BD%92%E6%8E%A8%E7%90%86%E7%9A%84%E6%A0%B8%E5%BF%83%E7%93%B6%E9%A2%88%E4%B8%8D%E5%9C%A8%E8%AE%A1%E7%AE%97%EF%BC%8C%E8%80%8C%E5%9C%A8%E6%98%BE%E5%AD%98%E3%80%82%E6%AF%8F%E4%B8%80%E6%AD%A5%20decode%20%E9%83%BD%E9%9C%80%E8%A6%81%E8%AF%BB%E5%8F%96%E4%B9%8B%E5%89%8D%E6%89%80%E6%9C%89%20token%20%E7%9A%84%20Key%20%E5%92%8C%20Value%20%E5%BC%A0%E9%87%8F%EF%BC%8C%E8%BF%99%E5%B0%B1%E6%98%AF%20KV%20Cache%E3%80%82%E5%AF%B9%E4%BA%8E%E4%B8%80%E4%B8%AA%E5%B1%82%E6%95%B0%20L%E3%80%81%E9%9A%90%E8%97%8F%E7%BB%B4%E5%BA%A6%20d%E3%80%81%E5%BA%8F%E5%88%97%E9%95%BF%E5%BA%A6%20n%E3%80%81%E6%89%B9%E6%AC%A1%E5%A4%A7%E5%B0%8F%20b%20%E7%9A%84%E6%A8%A1%E5%9E%8B%EF%BC%8CKV%20Cache%20%E7%9A%84%E6%98%BE%E5%AD%98%E5%8D%A0%E7%94%A8%E7%BA%A6%E4%B8%BA%EF%BC%9A%3C/p%3E%0A%3Cpre%3E%3Ccode%3EKV%20Cache%20%E2%89%88%202%20%C3%97%20L%20%C3%97%20d%20%C3%97%20n%20%C3%97%20b%20%C3%97%20sizeof%28dtype%29%3C/code%3E%3C/pre%3E%0A%3Cp%3E%E4%BB%A5%20Llama-2-70B%20%E4%B8%BA%E4%BE%8B%EF%BC%88L%3D80%2C%20d%3D8192%EF%BC%89%EF%BC%8C%E5%A4%84%E7%90%86%E4%B8%80%E4%B8%AA%202048%20token%20%E7%9A%84%E5%BA%8F%E5%88%97%EF%BC%8CFP16%20%E4%B8%8B%E5%8D%95%E5%8D%A1%20KV%20%E5%B0%B1%E9%9C%80%E8%A6%81%E7%BA%A6%20512MB%E3%80%82%E5%BD%93%E5%B9%B6%E5%8F%91%E8%AF%B7%E6%B1%82%E8%BE%BE%E5%88%B0%20100%20%E6%97%B6%EF%BC%8C%E4%BB%85%20KV%20Cache%20%E5%B0%B1%E9%9C%80%E8%A6%81%2050GB%E2%80%94%E2%80%94%E8%BF%99%E6%98%AF%E5%A4%A7%E5%A4%9A%E6%95%B0%E5%8D%95%E5%8D%A1%20GPU%20%E6%97%A0%E6%B3%95%E6%89%BF%E5%8F%97%E7%9A%84%E3%80%82%3C/p%3E%0A%3Cp%3E%E6%9B%B4%E6%A3%98%E6%89%8B%E7%9A%84%E6%98%AF%EF%BC%8C%E4%BC%A0%E7%BB%9F%E6%8E%A8%E7%90%86%E5%BC%95%E6%93%8E%E4%BC%9A%E4%B8%BA%E6%AF%8F%E4%B8%AA%E8%AF%B7%E6%B1%82%E9%A2%84%E7%95%99%E5%85%B6%E6%9C%80%E5%A4%A7%E9%95%BF%E5%BA%A6%E7%9A%84%E8%BF%9E%E7%BB%AD%E6%98%BE%E5%AD%98%E7%A9%BA%E9%97%B4%E3%80%82%E5%AE%9E%E9%99%85%E5%9C%BA%E6%99%AF%E4%B8%AD%EF%BC%8C%E5%BA%8F%E5%88%97%E9%95%BF%E5%BA%A6%E5%88%86%E5%B8%83%E9%AB%98%E5%BA%A6%E4%B8%8D%E5%9D%87%EF%BC%9A%E6%9C%89%E7%9A%84%E8%AF%B7%E6%B1%82%E5%8F%AA%E9%9C%80%E8%A6%81%E5%87%A0%E5%8D%81%E4%B8%AA%20token%EF%BC%8C%E6%9C%89%E7%9A%84%E9%9C%80%E8%A6%81%E5%87%A0%E5%8D%83%E4%B8%AA%20token%E3%80%82%E6%8F%90%E5%89%8D%E4%B8%BA%E6%AF%8F%E4%B8%AA%E8%AF%B7%E6%B1%82%E5%88%86%E9%85%8D%E6%9C%80%E5%A4%A7%E9%95%BF%E5%BA%A6%E7%9A%84%E6%98%BE%E5%AD%98%E7%A9%BA%E9%97%B4%EF%BC%8C%E6%84%8F%E5%91%B3%E7%9D%80%E5%A4%A7%E9%87%8F%E6%98%BE%E5%AD%98%E8%A2%AB%E6%B5%AA%E8%B4%B9%E5%9C%A8%E9%A2%84%E7%95%99%E4%BD%86%E6%9C%AA%E4%BD%BF%E7%94%A8%E7%9A%84%E7%A9%BA%E9%97%B4%E4%B8%8A%E3%80%82%E5%AE%9E%E6%B5%8B%E6%95%B0%E6%8D%AE%E6%98%BE%E7%A4%BA%EF%BC%8C%E4%BC%A0%E7%BB%9F%E6%96%B9%E6%A1%88%E4%B8%AD%20KV%20Cache%20%E7%9A%84%E6%9C%89%E6%95%88%E5%88%A9%E7%94%A8%E7%8E%87%E5%BE%80%E5%BE%80%E4%B8%8D%E8%B6%B3%2025%25%E3%80%82%3C/p%3E%0A%3Ch2%3E%E4%BA%8C%E3%80%81PagedAttention%EF%BC%9A%E4%BB%8E%E8%BF%9E%E7%BB%AD%E5%88%86%E9%85%8D%E5%88%B0%E8%99%9A%E6%8B%9F%E5%86%85%E5%AD%98%E7%AE%A1%E7%90%86%3C/h2%3E%0A%3Cp%3E2023%20%E5%B9%B4%EF%BC%8CUC%20Berkeley%20%E7%9A%84%20Kwon%20%E7%AD%89%E4%BA%BA%E5%9C%A8%E8%AE%BA%E6%96%87%20%22Efficient%20Memory%20Management%20for%20Large%20Language%20Model%20Serving%20with%20PagedAttention%22%20%E4%B8%AD%E6%8F%90%E5%87%BA%E4%BA%86%E9%9D%A9%E5%91%BD%E6%80%A7%E7%9A%84%E8%A7%A3%E5%86%B3%E6%96%B9%E6%A1%88%EF%BC%8C%E5%85%B6%E6%80%9D%E6%83%B3%E7%9B%B4%E6%8E%A5%E5%80%9F%E9%89%B4%E4%BA%86%E6%93%8D%E4%BD%9C%E7%B3%BB%E7%BB%9F%E4%B8%AD%E7%9A%84%E8%99%9A%E6%8B%9F%E5%86%85%E5%AD%98%E5%92%8C%E5%88%86%E9%A1%B5%E6%9C%BA%E5%88%B6%E3%80%82%3C/p%3E%0A%3Ch3%3E2.1%20%E6%A0%B8%E5%BF%83%E6%80%9D%E6%83%B3%3C/h3%3E%0A%3Cp%3EPagedAttention%20%E7%9A%84%E6%A0%B8%E5%BF%83%E6%98%AF%EF%BC%9A%3Cstrong%3E%E5%B0%86%20KV%20Cache%20%E8%A7%86%E4%B8%BA%22%E5%86%85%E5%AD%98%E9%A1%B5%22%EF%BC%8C%E6%8C%89%E9%9C%80%E5%88%86%E9%85%8D%EF%BC%8C%E7%89%A9%E7%90%86%E4%B8%8A%E4%B8%8D%E8%A6%81%E6%B1%82%E8%BF%9E%E7%BB%AD%3C/strong%3E%E3%80%82%3C/p%3E%0A%3Cp%3E%E5%85%B7%E4%BD%93%E5%AE%9E%E7%8E%B0%E4%B8%8A%EF%BC%8C%E7%B3%BB%E7%BB%9F%E9%A2%84%E5%85%88%E5%B0%86%20GPU%20%E6%98%BE%E5%AD%98%E5%88%92%E5%88%86%E4%B8%BA%E5%9B%BA%E5%AE%9A%E5%A4%A7%E5%B0%8F%E7%9A%84%20block%EF%BC%88%E9%80%9A%E5%B8%B8%2016%20%E6%88%96%2032%20%E4%B8%AA%20token%20%E7%9A%84%20KV%20%E6%95%B0%E6%8D%AE%EF%BC%89%E3%80%82%E6%AF%8F%E4%B8%AA%E8%AF%B7%E6%B1%82%E7%BB%B4%E6%8A%A4%E4%B8%80%E4%B8%AA%20block%20table%EF%BC%8C%E9%80%BB%E8%BE%91%E4%B8%8A%E8%BF%9E%E7%BB%AD%E7%9A%84%E5%BA%8F%E5%88%97%E5%9C%A8%E7%89%A9%E7%90%86%E4%B8%8A%E5%8F%AF%E4%BB%A5%E6%95%A3%E5%B8%83%E5%9C%A8%E4%BB%BB%E6%84%8F%20block%20%E4%B8%AD%E3%80%82%E5%BD%93%E5%BA%8F%E5%88%97%E9%95%BF%E5%BA%A6%E5%A2%9E%E9%95%BF%E6%97%B6%EF%BC%8C%E5%8F%AA%E9%9C%80%E8%BF%BD%E5%8A%A0%E6%96%B0%E7%9A%84%20block%EF%BC%9B%E5%BD%93%E8%AF%B7%E6%B1%82%E5%AE%8C%E6%88%90%E6%97%B6%EF%BC%8C%E5%9B%9E%E6%94%B6%E8%BF%99%E4%BA%9B%20block%20%E4%BE%9B%E5%85%B6%E4%BB%96%E8%AF%B7%E6%B1%82%E4%BD%BF%E7%94%A8%E3%80%82%3C/p%3E%0A%3Cpre%3E%3Ccode%3E%23%20PagedAttention%20%E4%BC%AA%E4%BB%A3%E7%A0%81%EF%BC%9Ablock%20table%20%E7%AE%A1%E7%90%86%0Aclass%20BlockTable%3A%0A%20%20%20%20%22%22%22%E6%AF%8F%E4%B8%AA%E8%AF%B7%E6%B1%82%E7%9A%84%E9%80%BB%E8%BE%91%E5%88%B0%E7%89%A9%E7%90%86%20block%20%E6%98%A0%E5%B0%84%22%22%22%0A%20%20%20%20def%20__init__%28self%2C%20block_size%3A%20int%20%3D%2016%29%3A%0A%20%20%20%20%20%20%20%20self.block_size%20%3D%20block_size%20%20%20%20%20%20%20%20%20%20%23%20%E6%AF%8F%E4%B8%AA%20block%20%E5%AD%98%E5%82%A8%E5%A4%9A%E5%B0%91%E4%B8%AA%20token%20%E7%9A%84%20KV%0A%20%20%20%20%20%20%20%20self.logical_blocks%3A%20list%5Bint%5D%20%3D%20%5B%5D%20%20%20%23%20%E7%89%A9%E7%90%86%20block%20%E7%B4%A2%E5%BC%95%E5%88%97%E8%A1%A8%0A%20%20%20%20%20%20%20%20self.num_tokens%3A%20int%20%3D%200%0A%20%20%20%20%0A%20%20%20%20def%20append_block%28self%2C%20physical_block%3A%20int%29%3A%0A%20%20%20%20%20%20%20%20%22%22%22%E5%BA%8F%E5%88%97%E5%A2%9E%E9%95%BF%E6%97%B6%E8%BF%BD%E5%8A%A0%E4%B8%80%E4%B8%AA%E7%89%A9%E7%90%86%20block%22%22%22%0A%20%20%20%20%20%20%20%20self.logical_blocks.append%28physical_block%29%0A%20%20%20%20%0A%20%20%20%20def%20get_physical_block%28self%2C%20logical_idx%3A%20int%29%20-%26gt%3B%20int%3A%0A%20%20%20%20%20%20%20%20%22%22%22%E6%A0%B9%E6%8D%AE%E9%80%BB%E8%BE%91%E4%BD%8D%E7%BD%AE%E8%8E%B7%E5%8F%96%E7%89%A9%E7%90%86%20block%22%22%22%0A%20%20%20%20%20%20%20%20return%20self.logical_blocks%5Blogical_idx%5D%0A%20%20%20%20%0A%20%20%20%20%40property%0A%20%20%20%20def%20num_blocks%28self%29%20-%26gt%3B%20int%3A%0A%20%20%20%20%20%20%20%20%22%22%22%E9%9C%80%E8%A6%81%E7%9A%84%20block%20%E6%95%B0%E9%87%8F%20%3D%20ceil%28num_tokens%20/%20block_size%29%22%22%22%0A%20%20%20%20%20%20%20%20return%20%28self.num_tokens%20%2B%20self.block_size%20-%201%29%20//%20self.block_size%3C/code%3E%3C/pre%3E%0A%3Ch3%3E2.2%20%E6%98%BE%E5%AD%98%E5%88%A9%E7%94%A8%E7%8E%87%E6%8F%90%E5%8D%87%3C/h3%3E%0A%3Cp%3E%E8%BF%99%E7%A7%8D%E5%88%86%E9%A1%B5%E7%AE%A1%E7%90%86%E5%B8%A6%E6%9D%A5%E7%9A%84%E6%9C%80%E7%9B%B4%E6%8E%A5%E7%9A%84%E6%94%B6%E7%9B%8A%E6%98%AF%E6%98%BE%E5%AD%98%E5%88%A9%E7%94%A8%E7%8E%87%E6%8E%A5%E8%BF%91%20100%25%E3%80%82%E5%9B%A0%E4%B8%BA%EF%BC%9A%3C/p%3E%0A%3Col%3E%0A%3Cli%3E%3Cstrong%3E%E6%B6%88%E9%99%A4%E5%86%85%E9%83%A8%E7%A2%8E%E7%89%87%3C/strong%3E%EF%BC%9A%E5%8F%AA%E6%9C%89%E6%9C%80%E5%90%8E%E4%B8%80%E4%B8%AA%20block%20%E5%8F%AF%E8%83%BD%E5%AD%98%E5%9C%A8%E6%9C%AA%E5%A1%AB%E6%BB%A1%E7%9A%84%E6%83%85%E5%86%B5%EF%BC%8C%E6%B5%AA%E8%B4%B9%E6%9C%80%E5%A4%9A%E4%B8%80%E4%B8%AA%20block%EF%BC%88%E9%80%9A%E5%B8%B8%E4%BB%85%E5%8D%A0%E6%80%BB%E5%88%86%E9%85%8D%E7%9A%84%203-6%25%EF%BC%89%E3%80%82%3C/li%3E%0A%3Cli%3E%3Cstrong%3E%E6%B6%88%E9%99%A4%E5%A4%96%E9%83%A8%E7%A2%8E%E7%89%87%3C/strong%3E%EF%BC%9A%E7%89%A9%E7%90%86%20block%20%E6%B1%A0%E5%8C%96%E4%BD%BF%E7%94%A8%EF%BC%8C%E4%B8%8D%E5%AD%98%E5%9C%A8%E4%BC%A0%E7%BB%9F%E6%96%B9%E6%A1%88%E4%B8%AD%E5%9B%A0%E8%AF%B7%E6%B1%82%E9%95%BF%E5%BA%A6%E5%B7%AE%E5%BC%82%E5%A4%A7%E5%AF%BC%E8%87%B4%E7%9A%84%E6%98%BE%E5%AD%98%22%E7%A9%BA%E6%B4%9E%22%E3%80%82%3C/li%3E%0A%3Cli%3E%3Cstrong%3E%E7%B2%BE%E7%A1%AE%E6%8C%89%E9%9C%80%E5%88%86%E9%85%8D%3C/strong%3E%EF%BC%9A%E5%8F%AA%E5%88%86%E9%85%8D%E5%AE%9E%E9%99%85%E9%9C%80%E8%A6%81%E7%9A%84%20block%EF%BC%8C%E4%B8%8D%E4%BD%BF%E7%94%A8%E9%A2%84%E5%88%86%E9%85%8D%E6%9C%80%E5%A4%A7%E9%95%BF%E5%BA%A6%E7%9A%84%E7%AD%96%E7%95%A5%E3%80%82%3C/li%3E%0A%3C/ul%3E%0A%3Cp%3E%E5%AE%9E%E6%B5%8B%E5%AF%B9%E6%AF%94%E6%98%BE%E7%A4%BA%EF%BC%8CPagedAttention%20%E5%8F%AF%E4%BB%A5%E5%B0%86%E7%B3%BB%E7%BB%9F%E5%90%9E%E5%90%90%E6%8F%90%E5%8D%87%20%3Cstrong%3E2-4%20%E5%80%8D%3C/strong%3E%EF%BC%8C%E5%90%8C%E6%97%B6%E4%BF%9D%E6%8C%81%E4%B8%8E%E5%9F%BA%E7%BA%BF%E6%96%B9%E6%A1%88%E7%9B%B8%E5%90%8C%E7%9A%84%E5%BB%B6%E8%BF%9F%E6%B0%B4%E5%B9%B3%E3%80%82%3C/p%3E%0A%3Ch3%3E2.3%20Copy-on-Write%20%E4%B8%8E%E5%85%B1%E4%BA%AB%E5%89%8D%E7%BC%80%3C/h3%3E%0A%3Cp%3EPagedAttention%20%E7%9A%84%E5%8F%A6%E4%B8%80%E5%A4%A7%E4%BA%AE%E7%82%B9%E6%98%AF%E6%94%AF%E6%8C%81%20Copy-on-Write%EF%BC%88COW%EF%BC%89%E6%9C%BA%E5%88%B6%E3%80%82%E5%9C%A8%20system%20prompt%20%E5%85%B1%E4%BA%AB%E7%9A%84%E5%9C%BA%E6%99%AF%E4%B8%8B%EF%BC%88%E5%A6%82%E5%A4%9A%E8%BD%AE%E5%AF%B9%E8%AF%9D%E3%80%81few-shot%20prompting%EF%BC%89%EF%BC%8C%E5%A4%9A%E4%B8%AA%E8%AF%B7%E6%B1%82%E7%9A%84%E5%89%8D%E7%BC%80%20KV%20Cache%20%E5%8F%AF%E4%BB%A5%E5%85%B1%E4%BA%AB%E5%90%8C%E4%B8%80%E4%B8%AA%E7%89%A9%E7%90%86%20block%E3%80%82%E5%8F%AA%E6%9C%89%E5%9C%A8%E6%9F%90%E4%B8%AA%E8%AF%B7%E6%B1%82%E9%9C%80%E8%A6%81%E4%BF%AE%E6%94%B9%E5%89%8D%E7%BC%80%E6%97%B6%EF%BC%8C%E6%89%8D%E8%A7%A6%E5%8F%91%E5%AE%9E%E9%99%85%E7%9A%84%E6%8B%B7%E8%B4%9D%E6%93%8D%E4%BD%9C%E3%80%82%3C/p%3E%0A%3Cpre%3E%3Ccode%3E%23%20%E5%85%B1%E4%BA%AB%20system%20prompt%20%E7%9A%84%20COW%20%E7%A4%BA%E4%BE%8B%0Asystem_prompt_blocks%20%3D%20%5B10%2C%2011%2C%2012%2C%2013%5D%20%20%23%20system%20prompt%20%E5%8D%A0%E7%94%A8%E7%9A%84%E7%89%A9%E7%90%86%20block%0A%0A%23%20%E4%B8%A4%E4%B8%AA%E7%94%A8%E6%88%B7%E8%AF%B7%E6%B1%82%E5%85%B1%E4%BA%AB%E7%9B%B8%E5%90%8C%E7%9A%84%20system%20prompt%0Arequest_a%20%3D%20BlockTable%28shared_blocks%3Dsystem_prompt_blocks.copy%28%29%29%0Arequest_b%20%3D%20BlockTable%28shared_blocks%3Dsystem_prompt_blocks.copy%28%29%29%0A%0A%23%20%E5%BD%93%20request_a%20%E7%9A%84%E7%94%A8%E6%88%B7%E8%BE%93%E5%85%A5%E9%83%A8%E5%88%86%E5%BC%80%E5%A7%8B%E5%86%99%E5%85%A5%E6%97%B6%EF%BC%8C%E5%8F%AA%E9%9C%80%E4%B8%BA%E6%96%B0%E5%86%85%E5%AE%B9%E5%88%86%E9%85%8D%20block%0Arequest_a.append_block%28alloc_new_block%28%29%29%20%20%20%23%20%E5%86%99%E5%85%A5%E7%94%A8%E6%88%B7%20token%20%E7%9A%84%20KV%0Arequest_b.append_block%28alloc_new_block%28%29%29%20%20%20%23%20%E5%86%99%E5%85%A5%E7%94%A8%E6%88%B7%20token%20%E7%9A%84%20KV%0A%0A%23%20system%20prompt%20%E9%83%A8%E5%88%86%E4%BB%8D%E7%84%B6%E5%85%B1%E4%BA%AB%EF%BC%8C%E6%97%A0%E9%9C%80%E6%8B%B7%E8%B4%9D%3C/code%3E%3C/pre%3E%0A%3Ch2%3E%E4%B8%89%E3%80%81Continuous%20Batching%EF%BC%9A%E7%AA%81%E7%A0%B4%E9%9D%99%E6%80%81%E6%89%B9%E5%A4%84%E7%90%86%E5%90%9E%E5%90%90%E7%93%B6%E9%A2%88%3C/h2%3E%0A%3Cp%3E%E5%9C%A8%20PagedAttention%20%E4%B9%8B%E5%89%8D%EF%BC%8C%E6%8E%A8%E7%90%86%E5%BC%95%E6%93%8E%E4%B8%BB%E8%A6%81%E4%BD%BF%E7%94%A8%20Static%20Batching%EF%BC%9A%E5%B0%86%E4%B8%80%E6%89%B9%E8%AF%B7%E6%B1%82%E4%B8%80%E8%B5%B7%E9%80%81%E5%85%A5%20GPU%EF%BC%8C%E7%AD%89%E6%89%80%E6%9C%89%E8%AF%B7%E6%B1%82%E9%83%BD%E5%AE%8C%E6%88%90%E8%A7%A3%E7%A0%81%E5%90%8E%E5%86%8D%E8%BF%94%E5%9B%9E%E7%BB%93%E6%9E%9C%E3%80%82%E5%81%87%E8%AE%BE%E4%B8%80%E6%89%B9%E4%B8%AD%E6%9C%89%2010%20%E4%B8%AA%E8%AF%B7%E6%B1%82%EF%BC%8C%E5%85%B6%E4%B8%AD%209%20%E4%B8%AA%E5%9C%A8%20100%20token%20%E5%86%85%E5%AE%8C%E6%88%90%EF%BC%8C%E7%AC%AC%2010%20%E4%B8%AA%E9%9C%80%E8%A6%81%202000%20token%E2%80%94%E2%80%94%E9%82%A3%E4%B9%88%E5%89%8D%209%20%E4%B8%AA%E8%AF%B7%E6%B1%82%E7%9A%84%20GPU%20%E8%AE%A1%E7%AE%97%E5%9C%A8%E7%AD%89%E5%BE%85%E4%B8%AD%E7%99%BD%E7%99%BD%E6%B5%AA%E8%B4%B9%EF%BC%8C%E6%95%B4%E4%BD%93%E5%90%9E%E5%90%90%E8%A2%AB%E6%9C%80%E6%85%A2%E7%9A%84%E8%AF%B7%E6%B1%82%E6%8B%96%E7%B4%AF%E3%80%82%3C/p%3E%0A%3Cp%3EContinuous%20Batching%EF%BC%88%E4%B9%9F%E7%A7%B0%E4%B8%BA%20Iteration-Level%20Scheduling%EF%BC%89%E5%9C%A8%E6%AF%8F%E4%B8%AA%20decode%20%E6%AD%A5%E9%AA%A4%E7%9A%84%E9%97%B4%E9%9A%94%E4%B8%AD%E6%A3%80%E6%9F%A5%E9%98%9F%E5%88%97%EF%BC%8C%E5%8A%A8%E6%80%81%E5%9C%B0%E5%B0%86%E5%B7%B2%E5%AE%8C%E6%88%90%E8%AF%B7%E6%B1%82%E7%A7%BB%E5%87%BA%E3%80%81%E5%B0%86%E6%96%B0%E8%AF%B7%E6%B1%82%E6%8F%92%E5%85%A5%E3%80%82%E5%85%B6%E6%95%88%E6%9E%9C%E5%8F%AF%E4%BB%A5%E9%80%9A%E8%BF%87%E4%BB%A5%E4%B8%8B%E5%AF%B9%E6%AF%94%E7%90%86%E8%A7%A3%EF%BC%9A%3C/p%3E%0A%3Cpre%3E%3Ccode%3EStatic%20Batching%3A%0A%20%20Batch-1%3A%20%5BReq1%3A%20100%20tokens%5D%20%5BReq2%3A%202000%20tokens%5D%20%5BReq3%3A%2050%20tokens%5D%0A%20%20%20%20%20%20%20%20%20%20%20GPU%20%E5%88%A9%E7%94%A8%E7%8E%87%3A%20%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%88%E2%96%91%E2%96%91%E2%96%91%E2%96%91%E2%96%91%E2%96%91%E2%96%91%E2%96%91%E2%96%91%E2%96%91%E2%96%91%E2%96%91%E2%96%91%E2%96%91%E2%96%91%20%28%E5%89%8D50%E6%AD%A5100%25%2C%20%E4%B8%AD%E9%97%B41500%E6%AD%A5%E5%8F%AA%E6%9C%89Req2%E5%9C%A8%E8%B7%91%29%0A%0AContinuous%20Batching%3A%0A%20%20Step%200-50%3A%20%20%20%5BReq1%5D%20%5BReq2%5D%20%5BReq3%5D%20%20%20%20%20%E2%86%90%203%E4%B8%AA%E8%AF%B7%E6%B1%82%E5%B9%B6%E8%A1%8C%0A%20%20Step%2050%3A%20%20%20%20%20%5BReq3%E5%AE%8C%E6%88%90%2C%20Req4%E5%8A%A0%E5%85%A5%5D%20%20%20%20%20%20%E2%86%90%20Req3%20%E9%80%80%E5%87%BA%2C%20Req4%20%E8%BF%9B%E5%85%A5%0A%20%20Step%2050-100%3A%20%5BReq1%5D%20%5BReq2%5D%20%5BReq4%5D%20%20%20%20%20%E2%86%90%20GPU%20%E6%8C%81%E7%BB%AD%E6%BB%A1%E8%BD%BD%0A%20%20Step%20100%3A%20%20%20%20%5BReq1%E5%AE%8C%E6%88%90%2C%20Req5%E5%8A%A0%E5%85%A5%5D%0A%20%20Step%20100%2B%3A%20%20%20%5BReq2%5D%20%5BReq5%5D%20%20%20%20%20%20%20%20%20%20%20%20%E2%86%90%20%E6%8C%81%E7%BB%AD%E9%AB%98%E5%88%A9%E7%94%A8%E7%8E%87%3C/code%3E%3C/pre%3E%0A%3Cp%3EvLLM%20%E7%9A%84%E5%AE%9E%E6%B5%8B%E6%95%B0%E6%8D%AE%E6%98%BE%E7%A4%BA%EF%BC%8C%E5%9C%A8%E7%8E%B0%E5%AE%9E%E6%B5%81%E9%87%8F%E6%A8%A1%E5%BC%8F%E4%B8%8B%EF%BC%88%E8%AF%B7%E6%B1%82%E5%88%B0%E8%BE%BE%E6%9C%8D%E4%BB%8E%E6%B3%8A%E6%9D%BE%E5%88%86%E5%B8%83%EF%BC%8C%E9%95%BF%E5%BA%A6%E6%9C%8D%E4%BB%8E%E9%87%8D%E5%B0%BE%E5%88%86%E5%B8%83%EF%BC%89%EF%BC%8CContinuous%20Batching%20%E6%AF%94%20Static%20Batching%20%E6%8F%90%E5%8D%87%E5%90%9E%E5%90%90%20%3Cstrong%3E23%20%E5%80%8D%3C/strong%3E%E3%80%82%3C/p%3E%0A%3Ch3%3E3.1%20%E8%B0%83%E5%BA%A6%E7%AD%96%E7%95%A5%E7%9A%84%E6%9D%83%E8%A1%A1%3C/h3%3E%0A%3Cp%3EContinuous%20Batching%20%E5%B9%B6%E9%9D%9E%E6%B2%A1%E6%9C%89%E8%B0%83%E5%BA%A6%E5%BC%80%E9%94%80%E3%80%82%E5%BD%93%20batch%20%E4%B8%AD%E6%B7%B7%E5%90%88%E4%BA%86%20Prefill%EF%BC%88%E9%A6%96%20token%20%E7%94%9F%E6%88%90%EF%BC%8C%E8%AE%A1%E7%AE%97%E5%AF%86%E9%9B%86%EF%BC%89%E5%92%8C%20Decode%EF%BC%88%E9%80%90%20token%20%E7%94%9F%E6%88%90%EF%BC%8C%E8%AE%BF%E5%AD%98%E5%AF%86%E9%9B%86%EF%BC%89%E9%98%B6%E6%AE%B5%E6%97%B6%EF%BC%8C%E9%9C%80%E8%A6%81%E4%BB%94%E7%BB%86%E8%AE%BE%E8%AE%A1%E8%B0%83%E5%BA%A6%E7%AD%96%E7%95%A5%EF%BC%9A%3C/p%3E%0A%3Cpre%3E%3Ccode%3E%23%20%E7%AE%80%E5%8D%95%E4%BD%86%E6%9C%89%E6%95%88%E7%9A%84%E8%B0%83%E5%BA%A6%E7%AD%96%E7%95%A5%3A%20Swapping-aware%20Scheduling%0Aclass%20ContinuousBatchScheduler%3A%0A%20%20%20%20def%20schedule%28self%2C%20waiting_queue%3A%20list%5BRequest%5D%2C%20running_batch%3A%20list%5BRequest%5D%29%3A%0A%20%20%20%20%20%20%20%20%22%22%22%0A%20%20%20%20%20%20%20%20%E8%B0%83%E5%BA%A6%E9%80%BB%E8%BE%91%3A%0A%20%20%20%20%20%20%20%201.%20%E4%BC%98%E5%85%88%E4%BF%9D%E8%AF%81%20running%20batch%20%E4%B8%AD%20decode%20%E8%AF%B7%E6%B1%82%E7%9A%84%20GPU%20%E6%97%B6%E9%97%B4%0A%20%20%20%20%20%20%20%202.%20%E7%A9%BA%E4%BD%99%E7%AE%97%E5%8A%9B%E7%94%A8%E4%BA%8E%E9%A2%84%E5%A1%AB%E5%85%85%20%28prefill%29%20%E7%AD%89%E5%BE%85%E9%98%9F%E5%88%97%E4%B8%AD%E7%9A%84%E8%AF%B7%E6%B1%82%0A%20%20%20%20%20%20%20%203.%20%E5%A6%82%E6%9E%9C%E6%98%BE%E5%AD%98%E4%B8%8D%E8%B6%B3%EF%BC%8Cswap%20out%20%E4%BD%8E%E4%BC%98%E5%85%88%E7%BA%A7%20decode%20%E8%AF%B7%E6%B1%82%E5%88%B0%20CPU%0A%20%20%20%20%20%20%20%20%22%22%22%0A%20%20%20%20%20%20%20%20gpu_budget%20%3D%20self.max_batch_size%0A%20%20%20%20%20%20%20%20kv_budget%20%3D%20self.available_kv_blocks%0A%20%20%20%20%20%20%20%20%0A%20%20%20%20%20%20%20%20%23%201.%20%E4%BF%9D%E7%95%99%20running%20batch%20%E4%B8%AD%E7%9A%84%20decode%20%E8%AF%B7%E6%B1%82%EF%BC%88%E4%B8%8D%E8%83%BD%E4%B8%A2%EF%BC%8C%E5%AE%83%E4%BB%AC%E5%B7%B2%E7%BB%8F%E5%9C%A8%E7%94%9F%E6%88%90%E4%B8%AD%EF%BC%89%0A%20%20%20%20%20%20%20%20scheduled%20%3D%20%5Breq%20for%20req%20in%20running_batch%20if%20req.phase%20%3D%3D%20%27decode%27%5D%0A%20%20%20%20%20%20%20%20remaining_slots%20%3D%20gpu_budget%20-%20len%28scheduled%29%0A%20%20%20%20%20%20%20%20remaining_blocks%20%3D%20kv_budget%20-%20sum%28req.kv_blocks%20for%20req%20in%20scheduled%29%0A%20%20%20%20%20%20%20%20%0A%20%20%20%20%20%20%20%20%23%202.%20%E4%BB%8E%E7%AD%89%E5%BE%85%E9%98%9F%E5%88%97%E4%B8%AD%E9%80%89%E6%8B%A9%E8%AF%B7%E6%B1%82%E8%BF%9B%E8%A1%8C%20prefill%0A%20%20%20%20%20%20%20%20for%20req%20in%20waiting_queue%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20if%20remaining_slots%20%26lt%3B%3D%200%20or%20remaining_blocks%20%26lt%3B%20req.estimated_kv_blocks%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20continue%0A%20%20%20%20%20%20%20%20%20%20%20%20scheduled.append%28req%29%0A%20%20%20%20%20%20%20%20%20%20%20%20remaining_slots%20-%3D%201%0A%20%20%20%20%20%20%20%20%20%20%20%20remaining_blocks%20-%3D%20req.estimated_kv_blocks%0A%20%20%20%20%20%20%20%20%0A%20%20%20%20%20%20%20%20return%20scheduled%3C/code%3E%3C/pre%3E%0A%3Ch2%3E%E5%9B%9B%E3%80%81Speculative%20Decoding%EF%BC%9A%E7%94%A8%E5%B0%8F%E6%A8%A1%E5%9E%8B%E8%B7%91%E8%B5%A2%E5%A4%A7%E6%A8%A1%E5%9E%8B%E7%9A%84%E8%A7%A3%E7%A0%81%E9%80%9F%E5%BA%A6%3C/h2%3E%0A%3Cp%3E%E5%8D%B3%E4%BE%BF%E8%A7%A3%E5%86%B3%E4%BA%86%E6%98%BE%E5%AD%98%E5%92%8C%E6%89%B9%E5%A4%84%E7%90%86%E9%97%AE%E9%A2%98%EF%BC%8C%E5%A4%A7%E6%A8%A1%E5%9E%8B%E6%8E%A8%E7%90%86%E8%BF%98%E6%9C%89%E4%B8%80%E4%B8%AA%E6%A0%B9%E6%9C%AC%E6%80%A7%E7%9A%84%E7%93%B6%E9%A2%88%EF%BC%9A%3Cstrong%3Edecode%20%E9%98%B6%E6%AE%B5%E7%9A%84%20GPU%20%E5%88%A9%E7%94%A8%E7%8E%87%E6%9E%81%E4%BD%8E%3C/strong%3E%E3%80%82%E6%AF%8F%E6%AD%A5%20decode%20%E5%8F%AA%E7%94%9F%E6%88%90%E4%B8%80%E4%B8%AA%20token%EF%BC%8C70B%20%E6%A8%A1%E5%9E%8B%E7%9A%84%E6%95%B0%E5%8D%81%E4%BA%BF%E5%8F%82%E6%95%B0%E5%8F%AA%E4%B8%BA%E4%BA%86%E8%AE%A1%E7%AE%97%E4%B8%80%E4%B8%AA%20token%20%E7%9A%84%20softmax%EF%BC%8C%E8%AE%A1%E7%AE%97%E8%AE%BF%E5%AD%98%E6%AF%94%EF%BC%88arithmetic%20intensity%EF%BC%89%E6%8E%A5%E8%BF%91%201%E2%80%94%E2%80%94%E8%BF%99%E6%98%AF%20GPU%20%E4%B8%8A%E6%9C%80%E7%B3%9F%E7%B3%95%E7%9A%84%E5%B7%A5%E4%BD%9C%E6%A8%A1%E5%BC%8F%E3%80%82%3C/p%3E%0A%3Cp%3ESpeculative%20Decoding%EF%BC%88Leviathan%20et%20al.%2C%202023%EF%BC%89%E7%9A%84%E6%A0%B8%E5%BF%83%E6%B4%9E%E5%AF%9F%E6%98%AF%EF%BC%9A%3Cstrong%3E%E8%AE%A9%E4%B8%80%E4%B8%AA%E5%B0%8F%E5%BE%97%E5%A4%9A%E7%9A%84%20Draft%20%E6%A8%A1%E5%9E%8B%E5%85%88%E7%94%9F%E6%88%90%E5%80%99%E9%80%89%20token%EF%BC%8C%E7%84%B6%E5%90%8E%E8%AE%A9%E5%A4%A7%E6%A8%A1%E5%9E%8B%E4%B8%80%E6%AC%A1%E6%80%A7%E9%AA%8C%E8%AF%81%E8%BF%99%E4%BA%9B%E5%80%99%E9%80%89%3C/strong%3E%E3%80%82%3C/p%3E%0A%3Ch3%3E4.1%20%E7%9B%B4%E8%A7%89%E7%90%86%E8%A7%A3%3C/h3%3E%0A%3Cp%3E%E8%BF%99%E5%B0%B1%E5%83%8F%E5%86%99%E6%96%87%E7%AB%A0%E6%97%B6%E5%85%88%E8%AE%A9%E5%8A%A9%E6%89%8B%E8%B5%B7%E8%8D%89%E8%8D%89%E7%A8%BF%EF%BC%8C%E7%84%B6%E5%90%8E%E4%BD%A0%E4%B8%80%E6%AC%A1%E6%80%A7%E5%AE%A1%E9%98%85%E4%BF%AE%E6%94%B9%E3%80%82%E8%99%BD%E7%84%B6%E4%BD%A0%EF%BC%88%E5%A4%A7%E6%A8%A1%E5%9E%8B%EF%BC%89%E6%AF%8F%E4%B8%80%E6%AD%A5%E8%83%BD%E5%81%9A%E7%9A%84%E5%88%A4%E6%96%AD%E6%AF%94%E5%8A%A9%E6%89%8B%EF%BC%88%E5%B0%8F%E6%A8%A1%E5%9E%8B%EF%BC%89%E5%87%86%EF%BC%8C%E4%BD%86%E4%BD%A0%E5%AE%A1%E9%98%85%20N%20%E4%B8%AA%E8%8D%89%E7%A8%BF%20token%20%E7%9A%84%E6%97%B6%E9%97%B4%EF%BC%8C%E5%8F%AF%E8%83%BD%E6%AF%94%E4%BD%A0%E8%87%AA%E5%B7%B1%E5%86%99%20N%20%E4%B8%AA%20token%20%E7%9A%84%E6%97%B6%E9%97%B4%E8%BF%98%E5%B0%91%E3%80%82%3C/p%3E%0A%3Cp%3E%E6%95%B0%E5%AD%A6%E8%A1%A8%E8%BE%BE%EF%BC%9A%E8%AE%BE%E5%A4%A7%E6%A8%A1%E5%9E%8B%E4%B8%80%E6%AD%A5%E8%80%97%E6%97%B6%20%24T_L%24%EF%BC%8C%E5%B0%8F%E6%A8%A1%E5%9E%8B%E4%B8%80%E6%AD%A5%E8%80%97%E6%97%B6%20%24T_S%24%EF%BC%88%24T_S%20%5Cll%20T_L%24%EF%BC%89%EF%BC%8C%E5%B0%8F%E6%A8%A1%E5%9E%8B%E7%94%9F%E6%88%90%E7%9A%84%E5%80%99%E9%80%89%E9%95%BF%E5%BA%A6%E4%B8%BA%20%24%5Cgamma%24%EF%BC%8C%E6%9C%80%E7%BB%88%E6%8E%A5%E5%8F%97%E7%8E%87%E4%B8%BA%20%24%5Calpha%24%E3%80%82%E5%88%99%E6%AF%8F%E6%AC%A1%20speculation%20%E7%9A%84%E6%9C%9F%E6%9C%9B%E6%97%B6%E9%97%B4%EF%BC%9A%3C/p%3E%0A%3Cp%3E%24%24E%5Bt%5D%20%3D%20T_S%20%5Ctimes%20%5Cgamma%20%2B%20T_L%24%24%3C/p%3E%0A%3Cp%3E%E8%80%8C%E7%AD%89%E6%95%88%E7%9A%84%E4%BC%A0%E7%BB%9F%20decode%20%E6%97%B6%E9%97%B4%E4%B8%BA%EF%BC%9A%3C/p%3E%0A%3Cp%3E%24%24E%5Bt_%7Bbaseline%7D%5D%20%3D%20T_L%20%5Ctimes%20%28%5Calpha%20%5Ctimes%20%5Cgamma%20%2B%201%29%24%24%3C/p%3E%0A%3Cp%3E%E5%BD%93%20%24%5Calpha%20%5Ctimes%20%5Cgamma%20%5Ctimes%20T_L%20%3E%20%5Cgamma%20%5Ctimes%20T_S%20%2B%20T_L%24%20%E6%97%B6%EF%BC%8Cspeculative%20decoding%20%E5%B0%B1%E5%B8%A6%E6%9D%A5%E4%BA%86%E5%8A%A0%E9%80%9F%E3%80%82%E7%94%B1%E4%BA%8E%20%24T_S%20%5Cll%20T_L%24%20%E4%B8%94%20%24%5Calpha%24%20%E9%80%9A%E5%B8%B8%E5%9C%A8%200.7-0.9%20%E4%B9%8B%E9%97%B4%EF%BC%8C%E8%BF%99%E7%A7%8D%E5%8A%A0%E9%80%9F%E5%9C%A8%E5%A4%A7%E5%A4%9A%E6%95%B0%E5%9C%BA%E6%99%AF%E4%B8%8B%E9%9D%9E%E5%B8%B8%E5%8F%AF%E8%A7%82%E3%80%82%3C/p%3E%0A%3Ch3%3E4.2%20%E9%AA%8C%E8%AF%81%E7%AE%97%E6%B3%95%3C/h3%3E%0A%3Cp%3ESpeculative%20Decoding%20%E7%9A%84%E5%85%B3%E9%94%AE%E5%9C%A8%E4%BA%8E%E9%AA%8C%E8%AF%81%E9%98%B6%E6%AE%B5%E5%A6%82%E4%BD%95%E9%AB%98%E6%95%88%E5%9C%B0%E4%BD%BF%E7%94%A8%E5%A4%A7%E6%A8%A1%E5%9E%8B%E3%80%82%E5%A4%A7%E6%A8%A1%E5%9E%8B%E4%B8%8D%E9%9C%80%E8%A6%81%E9%87%8D%E6%96%B0%E8%AE%A1%E7%AE%97%E6%95%B4%E4%B8%AA%E5%BA%8F%E5%88%97%E2%80%94%E2%80%94%E5%AE%83%E5%8F%AA%E9%9C%80%E8%A6%81%E4%B8%80%E6%AC%A1%E5%89%8D%E5%90%91%E4%BC%A0%E6%92%AD%EF%BC%8C%E5%90%8C%E6%97%B6%E9%AA%8C%E8%AF%81%E6%89%80%E6%9C%89%E5%80%99%E9%80%89%20token%EF%BC%9A%3C/p%3E%0A%3Cpre%3E%3Ccode%3Eimport%20torch%0A%0Adef%20speculative_verify%28draft_tokens%2C%20model_large%2C%20prefix_kv%2C%20%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20prefix_logits%3DNone%2C%20temperature%3D0.0%29%3A%0A%20%20%20%20%22%22%22%0A%20%20%20%20draft_tokens%3A%20%5Bbatch%2C%20gamma%5D%20-%20draft%20%E6%A8%A1%E5%9E%8B%E7%94%9F%E6%88%90%E7%9A%84%E5%80%99%E9%80%89%20token%0A%20%20%20%20model_large%3A%20%E7%9B%AE%E6%A0%87%E5%A4%A7%E6%A8%A1%E5%9E%8B%0A%20%20%20%20prefix_kv%3A%20%E5%89%8D%E7%BC%80%E7%9A%84%20KV%20Cache%EF%BC%88%E5%B7%B2%E8%AE%A1%E7%AE%97%E5%A5%BD%E7%9A%84%EF%BC%89%0A%20%20%20%20%0A%20%20%20%20%E8%BF%94%E5%9B%9E%3A%20%E6%AF%8F%E4%B8%AA%E4%BD%8D%E7%BD%AE%E6%8E%A5%E5%8F%97%E7%9A%84%20token%20%E6%95%B0%E9%87%8F%E5%92%8C%E5%85%B7%E4%BD%93%E7%9A%84%E6%8E%A5%E5%8F%97/%E6%8B%92%E7%BB%9D%E5%86%B3%E7%AD%96%0A%20%20%20%20%22%22%22%0A%20%20%20%20gamma%20%3D%20draft_tokens.shape%5B1%5D%0A%20%20%20%20%0A%20%20%20%20%23%201.%20%E5%A4%A7%E6%A8%A1%E5%9E%8B%E4%B8%80%E6%AC%A1%E6%80%A7%E8%AE%A1%E7%AE%97%E6%89%80%E6%9C%89%E5%80%99%E9%80%89%20token%20%E7%9A%84%20logits%0A%20%20%20%20%23%20%20%20%20%E4%BD%BF%E7%94%A8%20prefix_kv%20%E4%BD%9C%E4%B8%BA%E5%89%8D%E7%BC%80%EF%BC%8Cdraft_tokens%20%E4%BD%9C%E4%B8%BA%E5%BD%93%E5%89%8D%E8%BE%93%E5%85%A5%0A%20%20%20%20%23%20%20%20%20%E8%BF%99%E5%8F%AA%E9%9C%80%E8%A6%81%E4%B8%80%E6%AC%A1%20forward%20pass%0A%20%20%20%20logits_large%20%3D%20model_large.forward_with_kv%28%0A%20%20%20%20%20%20%20%20draft_tokens%2C%20past_key_values%3Dprefix_kv%0A%20%20%20%20%29%20%20%23%20%5Bbatch%2C%20gamma%2C%20vocab_size%5D%0A%20%20%20%20%0A%20%20%20%20%23%202.%20%E8%8E%B7%E5%8F%96%20draft%20%E6%A8%A1%E5%9E%8B%E5%AF%B9%E5%BA%94%E4%BD%8D%E7%BD%AE%E7%9A%84%20logits%EF%BC%88%E9%9C%80%E8%A6%81%E4%BF%9D%E7%95%99%E6%8E%A8%E7%90%86%E6%97%B6%E7%9A%84%E6%A6%82%E7%8E%87%E5%88%86%E5%B8%83%EF%BC%89%0A%20%20%20%20logits_draft%20%3D%20model_small.get_logits%28%29%20%20%23%20%5Bbatch%2C%20gamma%2C%20vocab_size%5D%0A%20%20%20%20%0A%20%20%20%20%23%203.%20%E9%80%90%E4%BD%8D%E7%BD%AE%E9%AA%8C%E8%AF%81%0A%20%20%20%20accepted%20%3D%200%0A%20%20%20%20for%20i%20in%20range%28gamma%29%3A%0A%20%20%20%20%20%20%20%20prob_large%20%3D%20softmax%28logits_large%5B%3A%2C%20i%2C%20%3A%5D%20/%20temperature%29%0A%20%20%20%20%20%20%20%20prob_draft%20%3D%20softmax%28logits_draft%5B%3A%2C%20i%2C%20%3A%5D%20/%20temperature%29%0A%20%20%20%20%20%20%20%20%0A%20%20%20%20%20%20%20%20%23%20%E6%8E%A5%E5%8F%97%E6%A6%82%E7%8E%87%EF%BC%88%E4%B8%A4%E7%A7%8D%E7%AD%89%E4%BB%B7%E7%9A%84%E9%87%87%E6%A0%B7%E6%96%B9%E6%B3%95%EF%BC%89%0A%20%20%20%20%20%20%20%20%23%20%E6%96%B9%E6%B3%95%20A%3A%20Rejection%20sampling%0A%20%20%20%20%20%20%20%20coin_flip%20%3D%20torch.rand%28prob_large.shape%5B0%5D%29%0A%20%20%20%20%20%20%20%20accept_prob%20%3D%20torch.clamp%28%0A%20%20%20%20%20%20%20%20%20%20%20%20%28prob_large.gather%281%2C%20draft_tokens%5B%3A%2C%20i%3Ai%2B1%5D%29%20-%20%0A%20%20%20%20%20%20%20%20%20%20%20%20%20prob_draft.gather%281%2C%20draft_tokens%5B%3A%2C%20i%3Ai%2B1%5D%29%29%20/%20%0A%20%20%20%20%20%20%20%20%20%20%20%20%28prob_draft.gather%281%2C%20draft_tokens%5B%3A%2C%20i%3Ai%2B1%5D%29%20%2B%201e-8%29%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20min%3D0.0%0A%20%20%20%20%20%20%20%20%29%0A%20%20%20%20%20%20%20%20%0A%20%20%20%20%20%20%20%20if%20%28coin_flip%20%26lt%3B%20accept_prob%29.all%28%29%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20accepted%20%2B%3D%201%0A%20%20%20%20%20%20%20%20else%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20%23%20%E5%9C%A8%E4%BD%8D%E7%BD%AE%20i%20%E6%8B%92%E7%BB%9D%EF%BC%9A%E4%BB%8E%E8%B0%83%E6%95%B4%E5%90%8E%E7%9A%84%E5%88%86%E5%B8%83%E4%B8%AD%E9%87%87%E6%A0%B7%E4%B8%80%E4%B8%AA%20token%0A%20%20%20%20%20%20%20%20%20%20%20%20adjusted_prob%20%3D%20torch.clamp%28prob_large%20-%20prob_draft%2C%20min%3D0%29%0A%20%20%20%20%20%20%20%20%20%20%20%20adjusted_prob%20%3D%20adjusted_prob%20/%20adjusted_prob.sum%28dim%3D-1%2C%20keepdim%3DTrue%29%0A%20%20%20%20%20%20%20%20%20%20%20%20%23%20%E9%87%8D%E9%87%87%E6%A0%B7...%0A%20%20%20%20%20%20%20%20%20%20%20%20break%0A%20%20%20%20%0A%20%20%20%20return%20accepted%2C%20final_tokens%3C/code%3E%3C/pre%3E%0A%3Ch3%3E4.3%20Medusa%EF%BC%9A%E5%A4%9A%E5%A4%B4%E8%A7%A3%E7%A0%81%E7%9A%84%E5%B7%A5%E7%A8%8B%E7%AA%81%E7%A0%B4%3C/h3%3E%0A%3Cp%3ESpeculative%20Decoding%20%E7%9A%84%E4%B8%80%E4%B8%AA%E9%99%90%E5%88%B6%E6%98%AF%20Draft%20%E6%A8%A1%E5%9E%8B%E7%9A%84%E9%80%89%E6%8B%A9%E3%80%82%E5%A6%82%E6%9E%9C%E5%B0%8F%E6%A8%A1%E5%9E%8B%E5%92%8C%E5%A4%A7%E6%A8%A1%E5%9E%8B%E5%B7%AE%E5%BC%82%E5%A4%AA%E5%A4%A7%EF%BC%8C%E6%8E%A5%E5%8F%97%E7%8E%87%20%24%5Calpha%24%20%E4%BC%9A%E5%BE%88%E4%BD%8E%EF%BC%8C%E5%AF%BC%E8%87%B4%E9%A2%9D%E5%A4%96%E7%9A%84%E8%AE%A1%E7%AE%97%E8%B4%9F%E6%8B%85%E5%8F%8D%E8%80%8C%E6%8B%96%E6%85%A2%E9%80%9F%E5%BA%A6%E3%80%82%3C/p%3E%0A%3Cp%3EMedusa%EF%BC%88Cai%20et%20al.%2C%202024%EF%BC%89%E6%8F%90%E5%87%BA%E4%BA%86%E4%B8%80%E7%A7%8D%E4%BC%98%E9%9B%85%E7%9A%84%E8%A7%A3%E6%B3%95%EF%BC%9A%3Cstrong%3E%E7%9B%B4%E6%8E%A5%E5%9C%A8%E5%A4%A7%E6%A8%A1%E5%9E%8B%E4%B9%8B%E4%B8%8A%E8%AE%AD%E7%BB%83%E5%A4%9A%E4%B8%AA%E5%B9%B6%E8%A1%8C%E7%9A%84%E9%A2%84%E6%B5%8B%E5%A4%B4%3C/strong%3E%EF%BC%8C%E4%B8%8D%E9%9C%80%E8%A6%81%E7%8B%AC%E7%AB%8B%E7%9A%84%20Draft%20%E6%A8%A1%E5%9E%8B%E3%80%82%3C/p%3E%0A%3Cpre%3E%3Ccode%3ENormal%20Decode%3A%0A%5BBase%20Model%5D%20%E2%86%92%20token%E2%82%81%20%E2%86%92%20token%E2%82%82%20%E2%86%92%20token%E2%82%83%20%E2%86%92%20token%E2%82%84%0A%0AMedusa%3A%0A%5BBase%20Model%20%2B%20Head%E2%82%81%5D%20%E2%86%92%20token%E2%82%81%0A%5BBase%20Model%20%2B%20Head%E2%82%82%5D%20%E2%86%92%20token%E2%82%82%20%28%E9%A2%84%E6%B5%8B%E4%BD%8D%E7%BD%AE%2B2%29%0A%5BBase%20Model%20%2B%20Head%E2%82%83%5D%20%E2%86%92%20token%E2%82%83%20%28%E9%A2%84%E6%B5%8B%E4%BD%8D%E7%BD%AE%2B3%29%20%20%0A%5BBase%20Model%20%2B%20Head%E2%82%84%5D%20%E2%86%92%20token%E2%82%84%20%28%E9%A2%84%E6%B5%8B%E4%BD%8D%E7%BD%AE%2B4%29%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%86%93%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%5BVerification%5D%20%E4%B8%80%E6%AC%A1%E6%80%A7%E9%AA%8C%E8%AF%81%3C/code%3E%3C/pre%3E%0A%3Cp%3E%E6%AF%8F%E4%B8%AA%20Medusa%20Head%20%E6%98%AF%E4%B8%80%E4%B8%AA%E8%BD%BB%E9%87%8F%E7%9A%84%20MLP%20%E6%A8%A1%E5%9D%97%EF%BC%8C%E9%99%84%E5%8A%A0%E5%9C%A8%E5%A4%A7%E6%A8%A1%E5%9E%8B%E7%9A%84%E6%9C%80%E5%90%8E%E4%B8%80%E5%B1%82%E4%B9%8B%E5%90%8E%E3%80%82%E5%AE%83%E6%A0%B9%E6%8D%AE%E5%BD%93%E5%89%8D%E4%BD%8D%E7%BD%AE%E7%9A%84%20hidden%20state%20%E7%9B%B4%E6%8E%A5%E9%A2%84%E6%B5%8B%E6%9C%AA%E6%9D%A5%E7%AC%AC%20k%20%E4%B8%AA%E4%BD%8D%E7%BD%AE%E7%9A%84%20token%E3%80%82%E6%9C%80%E7%BB%88%E9%AA%8C%E8%AF%81%E6%97%B6%EF%BC%8C%E5%A4%A7%E6%A8%A1%E5%9E%8B%E7%9A%84%E5%89%8D%E5%90%91%E4%BC%A0%E6%92%AD%E5%8F%AF%E4%BB%A5%E5%85%B1%E4%BA%AB%E7%BB%9D%E5%A4%A7%E9%83%A8%E5%88%86%E8%AE%A1%E7%AE%97%E2%80%94%E2%80%94%E9%AA%8C%E8%AF%81%20N%20%E4%B8%AA%E5%80%99%E9%80%89%20token%20%E4%B8%8E%E9%AA%8C%E8%AF%81%201%20%E4%B8%AA%20token%20%E7%9A%84%E8%AE%A1%E7%AE%97%E9%87%8F%E5%87%A0%E4%B9%8E%E7%9B%B8%E5%90%8C%E3%80%82%3C/p%3E%0A%3Cp%3EMedusa%20%E5%9C%A8%E4%BF%9D%E6%8C%81%E5%A4%A7%E6%A8%A1%E5%9E%8B%E8%BE%93%E5%87%BA%E5%88%86%E5%B8%83%E5%AE%8C%E5%85%A8%E4%B8%80%E8%87%B4%E7%9A%84%E5%89%8D%E6%8F%90%E4%B8%8B%EF%BC%8C%E5%AE%9E%E7%8E%B0%20%3Cstrong%3E2-3%20%E5%80%8D%3C/strong%3E%E7%9A%84%E8%A7%A3%E7%A0%81%E5%8A%A0%E9%80%9F%E3%80%82%3C/p%3E%0A%3Ch2%3E%E4%BA%94%E3%80%81%E5%B7%A5%E7%A8%8B%E5%AE%9E%E8%B7%B5%EF%BC%9AvLLM%20%E7%9A%84%E6%9E%B6%E6%9E%84%E8%A7%A3%E6%9E%90%3C/h2%3E%0A%3Cp%3EvLLM%20%E6%98%AF%E7%9B%AE%E5%89%8D%E6%9C%80%E6%B5%81%E8%A1%8C%E7%9A%84%E5%BC%80%E6%BA%90%20LLM%20%E6%8E%A8%E7%90%86%E5%BC%95%E6%93%8E%EF%BC%8C%E5%AE%83%E5%B0%86%20PagedAttention%20%E5%92%8C%20Continuous%20Batching%20%E6%B7%B1%E5%BA%A6%E9%9B%86%E6%88%90%E3%80%82%E7%90%86%E8%A7%A3%E5%85%B6%E6%9E%B6%E6%9E%84%E6%9C%89%E5%8A%A9%E4%BA%8E%E6%88%91%E4%BB%AC%E5%9C%A8%E5%AE%9E%E9%99%85%E9%83%A8%E7%BD%B2%E6%97%B6%E5%81%9A%E5%87%BA%E6%AD%A3%E7%A1%AE%E7%9A%84%E9%85%8D%E7%BD%AE%E9%80%89%E6%8B%A9%E3%80%82%3C/p%3E%0A%3Ch3%3E5.1%20%E6%A0%B8%E5%BF%83%E7%BB%84%E4%BB%B6%3C/h3%3E%0A%3Cpre%3E%3Ccode%3E%E2%94%8C%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%90%0A%E2%94%82%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20vLLM%20Architecture%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%94%82%0A%E2%94%9C%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%A4%0A%E2%94%82%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%94%82%0A%E2%94%82%20%20%E2%94%8C%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%90%20%20%20%20%E2%94%8C%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%90%20%20%20%20%E2%94%8C%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%90%20%20%E2%94%82%0A%E2%94%82%20%20%E2%94%82%20FastAPI%20%20%E2%94%82%E2%94%80%E2%94%80%E2%94%80%E2%96%B6%E2%94%82%20%20Scheduler%20%20%20%E2%94%82%E2%94%80%E2%94%80%E2%94%80%E2%96%B6%E2%94%82%20%20GPU%20Model%20%20%20%E2%94%82%20%20%E2%94%82%0A%E2%94%82%20%20%E2%94%82%20%20Server%20%20%E2%94%82%20%20%20%20%E2%94%82%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%94%82%20%20%20%20%E2%94%82%20%20Runner%20%20%20%20%20%20%E2%94%82%20%20%E2%94%82%0A%E2%94%82%20%20%E2%94%94%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%98%20%20%20%20%E2%94%82%20%E2%94%8C%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%90%20%E2%94%82%20%20%20%20%E2%94%82%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%94%82%20%20%E2%94%82%0A%E2%94%82%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%94%82%20%E2%94%82%20Prefill%20%20%E2%94%82%20%E2%94%82%20%20%20%20%E2%94%82%20%E2%94%8C%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%90%20%E2%94%82%20%20%E2%94%82%0A%E2%94%82%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%94%82%20%E2%94%82%20%20Queue%20%20%20%E2%94%82%20%E2%94%82%20%20%20%20%E2%94%82%20%E2%94%82%20Block%20%20%20%20%E2%94%82%20%E2%94%82%20%20%E2%94%82%0A%E2%94%82%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%94%82%20%E2%94%94%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%98%20%E2%94%82%20%20%20%20%E2%94%82%20%E2%94%82%20Manager%20%20%E2%94%82%20%E2%94%82%20%20%E2%94%82%0A%E2%94%82%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%94%82%20%E2%94%8C%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%90%20%E2%94%82%20%20%20%20%E2%94%82%20%E2%94%94%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%98%20%E2%94%82%20%20%E2%94%82%0A%E2%94%82%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%94%82%20%E2%94%82%20Decode%20%20%20%E2%94%82%20%E2%94%82%20%20%20%20%E2%94%82%20%E2%94%8C%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%90%20%E2%94%82%20%20%E2%94%82%0A%E2%94%82%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%94%82%20%E2%94%82%20%20Batch%20%20%20%E2%94%82%20%E2%94%82%20%20%20%20%E2%94%82%20%E2%94%82%20Paged%20%20%20%20%E2%94%82%20%E2%94%82%20%20%E2%94%82%0A%E2%94%82%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%94%82%20%E2%94%94%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%98%20%E2%94%82%20%20%20%20%E2%94%82%20%E2%94%82%20Attention%E2%94%82%20%E2%94%82%20%20%E2%94%82%0A%E2%94%82%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%94%94%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%98%20%20%20%20%E2%94%82%20%E2%94%94%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%98%20%E2%94%82%20%20%E2%94%82%0A%E2%94%82%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%E2%94%94%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%98%20%20%E2%94%82%0A%E2%94%94%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%80%E2%94%98%3C/code%3E%3C/pre%3E%0A%3Ch3%3E5.2%20%E5%85%B3%E9%94%AE%E9%85%8D%E7%BD%AE%E5%8F%82%E6%95%B0%3C/h3%3E%0A%3Cpre%3E%3Ccode%3E%23%20vLLM%20%E5%90%AF%E5%8A%A8%E9%85%8D%E7%BD%AE%E7%A4%BA%E4%BE%8B%EF%BC%9ALlama-2-70B%20%E9%83%A8%E7%BD%B2%0Apython%20-m%20vllm.entrypoints.openai.api_server%20%5C%0A%20%20%20%20--model%20meta-llama/Llama-2-70b-chat-hf%20%5C%0A%20%20%20%20--tensor-parallel-size%204%20%5C%20%20%20%20%20%20%20%20%20%23%204-way%20TP%20%E5%88%86%E5%B8%83%E5%9C%A8%204%20%E5%BC%A0%20A100-80G%0A%20%20%20%20--gpu-memory-utilization%200.90%20%5C%20%20%20%20%20%23%20%E9%A2%84%E7%95%99%2010%25%20%E6%98%BE%E5%AD%98%E7%BB%99%20system/PagedAttention%20%E5%BC%80%E9%94%80%0A%20%20%20%20--max-num-batched-tokens%208192%20%5C%20%20%20%20%20%23%20%E6%AF%8F%20batch%20%E6%9C%80%E5%A4%A7%20token%20%E6%95%B0%EF%BC%88%E6%8E%A7%E5%88%B6%20prefill%20%E7%B2%92%E5%BA%A6%EF%BC%89%0A%20%20%20%20--max-num-seqs%20256%20%5C%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%23%20%E6%9C%80%E5%A4%A7%E5%B9%B6%E5%8F%91%E5%BA%8F%E5%88%97%E6%95%B0%0A%20%20%20%20--enable-speculative-decoding%20%5C%20%20%20%20%20%23%20%E5%90%AF%E7%94%A8%20speculative%20decoding%0A%20%20%20%20--speculative-model%20%5C%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%23%20draft%20%E6%A8%A1%E5%9E%8B%E8%B7%AF%E5%BE%84%0A%20%20%20%20%22meta-llama/Llama-2-13b-chat-hf%22%20%5C%0A%20%20%20%20--num-speculative-tokens%205%20%5C%20%20%20%20%20%20%20%20%23%20%E6%AF%8F%E4%B8%AA%20speculation%20%E7%9A%84%E5%80%99%E9%80%89%20token%20%E6%95%B0%0A%20%20%20%20--block-size%2016%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%23%20PagedAttention%20%E7%9A%84%20page%20%E5%A4%A7%E5%B0%8F%3C/code%3E%3C/pre%3E%0A%3Ch3%3E5.3%20%E6%80%A7%E8%83%BD%E8%B0%83%E4%BC%98%E5%AE%9E%E8%B7%B5%3C/h3%3E%0A%3Cp%3E%E5%9C%A8%E5%AE%9E%E9%99%85%E9%83%A8%E7%BD%B2%E4%B8%AD%EF%BC%8C%E6%9C%89%E5%87%A0%E4%B8%AA%E5%AE%B9%E6%98%93%E8%A2%AB%E5%BF%BD%E8%A7%86%E4%BD%86%E5%AF%B9%E6%80%A7%E8%83%BD%E5%BD%B1%E5%93%8D%E5%B7%A8%E5%A4%A7%E7%9A%84%E5%8F%82%E6%95%B0%EF%BC%9A%3C/p%3E%0A%3Cp%3E%3Cstrong%3Egpu-memory-utilization%3C/strong%3E%EF%BC%9A%E4%B8%8D%E5%AE%9C%E8%AE%BE%E7%BD%AE%E8%BF%87%E9%AB%98%EF%BC%88%E5%BB%BA%E8%AE%AE%200.85-0.92%EF%BC%89%E3%80%82PagedBlock%20%E7%AE%A1%E7%90%86%E5%99%A8%E9%9C%80%E8%A6%81%E5%B0%91%E9%87%8F%E6%98%BE%E5%AD%98%E5%81%9A%20bookkeeping%EF%BC%8C%E5%90%8C%E6%97%B6%20CUDA%20%E4%B8%8A%E4%B8%8B%E6%96%87%E6%9C%AC%E8%BA%AB%E4%B9%9F%E9%9C%80%E8%A6%81%E5%87%A0%E7%99%BE%20MB%E3%80%82%E8%AE%BE%E7%BD%AE%E8%BF%87%E9%AB%98%E4%BC%9A%E5%AF%BC%E8%87%B4%E8%BF%90%E8%A1%8C%E6%97%B6%20OOM%E3%80%82%3C/p%3E%0A%3Cp%3E%3Cstrong%3Emax-num-batched-tokens%3C/strong%3E%EF%BC%9A%E8%BF%99%E6%98%AF%E5%8D%95%E6%AC%A1%20forward%20%E4%B8%AD%E6%9C%80%E5%A4%9A%E7%9A%84%20token%20%E6%80%BB%E6%95%B0%EF%BC%8C%E7%9B%B4%E6%8E%A5%E5%BD%B1%E5%93%8D%20prefill%20%E9%98%B6%E6%AE%B5%E7%9A%84%E5%BB%B6%E8%BF%9F%E3%80%82%E5%AF%B9%E4%BA%8E%20latency-sensitive%20%E5%9C%BA%E6%99%AF%EF%BC%8C%E5%BB%BA%E8%AE%AE%E8%AE%BE%E7%BD%AE%E4%B8%BA%202048-4096%EF%BC%9B%E5%AF%B9%E4%BA%8E%20throughput-only%20%E5%9C%BA%E6%99%AF%EF%BC%8C%E5%8F%AF%E4%BB%A5%E9%80%82%E5%BD%93%E5%A2%9E%E5%A4%A7%E3%80%82%3C/p%3E%0A%3Cp%3E%3Cstrong%3Echunked-prefill%3C/strong%3E%EF%BC%9AvLLS%202024%20%E5%BC%95%E5%85%A5%E7%9A%84%E6%96%B0%E7%89%B9%E6%80%A7%EF%BC%8C%E5%85%81%E8%AE%B8%E5%B0%86%E4%B8%80%E4%B8%AA%E9%95%BF%20prefill%20%E8%AF%B7%E6%B1%82%E6%8B%86%E5%88%86%E4%B8%BA%E5%A4%9A%E4%B8%AA%20chunk%EF%BC%8C%E4%B8%8E%20decode%20%E6%AD%A5%E9%AA%A4%E4%BA%A4%E9%94%99%E6%89%A7%E8%A1%8C%E3%80%82%E8%BF%99%E8%A7%A3%E5%86%B3%E4%BA%86%E9%95%BF%20prefill%20%22%E9%98%BB%E5%A1%9E%22%20decode%20%E8%AF%B7%E6%B1%82%E7%9A%84%E9%97%AE%E9%A2%98%EF%BC%8C%E5%AF%B9%20P99%20%E5%BB%B6%E8%BF%9F%E6%9C%89%E6%98%BE%E8%91%97%E6%94%B9%E5%96%84%E3%80%82%3C/p%3E%0A%3Ch2%3E%E5%85%AD%E3%80%81%E5%89%8D%E6%B2%BF%E8%B6%8B%E5%8A%BF%EF%BC%9A%E6%8E%A8%E7%90%86%E5%BC%95%E6%93%8E%E7%9A%84%E4%B8%8B%E4%B8%80%E4%B8%AA%E6%88%98%E5%9C%BA%3C/h2%3E%0A%3Ch3%3E6.1%20%E5%88%86%E5%B8%83%E5%BC%8F%E6%8E%A8%E7%90%86%E4%B8%8E%E5%BC%82%E6%9E%84%E8%B0%83%E5%BA%A6%3C/h3%3E%0A%3Cp%3E%E5%BD%93%E6%A8%A1%E5%9E%8B%E8%A7%84%E6%A8%A1%E4%BB%8E%2070B%20%E6%89%A9%E5%B1%95%E5%88%B0%20405B%20%E7%94%9A%E8%87%B3%E6%9B%B4%E5%A4%A7%E6%97%B6%EF%BC%8C%E5%8D%95%E8%8A%82%E7%82%B9%208%20%E5%8D%A1%E5%B7%B2%E7%BB%8F%E4%B8%8D%E5%A4%9F%E3%80%82%E6%9C%AA%E6%9D%A5%E7%9A%84%E6%8E%A8%E7%90%86%E7%B3%BB%E7%BB%9F%E9%9C%80%E8%A6%81%E5%9C%A8%E5%A4%9A%E8%8A%82%E7%82%B9%E9%97%B4%E5%88%86%E5%B8%83%20KV%20Cache%EF%BC%8C%E5%AE%9E%E7%8E%B0%E8%B7%A8%E8%8A%82%E7%82%B9%E7%9A%84%20PagedAttention%E3%80%82SGLang%E3%80%81%20LMDeploy%20%E7%AD%89%E5%BC%95%E6%93%8E%E5%B7%B2%E7%BB%8F%E5%9C%A8%E8%BF%99%E4%B8%AA%E6%96%B9%E5%90%91%E4%B8%8A%E6%9C%89%E4%BA%86%E5%88%9D%E6%AD%A5%E6%8E%A2%E7%B4%A2%E3%80%82%3C/p%3E%0A%3Ch3%3E6.2%20%E9%87%8F%E5%8C%96%E4%B8%8E%E5%8E%8B%E7%BC%A9%E7%9A%84%E5%8D%8F%E5%90%8C%E4%BC%98%E5%8C%96%3C/h3%3E%0A%3Cp%3EGPTQ%E3%80%81AWQ%E3%80%81SmoothQuant%20%E7%AD%89%E9%87%8F%E5%8C%96%E6%8A%80%E6%9C%AF%E5%8F%AF%E4%BB%A5%E5%B0%86%E6%A8%A1%E5%9E%8B%E4%BB%8E%20FP16%20%E5%8E%8B%E7%BC%A9%E5%88%B0%20INT4/INT8%EF%BC%8C%E7%9B%B4%E6%8E%A5%E9%99%8D%E4%BD%8E%20KV%20Cache%20%E7%9A%84%E6%98%BE%E5%AD%98%E5%8D%A0%E7%94%A8%E3%80%82%E5%B0%A4%E5%85%B6%E6%98%AF%20%3Cstrong%3EKV%20Cache%20%E9%87%8F%E5%8C%96%3C/strong%3E%E2%80%94%E2%80%94%E5%9C%A8%20Attention%20%E8%AE%A1%E7%AE%97%E4%B8%AD%E4%BD%BF%E7%94%A8%E4%BD%8E%E7%B2%BE%E5%BA%A6%E7%9A%84%20KV%EF%BC%8C%E7%B2%BE%E5%BA%A6%E6%8D%9F%E5%A4%B1%E6%9E%81%E5%B0%8F%EF%BC%88%3C0.1%20%E4%B8%AA%20perplexity%20%E7%82%B9%EF%BC%89%EF%BC%8C%E4%BD%86%E6%98%BE%E5%AD%98%E5%92%8C%E5%B8%A6%E5%AE%BD%E8%8A%82%E7%9C%81%E5%8F%AF%E8%BE%BE%204%20%E5%80%8D%E3%80%82%3C/p%3E%0A%3Ch3%3E6.3%20%E6%8A%95%E6%9C%BA%E8%A7%A3%E7%A0%81%E7%9A%84%E5%8F%98%E4%BD%93%E4%B8%8E%E8%BF%9B%E5%8C%96%3C/h3%3E%0A%3Cp%3ETree-based%20Speculative%20Decoding%20%E5%B0%86%E7%BA%BF%E6%80%A7%E7%9A%84%20candidate%20%E5%BA%8F%E5%88%97%E6%89%A9%E5%B1%95%E4%B8%BA%E5%80%99%E9%80%89%E6%A0%91%E7%BB%93%E6%9E%84%EF%BC%8C%E5%9C%A8%E4%B8%80%E6%AC%A1%E9%AA%8C%E8%AF%81%E4%B8%AD%E4%BB%8E%E5%A4%9A%E4%B8%AA%E5%88%86%E6%94%AF%E4%B8%AD%E9%80%89%E6%8B%A9%E6%9C%80%E4%BC%98%E8%B7%AF%E5%BE%84%E3%80%82%E8%80%8C%20Prompt%20Lookup%20Decoding%20%E5%88%A9%E7%94%A8%E7%94%A8%E6%88%B7%20prompt%20%E4%B8%AD%E5%B7%B2%E6%9C%89%E7%9A%84%E7%9F%AD%E8%AF%AD%E7%9B%B4%E6%8E%A5%E4%BD%9C%E4%B8%BA%20draft%EF%BC%8C%E5%9C%A8%E7%89%B9%E5%AE%9A%E5%9C%BA%E6%99%AF%E4%B8%8B%E5%AE%9E%E7%8E%B0%E9%9B%B6%E9%A2%9D%E5%A4%96%E5%BC%80%E9%94%80%E7%9A%84%E6%8A%95%E6%9C%BA%E8%A7%A3%E7%A0%81%E3%80%82%3C/p%3E%0A%3Ch2%3E%E4%B8%83%E3%80%81%E6%80%BB%E7%BB%93%3C/h2%3E%0A%3Cp%3ELLM%20%E6%8E%A8%E7%90%86%E4%BC%98%E5%8C%96%E7%9A%84%E6%A0%B8%E5%BF%83%E6%98%AF%E4%B8%80%E4%B8%AA%E7%B3%BB%E7%BB%9F%E5%B7%A5%E7%A8%8B%E9%97%AE%E9%A2%98%E2%80%94%E2%80%94%E5%AE%83%E4%B8%8D%E5%8F%AA%E6%98%AF%E7%AE%97%E6%B3%95%E5%B1%82%E9%9D%A2%E7%9A%84%E5%88%9B%E6%96%B0%EF%BC%8C%E6%9B%B4%E6%98%AF%E6%93%8D%E4%BD%9C%E7%B3%BB%E7%BB%9F%E5%8E%9F%E7%90%86%EF%BC%88%E5%88%86%E9%A1%B5%E3%80%81%E8%B0%83%E5%BA%A6%EF%BC%89%E4%B8%8E%E7%A1%AC%E4%BB%B6%E7%89%B9%E6%80%A7%EF%BC%88GPU%20%E8%AE%BF%E5%AD%98%E6%A8%A1%E5%BC%8F%E3%80%81Tensor%20Core%EF%BC%89%E7%9A%84%E6%B7%B1%E5%BA%A6%E7%BB%93%E5%90%88%E3%80%82%E4%BB%8E%20PagedAttention%20%E5%88%B0%20Continuous%20Batching%20%E5%86%8D%E5%88%B0%20Speculative%20Decoding%EF%BC%8C%E6%AF%8F%E4%B8%80%E6%AD%A5%E4%BC%98%E5%8C%96%E9%83%BD%E8%A7%A3%E5%86%B3%E4%BA%86%E4%B8%80%E4%B8%AA%E6%98%8E%E7%A1%AE%E7%9A%84%E5%B7%A5%E7%A8%8B%E7%93%B6%E9%A2%88%EF%BC%8C%E8%80%8C%E5%AE%83%E4%BB%AC%E7%BB%84%E5%90%88%E5%9C%A8%E4%B8%80%E8%B5%B7%E7%9A%84%E6%94%B6%E7%9B%8A%E8%BF%9C%E8%B6%85%E5%90%84%E9%83%A8%E5%88%86%E4%B9%8B%E5%92%8C%E3%80%82%3C/p%3E%0A%3Cp%3E%E5%AF%B9%E4%BA%8E%E5%AE%9E%E8%B7%B5%E8%80%85%E6%9D%A5%E8%AF%B4%EF%BC%8C%E7%90%86%E8%A7%A3%E8%BF%99%E4%BA%9B%E5%BA%95%E5%B1%82%E6%9C%BA%E5%88%B6%E7%9A%84%E4%BB%B7%E5%80%BC%E5%9C%A8%E4%BA%8E%EF%BC%9A%E5%BD%93%E4%BD%A0%E5%9C%A8%20vLLM%20%E4%B8%AD%E7%9C%8B%E5%88%B0%E4%B8%80%E4%B8%AA%20OOM%20%E9%94%99%E8%AF%AF%E6%97%B6%EF%BC%8C%E4%BD%A0%E4%BC%9A%E7%9F%A5%E9%81%93%E8%AF%A5%E8%B0%83%E9%AB%98%20%3Ccode%3Egpu-memory-utilization%3C/code%3E%20%E8%BF%98%E6%98%AF%E5%87%8F%E5%B0%8F%20%3Ccode%3Emax-num-seqs%3C/code%3E%EF%BC%9B%E5%BD%93%E4%BD%A0%E5%8F%91%E7%8E%B0%20P99%20%E5%BB%B6%E8%BF%9F%E6%8A%96%E5%8A%A8%E6%97%B6%EF%BC%8C%E4%BD%A0%E4%BC%9A%E6%80%80%E7%96%91%E6%98%AF%E4%B8%8D%E6%98%AF%20chunked-prefill%20%E6%B2%A1%E6%9C%89%E5%BC%80%E5%90%AF%EF%BC%9B%E5%BD%93%E4%BD%A0%E7%9A%84%20Draft%20%E6%A8%A1%E5%9E%8B%E6%8E%A5%E5%8F%97%E7%8E%87%E5%BE%88%E4%BD%8E%E6%97%B6%EF%BC%8C%E4%BD%A0%E4%BC%9A%E6%A3%80%E6%9F%A5%E4%B8%A4%E4%B8%AA%E6%A8%A1%E5%9E%8B%E7%9A%84%E5%88%86%E5%B8%83%E5%B7%AE%E5%BC%82%E6%98%AF%E5%90%A6%E8%BF%87%E5%A4%A7%E3%80%82%3C/p%3E%0A%3Cp%3E%E8%BF%99%E5%B0%B1%E6%98%AF%E7%B3%BB%E7%BB%9F%E5%B7%A5%E7%A8%8B%E7%9A%84%E9%AD%85%E5%8A%9B%EF%BC%9A%E5%8E%9F%E7%90%86%E5%91%8A%E8%AF%89%E4%BD%A0%E5%8E%9F%E5%9B%A0%EF%BC%8C%E6%95%B0%E6%8D%AE%E6%8C%87%E5%AF%BC%E4%BD%A0%E4%BC%98%E5%8C%96%E3%80%82%3C/p%3E