Linux内核cpufreq子系统与schedutil调控器深度工程实践：异构服务器能效优化全路径解析

概述

在现代数据中心中，能耗成本已占到TCO（总拥有成本）的30%-40%。Linux内核的cpufreq子系统配合intel_pstate/amd_pstate驱动，以及schedutil频率调控器，构成了CPU能效管理的核心基础设施。本文将从内核源码层面剖析cpufreq框架的架构设计、schedutil调控器的工作原理、EAS（Energy Aware Scheduling）与cpufreq的协同机制，以及在异构服务器（混合架构、热插拔、NUMA）场景下的工程实践和调优策略。

一、cpufreq子系统架构全景

1.1 核心数据结构

cpufreq子系统的核心围绕几个关键数据结构展开。struct cpufreq_policy 是每个调频策略的基本单元，封装了一组共享频率/电压域的CPU。struct cpufreq_governor 定义了调频策略的具体算法（如ondemand、performance、schedutil）。struct cpufreq_driver 则是底层硬件驱动接口，由intel_pstate、amd_pstate、acpi-cpufreq等实现。

// include/linux/cpufreq.h
struct cpufreq_policy {
    cpumask_var_t       cpus;       /* 该policy覆盖的CPU */
    cpumask_var_t       related_cpus;
    cpumask_var_t       realtime_cpus;

    unsigned int        min;
    unsigned int        max;
    unsigned int        cur;        /* 当前硬件频率(kHz) */
    unsigned int        restore_freq;
    unsigned int        cpuinfo.min_freq;
    unsigned int        cpuinfo.max_freq;
    unsigned int        cpuinfo.performance_states[64]; /* P states (Intel) */

    struct cpufreq_governor *governor;
    void               *governor_data;
    struct cpufreq_scaling_cur_limits limits;

    struct cpufreq_stats   *stats;
    struct cpufreq_freq_table *freq_table;

    struct cpufreq_driver  *driver;
    struct cpufreq_cpu_driver *cpu_driver;

    struct kobject      kobj;
    struct completion   kobj_unregister;

    bool                governor_enabled;
    unsigned int        last_policy;
    unsigned int        cached_target_freq;
    unsigned int        transition_delay_us;
    unsigned int        fast_switch_possible;
    bool                fast_switch_enabled;
};

1.2 驱动层：intel_pstate vs amd_pstate

intel_pstate自Linux 3.9引入，是一个独立的cpufreq驱动，跳过通用acpi-cpufreq，直接与CPU的MSR寄存器通信。其核心优势在于支持Hardware P-States（HWP），允许CPU硬件自主管理频率。

// drivers/cpufreq/intel_pstate.c
static struct cpufreq_driver intel_pstate_cpufreq_driver = {
    .flags          = CPUFREQ_CONST_LOOPS |
                      CPUFREQ_HAVE_GOVERNOR_PER_POLICY,
    .verify         = intel_pstate_verify_policy,
    .setpolicy      = intel_pstate_set_policy,
    .init           = intel_pstate_cpu_init,
    .exit           = intel_pstate_cpu_exit,
    .online         = intel_pstate_cpu_online,
    .offline        = intel_pstate_cpu_offline,
    .suspend        = intel_pstate_suspend,
    .resume         = intel_pstate_resume,
    .name           = "intel_pstate",
    .attr           = intel_pstate_attr,  /* perf-bias, no_turbo, etc. */
};

/* MSR读取当前P-state */
static int intel_pstate_get(unsigned int cpu, struct cpufreq_policy *policy)
{
    u64 val;
    rdmsrl_on_cpu(MSR_IA32_PERF_STATUS, cpu, &val);
    return (val >> 8) & 0xFF;  /* 当前倍频 */
}

amd_pstate是Linux 5.17引入的新一代AMD驱动，同样基于CPPC（Collaborative Processor Performance Control）协议，支持三种模式：active（硬件自主控制）、passive（软件调控）、guided（EPP引导的自适应）。

// drivers/cpufreq/amd-pstate.c
static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state)
{
    u64 cppc_max, cppc_min;

    cppc_get_perf_caps(policy->cpu, &cppc_max, &cppc_min);
    wrmsrl_on_cpu(policy->cpu, MSR_AMD_CPPC_REQ,
                  FIELD_PREP(AMD_CPPC_MAX_PERF, cppc_max) |
                  FIELD_PREP(AMD_CPPC_MIN_PERF, cppc_min));
    return 0;
}

二、schedutil调控器深度解析

2.1 设计理念

schedutil区别于传统的ondemand/conservative调控器，最大的创新在于直接从内核调度器获取CPU利用率，无需周期性采样。其基于CFS的PELT（Per-Entity Load Tracking）指标做频率决策，具有低延迟和高精度的特点。

// kernel/sched/cpufreq_schedutil.c
static void sugov_update_single(struct sugov_policy *sg_policy, u64 time,
                                unsigned long util, unsigned long max,
                                unsigned int flags)
{
    unsigned long freq_next, freq_new;

    /* 从调度器获取利用率（PELT计算） */
    /* util: 0 ~ max (arch_scale_cpu_capacity(cpu)) */
    freq_next = util * policy->cpuinfo.max_freq / max;

    /* 频率限制检查 */
    freq_next = clamp(freq_next, policy->min, policy->max);

    /* fast_switch路径（HWP支持时） */
    if (policy->fast_switch_enabled) {
        cpufreq_driver_fast_switch(policy, freq_next);
        return;
    }

    /* 慢速路径：触发IPI中断修改频率 */
    raw_spin_lock(&sg_policy->update_lock);
    sg_policy->next_freq = freq_next;
    cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_L);
    raw_spin_unlock(&sg_policy->update_lock);
}

2.2 PELT利用率计算

调度器通过PELT（Per-Entity Load Tracking）维护每cgroup/task的负载贡献度。PELT使用指数衰减模型跟踪负载历史，半衰期32ms（对应LOAD_AVG_PERIOD=32的负载平均周期）。

// kernel/sched/pelt.c
static __always_inline u64 decay_load(u64 val, u64 n)
{
    unsigned int local_n;

    if (unlikely(n > LOAD_AVG_PERIOD * 63))
        return 0;

    local_n = n;
    if (unlikely(local_n >= LOAD_AVG_PERIOD)) {
        val >>= local_n / LOAD_AVG_PERIOD;
        local_n %= LOAD_AVG_PERIOD;
    }

    return val * decay_to_leaky_table[local_n] >> 3; /* 折半系数 */
}

static u64 __accumulate_pelt_sectors(struct sched_avg *sa)
{
    u64 decayed, delta;

    decayed = decay_load(sa->runnable_load_sum, periods + 1);
    delta = sa->load_sum - decayed;

    return delta;
}

2.3 频率决策数学

schedutil的频率决策本质是线性映射：

freq_next = max_freq × util / capacity

其中util来自PELT跟踪的CPU运行占比，capacity来自arch_scale_cpu_capacity()（考虑arm_big_little异构的capa权重）。

对于混合架构（如Intel P-core+E-core），不同CPU的capa权重不同：

// kernel/arch_topology.c
void update_cpu_capacity(unsigned int cpu, unsigned long capacity)
{
    /* 考虑ITMT（Intel Turbo Boost Max 3.0）拓扑 */
    /* P-core capacity = 1024, E-core capacity = 512 (典型值) */
    per_cpu(cpu_scale, cpu) = capacity;
}

三、EAS（Energy Aware Scheduling）与cpufreq协同

3.1 能源模型

EAS通过CPUFreq获取每个OPP（Operating Performance Point）的功耗数据，建立"频率-功耗"模型（Power = C × V² × f）。内核中该模型存储在struct em_perf_state中：

// include/linux/energy_model.h
struct em_perf_state {
    unsigned long frequency;  /* kHz */
    unsigned long power;      /* mW */
    unsigned long cost;       /* 归一化能耗代价 */
    unsigned long flags;
};

struct em_perf_domain {
    struct em_perf_state *table;
    int nr_perf_states;
    unsigned long cpus[];     /* 共享能源策略的CPU */
};

3.2 EAS放置决策

EAS在task placement时会同时考虑性能和能耗。find_energy_efficient_cpu()遍历所有候选CPU，估算各选择的总能耗增量：

// kernel/sched/fair.c
static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu)
{
    unsigned long base_energy, best_energy = ULONG_MAX;
    int cpu, best_energy_cpu = prev_cpu;
    struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask);

    for_each_cpu_and(cpus, &p->cpus_mask, cpu_active_mask) {
        /* 估算放置该任务后的CPU利用率 */
        util = cpu_util_next(cpu, p, 0);

        /* 获取当前频率下的功耗 */
        pd = rcu_dereference(per_cpu(EM_DOMAIN, cpu));
        freq = cpu_freq(cpu, util);  /* schedutil公式反推 */
        base_energy = em_cpu_energy(pd, max_cap, prev_cpu);

        /* 选择能耗增量最小的CPU */
        if (energy < best_energy && fits_capacity(util, max_cap, cpu)) {
            best_energy_cpu = cpu;
            best_energy = energy;
        }
    }
    return best_energy_cpu;
}

3.3 EPP（Energy Performance Preference）

intel_pstate支持EPP（Energy per Preference）和EPB（Energy Performance Bias）接口，通过MSR_IA32_HWP_REQUEST将能效偏好下发给CPU硬件自主调频：

/* /sys/devices/system/cpu/cpu*/cpufreq/energy_performance_preference */
static const char *epp_strings[] = {
    [EPP_INDEX_PERFORMANCE]       = "performance",
    [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance",
    [EPP_INDEX_BALANCE_POWER]     = "balance_power",
    [EPP_INDEX_POWER]             = "power",
};
/* 对应值：0(性能) ~ 255(能效) */
/* HWP硬件依据此值在perf限制范围内自主调频 */

四、生产环境工程实践

4.1 频率震荡问题诊断

schedutil在高频负载变化时可能出现频率震荡。内核社区引入了双调用（dual-call）机制和最小频率保持时间来解决此问题：

# 查看频率状态
$ cat /proc/cpuinfo | grep "MHz"
$ cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq

# 查看cpufreq统计数据
$ cat /sys/devices/system/cpu/cpu0/cpufreq/stats/time_in_state
800000  12345
1200000 6789
2400000 2345
3600000 123

# 使用trace观测调频决策
$ trace-cmd record -e 'cpufreq_schedutil:*'
$ kernelshark trace.dat

4.2 热插拔后频率恢复策略

在异构服务器中，CPU热插拔后需要正确恢复cpufreq策略：

# 热插出时保存当前策略
echo 0 > /sys/devices/system/cpu/cpuN/online

# 热插回时恢复
echo 1 > /sys/devices/system/cpu/cpuN/online

# 内核处理路径：
# kernel/cpu.c -> cpu_down -> cpufreq_offline()
# 将policy->governor临时保存到percpu变量
# 重新上线时调用cpufreq_online() -> if (governor_enabled)

4.3 NUMA affinity的cpufreq隔离

当NUMA节点绑定时，可设置per-CPU频率策略：

# 设置numa node 0使用performance
for cpu in $(lscpu -p=CPU,Node | awk -F, '$2==0{print $1}'); do
    echo performance > /sys/devices/system/cpu/cpu$cpu/cpufreq/scaling_governor
done

# 节点1节能模式
for cpu in $(lscpu -p=CPU,Node | awk -F, '$2==1{print $1}'); do
    echo schedutil > /sys/devices/system/cpu/cpu$cpu/cpufreq/scaling_governor
    echo balance_power > /sys/devices/system/cpu/cpu$cpu/cpufreq/energy_performance_preference
done

4.4 性能基准测试与监控

使用eBPF可无侵入地实时观测cpufreq状态变化：

/* bpftrace脚本示例 */
kprobe:cpufreq_driver_target {
    $policy = (struct cpufreq_policy *)arg0;
    $freq = arg1;

    time("%H:%M:%S ");
    printf("CPU%d target_freq=%d kHz curr=%d kHz governor=%s\n",
           $policy->cpu, $freq, $policy->cur, $policy->governor->name);
}

tracepoint:power:cpu_frequency {
    time("%H:%M:%S ");
    printf("CPU%d freq_set=%d util=%d\n",
           args->cpu_id, args->state, 
           bpf_get_task_util(args->cpu_id));
}

$ bpftrace cpufreq_trace.bt -o cpufreq_trace.log &
$ stress-ng --cpu 16 --timeout 60s
$ kill %1 && cat cpufreq_trace.log

4.5 sysctl与tuned配置

# tuned-adm配置推荐
# /etc/tuned/profiles/throughput-performance/tuned.conf
[cpu]
governor=schedutil
energy_performance_preference=performance
sampling_down_factor=1

[sysfs]
/sys/devices/system/cpu/cpufreq/policy*/schedutil/rate_limit_us = 500
/sys/devices/system/cpu/intel_pstate/min_perf_pct = 100

# 应用
tuned-adm profile throughput-performance
tuned-adm active  # 确认当前profile

五、瓶颈与演进方向

5.1 当前瓶颈

1. PELT半衰期固定：32ms半衰期对突发负载响应滞后，社区正在探索自适应半衰期
2. HWP硬件局限：部分场景HWP的"激进升频+保守降频"策略导致尾延迟恶化
3. 混合架构调度冲突：P-core和E-core的capacity差异与EAS能耗模型存在拓扑盲区
4. 虚拟化环境：vcpu cpufreq在KVM/QEMU中依赖host pass-through，缺少细粒度guest调控

5.2 社区演进

• Linux 6.6+：引入cppc_perf内核驱动统一CPPC接口（AMD/部分ARM），取代acpi_cppc
• Linux 6.8+：AMD P-State EPP driver重构，支持per-CPU的EPP调优
• P-Core/E-Core协同：ITMT 2.0 + Thread Director硬件辅助，shim层感知核心能力调度
• Intel Speed Select Technology（SST）：基站/边缘场景下的实时频率硬件分区

总结

Linux内核cpufreq子系统从传统的周期性采样调频，演进到基于PELT实时利用率的schedutil，再到支持HWP硬件自主调频的intel_pstate/amd_pstate框架，形成了"硬件能力发现 + 软件策略引导 + 调度器协同"的完整能效管理栈。在异构服务器（ARM big.LITTLE、Intel AlderLake/Raptor Lake、AMD Bergamo）场景下，理解并正确配置cpufreq + EAS + EPP，是获得最佳每瓦性能（Perf/Watt）的关键。对于高性能计算场景，建议采用performance模式避免频率切换波动；对于密度优化的云原生场景，schedutil + EAS可在不牺牲SLA的前提下节省15%-30%能耗。

实战建议：生产环境中推荐先用tuned-adm profile throughput-performance统一管理cpufreq/CPUidle，再通过perf stat -e power/energy-pkg/验证实际能效收益。如遇QoS回退问题，优先检查energy_performance_preference和scaling_max_freq的互斥策略。