void trigger_load_balance(struct rq *rq, int cpu){        /* Don't need to rebalance while attached to NULL domain */        if (time_after_eq(jiffies, rq->next_balance) &&            likely(!on_null_domain(cpu)))                raise_softirq(SCHED_SOFTIRQ);#ifdef CONFIG_NO_HZ_COMMON        if (nohz_kick_needed(rq) && likely(!on_null_domain(cpu)))                nohz_balancer_kick(cpu);#endif}

static void __sched __schedule(void){        ...        if (unlikely(!rq->nr_running))                idle_balance(cpu, rq);
        ...}

/* * idle_balance is called by schedule() if this_cpu is about to become * idle. Attempts to pull tasks from other CPUs. */void idle_balance(int this_cpu, struct rq *this_rq){	unsigned long next_balance = jiffies + HZ;	struct sched_domain *sd;	int pulled_task = 0;	u64 curr_cost = 0;
	this_rq->idle_stamp = rq_clock(this_rq);
    /* 如果该CPU平均空闲时间小于/proc中的配置值或者该cpu调度域中所有cpu都是idle状态，那么不需要负载均衡了*/	if (this_rq->avg_idle < sysctl_sched_migration_cost ||	    !this_rq->rd->overload) {		rcu_read_lock();		sd = rcu_dereference_check_sched_domain(this_rq->sd);		if (sd)			update_next_balance(sd, 0, &next_balance);		rcu_read_unlock();
		goto out;	}
	/*	 * Drop the rq->lock, but keep IRQ/preempt disabled.	 */	raw_spin_unlock(&this_rq->lock);
	update_blocked_averages(this_cpu);	rcu_read_lock();    /* 从底向上遍历调度域，只要迁移成功一个进程就跳出循环*/	for_each_domain(this_cpu, sd) {		int should_balance;		u64 t0, domain_cost;
		if (!(sd->flags & SD_LOAD_BALANCE))			continue;
        /*          * 如果（当前累积的负载均衡开销时间 + 历史上该层级负载均衡开销最大值）已经大于CPU平均空闲时间了，         * 那么就没有必要负载均衡了。注意，sd->max_newidle_lb_cost会在load_balance()函数中缓慢减少。         */		if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) {			update_next_balance(sd, 0, &next_balance);			break;		}
        /* 我的机器上该标记总是设置了SD_BALANCE_NEWIDLE */		if (sd->flags & SD_BALANCE_NEWIDLE) {			t0 = sched_clock_cpu(this_cpu);
			pulled_task = load_balance(this_cpu, this_rq,						   sd, CPU_NEWLY_IDLE,						   &should_balance);           			domain_cost = sched_clock_cpu(this_cpu) - t0;			if (domain_cost > sd->max_newidle_lb_cost)				sd->max_newidle_lb_cost = domain_cost;
           /* 记录了当前负载均衡开销累计值 */			curr_cost += domain_cost;		}
		update_next_balance(sd, 0, &next_balance);
		/*		 * Stop searching for tasks to pull if there are		 * now runnable tasks on this rq.		 */        		if (pulled_task || this_rq->nr_running > 0) {			this_rq->idle_stamp = 0;			break;		}	}	rcu_read_unlock();
	raw_spin_lock(&this_rq->lock);
out:	/* Move the next balance forward */	if (time_after(this_rq->next_balance, next_balance))		this_rq->next_balance = next_balance;
	if (curr_cost > this_rq->max_idle_balance_cost)		this_rq->max_idle_balance_cost = curr_cost;}

/* * sched_balance_self: balance the current task (running on cpu) in domains * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and * SD_BALANCE_EXEC. * * Balance, ie. select the least loaded group. * * Returns the target CPU number, or the same CPU if no balancing is needed. * * preempt must be disabled. *//* A进程给自己或者B进程选择一个CPU运行， * 1: A唤醒B * 2: A fork()出B后让B运行 * 3: A execute()后重新选择自己将要运行的CPU */ static intselect_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags){	struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL;	int cpu = smp_processor_id();	int new_cpu = cpu;	int want_affine = 0;	int sync = wake_flags & WF_SYNC;
    /* 当A进程唤醒B进程时，从try_to_wake_up()进入本函数，这里会置位SD_BALANCE_WAKE。 */	if (sd_flag & SD_BALANCE_WAKE) {        /* B进程被唤醒时希望运行的CPU尽可能离A进程所在CPU近一点 */		if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))			want_affine = 1;		new_cpu = prev_cpu;		record_wakee(p);	}
	rcu_read_lock();    /*      * 如果是A唤醒B模式，则查找同时包含A所在cpu和B睡眠前所在prev_cpu的最低级别的调度域。因为A进程     * 和B进程大概率会有某种数据交换关系，唤醒B时让它们所在的CPU离的近一点会性能最优。     * 否则，查找包含了sd_flag的最高调度域。     */	for_each_domain(cpu, tmp) {		if (!(tmp->flags & SD_LOAD_BALANCE))			continue;
		/*		 * If both cpu and prev_cpu are part of this domain,		 * cpu is a valid SD_WAKE_AFFINE target.		 */        		if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&		    cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {			affine_sd = tmp;			break;		}
		if (tmp->flags & sd_flag)			sd = tmp;	}
    /* 如果是A唤醒B模式，则在同时包含A所在cpu和B睡眠前所在prev_cpu的最低级别的调度域中寻找合适的CPU */	if (affine_sd) {       /*         * wake_affine()计算A所在CPU和B睡眠前所在CPU的负载值，判断出B进程唤醒时是否        * 需要离A近一点。        */		if (cpu != prev_cpu && wake_affine(affine_sd, p, sync))			prev_cpu = cpu;
       /* 在与prev_cpu共享LLC的CPU中寻找空闲CPU，如果没有找到，则返回prev_cpu。这里将确定        * B进程唤醒后在哪个CPU运行。        */		new_cpu = select_idle_sibling(p, prev_cpu);		goto unlock;	}
    /* 到这里，A进程和B进程基本是没有啥亲缘关系的。不用考虑两个进程的Cache亲缘性 */	while (sd) {		int load_idx = sd->forkexec_idx;		struct sched_group *group;		int weight;
		if (!(sd->flags & sd_flag)) {			sd = sd->child;			continue;		}
		if (sd_flag & SD_BALANCE_WAKE)			load_idx = sd->wake_idx;
		group = find_idlest_group(sd, p, cpu, load_idx);		if (!group) {			sd = sd->child;			continue;		}
		new_cpu = find_idlest_cpu(group, p, cpu);		if (new_cpu == -1 || new_cpu == cpu) {			/* Now try balancing at a lower domain level of cpu */			sd = sd->child;			continue;		}
		/* Now try balancing at a lower domain level of new_cpu */		cpu = new_cpu;		weight = sd->span_weight;		sd = NULL;		for_each_domain(cpu, tmp) {			if (weight <= tmp->span_weight)				break;			if (tmp->flags & sd_flag)				sd = tmp;		}		/* while loop will break here if sd == NULL */	}unlock:	rcu_read_unlock();
	return new_cpu;}

/* * Try and locate an idle CPU in the sched_domain. */ /* 寻找离target CPU最近的空闲CPU(Cache或者内存距离最近)*/static int select_idle_sibling(struct task_struct *p, int target){	struct sched_domain *sd;	struct sched_group *sg;	int i = task_cpu(p);        /* target CPU正好空闲，自己跟自己当然最近*/	if (idle_cpu(target))		return target;
	/*	 * If the prevous cpu is cache affine and idle, don't be stupid.	 */    /*      * p进程所在的CPU跟target CPU有Cache共享关系(SMT,或者MC层才有这个关系)，并且是空闲的，那就用它了。     * Cache共享说明距离很近了      */	if (i != target && cpus_share_cache(i, target) && idle_cpu(i))		return i;
	/*	 * Otherwise, iterate the domains and find an elegible idle cpu.	 */    /*     * 在与target CPU有LLC Cache共享关系的调度域中寻找空闲CPU。注意，在X86体系中只有SMT和MC层的调度域才有Cache共享。     */	sd = rcu_dereference(per_cpu(sd_llc, target));        /* 在我的机器上是按MC，SMT调度域顺序遍历 */	for_each_lower_domain(sd) {		sg = sd->groups;		do {			if (!cpumask_intersects(sched_group_cpus(sg),						tsk_cpus_allowed(p)))				goto next;
           /* 调度组内所有CPU都是空闲状态，才能选定 */			for_each_cpu(i, sched_group_cpus(sg)) {				if (i == target || !idle_cpu(i))					goto next;			}
           /* 选择全部CPU都空闲的调度组中第一个CPU*/			target = cpumask_first_and(sched_group_cpus(sg),					tsk_cpus_allowed(p));			goto done;next:			sg = sg->next;		} while (sg != sd->groups);	}done:	return target;}

/* * sched_exec - execve() is a valuable balancing opportunity, because at * this point the task has the smallest effective memory and cache footprint. */void sched_exec(void){	struct task_struct *p = current;	unsigned long flags;	int dest_cpu;
	raw_spin_lock_irqsave(&p->pi_lock, flags);    /* 选择最合适的CPU，这里由于进程execve()后，之前的Cache就无意义了，因此选择目标CPU不用考虑Cache距离 */	dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0);	if (dest_cpu == smp_processor_id())		goto unlock;
	if (likely(cpu_active(dest_cpu))) {		struct migration_arg arg = { p, dest_cpu };
		raw_spin_unlock_irqrestore(&p->pi_lock, flags);		stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);		return;	}unlock:	raw_spin_unlock_irqrestore(&p->pi_lock, flags);}

/* * wake_up_new_task - wake up a newly created task for the first time. * * This function will do some initial scheduler statistics housekeeping * that must be done for every newly created context, then puts the task * on the runqueue and wakes it. */void wake_up_new_task(struct task_struct *p){	unsigned long flags;	struct rq *rq;
	raw_spin_lock_irqsave(&p->pi_lock, flags);#ifdef CONFIG_SMP	/*	 * Fork balancing, do it here and not earlier because:	 *  - cpus_allowed can change in the fork path	 *  - any previously selected cpu might disappear through hotplug	 */    /* 选择最合适的CPU，这里由于进程execve()后，之前的Cache就无意义了，因此选择目标CPU不用考虑Cache距离 */	set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0));#endif
	/* Initialize new task's runnable average */	init_task_runnable_average(p);	rq = __task_rq_lock(p);	activate_task(rq, p, 0);	p->on_rq = TASK_ON_RQ_QUEUED;	trace_sched_wakeup_new(p, true);	check_preempt_curr(rq, p, WF_FORK);#ifdef CONFIG_SMP	if (p->sched_class->task_woken)		p->sched_class->task_woken(rq, p);#endif	task_rq_unlock(rq, p, &flags);}