b39c2af9db
As cpu-partitioning doesn't use isolcpus=, isolated CPUs can incur
very high latency due to load balancing performed by the kernel
scheduler.
This patch introduces a new configuration option called
no_rebalance_cores=. This options allow users to set a range
of CPUs to have load balancing disabled from /proc. It is
expected that users will only disable load balancing on CPUs
which run PMD threads (or vCPU threads running PMD threads),
since disabling this for all isolated CPUs can cause certain
performance problems for OpenStack.
Here are some numbers:
- Without using no_rebalance_cores=, I can easily get 40us+
in load balancing isolated CPUs:
<...>-14140 [001] 1656.121155: funcgraph_entry: + 43.927 us | rebalance_domains();
<...>-14140 [001] 1660.117150: funcgraph_entry: + 41.587 us | rebalance_domains();
<...>-14140 [001] 1658.119152: funcgraph_entry: + 41.565 us | rebalance_domains();
<...>-14140 [001] 1668.110140: funcgraph_entry: + 40.539 us | rebalance_domains();
- When using no_rebalance_cores=, the maximum latency I
get is ~ 1us:
<...>-2440 [012] 1705.078076: funcgraph_entry: 1.024 us | rebalance_domains();
<...>-2438 [008] 2021.065218: funcgraph_entry: 0.995 us | rebalance_domains();
<...>-2447 [010] 2223.057289: funcgraph_entry: 0.991 us | rebalance_domains();
IMPORTANT: 10 minutes run only.
This solution was developed by Andrew Theurer <atheurer@redhat.com>,
I'm merely integrating it into the profile.
Signed-off-by: Luiz Capitulino <lcapitulino@redhat.com>
57 lines
2 KiB
Text
57 lines
2 KiB
Text
# tuned configuration
|
|
#
|
|
|
|
[main]
|
|
summary=Optimize for CPU partitioning
|
|
include=network-latency
|
|
|
|
[variables]
|
|
# User is responsible for updating variables.conf with variable content such as isolated_cores=X-Y
|
|
include=/etc/tuned/cpu-partitioning-variables.conf
|
|
|
|
isolated_cores_assert_check = \\${isolated_cores}
|
|
# Fail if isolated_cores are not set
|
|
assert1=${f:assertion_non_equal:isolated_cores are set:${isolated_cores}:${isolated_cores_assert_check}}
|
|
|
|
# tmpdir
|
|
tmpdir=${f:strip:${f:exec:/usr/bin/mktemp:-d}}
|
|
# Non-isolated cores cpumask including offline cores
|
|
isolated_cores_expanded=${f:cpulist_unpack:${isolated_cores}}
|
|
isolated_cpumask=${f:cpulist2hex:${isolated_cores_expanded}}
|
|
not_isolated_cores_expanded=${f:cpulist_invert:${isolated_cores_expanded}}
|
|
isolated_cores_present_expanded=${f:cpulist_present:${isolated_cores}}
|
|
not_isolated_cores_present_expanded=${f:cpulist_present:${not_isolated_cores_expanded}}
|
|
not_isolated_cpumask=${f:cpulist2hex:${not_isolated_cores_expanded}}
|
|
no_balance_cores_expanded=${f:cpulist_unpack:${no_balance_cores}}
|
|
|
|
# Fail if isolated_cores contains CPUs which are not present
|
|
assert2=${f:assertion:isolated_cores contains present CPU(s):${isolated_cores_expanded}:${isolated_cores_present_expanded}}
|
|
|
|
[sysctl]
|
|
kernel.hung_task_timeout_secs = 600
|
|
kernel.nmi_watchdog = 0
|
|
vm.stat_interval = 10
|
|
kernel.timer_migration = 1
|
|
|
|
[sysfs]
|
|
/sys/bus/workqueue/devices/writeback/cpumask = ${not_isolated_cpumask}
|
|
/sys/devices/virtual/workqueue/cpumask = ${not_isolated_cpumask}
|
|
/sys/devices/system/machinecheck/machinecheck*/ignore_ce = 1
|
|
|
|
[systemd]
|
|
cpu_affinity=${not_isolated_cores_expanded}
|
|
|
|
[script]
|
|
priority=5
|
|
script=${i:PROFILE_DIR}/script.sh
|
|
|
|
[scheduler]
|
|
isolated_cores=${isolated_cores}
|
|
ps_blacklist=.*pmd.*;.*PMD.*;^DPDK;.*qemu-kvm.*
|
|
|
|
[bootloader]
|
|
priority=10
|
|
initrd_remove_dir=True
|
|
initrd_dst_img=tuned-initrd.img
|
|
initrd_add_dir=${tmpdir}
|
|
cmdline_cpu_part=+nohz=on nohz_full=${isolated_cores} rcu_nocbs=${isolated_cores} tuned.non_isolcpus=${not_isolated_cpumask} intel_pstate=disable nosoftlockup
|