diff --git a/profiles/cpu-partitioning/script.sh b/profiles/cpu-partitioning/script.sh index fe725ca..0e94d3a 100755 --- a/profiles/cpu-partitioning/script.sh +++ b/profiles/cpu-partitioning/script.sh @@ -39,10 +39,8 @@ start() { mkdir -p "${TUNED_tmpdir}/usr/lib/dracut/hooks/pre-udev" cp /etc/systemd/system.conf "${TUNED_tmpdir}/etc/systemd/" cp 00-tuned-pre-udev.sh "${TUNED_tmpdir}/usr/lib/dracut/hooks/pre-udev/" - if python /usr/libexec/tuned/defirqaffinity.py "remove" "$TUNED_isolated_cores_expanded"; then - sed -i '/^IRQBALANCE_BANNED_CPUS=/d' /etc/sysconfig/irqbalance - echo "IRQBALANCE_BANNED_CPUS=$TUNED_isolated_cpumask" >>/etc/sysconfig/irqbalance - fi + sed -i '/^IRQBALANCE_BANNED_CPUS=/d' /etc/sysconfig/irqbalance + echo "IRQBALANCE_BANNED_CPUS=$TUNED_isolated_cpumask" >>/etc/sysconfig/irqbalance setup_kvm_mod_low_latency disable_ksm @@ -52,7 +50,6 @@ start() { } stop() { - python /usr/libexec/tuned/defirqaffinity.py "add" "$TUNED_isolated_cores_expanded" if [ "$1" = "full_rollback" ] then sed -i '/^IRQBALANCE_BANNED_CPUS=/d' /etc/sysconfig/irqbalance @@ -63,9 +60,4 @@ stop() { return "$?" } -verify() { - python /usr/libexec/tuned/defirqaffinity.py "verify" "$TUNED_isolated_cores_expanded" - return "$?" -} - process $@ diff --git a/profiles/realtime-virtual-guest/script.sh b/profiles/realtime-virtual-guest/script.sh deleted file mode 100755 index f3d7443..0000000 --- a/profiles/realtime-virtual-guest/script.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh - -. /usr/lib/tuned/functions - -start() { - python /usr/libexec/tuned/defirqaffinity.py "remove" "$TUNED_isolated_cores_expanded" && - return "$?" -} - -stop() { - python /usr/libexec/tuned/defirqaffinity.py "add" "$TUNED_isolated_cores_expanded" - return "$?" -} - -verify() { - python /usr/libexec/tuned/defirqaffinity.py "verify" "$TUNED_isolated_cores_expanded" - return "$?" -} - -process $@ diff --git a/profiles/realtime-virtual-guest/tuned.conf b/profiles/realtime-virtual-guest/tuned.conf index cf349ef..b90e76f 100644 --- a/profiles/realtime-virtual-guest/tuned.conf +++ b/profiles/realtime-virtual-guest/tuned.conf @@ -34,8 +34,7 @@ group.rcub=0:f:4:*:rcub.* # for i in `pgrep ktimersoftd` ; do grep Cpus_allowed_list /proc/$i/status ; done group.ktimersoftd=0:f:3:*:ktimersoftd.* -[script] -script=${i:PROFILE_DIR}/script.sh +ps_blacklist=ksoftirqd.*;rcuc.*;rcub.*;ktimersoftd.* [bootloader] cmdline_rvg=+nohz=on nohz_full=${isolated_cores} rcu_nocbs=${isolated_cores} diff --git a/profiles/realtime-virtual-host/script.sh b/profiles/realtime-virtual-host/script.sh index e821bef..515d254 100755 --- a/profiles/realtime-virtual-host/script.sh +++ b/profiles/realtime-virtual-host/script.sh @@ -44,10 +44,6 @@ run_tsc_deadline_latency() } start() { - if ! /usr/libexec/tuned/defirqaffinity.py "remove" "$TUNED_isolated_cores_expanded"; then - die defirqaffinity.py remove failed - fi - setup_kvm_mod_low_latency disable_ksm @@ -97,19 +93,18 @@ start() { stop() { [ "$1" = "full_rollback" ] && teardown_kvm_mod_low_latency - python /usr/libexec/tuned/defirqaffinity.py "add" "$TUNED_isolated_cores_expanded" enable_ksm return "$?" } verify() { - python /usr/libexec/tuned/defirqaffinity.py "verify" "$TUNED_isolated_cores_expanded" - retval = "$?" - if [ $retval -eq 0 -a -f /sys/module/kvm/parameters/kvmclock_periodic_sync ]; then - retval = `cat /sys/module/kvm/parameters/kvmclock_periodic_sync` + if [ -f /sys/module/kvm/parameters/kvmclock_periodic_sync ]; then + test "$(cat /sys/module/kvm/parameters/kvmclock_periodic_sync)" = 0 + retval=$? fi if [ $retval -eq 0 -a -f /sys/module/kvm_intel/parameters/ple_gap ]; then - retval = `cat /sys/module/kvm_intel/parameters/ple_gap` + test "$(cat /sys/module/kvm_intel/parameters/ple_gap)" = 0 + retval=$? fi return $retval } diff --git a/profiles/realtime-virtual-host/tuned.conf b/profiles/realtime-virtual-host/tuned.conf index cab2b73..0346fff 100644 --- a/profiles/realtime-virtual-host/tuned.conf +++ b/profiles/realtime-virtual-host/tuned.conf @@ -39,6 +39,8 @@ group.rcub=0:f:4:*:rcub.* # for i in `pgrep ktimersoftd` ; do grep Cpus_allowed_list /proc/$i/status ; done group.ktimersoftd=0:f:3:*:ktimersoftd.* +ps_blacklist=ksoftirqd.*;rcuc.*;rcub.*;ktimersoftd.* + [script] script=${i:PROFILE_DIR}/script.sh diff --git a/profiles/realtime/script.sh b/profiles/realtime/script.sh index f0ea4d8..4151731 100755 --- a/profiles/realtime/script.sh +++ b/profiles/realtime/script.sh @@ -3,15 +3,7 @@ . /usr/lib/tuned/functions start() { - if [ -z "$TUNED_isolated_cores" ]; then - echo "no isolated cores set, realtime profile not correctly activated" >&2 - exit 1 - fi - - # move threads off the selected cpu cores - tuna -c "$TUNED_isolated_cores" -i - - return "$?" + return 0 } stop() { diff --git a/profiles/realtime/tuned.conf b/profiles/realtime/tuned.conf index 6feee3f..b2273cf 100644 --- a/profiles/realtime/tuned.conf +++ b/profiles/realtime/tuned.conf @@ -40,3 +40,6 @@ cmdline_realtime=+isolcpus=${isolated_cores} intel_pstate=disable nosoftlockup [script] script = ${i:PROFILE_DIR}/script.sh + +[scheduler] +isolated_cores=${isolated_cores} diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 3949cb2..93b6d91 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -561,13 +561,39 @@ class SchedulerPlugin(base.Plugin): log.error("error applying tuning, cannot get information about running processes: %s" % e) - def _set_irq_affinity(self, irq, affinity_hex): - self._cmd.write_to_file("/proc/irq/%s/smp_affinity" % irq, - affinity_hex, no_error = True) + def _set_irq_affinity(self, irq, affinity, restoring): + try: + affinity_hex = self._cmd.cpulist2hex(affinity) + log.debug("Setting SMP affinity of IRQ %s to '%s'" + % (irq, affinity_hex)) + filename = "/proc/irq/%s/smp_affinity" % irq + with open(filename, "w") as f: + f.write(affinity_hex) + return True + except (OSError, IOError) as e: + # EIO is returned by + # kernel/irq/proc.c:write_irq_affinity() if changing + # the affinity is not supported + # (at least on kernels 3.10 and 4.18) + if hasattr(e, "errno") and e.errno == errno.EIO \ + and not restoring: + log.debug("Setting SMP affinity of IRQ %s is not supported" + % irq) + else: + log.error("Failed to set SMP affinity of IRQ %s to '%s': %s" + % (irq, affinity_hex, e)) + return False - def _set_default_irq_affinity(self, affinity_hex): - self._cmd.write_to_file("/proc/irq/default_smp_affinity", - affinity_hex) + def _set_default_irq_affinity(self, affinity): + try: + affinity_hex = self._cmd.cpulist2hex(affinity) + log.debug("Setting default SMP IRQ affinity to '%s'" + % affinity_hex) + with open("/proc/irq/default_smp_affinity", "w") as f: + f.write(affinity_hex) + except (OSError, IOError) as e: + log.error("Failed to set default SMP IRQ affinity to '%s': %s" + % (affinity_hex, e)) def _set_all_irq_affinity(self, affinity): irq_original = IRQAffinities() @@ -580,16 +606,16 @@ class SchedulerPlugin(base.Plugin): except KeyError: continue _affinity = self._get_intersect_affinity(prev_affinity, affinity, affinity) - affinity_hex = self._cmd.cpulist2hex(_affinity) - self._set_irq_affinity(irq, affinity_hex) - irq_original.irqs[irq] = prev_affinity + if set(_affinity) == set(prev_affinity): + continue + if self._set_irq_affinity(irq, _affinity, False): + irq_original.irqs[irq] = prev_affinity # default affinity prev_affinity_hex = self._cmd.read_file("/proc/irq/default_smp_affinity") prev_affinity = self._cmd.hex2cpulist(prev_affinity_hex) _affinity = self._get_intersect_affinity(prev_affinity, affinity, affinity) - affinity_hex = self._cmd.cpulist2hex(_affinity) - self._set_default_irq_affinity(affinity_hex) + self._set_default_irq_affinity(_affinity) irq_original.default = prev_affinity self._storage.set(self._irq_storage_key, irq_original) @@ -598,28 +624,68 @@ class SchedulerPlugin(base.Plugin): if irq_original is None: return for irq, affinity in irq_original.irqs.items(): - affinity_hex = self._cmd.cpulist2hex(affinity) - self._set_irq_affinity(irq, affinity_hex) + self._set_irq_affinity(irq, affinity, True) affinity = irq_original.default - affinity_hex = self._cmd.cpulist2hex(affinity) - self._set_default_irq_affinity(affinity_hex) + self._set_default_irq_affinity(affinity) self._storage.unset(self._irq_storage_key) + def _verify_irq_affinity(self, irq_description, correct_affinity, + current_affinity): + res = set(current_affinity).issubset(set(correct_affinity)) + if res: + log.info(consts.STR_VERIFY_PROFILE_VALUE_OK + % (irq_description, current_affinity)) + else: + log.error(consts.STR_VERIFY_PROFILE_VALUE_FAIL + % (irq_description, current_affinity, + correct_affinity)) + return res + + def _verify_all_irq_affinity(self, correct_affinity): + irqs = procfs.interrupts() + res = True + for irq in irqs.keys(): + try: + current_affinity = irqs[irq]["affinity"] + log.debug("Read SMP affinity of IRQ '%s': '%s'" + % (irq, current_affinity)) + irq_description = "SMP affinity of IRQ %s" % irq + if not self._verify_irq_affinity( + irq_description, + correct_affinity, + current_affinity): + res = False + except KeyError: + continue + + current_affinity_hex = self._cmd.read_file( + "/proc/irq/default_smp_affinity") + current_affinity = self._cmd.hex2cpulist(current_affinity_hex) + if not self._verify_irq_affinity("default IRQ SMP affinity", + current_affinity, correct_affinity): + res = False + return res + @command_custom("isolated_cores", per_device = False, priority = 10) def _isolated_cores(self, enabling, value, verify, ignore_missing): - # currently unsupported - if verify: + affinity = None + if value is not None: + isolated = set(self._cmd.cpulist_unpack(value)) + present = set(self._cpus) + if isolated.issubset(present): + affinity = list(present - isolated) + else: + str_cpus = ",".join([str(x) for x in self._cpus]) + log.error("Invalid isolated_cores specified, '%s' does not match available cores '%s'" + % (value, str_cpus)) + if (enabling or verify) and affinity is None: return None - if enabling: - if value is not None: - affinity = self._cmd.cpulist_invert(value) - sa = set(affinity) - if set(self._cpus).intersection(sa) != sa: - str_cpus = ",".join([str(x) for x in self._cpus]) - log.error("invalid isolated_cores specified, '%s' don't match available cores '%s'" % (value, str_cpus)) - return None - self._set_ps_affinity(affinity) - self._set_all_irq_affinity(affinity) + # currently only IRQ affinity verification is supported + if verify: + return self._verify_all_irq_affinity(affinity) + elif enabling: + self._set_ps_affinity(affinity) + self._set_all_irq_affinity(affinity) else: # Restoring processes' affinity is done in # _instance_unapply_static()