From e511d70d40679f27aac84fe0b1574d039ebe9f35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Fri, 1 Jun 2018 23:49:34 +0200 Subject: [PATCH 01/28] scheduler: Fix reverting CPU affinity in process groups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously CPU affinity of tasks created while Tuned was running was not correctly reverted on Tuned shutdown. Reproducer (on a 4 core machine): $ mkdir /etc/tuned/test $ cat > /etc/tuned/test/tuned.conf << EOF [scheduler] group.foo=0:o:0:1:a.out EOF $ cat > a.c << EOF #include int main(void) { pause(); return 0; } EOF $ gcc a.c $ systemctl start tuned $ ./a.out & $ systemctl stop tuned $ taskset -p $(pgrep a.out) pid 9950's current affinity mask: 1 <<< should be "f" Known issue: if you run the above reproducer with the config below after this commit is applied, the affinity of the task after stopping tuned will be 0xd, not 0xf. This is because after starting tuned, first the affinity of the shell session is set to the non-isolated cores (0xd). Then when ./a.out starts, it inherits that affinity. Then when tuned tunes the affinity of the ./a.out process, it remembers 0xd as its old affinity rather than 0xf. It is unclear to me at this point whether we should do anything about this issue. Properly fixing it would require tracing where processes get their affinity from. [scheduler] group.foo=0:o:0:1:a.out ps_blacklist=.*a.out.* isolated_cores=1 Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index babe108..890c2b3 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -284,7 +284,7 @@ class SchedulerPlugin(base.Plugin): for pid, vals in list(instance._scheduler_original.items()): # if command line for the pid didn't change, it's very probably the same process try: - if ps[pid] == vals[0]: + if ps[int(pid)] == vals[0]: self._set_rt(pid, self._sched2param(vals[1]), vals[2]) self._set_affinity(pid, vals[3]) except KeyError as e: From f3f05e426f05a5418fda1d0f8eb87a673e08ae85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Sun, 3 Jun 2018 10:58:32 +0200 Subject: [PATCH 02/28] scheduler: Prevent a traceback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new method for restoring original CPU affinity of processes _restore_ps_affinity() and call that in _instance_init() instead of _instance_unapply_static(). _instance_unapply_static() touches instance._terminate, which does not yet exist at that point (_instance_init always gets a fresh instance object). The reason this was not a problem in the past is that the true branch of "if len(instance._scheduler_original) > 0:" was never executed, because instance._scheduler_original was never correctly saved to storage. The next commit in this patch series fixes that. Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 890c2b3..6905bd1 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -57,7 +57,7 @@ class SchedulerPlugin(base.Plugin): instance._scheduler_original = self._storage.get(self._scheduler_storage_key(instance), {}) if len(instance._scheduler_original) > 0: log.info("recovering scheduling settings from previous run") - self._instance_unapply_static(instance) + self._restore_ps_affinity(instance) instance._scheduler_original = {} self._storage.unset(self._scheduler_storage_key(instance)) @@ -274,13 +274,8 @@ class SchedulerPlugin(base.Plugin): instance._thread = threading.Thread(target = self._thread_code, args = [instance]) instance._thread.start() - def _instance_unapply_static(self, instance, full_rollback = False): - super(SchedulerPlugin, self)._instance_unapply_static(instance, full_rollback) + def _restore_ps_affinity(self, instance): ps = self.get_processes() - if self._daemon and instance._runtime_tuning: - instance._terminate.set() - instance._thread.join() - for pid, vals in list(instance._scheduler_original.items()): # if command line for the pid didn't change, it's very probably the same process try: @@ -289,6 +284,15 @@ class SchedulerPlugin(base.Plugin): self._set_affinity(pid, vals[3]) except KeyError as e: pass + instance._scheduler_original = {} + self._storage.unset(self._scheduler_storage_key(instance)) + + def _instance_unapply_static(self, instance, full_rollback = False): + super(SchedulerPlugin, self)._instance_unapply_static(instance, full_rollback) + if self._daemon and instance._runtime_tuning: + instance._terminate.set() + instance._thread.join() + self._restore_ps_affinity(instance) def _add_pid(self, instance, pid, r): cmd = self.get_process(pid) From e9fcd4525ddab44890ab58ab347dadddc6f8105e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Thu, 31 May 2018 14:48:32 +0200 Subject: [PATCH 03/28] Fix storage access MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 9 ++++++--- tuned/plugins/plugin_sysctl.py | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 6905bd1..e06a5f3 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -269,7 +269,8 @@ class SchedulerPlugin(base.Plugin): #vals[0] - process name, vals[1] - rule prio, vals[2] - sched, vals[3] - prio, vals[4] - affinity, #vals[5] - regex self._tune_process(instance, pid, vals[0], vals[2], vals[3], vals[4]) - self._storage.set("options", instance._scheduler_original) + storage_key = self._scheduler_storage_key(instance) + self._storage.set(storage_key, instance._scheduler_original) if self._daemon and instance._runtime_tuning: instance._thread = threading.Thread(target = self._thread_code, args = [instance]) instance._thread.start() @@ -304,13 +305,15 @@ class SchedulerPlugin(base.Plugin): log.debug("tuning new process '%s' with pid '%s' by '%s'" % (cmd, pid, str(v))) #v[0] - sched, v[1] - prio, v[2] - affinity self._tune_process(instance, pid, cmd, v[0], v[1], v[2], no_error = True) - self._storage.set("options", instance._scheduler_original) + storage_key = self._scheduler_storage_key(instance) + self._storage.set(storage_key, instance._scheduler_original) def _remove_pid(self, instance, pid): if pid in instance._scheduler_original: del instance._scheduler_original[pid] log.debug("removed PID %s from the rollback database" % pid) - self._storage.set("options", instance._scheduler_original) + storage_key = self._scheduler_storage_key(instance) + self._storage.set(storage_key, instance._scheduler_original) def _thread_code(self, instance): r = self._cmd.re_lookup_compile(instance._sched_lookup) diff --git a/tuned/plugins/plugin_sysctl.py b/tuned/plugins/plugin_sysctl.py index f98332b..18e419a 100644 --- a/tuned/plugins/plugin_sysctl.py +++ b/tuned/plugins/plugin_sysctl.py @@ -46,7 +46,8 @@ class SysctlPlugin(base.Plugin): instance._sysctl_original[option] = original_value self._write_sysctl(option, self._process_assignment_modifiers(self._variables.expand(self._cmd.unquote(value)), original_value)) - self._storage.set("options", instance._sysctl_original) + storage_key = self._sysctl_storage_key(instance) + self._storage.set(storage_key, instance._sysctl_original) if self._global_cfg.get_bool(consts.CFG_REAPPLY_SYSCTL, consts.CFG_DEF_REAPPLY_SYSCTL): log.info("reapplying system sysctl") From 5bfb596a7c85326dd3c1de01cc89c371e7b8f774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Thu, 31 May 2018 16:44:17 +0200 Subject: [PATCH 04/28] Unify storage keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ondřej Lysoněk --- tuned/plugins/base.py | 13 ++++++++----- tuned/plugins/plugin_disk.py | 4 +++- tuned/plugins/plugin_mounts.py | 4 +++- tuned/plugins/plugin_net.py | 4 +++- tuned/plugins/plugin_scheduler.py | 16 +++++++--------- tuned/plugins/plugin_sysctl.py | 13 ++++++------- 6 files changed, 30 insertions(+), 24 deletions(-) diff --git a/tuned/plugins/base.py b/tuned/plugins/base.py index a3c9814..88cb548 100644 --- a/tuned/plugins/base.py +++ b/tuned/plugins/base.py @@ -379,11 +379,14 @@ class Plugin(object): # Operations with persistent storage for status data. # - def _storage_key(self, instance_name, command_name, device_name=None): - if device_name is not None: - return "%s/%s/%s" % (command_name, instance_name, device_name) - else: - return "%s/%s" % (command_name, instance_name) + def _storage_key(self, instance_name = None, command_name = None, + device_name = None): + class_name = type(self).__name__ + instance_name = "" if instance_name is None else instance_name + command_name = "" if command_name is None else command_name + device_name = "" if device_name is None else device_name + return "%s/%s/%s/%s" % (class_name, instance_name, + command_name, device_name) def _storage_set(self, instance, command, value, device_name=None): key = self._storage_key(instance.name, command["name"], device_name) diff --git a/tuned/plugins/plugin_disk.py b/tuned/plugins/plugin_disk.py index 04f2dee..1de1257 100644 --- a/tuned/plugins/plugin_disk.py +++ b/tuned/plugins/plugin_disk.py @@ -319,7 +319,9 @@ class DiskPlugin(hotplug.Plugin): def _multiply_readahead(self, enabling, multiplier, device, verify, ignore_missing): if verify: return None - storage_key = self._storage_key("readahead_multiply", device) + storage_key = self._storage_key( + command_name = "readahead_multiply", + device_name = device) if enabling: old_readahead = self._get_readahead(device) if old_readahead is None: diff --git a/tuned/plugins/plugin_mounts.py b/tuned/plugins/plugin_mounts.py index 8e2eacc..7109427 100644 --- a/tuned/plugins/plugin_mounts.py +++ b/tuned/plugins/plugin_mounts.py @@ -125,7 +125,9 @@ class MountsPlugin(base.Plugin): @command_custom("disable_barriers", per_device=True) def _disable_barriers(self, start, value, mountpoint, verify, ignore_missing): - storage_key = self._storage_key("disable_barriers", mountpoint) + storage_key = self._storage_key( + command_name = "disable_barriers", + device_name = mountpoint) force = str(value).lower() == "force" value = force or self._option_bool(value) diff --git a/tuned/plugins/plugin_net.py b/tuned/plugins/plugin_net.py index d4a3dac..9e2329a 100644 --- a/tuned/plugins/plugin_net.py +++ b/tuned/plugins/plugin_net.py @@ -332,7 +332,9 @@ class NetTuningPlugin(base.Plugin): return d def _custom_parameters(self, context, start, value, device, verify): - storage_key = self._storage_key(context, device) + storage_key = self._storage_key( + command_name = context, + device_name = device) if start: cd = self._get_device_parameters(context, device) d = self._set_device_parameters(context, value, device, verify) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index e06a5f3..fa533a6 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -41,9 +41,6 @@ class SchedulerPlugin(base.Plugin): self._ps_whitelist = ".*" self._ps_blacklist = "" - def _scheduler_storage_key(self, instance): - return "%s/options" % instance.name - def _instance_init(self, instance): instance._has_dynamic_tuning = False instance._has_static_tuning = True @@ -54,12 +51,13 @@ class SchedulerPlugin(base.Plugin): # FIXME: do we want to do this here? # recover original values in case of crash - instance._scheduler_original = self._storage.get(self._scheduler_storage_key(instance), {}) + storage_key = self._storage_key(instance.name) + instance._scheduler_original = self._storage.get(storage_key, {}) if len(instance._scheduler_original) > 0: log.info("recovering scheduling settings from previous run") self._restore_ps_affinity(instance) instance._scheduler_original = {} - self._storage.unset(self._scheduler_storage_key(instance)) + self._storage.unset(storage_key) instance._scheduler = instance.options for k in instance._scheduler: @@ -269,7 +267,7 @@ class SchedulerPlugin(base.Plugin): #vals[0] - process name, vals[1] - rule prio, vals[2] - sched, vals[3] - prio, vals[4] - affinity, #vals[5] - regex self._tune_process(instance, pid, vals[0], vals[2], vals[3], vals[4]) - storage_key = self._scheduler_storage_key(instance) + storage_key = self._storage_key(instance.name) self._storage.set(storage_key, instance._scheduler_original) if self._daemon and instance._runtime_tuning: instance._thread = threading.Thread(target = self._thread_code, args = [instance]) @@ -286,7 +284,7 @@ class SchedulerPlugin(base.Plugin): except KeyError as e: pass instance._scheduler_original = {} - self._storage.unset(self._scheduler_storage_key(instance)) + self._storage.unset(self._storage_key(instance.name)) def _instance_unapply_static(self, instance, full_rollback = False): super(SchedulerPlugin, self)._instance_unapply_static(instance, full_rollback) @@ -305,14 +303,14 @@ class SchedulerPlugin(base.Plugin): log.debug("tuning new process '%s' with pid '%s' by '%s'" % (cmd, pid, str(v))) #v[0] - sched, v[1] - prio, v[2] - affinity self._tune_process(instance, pid, cmd, v[0], v[1], v[2], no_error = True) - storage_key = self._scheduler_storage_key(instance) + storage_key = self._storage_key(instance.name) self._storage.set(storage_key, instance._scheduler_original) def _remove_pid(self, instance, pid): if pid in instance._scheduler_original: del instance._scheduler_original[pid] log.debug("removed PID %s from the rollback database" % pid) - storage_key = self._scheduler_storage_key(instance) + storage_key = self._storage_key(instance.name) self._storage.set(storage_key, instance._scheduler_original) def _thread_code(self, instance): diff --git a/tuned/plugins/plugin_sysctl.py b/tuned/plugins/plugin_sysctl.py index 18e419a..71d35f3 100644 --- a/tuned/plugins/plugin_sysctl.py +++ b/tuned/plugins/plugin_sysctl.py @@ -18,26 +18,25 @@ class SysctlPlugin(base.Plugin): self._has_dynamic_options = True self._cmd = commands() - def _sysctl_storage_key(self, instance): - return "%s/options" % instance.name - def _instance_init(self, instance): instance._has_dynamic_tuning = False instance._has_static_tuning = True # FIXME: do we want to do this here? # recover original values in case of crash - instance._sysctl_original = self._storage.get(self._sysctl_storage_key(instance), {}) + storage_key = self._storage_key(instance.name) + instance._sysctl_original = self._storage.get(storage_key, {}) if len(instance._sysctl_original) > 0: log.info("recovering old sysctl settings from previous run") self._instance_unapply_static(instance) instance._sysctl_original = {} - self._storage.unset(self._sysctl_storage_key(instance)) + self._storage.unset(storage_key) instance._sysctl = instance.options def _instance_cleanup(self, instance): - self._storage.unset(self._sysctl_storage_key(instance)) + storage_key = self._storage_key(instance.name) + self._storage.unset(storage_key) def _instance_apply_static(self, instance): for option, value in list(instance._sysctl.items()): @@ -46,7 +45,7 @@ class SysctlPlugin(base.Plugin): instance._sysctl_original[option] = original_value self._write_sysctl(option, self._process_assignment_modifiers(self._variables.expand(self._cmd.unquote(value)), original_value)) - storage_key = self._sysctl_storage_key(instance) + storage_key = self._storage_key(instance.name) self._storage.set(storage_key, instance._sysctl_original) if self._global_cfg.get_bool(consts.CFG_REAPPLY_SYSCTL, consts.CFG_DEF_REAPPLY_SYSCTL): From 625d219f9e7ebf5f3ed5cffcb4ae2783c90c0803 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Thu, 31 May 2018 16:51:18 +0200 Subject: [PATCH 05/28] scheduler: Make storage global for the whole plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It doesn't make much sense to have multiple instances of the scheduler plugin, so let's make storage global for the plugin. Later we should make Tuned report an error if the user defines multiple instances of the scheduler plugin. Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 46 +++++++++++++++---------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index fa533a6..bb14076 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -51,12 +51,12 @@ class SchedulerPlugin(base.Plugin): # FIXME: do we want to do this here? # recover original values in case of crash - storage_key = self._storage_key(instance.name) - instance._scheduler_original = self._storage.get(storage_key, {}) - if len(instance._scheduler_original) > 0: + storage_key = self._storage_key() + self._scheduler_original = self._storage.get(storage_key, {}) + if len(self._scheduler_original) > 0: log.info("recovering scheduling settings from previous run") - self._restore_ps_affinity(instance) - instance._scheduler_original = {} + self._restore_ps_affinity() + self._scheduler_original = {} self._storage.unset(storage_key) instance._scheduler = instance.options @@ -227,12 +227,12 @@ class SchedulerPlugin(base.Plugin): log.error(err_msg) #tune process and store previous values - def _tune_process(self, instance, pid, cmd, sched, prio, affinity, no_error = False): + def _tune_process(self, pid, cmd, sched, prio, affinity, no_error = False): #rt[0] - prev_sched, rt[1] - prev_prio rt = self._get_rt(pid) prev_affinity = self._get_affinity(pid, no_error) if prev_affinity is not None and rt is not None and len(rt) == 2 and rt[0] is not None and rt[1] is not None: - instance._scheduler_original[pid] = (cmd, rt[0], rt[1], prev_affinity) + self._scheduler_original[pid] = (cmd, rt[0], rt[1], prev_affinity) self._set_rt(pid, self._schedcfg2param(sched), prio, no_error) if affinity != "*": self._set_affinity(pid, affinity, no_error) @@ -266,16 +266,16 @@ class SchedulerPlugin(base.Plugin): for pid, vals in list(sched_all.items()): #vals[0] - process name, vals[1] - rule prio, vals[2] - sched, vals[3] - prio, vals[4] - affinity, #vals[5] - regex - self._tune_process(instance, pid, vals[0], vals[2], vals[3], vals[4]) - storage_key = self._storage_key(instance.name) - self._storage.set(storage_key, instance._scheduler_original) + self._tune_process(pid, vals[0], vals[2], vals[3], vals[4]) + storage_key = self._storage_key() + self._storage.set(storage_key, self._scheduler_original) if self._daemon and instance._runtime_tuning: instance._thread = threading.Thread(target = self._thread_code, args = [instance]) instance._thread.start() - def _restore_ps_affinity(self, instance): + def _restore_ps_affinity(self): ps = self.get_processes() - for pid, vals in list(instance._scheduler_original.items()): + for pid, vals in list(self._scheduler_original.items()): # if command line for the pid didn't change, it's very probably the same process try: if ps[int(pid)] == vals[0]: @@ -283,15 +283,15 @@ class SchedulerPlugin(base.Plugin): self._set_affinity(pid, vals[3]) except KeyError as e: pass - instance._scheduler_original = {} - self._storage.unset(self._storage_key(instance.name)) + self._scheduler_original = {} + self._storage.unset(self._storage_key()) def _instance_unapply_static(self, instance, full_rollback = False): super(SchedulerPlugin, self)._instance_unapply_static(instance, full_rollback) if self._daemon and instance._runtime_tuning: instance._terminate.set() instance._thread.join() - self._restore_ps_affinity(instance) + self._restore_ps_affinity() def _add_pid(self, instance, pid, r): cmd = self.get_process(pid) @@ -299,19 +299,19 @@ class SchedulerPlugin(base.Plugin): if cmd == "": return v = self._cmd.re_lookup(instance._sched_lookup, cmd, r) - if v is not None and not pid in instance._scheduler_original: + if v is not None and not pid in self._scheduler_original: log.debug("tuning new process '%s' with pid '%s' by '%s'" % (cmd, pid, str(v))) #v[0] - sched, v[1] - prio, v[2] - affinity - self._tune_process(instance, pid, cmd, v[0], v[1], v[2], no_error = True) - storage_key = self._storage_key(instance.name) - self._storage.set(storage_key, instance._scheduler_original) + self._tune_process(pid, cmd, v[0], v[1], v[2], no_error = True) + storage_key = self._storage_key() + self._storage.set(storage_key, self._scheduler_original) def _remove_pid(self, instance, pid): - if pid in instance._scheduler_original: - del instance._scheduler_original[pid] + if pid in self._scheduler_original: + del self._scheduler_original[pid] log.debug("removed PID %s from the rollback database" % pid) - storage_key = self._storage_key(instance.name) - self._storage.set(storage_key, instance._scheduler_original) + storage_key = self._storage_key() + self._storage.set(storage_key, self._scheduler_original) def _thread_code(self, instance): r = self._cmd.re_lookup_compile(instance._sched_lookup) From 02541dfb5cf43f8c6436302dcd359cdec945677c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Sat, 2 Jun 2018 08:26:29 +0200 Subject: [PATCH 06/28] scheduler: Catch OSError in addition to IOError MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In python3, functions such as open() or readline() raise OSError instead of IOError. Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index bb14076..f504dc7 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -204,14 +204,14 @@ class SchedulerPlugin(base.Plugin): return 0 else: return 1 - except IOError as e: + except (OSError, IOError) as e: if e.errno == errno.ENOENT or e.errno == errno.ESRCH: log.debug("Unable to set affinity for PID %s, the task vanished." % pid) return -1 else: log.error("Failed to get task info for PID %s: %s" % (pid, str(e))) return -2 - except (OSError, AttributeError, KeyError) as e: + except (AttributeError, KeyError) as e: log.error("Failed to get task info for PID %s: %s" % (pid, str(e))) return -2 From 8e68e22757178879993d631dc514cf243a3d63ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Thu, 17 May 2018 10:58:17 +0200 Subject: [PATCH 07/28] Use the errno attribute of an exception instead of indices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Obtaining the errno from an exception using e[0] is not possible in Python3. Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index f504dc7..ee9f8b0 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -355,7 +355,7 @@ class SchedulerPlugin(base.Plugin): return schedutils.get_affinity(pid) # Workaround for old python-schedutils which incorrectly raised error except (SystemError, OSError) as e: - if e[0] == 3: + if hasattr(e, "errno") and e.errno == errno.ESRCH: log.debug("Unable to read affinity for PID %s, the task vanished." % pid) return None log.error("unable to get affinity for PID '%s': %s" % (str(pid), e)) @@ -367,7 +367,7 @@ class SchedulerPlugin(base.Plugin): schedutils.set_affinity(pid, affinity) # Workaround for old python-schedutils which incorrectly raised error except (SystemError, OSError) as e: - if e[0] == 3: + if hasattr(e, "errno") and e.errno == errno.ESRCH: log.debug("Unable to set affinity for PID %s, the task vanished." % pid) return False log.error("unable to set affinity '%s' for PID '%s': %s" % (str(affinity), str(pid), e)) From 673d253ac138f67c76c0d3127a7ebb2013d8ebda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Sun, 20 May 2018 12:30:38 +0200 Subject: [PATCH 08/28] scheduler: Drop no_error argument in _get_affinity() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I believe it was originally added so that errors are not logged when we fail to read the affinity of a short-lived process which has already disappeared. This is no longer necessary, because we always check if the process has disappeared, and log debug messages instead of errors if it has. Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index ee9f8b0..f3e26b0 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -142,14 +142,13 @@ class SchedulerPlugin(base.Plugin): log.debug("read scheduler policy '%s' and priority '%s' for pid '%s'" % (sched, prio, pid)) return (sched, prio) - def _get_affinity(self, pid, no_error = False): - (rc, out, err_msg) = self._cmd.execute(["taskset", "-p", str(pid)], no_errors = [1] if no_error else [], return_err = True) + def _get_affinity(self, pid): + (rc, out, err_msg) = self._cmd.execute(["taskset", "-p", str(pid)], no_errors = [], return_err = True) if rc != 0: - if rc != 1 or not no_error: - if self._pid_exists(pid): - log.error(err_msg) - else: - log.debug("Unable to read affinity for PID %s, the task vanished." % pid) + if rc != 1 or self._pid_exists(pid): + log.error(err_msg) + else: + log.debug("Unable to read affinity for PID %s, the task vanished." % pid) return None v = self._parse_val(out.split("\n", 1)[0]) log.debug("read affinity '%s' for pid '%s'" % (v, pid)) @@ -230,7 +229,7 @@ class SchedulerPlugin(base.Plugin): def _tune_process(self, pid, cmd, sched, prio, affinity, no_error = False): #rt[0] - prev_sched, rt[1] - prev_prio rt = self._get_rt(pid) - prev_affinity = self._get_affinity(pid, no_error) + prev_affinity = self._get_affinity(pid) if prev_affinity is not None and rt is not None and len(rt) == 2 and rt[0] is not None and rt[1] is not None: self._scheduler_original[pid] = (cmd, rt[0], rt[1], prev_affinity) self._set_rt(pid, self._schedcfg2param(sched), prio, no_error) From a897259eec4cdc925f0bcb44fa391ebfdc6f4f16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Mon, 21 May 2018 11:59:30 +0200 Subject: [PATCH 09/28] scheduler: Drop no_error argument in _set_affinity() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I believe it was originally added so that errors are not logged when we fail to set the affinity of a short-lived process which has already disappeared. This is no longer necessary, because we always check if the process has disappeared, and log debug messages instead of errors if it has. Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index f3e26b0..7e78851 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -214,12 +214,12 @@ class SchedulerPlugin(base.Plugin): log.error("Failed to get task info for PID %s: %s" % (pid, str(e))) return -2 - def _set_affinity(self, pid, affinity, no_error = False): + def _set_affinity(self, pid, affinity): if pid is None or affinity is None: return log.debug("setting affinity to '%s' for PID '%s'" % (affinity, pid)) - (ret, out, err_msg) = self._cmd.execute(["taskset", "-p", str(affinity), str(pid)], no_errors = [1] if no_error else [], return_err = True) - if ret == 0 or (ret == 1 and no_error): + (ret, out, err_msg) = self._cmd.execute(["taskset", "-p", str(affinity), str(pid)], no_errors = [], return_err = True) + if ret == 0: return res = self._affinity_changeable(pid) if res == 1 or res == -2: @@ -234,7 +234,7 @@ class SchedulerPlugin(base.Plugin): self._scheduler_original[pid] = (cmd, rt[0], rt[1], prev_affinity) self._set_rt(pid, self._schedcfg2param(sched), prio, no_error) if affinity != "*": - self._set_affinity(pid, affinity, no_error) + self._set_affinity(pid, affinity) def _instance_apply_static(self, instance): super(SchedulerPlugin, self)._instance_apply_static(instance) From a0c906bff1dc07afe549de41f1997c8b87212baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Mon, 21 May 2018 15:18:54 +0200 Subject: [PATCH 10/28] scheduler: Drop no_error argument in _set_rt() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I believe it was originally added so that errors are not logged when we fail to set the scheduling parameters of a short-lived process which has already disappeared. This is no longer necessary, because we always check if the process has disappeared, and log debug messages instead of errors if it has. Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 7e78851..1e8242f 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -167,7 +167,7 @@ class SchedulerPlugin(base.Plugin): except KeyError: return "" - def _set_rt(self, pid, sched, prio, no_error = False): + def _set_rt(self, pid, sched, prio): if pid is None or prio is None: return if sched is not None and len(sched) > 0: @@ -176,8 +176,8 @@ class SchedulerPlugin(base.Plugin): else: schedl = [] log.debug("setting scheduler priority to '%s' for PID '%s'" % (prio, pid)) - (ret, out, err_msg) = self._cmd.execute(["chrt"] + schedl + ["-p", str(prio), str(pid)], no_errors = [1] if no_error else [], return_err = True) - if ret == 0 or (ret == 1 and no_error): + (ret, out, err_msg) = self._cmd.execute(["chrt"] + schedl + ["-p", str(prio), str(pid)], no_errors = [], return_err = True) + if ret == 0: return if self._pid_exists(pid): log.error(err_msg) @@ -226,13 +226,13 @@ class SchedulerPlugin(base.Plugin): log.error(err_msg) #tune process and store previous values - def _tune_process(self, pid, cmd, sched, prio, affinity, no_error = False): + def _tune_process(self, pid, cmd, sched, prio, affinity): #rt[0] - prev_sched, rt[1] - prev_prio rt = self._get_rt(pid) prev_affinity = self._get_affinity(pid) if prev_affinity is not None and rt is not None and len(rt) == 2 and rt[0] is not None and rt[1] is not None: self._scheduler_original[pid] = (cmd, rt[0], rt[1], prev_affinity) - self._set_rt(pid, self._schedcfg2param(sched), prio, no_error) + self._set_rt(pid, self._schedcfg2param(sched), prio) if affinity != "*": self._set_affinity(pid, affinity) @@ -301,7 +301,7 @@ class SchedulerPlugin(base.Plugin): if v is not None and not pid in self._scheduler_original: log.debug("tuning new process '%s' with pid '%s' by '%s'" % (cmd, pid, str(v))) #v[0] - sched, v[1] - prio, v[2] - affinity - self._tune_process(pid, cmd, v[0], v[1], v[2], no_error = True) + self._tune_process(pid, cmd, v[0], v[1], v[2]) storage_key = self._storage_key() self._storage.set(storage_key, self._scheduler_original) From 7e3819b4468568ce3af4fffaf0dd605c725ec932 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Tue, 29 May 2018 17:09:01 +0200 Subject: [PATCH 11/28] scheduler: Refactor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 1e8242f..42bbbae 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -276,12 +276,10 @@ class SchedulerPlugin(base.Plugin): ps = self.get_processes() for pid, vals in list(self._scheduler_original.items()): # if command line for the pid didn't change, it's very probably the same process - try: - if ps[int(pid)] == vals[0]: - self._set_rt(pid, self._sched2param(vals[1]), vals[2]) - self._set_affinity(pid, vals[3]) - except KeyError as e: - pass + if pid not in ps or ps[int(pid)] != vals[0]: + continue + self._set_rt(pid, self._sched2param(vals[1]), vals[2]) + self._set_affinity(pid, vals[3]) self._scheduler_original = {} self._storage.unset(self._storage_key()) From 7ec58f779a3c4b19125c8d9687a63cea5361dcd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Mon, 21 May 2018 14:43:15 +0200 Subject: [PATCH 12/28] scheduler: Improve code readability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 65 +++++++++++++++++++------------ 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 42bbbae..07aa00b 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -18,6 +18,14 @@ import errno log = tuned.logs.get() +class SchedulerParams(object): + def __init__(self, cmdline = None, scheduler = None, priority = None, + affinity = None): + self.cmdline = cmdline + self.scheduler = scheduler + self.priority = priority + self.affinity = affinity + # TODO move from cmdline tools to schedutils and consolidate the code class SchedulerPlugin(base.Plugin): """ @@ -108,7 +116,9 @@ class SchedulerPlugin(base.Plugin): (rc, out) = self._cmd.execute(["ps", "-eopid,cmd", "--no-headers"]) if rc != 0 or len(out) <= 0: return None - return dict([(int(pid_cmd1[0].lstrip()), pid_cmd1[1].lstrip()) for pid_cmd1 in [i for i in [s.split(None, 1) for s in out.split("\n")] if len(i) == 2]]) + lines = out.split("\n") + pid_cmd_list = [i for i in [s.split(None, 1) for s in lines] if len(i) == 2] + return dict([(int(pid.lstrip()), cmd.lstrip()) for pid, cmd in pid_cmd_list]) def _parse_val(self, val): v = val.split(":", 1) @@ -231,7 +241,11 @@ class SchedulerPlugin(base.Plugin): rt = self._get_rt(pid) prev_affinity = self._get_affinity(pid) if prev_affinity is not None and rt is not None and len(rt) == 2 and rt[0] is not None and rt[1] is not None: - self._scheduler_original[pid] = (cmd, rt[0], rt[1], prev_affinity) + self._scheduler_original[pid] = SchedulerParams( + cmdline = cmd, + scheduler = rt[0], + priority = rt[1], + affinity = prev_affinity) self._set_rt(pid, self._schedcfg2param(sched), prio) if affinity != "*": self._set_affinity(pid, affinity) @@ -242,30 +256,31 @@ class SchedulerPlugin(base.Plugin): if ps is None: log.error("error applying tuning, cannot get information about running processes") return - instance._sched_cfg = [(option_value[0], str(option_value[1]).split(":", 4)) for option_value in list(instance._scheduler.items())] - buf = [option_vals3 for option_vals3 in instance._sched_cfg if re.match(r"group\.", option_vals3[0]) and len(option_vals3[1]) == 5] + instance._sched_cfg = [(option, str(value).split(":", 4)) for option, value in instance._scheduler.items()] + buf = [(option, vals) for option, vals in instance._sched_cfg if re.match(r"group\.", option) and len(vals) == 5] instance._sched_cfg = sorted(buf, key=lambda option_vals: option_vals[1][0]) sched_all = dict() # for runtime tunning instance._sched_lookup = {} - for option, vals in instance._sched_cfg: + for option, (rule_prio, scheduler, priority, affinity, regex) \ + in instance._sched_cfg: try: - r = re.compile(vals[4]) + r = re.compile(regex) except re.error as e: - log.error("error compiling regular expression: '%s'" % str(vals[4])) + log.error("error compiling regular expression: '%s'" % str(regex)) continue - processes = [pid_cmd2 for pid_cmd2 in list(ps.items()) if re.search(r, pid_cmd2[1]) is not None] - #cmd - process name, option - group name, vals[0] - rule prio, vals[1] - sched, vals[2] - prio, - #vals[3] - affinity, vals[4] - regex - sched = dict([(pid_cmd[0], (pid_cmd[1], option, vals[1], vals[2], vals[3], vals[4])) for pid_cmd in processes]) + processes = [(pid, cmd) for pid, cmd in ps.items() if re.search(r, cmd) is not None] + #cmd - process name, option - group name + sched = dict([(pid, (cmd, option, scheduler, priority, affinity, regex)) + for pid, cmd in processes]) sched_all.update(sched) - v4 = str(vals[4]).replace("(", r"\(") - v4 = v4.replace(")", r"\)") - instance._sched_lookup[v4] = [vals[1], vals[2], vals[3]] - for pid, vals in list(sched_all.items()): - #vals[0] - process name, vals[1] - rule prio, vals[2] - sched, vals[3] - prio, vals[4] - affinity, - #vals[5] - regex - self._tune_process(pid, vals[0], vals[2], vals[3], vals[4]) + regex = str(regex).replace("(", r"\(") + regex = regex.replace(")", r"\)") + instance._sched_lookup[regex] = [scheduler, priority, affinity] + for pid, (cmd, option, scheduler, priority, affinity, regex) \ + in sched_all.items(): + self._tune_process(pid, cmd, scheduler, + priority, affinity) storage_key = self._storage_key() self._storage.set(storage_key, self._scheduler_original) if self._daemon and instance._runtime_tuning: @@ -274,12 +289,13 @@ class SchedulerPlugin(base.Plugin): def _restore_ps_affinity(self): ps = self.get_processes() - for pid, vals in list(self._scheduler_original.items()): + for pid, orig_params in self._scheduler_original.items(): # if command line for the pid didn't change, it's very probably the same process - if pid not in ps or ps[int(pid)] != vals[0]: + if pid not in ps or ps[int(pid)] != orig_params.cmdline: continue - self._set_rt(pid, self._sched2param(vals[1]), vals[2]) - self._set_affinity(pid, vals[3]) + self._set_rt(pid, self._sched2param(orig_params.scheduler), + orig_params.priority) + self._set_affinity(pid, orig_params.affinity) self._scheduler_original = {} self._storage.unset(self._storage_key()) @@ -298,8 +314,9 @@ class SchedulerPlugin(base.Plugin): v = self._cmd.re_lookup(instance._sched_lookup, cmd, r) if v is not None and not pid in self._scheduler_original: log.debug("tuning new process '%s' with pid '%s' by '%s'" % (cmd, pid, str(v))) - #v[0] - sched, v[1] - prio, v[2] - affinity - self._tune_process(pid, cmd, v[0], v[1], v[2]) + (sched, prio, affinity) = v + self._tune_process(pid, cmd, sched, prio, + affinity) storage_key = self._storage_key() self._storage.set(storage_key, self._scheduler_original) From 474a6eff90f7d5159b68b6c7a0fcf5e1e06fde0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Mon, 21 May 2018 23:22:40 +0200 Subject: [PATCH 13/28] scheduler: Always store PIDs as integers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Always store PIDs as integers as that is their natural form. Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 50 +++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 07aa00b..1b115f1 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -102,11 +102,11 @@ class SchedulerPlugin(base.Plugin): } def get_process(self, pid): - cmd = self._cmd.read_file("/proc/" + pid + "/comm", no_error = True) + cmd = self._cmd.read_file("/proc/" + str(pid) + "/comm", no_error = True) if cmd == "": return "" cmd = cmd.strip() - cmdline = self._cmd.read_file("/proc/" + pid + "/cmdline", no_error = True) + cmdline = self._cmd.read_file("/proc/" + str(pid) + "/cmdline", no_error = True) if cmdline == "": return "[" + cmd + "]" else: @@ -140,7 +140,7 @@ class SchedulerPlugin(base.Plugin): if self._pid_exists(pid): log.error(err_msg) else: - log.debug("Unable to read scheduling parameters for PID %s, the task vanished." % pid) + log.debug("Unable to read scheduling parameters for PID %d, the task vanished." % pid) return None vals = out.split("\n", 1) if len(vals) > 1: @@ -149,7 +149,7 @@ class SchedulerPlugin(base.Plugin): else: sched = None prio = None - log.debug("read scheduler policy '%s' and priority '%s' for pid '%s'" % (sched, prio, pid)) + log.debug("read scheduler policy '%s' and priority '%s' for pid '%d'" % (sched, prio, pid)) return (sched, prio) def _get_affinity(self, pid): @@ -158,10 +158,10 @@ class SchedulerPlugin(base.Plugin): if rc != 1 or self._pid_exists(pid): log.error(err_msg) else: - log.debug("Unable to read affinity for PID %s, the task vanished." % pid) + log.debug("Unable to read affinity for PID %d, the task vanished." % pid) return None v = self._parse_val(out.split("\n", 1)[0]) - log.debug("read affinity '%s' for pid '%s'" % (v, pid)) + log.debug("read affinity '%s' for pid '%d'" % (v, pid)) return v def _schedcfg2param(self, sched): @@ -182,17 +182,17 @@ class SchedulerPlugin(base.Plugin): return if sched is not None and len(sched) > 0: schedl = [sched] - log.debug("setting scheduler policy to '%s' for PID '%s'" % (sched, pid)) + log.debug("setting scheduler policy to '%s' for PID '%d'" % (sched, pid)) else: schedl = [] - log.debug("setting scheduler priority to '%s' for PID '%s'" % (prio, pid)) + log.debug("setting scheduler priority to '%s' for PID '%d'" % (prio, pid)) (ret, out, err_msg) = self._cmd.execute(["chrt"] + schedl + ["-p", str(prio), str(pid)], no_errors = [], return_err = True) if ret == 0: return if self._pid_exists(pid): log.error(err_msg) else: - log.debug("Unable to set scheduling parameters for PID %s, the task vanished." % pid) + log.debug("Unable to set scheduling parameters for PID %d, the task vanished." % pid) # Return codes: # 0 - Affinity is fixed @@ -205,29 +205,29 @@ class SchedulerPlugin(base.Plugin): process = procfs.process(pid) if process["stat"].is_bound_to_cpu(): if process["stat"]["state"] == "Z": - log.debug("Affinity of zombie task with PID %s cannot be changed, the task's affinity mask is fixed." % pid) + log.debug("Affinity of zombie task with PID %d cannot be changed, the task's affinity mask is fixed." % pid) elif process["stat"]["flags"] & procfs.pidstat.PF_KTHREAD != 0: - log.debug("Affinity of kernel thread with PID %s cannot be changed, the task's affinity mask is fixed." % pid) + log.debug("Affinity of kernel thread with PID %d cannot be changed, the task's affinity mask is fixed." % pid) else: - log.warn("Affinity of task with PID %s cannot be changed, the task's affinity mask is fixed." % pid) + log.warn("Affinity of task with PID %d cannot be changed, the task's affinity mask is fixed." % pid) return 0 else: return 1 except (OSError, IOError) as e: if e.errno == errno.ENOENT or e.errno == errno.ESRCH: - log.debug("Unable to set affinity for PID %s, the task vanished." % pid) + log.debug("Unable to set affinity for PID %d, the task vanished." % pid) return -1 else: - log.error("Failed to get task info for PID %s: %s" % (pid, str(e))) + log.error("Failed to get task info for PID %d: %s" % (pid, str(e))) return -2 except (AttributeError, KeyError) as e: - log.error("Failed to get task info for PID %s: %s" % (pid, str(e))) + log.error("Failed to get task info for PID %d: %s" % (pid, str(e))) return -2 def _set_affinity(self, pid, affinity): if pid is None or affinity is None: return - log.debug("setting affinity to '%s' for PID '%s'" % (affinity, pid)) + log.debug("setting affinity to '%s' for PID '%d'" % (affinity, pid)) (ret, out, err_msg) = self._cmd.execute(["taskset", "-p", str(affinity), str(pid)], no_errors = [], return_err = True) if ret == 0: return @@ -291,7 +291,7 @@ class SchedulerPlugin(base.Plugin): ps = self.get_processes() for pid, orig_params in self._scheduler_original.items(): # if command line for the pid didn't change, it's very probably the same process - if pid not in ps or ps[int(pid)] != orig_params.cmdline: + if pid not in ps or ps[pid] != orig_params.cmdline: continue self._set_rt(pid, self._sched2param(orig_params.scheduler), orig_params.priority) @@ -313,7 +313,7 @@ class SchedulerPlugin(base.Plugin): return v = self._cmd.re_lookup(instance._sched_lookup, cmd, r) if v is not None and not pid in self._scheduler_original: - log.debug("tuning new process '%s' with pid '%s' by '%s'" % (cmd, pid, str(v))) + log.debug("tuning new process '%s' with PID '%d' by '%s'" % (cmd, pid, str(v))) (sched, prio, affinity) = v self._tune_process(pid, cmd, sched, prio, affinity) @@ -323,7 +323,7 @@ class SchedulerPlugin(base.Plugin): def _remove_pid(self, instance, pid): if pid in self._scheduler_original: del self._scheduler_original[pid] - log.debug("removed PID %s from the rollback database" % pid) + log.debug("removed PID %d from the rollback database" % pid) storage_key = self._storage_key() self._storage.set(storage_key, self._scheduler_original) @@ -343,9 +343,9 @@ class SchedulerPlugin(base.Plugin): if event: read_events = True if event.type == perf.RECORD_COMM: - self._add_pid(instance, str(event.pid), r) + self._add_pid(instance, int(event.pid), r) elif event.type == perf.RECORD_EXIT: - self._remove_pid(instance, str(event.pid)) + self._remove_pid(instance, int(event.pid)) @command_custom("ps_whitelist", per_device = False) def _ps_whitelist(self, enabling, value, verify, ignore_missing): @@ -370,9 +370,9 @@ class SchedulerPlugin(base.Plugin): # Workaround for old python-schedutils which incorrectly raised error except (SystemError, OSError) as e: if hasattr(e, "errno") and e.errno == errno.ESRCH: - log.debug("Unable to read affinity for PID %s, the task vanished." % pid) + log.debug("Unable to read affinity for PID %d, the task vanished." % pid) return None - log.error("unable to get affinity for PID '%s': %s" % (str(pid), e)) + log.error("unable to get affinity for PID '%d': %s" % (pid, e)) return None # TODO: merge with _set_affinity @@ -382,9 +382,9 @@ class SchedulerPlugin(base.Plugin): # Workaround for old python-schedutils which incorrectly raised error except (SystemError, OSError) as e: if hasattr(e, "errno") and e.errno == errno.ESRCH: - log.debug("Unable to set affinity for PID %s, the task vanished." % pid) + log.debug("Unable to set affinity for PID %d, the task vanished." % pid) return False - log.error("unable to set affinity '%s' for PID '%s': %s" % (str(affinity), str(pid), e)) + log.error("unable to set affinity '%s' for PID '%d': %s" % (str(affinity), pid, e)) return False return True From 5ad1f42beb6c606a78ffdcf8cba2b9c6aab32bc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Tue, 22 May 2018 13:03:54 +0200 Subject: [PATCH 14/28] scheduler: Make instance._sched_cfg a local variable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's not needed anywhere else. Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 1b115f1..9ff7d7a 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -256,14 +256,14 @@ class SchedulerPlugin(base.Plugin): if ps is None: log.error("error applying tuning, cannot get information about running processes") return - instance._sched_cfg = [(option, str(value).split(":", 4)) for option, value in instance._scheduler.items()] - buf = [(option, vals) for option, vals in instance._sched_cfg if re.match(r"group\.", option) and len(vals) == 5] - instance._sched_cfg = sorted(buf, key=lambda option_vals: option_vals[1][0]) + sched_cfg = [(option, str(value).split(":", 4)) for option, value in instance._scheduler.items()] + buf = [(option, vals) for option, vals in sched_cfg if re.match(r"group\.", option) and len(vals) == 5] + sched_cfg = sorted(buf, key=lambda option_vals: option_vals[1][0]) sched_all = dict() # for runtime tunning instance._sched_lookup = {} for option, (rule_prio, scheduler, priority, affinity, regex) \ - in instance._sched_cfg: + in sched_cfg: try: r = re.compile(regex) except re.error as e: From bfc670dffc98eaa905f6d154ab6333b499af2cb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Tue, 22 May 2018 21:36:22 +0200 Subject: [PATCH 15/28] scheduler: Don't tune a process if reading original parameters failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We should not touch a process if we cannot later revert to it's original state. Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 9ff7d7a..7c90592 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -246,6 +246,11 @@ class SchedulerPlugin(base.Plugin): scheduler = rt[0], priority = rt[1], affinity = prev_affinity) + else: + if self._pid_exists(pid): + log.error("Refusing to tune PID %d, reading original scheduling parameters failed." + % pid) + return self._set_rt(pid, self._schedcfg2param(sched), prio) if affinity != "*": self._set_affinity(pid, affinity) From ea896e350186a5c90f6e055d6ad038c5d4d32dea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Tue, 22 May 2018 14:21:11 +0200 Subject: [PATCH 16/28] scheduler: Tune scheduler and affinity independently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set scheduler policy and affinity independently so that when reading original state of one of the parameters fails, we can still set the other one. Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 45 ++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 7c90592..d32bd18 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -237,23 +237,36 @@ class SchedulerPlugin(base.Plugin): #tune process and store previous values def _tune_process(self, pid, cmd, sched, prio, affinity): - #rt[0] - prev_sched, rt[1] - prev_prio rt = self._get_rt(pid) + (prev_sched, prev_prio) = (None, None) if rt is None else rt + if prev_sched is not None and prev_prio is not None: + sched = self._schedcfg2param(sched) + if sched == "": + prev_sched = None + self._set_rt(pid, sched, prio) + elif not self._pid_exists(pid): + return + else: + log.error("Refusing to set scheduler and priority of PID %d, reading original scheduling parameters failed." + % pid) prev_affinity = self._get_affinity(pid) - if prev_affinity is not None and rt is not None and len(rt) == 2 and rt[0] is not None and rt[1] is not None: + if prev_affinity is not None: + if affinity == "*": + prev_affinity = None + else: + self._set_affinity(pid, affinity) + elif not self._pid_exists(pid): + return + else: + log.error("Refusing to set CPU affinity of PID %d, reading original affinity failed." + % pid) + if prev_sched is not None or prev_prio is not None \ + or prev_affinity is not None: self._scheduler_original[pid] = SchedulerParams( cmdline = cmd, - scheduler = rt[0], - priority = rt[1], + scheduler = prev_sched, + priority = prev_prio, affinity = prev_affinity) - else: - if self._pid_exists(pid): - log.error("Refusing to tune PID %d, reading original scheduling parameters failed." - % pid) - return - self._set_rt(pid, self._schedcfg2param(sched), prio) - if affinity != "*": - self._set_affinity(pid, affinity) def _instance_apply_static(self, instance): super(SchedulerPlugin, self)._instance_apply_static(instance) @@ -298,9 +311,11 @@ class SchedulerPlugin(base.Plugin): # if command line for the pid didn't change, it's very probably the same process if pid not in ps or ps[pid] != orig_params.cmdline: continue - self._set_rt(pid, self._sched2param(orig_params.scheduler), - orig_params.priority) - self._set_affinity(pid, orig_params.affinity) + if orig_params.priority is not None: + sched = self._sched2param(orig_params.scheduler) + self._set_rt(pid, sched, orig_params.priority) + if orig_params.affinity is not None: + self._set_affinity(pid, orig_params.affinity) self._scheduler_original = {} self._storage.unset(self._storage_key()) From 00cda573e3208389cc1023a1e82cf1d87f83bbf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Tue, 22 May 2018 15:58:01 +0200 Subject: [PATCH 17/28] scheduler: Use schedutils in _set_rt() and _get_rt() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrite _set_rt() and _get_rt() to use schedutils instead of command line tools. Behaviour change: previously if the scheduling policy was set to "*" in a profile, e.g. [scheduler] group.foo=0:*:1:1:a.out then Tuned would set the scheduling policy to SCHED_OTHER on RHEL-7, and to SCHED_RR on Fedora (at least Fedora 27). This is due to the behaviour of the "chrt" tool. It behaved this way despite what it says in the following commit, which says the scheduler will not be changed if set to "*": https://github.com/redhat-performance/tuned/commit/bf42cb9ba3c This commit changes/fixes that, so that the scheduling policy is in fact not changed if "*" is specified. Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 205 +++++++++++++++++++----------- 1 file changed, 130 insertions(+), 75 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index d32bd18..35fb550 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -33,8 +33,13 @@ class SchedulerPlugin(base.Plugin): priorities of system threads (it is substitution for the rtctl tool). """ - _dict_sched2param = {"SCHED_FIFO":"f", "SCHED_BATCH":"b", "SCHED_RR":"r", - "SCHED_OTHER":"o", "SCHED_IDLE":"i"} + _dict_schedcfg2num = { + "f": schedutils.SCHED_FIFO, + "b": schedutils.SCHED_BATCH, + "r": schedutils.SCHED_RR, + "o": schedutils.SCHED_OTHER, + "i": schedutils.SCHED_IDLE, + } def __init__(self, monitor_repository, storage_factory, hardware_inventory, device_matcher, device_matcher_udev, plugin_instance_factory, global_cfg, variables): super(SchedulerPlugin, self).__init__(monitor_repository, storage_factory, hardware_inventory, device_matcher, device_matcher_udev, plugin_instance_factory, global_cfg, variables) @@ -134,23 +139,17 @@ class SchedulerPlugin(base.Plugin): except: return False + # Raises OSError + # Raises SystemError with old (pre-0.4) python-schedutils + # instead of OSError + # If PID doesn't exist, errno == ESRCH def _get_rt(self, pid): - (rc, out, err_msg) = self._cmd.execute(["chrt", "-p", str(pid)], return_err = True) - if rc != 0: - if self._pid_exists(pid): - log.error(err_msg) - else: - log.debug("Unable to read scheduling parameters for PID %d, the task vanished." % pid) - return None - vals = out.split("\n", 1) - if len(vals) > 1: - sched = self._parse_val(vals[0]) - prio = self._parse_val(vals[1]) - else: - sched = None - prio = None - log.debug("read scheduler policy '%s' and priority '%s' for pid '%d'" % (sched, prio, pid)) - return (sched, prio) + scheduler = schedutils.get_scheduler(pid) + sched_str = schedutils.schedstr(scheduler) + priority = schedutils.get_priority(pid) + log.debug("Read scheduler policy '%s' and priority '%d' of PID '%d'" + % (sched_str, priority, pid)) + return (scheduler, priority) def _get_affinity(self, pid): (rc, out, err_msg) = self._cmd.execute(["taskset", "-p", str(pid)], no_errors = [], return_err = True) @@ -164,35 +163,31 @@ class SchedulerPlugin(base.Plugin): log.debug("read affinity '%s' for pid '%d'" % (v, pid)) return v - def _schedcfg2param(self, sched): - if sched in ["f", "b", "r", "o"]: - return "-" + sched - # including '*' - else: - return "" - - def _sched2param(self, sched): - try: - return "-" + self._dict_sched2param[sched] - except KeyError: - return "" - def _set_rt(self, pid, sched, prio): - if pid is None or prio is None: - return - if sched is not None and len(sched) > 0: - schedl = [sched] - log.debug("setting scheduler policy to '%s' for PID '%d'" % (sched, pid)) - else: - schedl = [] - log.debug("setting scheduler priority to '%s' for PID '%d'" % (prio, pid)) - (ret, out, err_msg) = self._cmd.execute(["chrt"] + schedl + ["-p", str(prio), str(pid)], no_errors = [], return_err = True) - if ret == 0: - return - if self._pid_exists(pid): - log.error(err_msg) - else: - log.debug("Unable to set scheduling parameters for PID %d, the task vanished." % pid) + sched_str = schedutils.schedstr(sched) + log.debug("Setting scheduler policy to '%s' and priority to '%d' of PID '%d'." + % (sched_str, prio, pid)) + try: + prio_min = schedutils.get_priority_min(sched) + prio_max = schedutils.get_priority_max(sched) + if prio < prio_min or prio > prio_max: + log.error("Priority for %s must be in range %d - %d. '%d' was given." + % (sched_str, prio_min, + prio_max, prio)) + # Workaround for old (pre-0.4) python-schedutils which raised + # SystemError instead of OSError + except (SystemError, OSError) as e: + log.error("Failed to get allowed priority range: %s" + % e) + try: + schedutils.set_scheduler(pid, sched, prio) + except (SystemError, OSError) as e: + if hasattr(e, "errno") and e.errno == errno.ESRCH: + log.debug("Failed to set scheduling parameters of PID %d, the task vanished." + % pid) + else: + log.error("Failed to set scheduling parameters of PID %d: %s" + % (pid, e)) # Return codes: # 0 - Affinity is fixed @@ -235,38 +230,94 @@ class SchedulerPlugin(base.Plugin): if res == 1 or res == -2: log.error(err_msg) - #tune process and store previous values - def _tune_process(self, pid, cmd, sched, prio, affinity): - rt = self._get_rt(pid) - (prev_sched, prev_prio) = (None, None) if rt is None else rt - if prev_sched is not None and prev_prio is not None: - sched = self._schedcfg2param(sched) - if sched == "": - prev_sched = None + def _store_orig_process_rt(self, pid, scheduler, priority): + try: + params = self._scheduler_original[pid] + except KeyError: + params = SchedulerParams() + self._scheduler_original[pid] = params + if params.scheduler is None and params.priority is None: + params.scheduler = scheduler + params.priority = priority + + def _tune_process_rt(self, pid, sched, prio): + cont = True + if sched is None and prio is None: + return cont + try: + (prev_sched, prev_prio) = self._get_rt(pid) + if sched is None: + sched = prev_sched self._set_rt(pid, sched, prio) - elif not self._pid_exists(pid): - return - else: - log.error("Refusing to set scheduler and priority of PID %d, reading original scheduling parameters failed." - % pid) + self._store_orig_process_rt(pid, prev_sched, prev_prio) + except (SystemError, OSError) as e: + if hasattr(e, "errno") and e.errno == errno.ESRCH: + log.debug("Failed to read scheduler policy of PID %d, the task vanished." + % pid) + if pid in self._scheduler_original: + del self._scheduler_original[pid] + cont = False + else: + log.error("Refusing to set scheduler and priority of PID %d, reading original scheduling parameters failed: %s" + % (pid, e)) + return cont + + def _store_orig_process_affinity(self, pid, affinity): + try: + params = self._scheduler_original[pid] + except KeyError: + params = SchedulerParams() + self._scheduler_original[pid] = params + if params.affinity is None: + params.affinity = affinity + + def _tune_process_affinity(self, pid, affinity): + cont = True + if affinity == "*": + return cont prev_affinity = self._get_affinity(pid) if prev_affinity is not None: - if affinity == "*": - prev_affinity = None - else: - self._set_affinity(pid, affinity) + self._set_affinity(pid, affinity) + self._store_orig_process_affinity(pid, prev_affinity) elif not self._pid_exists(pid): - return + if pid in self._scheduler_original: + del self._scheduler_original[pid] + cont = False else: log.error("Refusing to set CPU affinity of PID %d, reading original affinity failed." % pid) - if prev_sched is not None or prev_prio is not None \ - or prev_affinity is not None: - self._scheduler_original[pid] = SchedulerParams( - cmdline = cmd, - scheduler = prev_sched, - priority = prev_prio, - affinity = prev_affinity) + return cont + + #tune process and store previous values + def _tune_process(self, pid, cmd, sched, prio, affinity): + cont = self._tune_process_rt(pid, sched, prio) + if not cont: + return + cont = self._tune_process_affinity(pid, affinity) + if not cont or pid not in self._scheduler_original: + return + self._scheduler_original[pid].cmdline = cmd + + def _convert_sched_params(self, str_scheduler, str_priority): + scheduler = self._dict_schedcfg2num.get(str_scheduler) + if scheduler is None and str_scheduler != "*": + log.error("Invalid scheduler: %s. Scheduler and priority will be ignored." + % str_scheduler) + return (None, None) + else: + try: + priority = int(str_priority) + except ValueError: + log.error("Invalid priority: %s. Scheduler and priority will be ignored." + % str_priority) + return (None, None) + return (scheduler, priority) + + def _convert_sched_cfg(self, vals): + (rule_prio, scheduler, priority, affinity, regex) = vals + (scheduler, priority) = self._convert_sched_params( + scheduler, priority) + return (rule_prio, scheduler, priority, affinity, regex) def _instance_apply_static(self, instance): super(SchedulerPlugin, self)._instance_apply_static(instance) @@ -275,7 +326,10 @@ class SchedulerPlugin(base.Plugin): log.error("error applying tuning, cannot get information about running processes") return sched_cfg = [(option, str(value).split(":", 4)) for option, value in instance._scheduler.items()] - buf = [(option, vals) for option, vals in sched_cfg if re.match(r"group\.", option) and len(vals) == 5] + buf = [(option, self._convert_sched_cfg(vals)) + for option, vals in sched_cfg + if re.match(r"group\.", option) + and len(vals) == 5] sched_cfg = sorted(buf, key=lambda option_vals: option_vals[1][0]) sched_all = dict() # for runtime tunning @@ -311,9 +365,10 @@ class SchedulerPlugin(base.Plugin): # if command line for the pid didn't change, it's very probably the same process if pid not in ps or ps[pid] != orig_params.cmdline: continue - if orig_params.priority is not None: - sched = self._sched2param(orig_params.scheduler) - self._set_rt(pid, sched, orig_params.priority) + if orig_params.scheduler is not None \ + and orig_params.priority is not None: + self._set_rt(pid, orig_params.scheduler, + orig_params.priority) if orig_params.affinity is not None: self._set_affinity(pid, orig_params.affinity) self._scheduler_original = {} From b4ddf474480d0398b216f4a56b6a12df592ee582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Mon, 28 May 2018 14:30:06 +0200 Subject: [PATCH 18/28] scheduler: Add a function for checking if a task is a kthread MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a function for checking if a task is a kernel thread. Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 35fb550..f5325ca 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -189,6 +189,11 @@ class SchedulerPlugin(base.Plugin): log.error("Failed to set scheduling parameters of PID %d: %s" % (pid, e)) + # process is a procfs.process object + # Raises OSError, IOError + def _is_kthread(self, process): + return process["stat"]["flags"] & procfs.pidstat.PF_KTHREAD != 0 + # Return codes: # 0 - Affinity is fixed # 1 - Affinity is changeable @@ -201,7 +206,7 @@ class SchedulerPlugin(base.Plugin): if process["stat"].is_bound_to_cpu(): if process["stat"]["state"] == "Z": log.debug("Affinity of zombie task with PID %d cannot be changed, the task's affinity mask is fixed." % pid) - elif process["stat"]["flags"] & procfs.pidstat.PF_KTHREAD != 0: + elif self._is_kthread(process): log.debug("Affinity of kernel thread with PID %d cannot be changed, the task's affinity mask is fixed." % pid) else: log.warn("Affinity of task with PID %d cannot be changed, the task's affinity mask is fixed." % pid) From 7da6e4adc96127f84ab86a69a8eb9d743508c3e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Wed, 23 May 2018 15:23:39 +0200 Subject: [PATCH 19/28] scheduler: Use procfs to list processes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use procfs instead of the ps command to list processes. Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 58 +++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index f5325ca..3c11e97 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -117,13 +117,33 @@ class SchedulerPlugin(base.Plugin): else: return cmdline.replace("\0", " ").strip() + # Raises OSError, IOError + def _get_cmdline(self, process): + if not isinstance(process, procfs.process): + pid = process + process = procfs.process(pid) + cmdline = procfs.process_cmdline(process) + if self._is_kthread(process): + cmdline = "[" + cmdline + "]" + return cmdline + + # Raises OSError, IOError def get_processes(self): - (rc, out) = self._cmd.execute(["ps", "-eopid,cmd", "--no-headers"]) - if rc != 0 or len(out) <= 0: - return None - lines = out.split("\n") - pid_cmd_list = [i for i in [s.split(None, 1) for s in lines] if len(i) == 2] - return dict([(int(pid.lstrip()), cmd.lstrip()) for pid, cmd in pid_cmd_list]) + ps = procfs.pidstats() + ps.reload_threads() + processes = {} + for proc in ps.values(): + try: + cmd = self._get_cmdline(proc) + pid = proc["pid"] + processes[pid] = cmd + except (OSError, IOError) as e: + if e.errno == errno.ENOENT \ + or e.errno == errno.ESRCH: + continue + else: + raise + return processes def _parse_val(self, val): v = val.split(":", 1) @@ -326,9 +346,11 @@ class SchedulerPlugin(base.Plugin): def _instance_apply_static(self, instance): super(SchedulerPlugin, self)._instance_apply_static(instance) - ps = self.get_processes() - if ps is None: - log.error("error applying tuning, cannot get information about running processes") + try: + ps = self.get_processes() + except (OSError, IOError) as e: + log.error("error applying tuning, cannot get information about running processes: %s" + % e) return sched_cfg = [(option, str(value).split(":", 4)) for option, value in instance._scheduler.items()] buf = [(option, self._convert_sched_cfg(vals)) @@ -365,7 +387,12 @@ class SchedulerPlugin(base.Plugin): instance._thread.start() def _restore_ps_affinity(self): - ps = self.get_processes() + try: + ps = self.get_processes() + except (OSError, IOError) as e: + log.error("error unapplying tuning, cannot get information about running processes: %s" + % e) + return for pid, orig_params in self._scheduler_original.items(): # if command line for the pid didn't change, it's very probably the same process if pid not in ps or ps[pid] != orig_params.cmdline: @@ -505,10 +532,13 @@ class SchedulerPlugin(base.Plugin): def _set_ps_affinity(self, affinity, intersect = False): _affinity = affinity affinity_hex = self._cmd.cpulist2hex(_affinity) - ps = procfs.pidstats() - ps.reload_threads() - self._set_all_obj_affinity(ps.values(), affinity, False, intersect) - + try: + ps = procfs.pidstats() + ps.reload_threads() + self._set_all_obj_affinity(ps.values(), affinity, False, intersect) + except (OSError, IOError) as e: + log.error("error applying tuning, cannot get information about running processes: %s" + % e) # process IRQs irqs = procfs.interrupts() for irq in list(irqs.keys()): From c1bc25756337bd6477744fcdce97096b46f47607 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Mon, 28 May 2018 15:37:44 +0200 Subject: [PATCH 20/28] scheduler: Replace get_process with _get_cmdline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 3c11e97..32612f9 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -106,17 +106,6 @@ class SchedulerPlugin(base.Plugin): "ps_blacklist": None, } - def get_process(self, pid): - cmd = self._cmd.read_file("/proc/" + str(pid) + "/comm", no_error = True) - if cmd == "": - return "" - cmd = cmd.strip() - cmdline = self._cmd.read_file("/proc/" + str(pid) + "/cmdline", no_error = True) - if cmdline == "": - return "[" + cmd + "]" - else: - return cmdline.replace("\0", " ").strip() - # Raises OSError, IOError def _get_cmdline(self, process): if not isinstance(process, procfs.process): @@ -414,9 +403,16 @@ class SchedulerPlugin(base.Plugin): self._restore_ps_affinity() def _add_pid(self, instance, pid, r): - cmd = self.get_process(pid) - # check to filter short living process - if cmd == "": + try: + cmd = self._get_cmdline(pid) + except (OSError, IOError) as e: + if e.errno == errno.ENOENT \ + or e.errno == errno.ESRCH: + log.debug("Failed to get cmdline of PID %d, the task vanished." + % pid) + else: + log.error("Failed to get cmdline of PID %d: %s" + % (pid, e)) return v = self._cmd.re_lookup(instance._sched_lookup, cmd, r) if v is not None and not pid in self._scheduler_original: From 7bb6ff4d4da42de7a51466ba58cc94d5effe3cba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Sat, 2 Jun 2018 17:08:04 +0200 Subject: [PATCH 21/28] scheduler: group.* options: Process threads as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously threads were not touched when processing group.* options. Reproducer: $ cat > a.c << EOF #define _GNU_SOURCE #include #include void *foo(void *unused) { pause(); return NULL; } int main(void) { pthread_t t; pthread_create(&t, NULL, foo, NULL); pthread_setname_np(t, "a.out"); pause(); return 0; } EOF $ gcc a.c -pthread $ mkdir /etc/tuned/test $ cat > /etc/tuned/test/tuned.conf << EOF [scheduler] group.foo=0:o:0:1:a.out EOF $ ./a.out & $ tuned-adm profile test $ for proc in /proc/$(pgrep a.out)/task/*; do taskset -p $(basename $proc); done pid 29052's current affinity mask: 1 pid 29053's current affinity mask: f <<< should be 1 Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 32612f9..582f0ad 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -126,6 +126,10 @@ class SchedulerPlugin(base.Plugin): cmd = self._get_cmdline(proc) pid = proc["pid"] processes[pid] = cmd + if "threads" in proc: + for pid in proc["threads"].keys(): + cmd = self._get_cmdline(proc) + processes[pid] = cmd except (OSError, IOError) as e: if e.errno == errno.ENOENT \ or e.errno == errno.ESRCH: @@ -446,9 +450,9 @@ class SchedulerPlugin(base.Plugin): if event: read_events = True if event.type == perf.RECORD_COMM: - self._add_pid(instance, int(event.pid), r) + self._add_pid(instance, int(event.tid), r) elif event.type == perf.RECORD_EXIT: - self._remove_pid(instance, int(event.pid)) + self._remove_pid(instance, int(event.tid)) @command_custom("ps_whitelist", per_device = False) def _ps_whitelist(self, enabling, value, verify, ignore_missing): From 74e87259058e3ae2268a64c523334ab4b3fbdade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Tue, 29 May 2018 12:31:09 +0200 Subject: [PATCH 22/28] scheduler: Use only one global CPU map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 582f0ad..9809bd6 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -53,6 +53,7 @@ class SchedulerPlugin(base.Plugin): # default is to whitelist all and blacklist none self._ps_whitelist = ".*" self._ps_blacklist = "" + self._cpus = perf.cpu_map() def _instance_init(self, instance): instance._has_dynamic_tuning = False @@ -80,15 +81,14 @@ class SchedulerPlugin(base.Plugin): instance._terminate = threading.Event() if self._daemon and instance._runtime_tuning: try: - instance._cpus = perf.cpu_map() instance._threads = perf.thread_map() evsel = perf.evsel(type = perf.TYPE_SOFTWARE, config = perf.COUNT_SW_DUMMY, task = 1, comm = 1, mmap = 0, freq = 0, wakeup_events = 1, watermark = 1, sample_type = perf.SAMPLE_TID | perf.SAMPLE_CPU) - evsel.open(cpus = instance._cpus, threads = instance._threads) - instance._evlist = perf.evlist(instance._cpus, instance._threads) + evsel.open(cpus = self._cpus, threads = instance._threads) + instance._evlist = perf.evlist(self._cpus, instance._threads) instance._evlist.add(evsel) instance._evlist.mmap() # no perf @@ -445,7 +445,7 @@ class SchedulerPlugin(base.Plugin): read_events = True while read_events: read_events = False - for cpu in instance._cpus: + for cpu in self._cpus: event = instance._evlist.read_on_cpu(cpu) if event: read_events = True @@ -564,16 +564,14 @@ class SchedulerPlugin(base.Plugin): # currently unsupported if verify: return None - # TODO merge with instance._cpus - cpus = list(perf.cpu_map()) if enabling: if value is not None: affinity = self._cmd.cpulist_invert(value) sa = set(affinity) - if set(cpus).intersection(sa) != sa: - str_cpus = ",".join([str(x) for x in cpus]) + if set(self._cpus).intersection(sa) != sa: + str_cpus = ",".join([str(x) for x in self._cpus]) log.error("invalid isolated_cores specified, '%s' don't match available cores '%s'" % (value, str_cpus)) return None self._set_ps_affinity(affinity, True) else: - self._set_ps_affinity(cpus, False) + self._set_ps_affinity(list(self._cpus), False) From 3ec410fdb1c1e3a36bb630c1e762ef6b61261f36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Tue, 29 May 2018 13:00:49 +0200 Subject: [PATCH 23/28] scheduler: Refactor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 9809bd6..83a42d3 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -504,24 +504,31 @@ class SchedulerPlugin(base.Plugin): def _set_all_obj_affinity(self, objs, affinity, threads = False, intersect = False): _affinity = affinity - psl = [v for v in objs if re.search(self._ps_whitelist, self._get_stat_comm(v)) is not None] + psl = [v for v in objs if re.search(self._ps_whitelist, + self._get_stat_comm(v)) is not None] if self._ps_blacklist != "": - psl = [v for v in psl if re.search(self._ps_blacklist, self._get_stat_comm(v)) is None] + psl = [v for v in psl if re.search(self._ps_blacklist, + self._get_stat_comm(v)) is None] psd = dict([(v.pid, v) for v in psl]) - for obj in psd: - if self._affinity_changeable(obj, process = psd[obj]) != 1: + for pid in psd: + if self._affinity_changeable(pid, process = psd[pid]) \ + != 1: continue - prev_affinity = self._get_affinity2(obj) + prev_affinity = self._get_affinity2(pid) if prev_affinity is None: continue if intersect: - _affinity = self._get_intersect_affinity(prev_affinity, affinity, affinity) + _affinity = self._get_intersect_affinity( + prev_affinity, affinity, + affinity) if set(_affinity) != set(prev_affinity): - if not self._set_affinity2(obj, _affinity): + if not self._set_affinity2(pid, _affinity): continue # process threads - if not threads and "threads" in psd[obj]: - self._set_all_obj_affinity(psd[obj]["threads"].values(), affinity, True, intersect) + if not threads and "threads" in psd[pid]: + self._set_all_obj_affinity( + psd[pid]["threads"].values(), + affinity, True, intersect) def _get_stat_comm(self, o): try: From ae537661532bfd5b3d6b20d21276d5d1fdc98e03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Fri, 1 Jun 2018 13:12:02 +0200 Subject: [PATCH 24/28] scheduler: Use separate storage for scheduling parameters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 83a42d3..a725b1b 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -54,6 +54,8 @@ class SchedulerPlugin(base.Plugin): self._ps_whitelist = ".*" self._ps_blacklist = "" self._cpus = perf.cpu_map() + self._scheduler_storage_key = self._storage_key( + command_name = "scheduler") def _instance_init(self, instance): instance._has_dynamic_tuning = False @@ -65,13 +67,13 @@ class SchedulerPlugin(base.Plugin): # FIXME: do we want to do this here? # recover original values in case of crash - storage_key = self._storage_key() - self._scheduler_original = self._storage.get(storage_key, {}) + self._scheduler_original = self._storage.get( + self._scheduler_storage_key, {}) if len(self._scheduler_original) > 0: log.info("recovering scheduling settings from previous run") self._restore_ps_affinity() self._scheduler_original = {} - self._storage.unset(storage_key) + self._storage.unset(self._scheduler_storage_key) instance._scheduler = instance.options for k in instance._scheduler: @@ -373,8 +375,8 @@ class SchedulerPlugin(base.Plugin): in sched_all.items(): self._tune_process(pid, cmd, scheduler, priority, affinity) - storage_key = self._storage_key() - self._storage.set(storage_key, self._scheduler_original) + self._storage.set(self._scheduler_storage_key, + self._scheduler_original) if self._daemon and instance._runtime_tuning: instance._thread = threading.Thread(target = self._thread_code, args = [instance]) instance._thread.start() @@ -397,7 +399,7 @@ class SchedulerPlugin(base.Plugin): if orig_params.affinity is not None: self._set_affinity(pid, orig_params.affinity) self._scheduler_original = {} - self._storage.unset(self._storage_key()) + self._storage.unset(self._scheduler_storage_key) def _instance_unapply_static(self, instance, full_rollback = False): super(SchedulerPlugin, self)._instance_unapply_static(instance, full_rollback) @@ -424,15 +426,15 @@ class SchedulerPlugin(base.Plugin): (sched, prio, affinity) = v self._tune_process(pid, cmd, sched, prio, affinity) - storage_key = self._storage_key() - self._storage.set(storage_key, self._scheduler_original) + self._storage.set(self._scheduler_storage_key, + self._scheduler_original) def _remove_pid(self, instance, pid): if pid in self._scheduler_original: del self._scheduler_original[pid] log.debug("removed PID %d from the rollback database" % pid) - storage_key = self._storage_key() - self._storage.set(storage_key, self._scheduler_original) + self._storage.set(self._scheduler_storage_key, + self._scheduler_original) def _thread_code(self, instance): r = self._cmd.re_lookup_compile(instance._sched_lookup) From 32aab15530d28943fe1bdd8d1b3b69db1d40eab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Fri, 1 Jun 2018 14:49:46 +0200 Subject: [PATCH 25/28] scheduler: Do true rollback of IRQs affinity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Do true rollback of IRQs affinity. Previously the affinity of IRQs was set to all cores on rollback. Related: rhbz#1512295 https://bugzilla.redhat.com/show_bug.cgi?id=1512295#c19 Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 56 ++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index a725b1b..5e9a5c7 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -26,6 +26,11 @@ class SchedulerParams(object): self.priority = priority self.affinity = affinity +class IRQAffinities(object): + def __init__(self): + self.irqs = {} + self.default = None + # TODO move from cmdline tools to schedutils and consolidate the code class SchedulerPlugin(base.Plugin): """ @@ -56,6 +61,8 @@ class SchedulerPlugin(base.Plugin): self._cpus = perf.cpu_map() self._scheduler_storage_key = self._storage_key( command_name = "scheduler") + self._irq_storage_key = self._storage_key( + command_name = "irq") def _instance_init(self, instance): instance._has_dynamic_tuning = False @@ -539,8 +546,6 @@ class SchedulerPlugin(base.Plugin): return "" def _set_ps_affinity(self, affinity, intersect = False): - _affinity = affinity - affinity_hex = self._cmd.cpulist2hex(_affinity) try: ps = procfs.pidstats() ps.reload_threads() @@ -548,25 +553,50 @@ class SchedulerPlugin(base.Plugin): except (OSError, IOError) as e: log.error("error applying tuning, cannot get information about running processes: %s" % e) - # process IRQs + + def _set_irq_affinity(self, irq, affinity_hex): + self._cmd.write_to_file("/proc/irq/%s/smp_affinity" % irq, + affinity_hex, no_error = True) + + def _set_default_irq_affinity(self, affinity_hex): + self._cmd.write_to_file("/proc/irq/default_smp_affinity", + affinity_hex) + + def _set_all_irq_affinity(self, affinity): + irq_original = IRQAffinities() irqs = procfs.interrupts() - for irq in list(irqs.keys()): + for irq in irqs.keys(): try: prev_affinity = irqs[irq]["affinity"] + log.debug("Read affinity of IRQ '%s': '%s'" + % (irq, prev_affinity)) except KeyError: continue - if intersect: - _affinity = self._get_intersect_affinity(prev_affinity, affinity, affinity) - affinity_hex = self._cmd.cpulist2hex(_affinity) - self._cmd.write_to_file("/proc/irq/%s/smp_affinity" % irq, affinity_hex, no_error = True) + _affinity = self._get_intersect_affinity(prev_affinity, affinity, affinity) + affinity_hex = self._cmd.cpulist2hex(_affinity) + self._set_irq_affinity(irq, affinity_hex) + irq_original.irqs[irq] = prev_affinity # default affinity prev_affinity_hex = self._cmd.read_file("/proc/irq/default_smp_affinity") prev_affinity = self._cmd.hex2cpulist(prev_affinity_hex) - if intersect: - _affinity = self._get_intersect_affinity(prev_affinity, affinity, affinity) - affinity_hex = self._cmd.cpulist2hex(_affinity) - self._cmd.write_to_file("/proc/irq/default_smp_affinity", affinity_hex) + _affinity = self._get_intersect_affinity(prev_affinity, affinity, affinity) + affinity_hex = self._cmd.cpulist2hex(_affinity) + self._set_default_irq_affinity(affinity_hex) + irq_original.default = prev_affinity + self._storage.set(self._irq_storage_key, irq_original) + + def _restore_all_irq_affinity(self): + irq_original = self._storage.get(self._irq_storage_key, None) + if irq_original is None: + return + for irq, affinity in irq_original.irqs.items(): + affinity_hex = self._cmd.cpulist2hex(affinity) + self._set_irq_affinity(irq, affinity_hex) + affinity = irq_original.default + affinity_hex = self._cmd.cpulist2hex(affinity) + self._set_default_irq_affinity(affinity_hex) + self._storage.unset(self._irq_storage_key) @command_custom("isolated_cores", per_device = False, priority = 10) def _isolated_cores(self, enabling, value, verify, ignore_missing): @@ -582,5 +612,7 @@ class SchedulerPlugin(base.Plugin): log.error("invalid isolated_cores specified, '%s' don't match available cores '%s'" % (value, str_cpus)) return None self._set_ps_affinity(affinity, True) + self._set_all_irq_affinity(affinity) else: self._set_ps_affinity(list(self._cpus), False) + self._restore_all_irq_affinity() From f8119fa6c64fd3c2b466d4089c3c314dc2842edf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Sun, 3 Jun 2018 15:47:55 +0200 Subject: [PATCH 26/28] scheduler: Merge (get|set)_affinity.* methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 156 ++++++++++++++---------------- 1 file changed, 72 insertions(+), 84 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 5e9a5c7..7afbbbb 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -31,7 +31,6 @@ class IRQAffinities(object): self.irqs = {} self.default = None -# TODO move from cmdline tools to schedutils and consolidate the code class SchedulerPlugin(base.Plugin): """ Plugin for tuning of scheduler. Currently it can control scheduling @@ -147,20 +146,6 @@ class SchedulerPlugin(base.Plugin): raise return processes - def _parse_val(self, val): - v = val.split(":", 1) - if len(v) == 2: - return v[1].strip() - else: - return None - - def _pid_exists(self, pid): - try: - p = procfs.pidstat(pid) - return True - except: - return False - # Raises OSError # Raises SystemError with old (pre-0.4) python-schedutils # instead of OSError @@ -173,18 +158,6 @@ class SchedulerPlugin(base.Plugin): % (sched_str, priority, pid)) return (scheduler, priority) - def _get_affinity(self, pid): - (rc, out, err_msg) = self._cmd.execute(["taskset", "-p", str(pid)], no_errors = [], return_err = True) - if rc != 0: - if rc != 1 or self._pid_exists(pid): - log.error(err_msg) - else: - log.debug("Unable to read affinity for PID %d, the task vanished." % pid) - return None - v = self._parse_val(out.split("\n", 1)[0]) - log.debug("read affinity '%s' for pid '%d'" % (v, pid)) - return v - def _set_rt(self, pid, sched, prio): sched_str = schedutils.schedstr(sched) log.debug("Setting scheduler policy to '%s' and priority to '%d' of PID '%d'." @@ -221,42 +194,36 @@ class SchedulerPlugin(base.Plugin): # 1 - Affinity is changeable # -1 - Task vanished # -2 - Error - def _affinity_changeable(self, pid, process = None): + def _affinity_changeable(self, pid): try: - if process is None: - process = procfs.process(pid) + process = procfs.process(pid) if process["stat"].is_bound_to_cpu(): if process["stat"]["state"] == "Z": - log.debug("Affinity of zombie task with PID %d cannot be changed, the task's affinity mask is fixed." % pid) + log.debug("Affinity of zombie task with PID %d cannot be changed, the task's affinity mask is fixed." + % pid) elif self._is_kthread(process): - log.debug("Affinity of kernel thread with PID %d cannot be changed, the task's affinity mask is fixed." % pid) + log.debug("Affinity of kernel thread with PID %d cannot be changed, the task's affinity mask is fixed." + % pid) else: - log.warn("Affinity of task with PID %d cannot be changed, the task's affinity mask is fixed." % pid) + log.warn("Affinity of task with PID %d cannot be changed, the task's affinity mask is fixed." + % pid) return 0 else: return 1 except (OSError, IOError) as e: if e.errno == errno.ENOENT or e.errno == errno.ESRCH: - log.debug("Unable to set affinity for PID %d, the task vanished." % pid) + log.debug("Failed to get task info for PID %d, the task vanished." + % pid) return -1 else: - log.error("Failed to get task info for PID %d: %s" % (pid, str(e))) + log.error("Failed to get task info for PID %d: %s" + % (pid, e)) return -2 except (AttributeError, KeyError) as e: - log.error("Failed to get task info for PID %d: %s" % (pid, str(e))) + log.error("Failed to get task info for PID %d: %s" + % (pid, e)) return -2 - def _set_affinity(self, pid, affinity): - if pid is None or affinity is None: - return - log.debug("setting affinity to '%s' for PID '%d'" % (affinity, pid)) - (ret, out, err_msg) = self._cmd.execute(["taskset", "-p", str(affinity), str(pid)], no_errors = [], return_err = True) - if ret == 0: - return - res = self._affinity_changeable(pid) - if res == 1 or res == -2: - log.error(err_msg) - def _store_orig_process_rt(self, pid, scheduler, priority): try: params = self._scheduler_original[pid] @@ -300,19 +267,23 @@ class SchedulerPlugin(base.Plugin): def _tune_process_affinity(self, pid, affinity): cont = True - if affinity == "*": + if affinity is None: return cont - prev_affinity = self._get_affinity(pid) - if prev_affinity is not None: + try: + prev_affinity = self._get_affinity(pid) self._set_affinity(pid, affinity) - self._store_orig_process_affinity(pid, prev_affinity) - elif not self._pid_exists(pid): - if pid in self._scheduler_original: - del self._scheduler_original[pid] - cont = False - else: - log.error("Refusing to set CPU affinity of PID %d, reading original affinity failed." - % pid) + self._store_orig_process_affinity(pid, + prev_affinity) + except (SystemError, OSError) as e: + if hasattr(e, "errno") and e.errno == errno.ESRCH: + log.debug("Failed to read affinity of PID %d, the task vanished." + % pid) + if pid in self._scheduler_original: + del self._scheduler_original[pid] + cont = False + else: + log.error("Refusing to set CPU affinity of PID %d, reading original affinity failed: %s" + % (pid, e)) return cont #tune process and store previous values @@ -340,10 +311,22 @@ class SchedulerPlugin(base.Plugin): return (None, None) return (scheduler, priority) + def _convert_affinity(self, str_affinity): + if str_affinity == "*": + affinity = None + else: + affinity = self._cmd.hex2cpulist(str_affinity) + if not affinity: + log.error("Invalid affinity: %s. It will be ignored." + % str_affinity) + affinity = None + return affinity + def _convert_sched_cfg(self, vals): (rule_prio, scheduler, priority, affinity, regex) = vals (scheduler, priority) = self._convert_sched_params( scheduler, priority) + affinity = self._convert_affinity(affinity) return (rule_prio, scheduler, priority, affinity, regex) def _instance_apply_static(self, instance): @@ -479,30 +462,32 @@ class SchedulerPlugin(base.Plugin): if enabling and value is not None: self._ps_blacklist = "|".join(["(%s)" % v for v in re.split(r"(? Date: Sun, 3 Jun 2018 18:46:55 +0200 Subject: [PATCH 27/28] scheduler: Store saved affinity as a bitmask rather than list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Store saved CPU affinity of processes as a bitmask rather than a CPU list. It should be much more memory-efficient. Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 23 +++++++++++++++++++---- tuned/utils/commands.py | 24 ++++++++++++++++-------- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index 7afbbbb..c310762 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -19,13 +19,28 @@ import errno log = tuned.logs.get() class SchedulerParams(object): - def __init__(self, cmdline = None, scheduler = None, priority = None, - affinity = None): + def __init__(self, cmd, cmdline = None, scheduler = None, + priority = None, affinity = None): + self._cmd = cmd self.cmdline = cmdline self.scheduler = scheduler self.priority = priority self.affinity = affinity + @property + def affinity(self): + if self._affinity is None: + return None + else: + return self._cmd.bitmask2cpulist(self._affinity) + + @affinity.setter + def affinity(self, value): + if value is None: + self._affinity = None + else: + self._affinity = self._cmd.cpulist2bitmask(value) + class IRQAffinities(object): def __init__(self): self.irqs = {} @@ -228,7 +243,7 @@ class SchedulerPlugin(base.Plugin): try: params = self._scheduler_original[pid] except KeyError: - params = SchedulerParams() + params = SchedulerParams(self._cmd) self._scheduler_original[pid] = params if params.scheduler is None and params.priority is None: params.scheduler = scheduler @@ -260,7 +275,7 @@ class SchedulerPlugin(base.Plugin): try: params = self._scheduler_original[pid] except KeyError: - params = SchedulerParams() + params = SchedulerParams(self._cmd) self._scheduler_original[pid] = params if params.affinity is None: params.affinity = affinity diff --git a/tuned/utils/commands.py b/tuned/utils/commands.py index 533726a..8b7df57 100644 --- a/tuned/utils/commands.py +++ b/tuned/utils/commands.py @@ -259,17 +259,21 @@ class commands: if mask is None: return None mask = str(mask).replace(",", "") - cpu = 0 - cpus = [] try: m = int(mask, 16) except ValueError: log.error("invalid hexadecimal mask '%s'" % str(mask)) return [] - while m > 0: - if m & 1: + return self.bitmask2cpulist(m) + + # Converts an integer bitmask to a list of cpus (e.g. [0,3,4]) + def bitmask2cpulist(self, mask): + cpu = 0 + cpus = [] + while mask > 0: + if mask & 1: cpus.append(cpu) - m >>= 1 + mask >>= 1 cpu += 1 return cpus @@ -366,12 +370,10 @@ class commands: def cpulist2hex(self, l): if l is None: return None - m = 0 ul = self.cpulist_unpack(l) if ul is None: return None - for v in ul: - m |= pow(2, v) + m = self.cpulist2bitmask(ul) s = "%x" % m ls = len(s) if ls % 8 != 0: @@ -379,6 +381,12 @@ class commands: s = s.zfill(ls) return ",".join(s[i:i + 8] for i in range(0, len(s), 8)) + def cpulist2bitmask(self, l): + m = 0 + for v in l: + m |= pow(2, v) + return m + def process_recommend_file(self, fname): matching_profile = None try: From de5c4432058bd2fcab43874dc122cde9dcb40f2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Lyson=C4=9Bk?= Date: Sun, 3 Jun 2018 16:12:41 +0200 Subject: [PATCH 28/28] scheduler: Do true rollback of processes' affinity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Properly revert affinity of processes whose affinity was changed due to the isolated_cores setting. Resolves: rhbz#1512295 https://bugzilla.redhat.com/show_bug.cgi?id=1512295#c19 Known regression: reverting affinity of processes started after Tuned was started does not work as one might expect. See the following reproducer (run on a 4 core machine): $ mkdir /etc/tuned/test $ cat > /etc/tuned/test/tuned.conf << EOF [scheduler] isolated_cores=1 EOF $ cat > a.c << EOF #include int main(void) { pause(); return 0; } EOF $ gcc a.c $ systemctl start tuned $ ./a.out & $ systemctl stop tuned $ taskset -p $(pgrep a.out) pid 9950's current affinity mask: d <<< *maybe* should be "f" The affinity of the ./a.out process is 0xd after tuned is stopped, not 0xf as one might expect. This is because after starting tuned, first the affinity of the shell session is set to the non-isolated cores (0xd). Then when ./a.out starts, it inherits that affinity. That is, tuned doesn't explicitly change affinity of the process. The affinity gets inherited. So tuned will not change affinity of the process upon rollback, because it hasn't ever touched that process. And currently, tuned would not event know what the affinity should be reverted to. It is unclear to me at this point whether we should attempt to address this issue, or if we should leave it be. Properly fixing it would require tracing where processes get their affinity from. Signed-off-by: Ondřej Lysoněk --- tuned/plugins/plugin_scheduler.py | 41 +++++++++++++++++-------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/tuned/plugins/plugin_scheduler.py b/tuned/plugins/plugin_scheduler.py index c310762..3949cb2 100644 --- a/tuned/plugins/plugin_scheduler.py +++ b/tuned/plugins/plugin_scheduler.py @@ -280,12 +280,16 @@ class SchedulerPlugin(base.Plugin): if params.affinity is None: params.affinity = affinity - def _tune_process_affinity(self, pid, affinity): + def _tune_process_affinity(self, pid, affinity, intersect = False): cont = True if affinity is None: return cont try: prev_affinity = self._get_affinity(pid) + if intersect: + affinity = self._get_intersect_affinity( + prev_affinity, affinity, + affinity) self._set_affinity(pid, affinity) self._store_orig_process_affinity(pid, prev_affinity) @@ -511,8 +515,7 @@ class SchedulerPlugin(base.Plugin): return list(aff) return affinity3 - def _set_all_obj_affinity(self, objs, affinity, threads = False, intersect = False): - _affinity = affinity + def _set_all_obj_affinity(self, objs, affinity, threads = False): psl = [v for v in objs if re.search(self._ps_whitelist, self._get_stat_comm(v)) is not None] if self._ps_blacklist != "": @@ -521,26 +524,27 @@ class SchedulerPlugin(base.Plugin): psd = dict([(v.pid, v) for v in psl]) for pid in psd: try: - prev_affinity = self._get_affinity(pid) - except (SystemError, OSError) as e: - if hasattr(e, "errno") and e.errno == errno.ESRCH: - log.debug("Failed to read affinity of PID %d, the task vanished." + cmd = self._get_cmdline(psd[pid]) + except (OSError, IOError) as e: + if e.errno == errno.ENOENT \ + or e.errno == errno.ESRCH: + log.debug("Failed to get cmdline of PID %d, the task vanished." % pid) else: - log.error("Refusing to set CPU affinity of PID %d, reading original affinity failed: %s" + log.error("Refusing to set affinity of PID %d, failed to get its cmdline: %s" % (pid, e)) continue - if intersect: - _affinity = self._get_intersect_affinity( - prev_affinity, affinity, - affinity) - if not self._set_affinity(pid, _affinity): + cont = self._tune_process_affinity(pid, affinity, + intersect = True) + if not cont: continue + if pid in self._scheduler_original: + self._scheduler_original[pid].cmdline = cmd # process threads if not threads and "threads" in psd[pid]: self._set_all_obj_affinity( psd[pid]["threads"].values(), - affinity, True, intersect) + affinity, True) def _get_stat_comm(self, o): try: @@ -548,11 +552,11 @@ class SchedulerPlugin(base.Plugin): except (OSError, IOError, KeyError): return "" - def _set_ps_affinity(self, affinity, intersect = False): + def _set_ps_affinity(self, affinity): try: ps = procfs.pidstats() ps.reload_threads() - self._set_all_obj_affinity(ps.values(), affinity, False, intersect) + self._set_all_obj_affinity(ps.values(), affinity, False) except (OSError, IOError) as e: log.error("error applying tuning, cannot get information about running processes: %s" % e) @@ -614,8 +618,9 @@ class SchedulerPlugin(base.Plugin): str_cpus = ",".join([str(x) for x in self._cpus]) log.error("invalid isolated_cores specified, '%s' don't match available cores '%s'" % (value, str_cpus)) return None - self._set_ps_affinity(affinity, True) + self._set_ps_affinity(affinity) self._set_all_irq_affinity(affinity) else: - self._set_ps_affinity(list(self._cpus), False) + # Restoring processes' affinity is done in + # _instance_unapply_static() self._restore_all_irq_affinity()