feat: persistence of dynamic tuning changes

Currently, any changes made to the tuning via the `instance_*` dbus calls
are lost when tuning is stopped by the service, or when the TuneD service
itself is stopped/restarted, or when the service crashes.

This commit:

* implements a sync of Plugin Instances and Profile Units that is currently missing.
  This way, dynamic instances and device assignments are persistent across stop/start
  dbus calls to TuneD.
* calculates a hash of the current profile after loading it from disk
  (after processing all includes, so we have a "flat" representation)
* creates snapshots of the current profile whenever instances or assigned
  devices change. the snapshot includes the hash of the profile as
  it was initially loaded. for each instance it stores the devices that
  are currently attached.
* restores a snapshot found at startup, if the hashes match (i.e. there
  have been no profile switches and no changes to the profile or any of
  its includes on disk)

  snapshots are restored in case of
  - daemon restarts (systemctl restart/stop/start)
  - daemon crashes
  snapshots are NOT restored in case of
  - reboots (snapshots are stored in /var/run)
  - profile changes (snapshots are explicitly deleted when switching profiles,
    even when "switching" to the same/current profile)

Signed-off-by: Adriaan Schmidt <adriaan.schmidt@siemens.com>

tests/unit/profiles/test_profile.py

@@ -46,7 +46,7 @@ class ProfileTestCase(unittest.TestCase):
 			"network" : { "type": "net", "devices": "*" },
 		})
 
-		self.assertIs(type(profile.options), dict)
+		self.assertIs(type(profile.options), collections.OrderedDict)
 		self.assertEqual(profile.options["anything"], 10)
 
 	def test_sets_options_empty(self):

tuned/consts.py

@@ -19,6 +19,7 @@ DEFAULT_PROFILE = "balanced"
 DEFAULT_STORAGE_FILE = "/run/tuned/save.pickle"
 USER_PROFILES_DIR = "/etc/tuned/profiles"
 SYSTEM_PROFILES_DIR = "/usr/lib/tuned/profiles"
+PROFILE_SNAPSHOT_FILE = "/run/tuned/profile-snapshot.conf"
 PERSISTENT_STORAGE_DIR = "/var/lib/tuned"
 PLUGIN_MAIN_UNIT_NAME = "main"
 PLUGIN_VARIABLES_UNIT_NAME = "variables"

tuned/daemon/controller.py

@@ -417,6 +417,7 @@ class Controller(tuned.exports.interfaces.ExportableInterface):
 			rets = "Ignoring devices not handled by any instance '%s'." % str(devs)
 			log.info(rets)
 			return (False, rets)
+		self._daemon.sync_instances()
 		return (True, "OK")
 
 	@exports.export("s", "(bsa(ss))")
@@ -482,6 +483,8 @@ class Controller(tuned.exports.interfaces.ExportableInterface):
 		"""
 		if caller == "":
 			return (False, "Unauthorized")
+		plugin_name = str(plugin_name)
+		instance_name = str(instance_name)
 		if not self._cmd.is_valid_name(plugin_name):
 			return (False, "Invalid plugin_name")
 		if not self._cmd.is_valid_name(instance_name):
@@ -529,6 +532,7 @@ class Controller(tuned.exports.interfaces.ExportableInterface):
 					other_instance.name, instance.name))
 				plugin._remove_devices_nocheck(other_instance, devs_moving)
 				plugin._add_devices_nocheck(instance, devs_moving)
+		self._daemon.sync_instances()
 		return (True, "OK")
 
 	@exports.export("s", "(bs)")
@@ -571,4 +575,5 @@ class Controller(tuned.exports.interfaces.ExportableInterface):
 		for device in devices:
 			# _add_device() will find a suitable plugin instance
 			plugin._add_device(device)
+		self._daemon.sync_instances()
 		return (True, "OK")

tuned/daemon/daemon.py

@@ -123,6 +123,11 @@ class Daemon(object):
 			self._notify_profile_changed(profile_names, False, errstr)
 			raise TunedException(errstr)
 
+		# restore profile snapshot (if there is one)
+		snapshot = self._profile_loader.restore_snapshot(self._profile)
+		if snapshot is not None:
+			self._profile = snapshot
+
 	def set_profile(self, profile_names, manual):
 		if self.is_running():
 			errstr = "Cannot set profile while the daemon is running."
@@ -156,6 +161,40 @@ class Daemon(object):
 			self._save_active_profile(active_profiles, manual)
 			self._save_post_loaded_profile(post_loaded_profile)
 
+	def sync_instances(self):
+		# NOTE: currently, Controller creates the new instances, and here in Daemon
+		#       we discover what happened, and update the profile accordingly.
+		#       a potentially better approach would be to move some of the logic
+		#       from Controller to Daemon, and create/destroy the instances here,
+		#       and at the same time update the profile.
+
+		# remove all units that don't have an instance
+		instance_names = [i.name for i in self._unit_manager.instances]
+		for unit in list(self._profile.units.keys()):
+			if unit in instance_names:
+				continue
+			log.debug("snapshot sync: removing unit '%s'" % unit)
+			del self._profile.units[unit]
+		# create units for new instances
+		for instance in self._unit_manager.instances:
+			if instance.name in self._profile.units:
+				continue
+			log.debug("snapshot sync: creating unit '%s'" % instance.name)
+			config = {
+				"priority": instance.priority,
+				"type": instance._plugin.name,
+				"enabled": instance.active,
+				"devices": instance.devices_expression,
+				"devices_udev_regex": instance.devices_udev_regex,
+				"script_pre": instance.script_pre,
+				"script_post": instance.script_post,
+			}
+			for k, v in instance.options.items():
+				config[k] = v
+			self._profile.units[instance.name] = self._profile._create_unit(instance.name, config)
+		# create profile snapshot
+		self._profile_loader.create_snapshot(self._profile, self._unit_manager.instances)
+
 	@property
 	def profile(self):
 		return self._profile
@@ -202,6 +241,8 @@ class Daemon(object):
 		self._save_active_profile(" ".join(self._active_profiles),
 					  self._manual)
 		self._save_post_loaded_profile(self._post_loaded_profile)
+		# trigger a profile snapshot
+		self.sync_instances()
 		self._unit_manager.start_tuning()
 		self._profile_applied.set()
 		log.info("static tuning from profile '%s' applied" % self._profile.name)
@@ -370,6 +411,7 @@ class Daemon(object):
 			return False
 		log.info("stopping tuning")
 		if profile_switch:
+			self._profile_loader.remove_snapshot()
 			self._terminate_profile_switch.set()
 		self._terminate.set()
 		self._thread.join()

tuned/plugins/base.py

@@ -164,16 +164,25 @@ class Plugin(object):
 			udev_devices = self._device_matcher_udev.match_list(instance.devices_udev_regex, udev_devices)
 			return set([x.sys_name for x in udev_devices])
 
+	def restore_devices(self, instance, devices):
+		if not self._devices_supported:
+			return
+
+		log.debug("Restoring devices of instance %s: %s" % (instance.name, " ".join(devices)))
+		for device in devices:
+			if device not in self._free_devices:
+				continue
+			self._free_devices.remove(device)
+			instance.assigned_devices.add(device)
+			self._assigned_devices.add(device)
+
 	def assign_free_devices(self, instance):
 		if not self._devices_supported:
 			return
 
 		log.debug("assigning devices to instance %s" % instance.name)
 		to_assign = self._get_matching_devices(instance, self._free_devices)
-		instance.active = len(to_assign) > 0
-		if not instance.active:
-			log.warning("instance %s: no matching devices available" % instance.name)
-		else:
+		if len(to_assign) > 0:
 			name = instance.name
 			if instance.name != self.name:
 				name += " (%s)" % self.name
@@ -181,6 +190,9 @@ class Plugin(object):
 			instance.assigned_devices.update(to_assign) # cannot use |=
 			self._assigned_devices |= to_assign
 			self._free_devices -= to_assign
+		instance.active = len(instance.assigned_devices) > 0
+		if not instance.active:
+			log.warning("instance %s: no matching devices available" % instance.name)
 
 	def release_devices(self, instance):
 		if not self._devices_supported:

tuned/profiles/loader.py

@@ -24,9 +24,6 @@ class Loader(object):
 		self._global_config = global_config
 		self._variables = variables
 
-	def _create_profile(self, profile_name, config):
-		return tuned.profiles.profile.Profile(profile_name, config)
-
 	@classmethod
 	def safe_name(cls, profile_name):
 		return re.match(r'^[a-zA-Z0-9_.-]+$', profile_name)
@@ -57,6 +54,7 @@ class Loader(object):
 		# FIXME hack, do all variable expansions in one place
 		self._expand_vars_in_devices(final_profile)
 		self._expand_vars_in_regexes(final_profile)
+		final_profile.calculate_hash()
 		return final_profile
 
 	def _expand_vars_in_devices(self, profile):
@@ -68,6 +66,47 @@ class Loader(object):
 			profile.units[unit].cpuinfo_regex = self._variables.expand(profile.units[unit].cpuinfo_regex)
 			profile.units[unit].uname_regex = self._variables.expand(profile.units[unit].uname_regex)
 
+	def create_snapshot(self, profile, instances):
+		snapshot = profile.snapshot(instances)
+		log.debug("Storing profile snapshot in %s:\n%s" % (consts.PROFILE_SNAPSHOT_FILE, snapshot))
+		with open(consts.PROFILE_SNAPSHOT_FILE, "w") as f:
+			f.write(snapshot)
+
+	def restore_snapshot(self, profile):
+		if profile is None:
+			# When tuning is stopped, we are called with profile==None -> skip
+			return None
+		snapshot = None
+		if os.path.isfile(consts.PROFILE_SNAPSHOT_FILE):
+			log.debug("Found profile snapshot '%s'" % consts.PROFILE_SNAPSHOT_FILE)
+			try:
+				config = self._load_config_data(consts.PROFILE_SNAPSHOT_FILE)
+				snapshot_hash = config.get("main", {}).get("profile_base_hash", None)
+				if snapshot_hash == profile._base_hash:
+					snapshot = self._profile_factory.create("restore", config)
+					snapshot.name = profile.name
+					# the snapshot is created directly (not via the merger),
+					# so extract its [variables] section manually
+					if consts.PLUGIN_VARIABLES_UNIT_NAME in snapshot.units:
+						snapshot.variables.update(snapshot.units[consts.PLUGIN_VARIABLES_UNIT_NAME].options)
+						del snapshot.units[consts.PLUGIN_VARIABLES_UNIT_NAME]
+					self._variables.add_from_cfg(snapshot.variables)
+					self._expand_vars_in_devices(snapshot)
+					self._expand_vars_in_regexes(snapshot)
+					log.info("Restored profile snapshot: %s" % snapshot.name)
+				else:
+					log.debug("Snapshot hash '%s' does not match current base hash '%s'. Not restoring." % (snapshot_hash, profile._base_hash))
+					os.remove(consts.PROFILE_SNAPSHOT_FILE)
+			except InvalidProfileException as e:
+				log.error("Could not process profile snapshot: %s" % e)
+		return snapshot
+
+	def remove_snapshot(self):
+		try:
+			os.remove(consts.PROFILE_SNAPSHOT_FILE)
+		except FileNotFoundError:
+			pass
+
 	def _load_profile(self, profile_names, profiles, processed_files):
 		for name in profile_names:
 			filename = self._profile_locator.get_config(name, processed_files)

tuned/profiles/profile.py

@@ -1,24 +1,27 @@
 import tuned.profiles.unit
 import tuned.consts as consts
 import collections
+import hashlib
+import json
 
 class Profile(object):
 	"""
 	Representation of a tuning profile.
 	"""
 
-	__slots__ = ["_name", "_options", "_variables", "_units"]
+	__slots__ = ["_name", "_options", "_variables", "_units", "_base_hash"]
 
 	def __init__(self, name=None, config={}):
 		self._name = name
 		self._variables = collections.OrderedDict()
 		self._init_options(config)
 		self._init_units(config)
+		self._base_hash = config.get("main", {}).get("profile_base_hash", None)
 
 	def _init_options(self, config):
 		self._options = {}
 		if consts.PLUGIN_MAIN_UNIT_NAME in config:
-			self._options = dict(config[consts.PLUGIN_MAIN_UNIT_NAME])
+			self._options = collections.OrderedDict(config[consts.PLUGIN_MAIN_UNIT_NAME])
 
 	def _init_units(self, config):
 		self._units = collections.OrderedDict()
@@ -30,6 +33,35 @@ class Profile(object):
 	def _create_unit(self, name, config):
 		return tuned.profiles.unit.Unit(name, config)
 
+	def as_ordered_dict(self):
+		"""generate serializable (with json.dumps()) representation for hashing"""
+		profile_dict = collections.OrderedDict()
+		profile_dict["main"] = self.options
+		profile_dict["variables"] = self._variables
+		for name, unit in self._units.items():
+			profile_dict[name] = unit.as_ordered_dict()
+		return profile_dict
+
+	def calculate_hash(self):
+		serialized = json.dumps(self.as_ordered_dict())
+		self._base_hash = hashlib.md5(serialized.encode(), usedforsecurity=False).hexdigest()
+
+	def snapshot(self, instances):
+		"""generate config representation that will re-create the data when read as a profile"""
+		snapshot = "[main]\n"
+		snapshot += "active_profile=%s\n" % self.name
+		snapshot += "profile_base_hash=%s\n" % self._base_hash
+		snapshot += "\n[variables]\n"
+		for key, value in self._variables.items():
+			snapshot += "%s=%s\n" % (key, value)
+		for unit in self.units.values():
+			snapshot += "\n" + unit.snapshot()
+			for instance in instances:
+				if instance.name == unit.name:
+					snapshot += "__devices__=%s\n" % " ".join(instance.assigned_devices | instance.processed_devices)
+					break
+		return snapshot
+
 	@property
 	def name(self):
 		"""

tuned/profiles/unit.py

@@ -27,6 +27,55 @@ class Unit(object):
 		self._script_post = config.pop("script_post", None)
 		self._options = collections.OrderedDict(config)
 
+	def as_ordered_dict(self):
+		"""generate serializable (with json.dumps()) representation for hashing"""
+		ret = collections.OrderedDict()
+		ret["name"] = self.name
+		ret["priority"] = self.priority
+		ret["type"] = self.type
+		ret["enabled"] = self.enabled
+		ret["replace"] = self.replace
+		ret["drop"] = self.drop
+		ret["devices"] = self.devices
+		ret["devices_udev_regex"] = self.devices_udev_regex
+		ret["cpuinfo_regex"] = self.cpuinfo_regex
+		ret["uname_regex"] = self.uname_regex
+		ret["script_pre"] = self.script_pre
+		ret["script_post"] = self.script_post
+		for k, v in self.options.items():
+			ret[k] = v
+		return ret
+
+	@staticmethod
+	def _snapshot_value(value):
+		"""serialize an option value into a form that round-trips through the profile loader"""
+		# some options (e.g. the script plugin's "script") are stored as lists of
+		# absolute paths; emit them space-joined so they are not rendered as a list repr
+		if isinstance(value, list):
+			return " ".join(str(v) for v in value)
+		return str(value)
+
+	def snapshot(self):
+		"""generate config representation that will re-create the data when read as a profile"""
+		snapshot = "[%s]\n" % self.name
+		snapshot += "priority=%s\n" % self.priority
+		snapshot += "type=%s\n" % self.type
+		snapshot += "enabled=%s\n" % self.enabled
+		snapshot += "devices=%s\n" % self.devices
+		if self.devices_udev_regex is not None:
+			snapshot += "devices_udev_regex=%s\n" % self.devices_udev_regex
+		if self.cpuinfo_regex is not None:
+			snapshot += "cpuinfo_regex=%s\n" % self.cpuinfo_regex
+		if self.uname_regex is not None:
+			snapshot += "uname_regex=%s\n" % self.uname_regex
+		if self.script_pre is not None:
+			snapshot += "script_pre=%s\n" % self._snapshot_value(self.script_pre)
+		if self.script_post is not None:
+			snapshot += "script_post=%s\n" % self._snapshot_value(self.script_post)
+		for k, v in self.options.items():
+			snapshot += "%s=%s\n" % (k, self._snapshot_value(v))
+		return snapshot
+
 	@property
 	def name(self):
 		return self._name

tuned/units/manager.py

@@ -98,17 +98,24 @@ class Manager(object):
 				continue
 
 		instances = []
+		instance_restore_devices = {}
 		for instance_info in instance_info_list:
 			plugin = plugins_by_name[instance_info.type]
 			if plugin is None:
 				continue
 			log.debug("creating '%s' (%s)" % (instance_info.name, instance_info.type))
+			restore = instance_info.options.pop("__devices__", None)
+			if restore:
+				instance_restore_devices[instance_info.name] = restore.split()
 			new_instance = plugin.create_instance(instance_info.name, instance_info.priority, \
 				instance_info.devices, instance_info.devices_udev_regex, \
 				instance_info.script_pre, instance_info.script_post, instance_info.options)
 			instances.append(new_instance)
 		for instance in instances:
 			instance.plugin.init_devices()
+			if instance.name in instance_restore_devices:
+				instance.plugin.restore_devices(instance, instance_restore_devices[instance.name])
+		for instance in instances:
 			instance.plugin.assign_free_devices(instance)
 			instance.plugin.initialize_instance(instance)
 		# At this point we should be able to start the HW events