From 8968280a4fcfcb086f19b829057c20e09c2d1085 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaroslav=20=C5=A0karvada?= <jskarvad@redhat.com>
Date: Fri, 5 Jun 2015 15:53:59 +0200
Subject: [PATCH] tuned: added support for built-in functions
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Built-in functions are defined as plugins in tuned/profiles/functions.
Each function (plugin) is python code named function_NAME.py, where
NAME is the name of the built-in function defined. For example
how to implement own function see provided functions.

Functions are expanded in tuned profile configuration file during profile
load. Functions are expanded after variables. The syntax for function call is
${f:NAME:ARG1:ARG2...}. The function NAME will be called and it's result
substituted. It is possible to escape $ by \$ and : by \:. Functions are
loaded and executed on demand. Function which is once loaded during profile
application is not reloaded on multiple executions. Variables
in profile arguments are supported, but function calls aren't.

The following functions have been implemented so far:

exec:ARG1:...:ARGN - execute external command with arguments and subtitute
                     its output.
kb2s:ARG           - converts kbytes specified as ARG to sectors.
s2kb:ARG           - converts sectors specified as ARG to kbytes.

Example:

[variables]
cmd = date
curr_date = ${f:exec:${cmd}}

This commit also changed escaping of variables to be consistent with
functions. They can be now escaped by \, i.e. \${VAR} will not be expanded.
Escaping by double '$' is no more supported.

This commit also removes handler for all exceptions during profile loading.
This is not to mask real errors and ease development.

Resolves: rhbz#1225135

Signed-off-by: Jaroslav Škarvada <jskarvad@redhat.com>
---
 tuned/consts.py                           |  2 +
 tuned/daemon/daemon.py                    |  3 +-
 tuned/profiles/__init__.py                |  1 +
 tuned/profiles/functions/__init__.py      |  1 +
 tuned/profiles/functions/base.py          | 34 ++++++++++++++++
 tuned/profiles/functions/function_exec.py | 20 ++++++++++
 tuned/profiles/functions/function_kb2s.py | 20 ++++++++++
 tuned/profiles/functions/function_s2kb.py | 20 ++++++++++
 tuned/profiles/functions/functions.py     | 48 +++++++++++++++++++++++
 tuned/profiles/functions/repository.py    | 43 ++++++++++++++++++++
 tuned/profiles/loader.py                  |  1 -
 tuned/profiles/variables.py               | 14 +++++--
 12 files changed, 202 insertions(+), 5 deletions(-)
 create mode 100644 tuned/profiles/functions/__init__.py
 create mode 100644 tuned/profiles/functions/base.py
 create mode 100644 tuned/profiles/functions/function_exec.py
 create mode 100644 tuned/profiles/functions/function_kb2s.py
 create mode 100644 tuned/profiles/functions/function_s2kb.py
 create mode 100644 tuned/profiles/functions/functions.py
 create mode 100644 tuned/profiles/functions/repository.py

diff --git a/tuned/consts.py b/tuned/consts.py
index cce6862..1969a1b 100644
--- a/tuned/consts.py
+++ b/tuned/consts.py
@@ -30,6 +30,8 @@ LOG_FILE_MAXBYTES = 100*1000
 LOG_FILE = "/var/log/tuned/tuned.log"
 PID_FILE = "/run/tuned/tuned.pid"
 SYSTEM_RELEASE_FILE = "/etc/system-release-cpe"
+# prefix for functions plugins
+FUNCTION_PREFIX = "function_"
 # prefix for exported environment variables when calling scripts
 ENV_PREFIX = "TUNED_"
 
diff --git a/tuned/daemon/daemon.py b/tuned/daemon/daemon.py
index 29869a3..ba744a5 100644
--- a/tuned/daemon/daemon.py
+++ b/tuned/daemon/daemon.py
@@ -3,6 +3,7 @@ import errno
 import threading
 import tuned.logs
 from tuned.exceptions import TunedException
+from tuned.profiles.exceptions import InvalidProfileException
 import tuned.consts as consts
 from tuned.utils.commands import commands
 
@@ -68,7 +69,7 @@ class Daemon(object):
 		else:
 			try:
 				self._profile = self._profile_loader.load(profile_name)
-			except:
+			except InvalidProfileException:
 				raise TunedException("Cannot load profile '%s'." % profile_name)
 
 		if save_instantly:
diff --git a/tuned/profiles/__init__.py b/tuned/profiles/__init__.py
index 22cd0df..5c34319 100644
--- a/tuned/profiles/__init__.py
+++ b/tuned/profiles/__init__.py
@@ -5,3 +5,4 @@ from tuned.profiles.unit import *
 from tuned.profiles.exceptions import *
 from tuned.profiles.factory import *
 from tuned.profiles.merger import *
+import functions
\ No newline at end of file
diff --git a/tuned/profiles/functions/__init__.py b/tuned/profiles/functions/__init__.py
new file mode 100644
index 0000000..684a4b6
--- /dev/null
+++ b/tuned/profiles/functions/__init__.py
@@ -0,0 +1 @@
+from repository import Repository
diff --git a/tuned/profiles/functions/base.py b/tuned/profiles/functions/base.py
new file mode 100644
index 0000000..5792165
--- /dev/null
+++ b/tuned/profiles/functions/base.py
@@ -0,0 +1,34 @@
+import os
+import tuned.logs
+from tuned.utils.commands import commands
+
+log = tuned.logs.get()
+
+class Function(object):
+	"""
+	Built-in function
+	"""
+	def __init__(self, name, nargs_max, nargs_min = None):
+		self._name = name
+		self._nargs_max = nargs_max
+		self._nargs_min = nargs_min
+		self._cmd = commands()
+
+	# checks arguments
+	# nargs_max - maximal number of arguments, there mustn't be more arguments,
+	#             if nargs_max is 0, number of arguments is unlimited
+	# nargs_min - minimal number of arguments, if not None there must
+	#             be the same number of arguments or more
+	@classmethod
+	def _check_args(cls, args, nargs_max, nargs_min = None):
+		if args is None or nargs_max is None:
+			return False
+		la = len(args)
+		return (nargs_max == 0 or nargs_max == la) and (nargs_min is None or nargs_min <= la)
+
+	def execute(self, args):
+		if self._check_args(args, self._nargs_max, self._nargs_min):
+			return True
+		else:
+			log.error("invalid number of arguments for builtin function '%s'" % self._name)
+		return False
diff --git a/tuned/profiles/functions/function_exec.py b/tuned/profiles/functions/function_exec.py
new file mode 100644
index 0000000..9d81971
--- /dev/null
+++ b/tuned/profiles/functions/function_exec.py
@@ -0,0 +1,20 @@
+import os
+import tuned.logs
+import base
+from tuned.utils.commands import commands
+
+class s2kb(base.Function):
+	"""
+	Conversion function: sectors to kbytes
+	"""
+	def __init__(self):
+		# unlimited number of arguments, min 1 argument (the name of executable)
+		super(self.__class__, self).__init__("s2kb", 0, 1)
+
+	def execute(self, args):
+		if not super(self.__class__, self).execute(args):
+			return None
+		(ret, out) = self._cmd.execute(args)
+		if ret == 0:
+			return out
+		return None
diff --git a/tuned/profiles/functions/function_kb2s.py b/tuned/profiles/functions/function_kb2s.py
new file mode 100644
index 0000000..7bdb381
--- /dev/null
+++ b/tuned/profiles/functions/function_kb2s.py
@@ -0,0 +1,20 @@
+import os
+import tuned.logs
+import base
+from tuned.utils.commands import commands
+
+class kb2s(base.Function):
+	"""
+	Conversion function: kbytes to sectors
+	"""
+	def __init__(self):
+		# 1 argument
+		super(self.__class__, self).__init__("kb2s", 1)
+
+	def execute(self, args):
+		if not super(self.__class__, self).execute(args):
+			return None
+		try:
+			return str(int(args[0]) * 2)
+		except ValueError:
+			return None
diff --git a/tuned/profiles/functions/function_s2kb.py b/tuned/profiles/functions/function_s2kb.py
new file mode 100644
index 0000000..f7e03da
--- /dev/null
+++ b/tuned/profiles/functions/function_s2kb.py
@@ -0,0 +1,20 @@
+import os
+import tuned.logs
+import base
+from tuned.utils.commands import commands
+
+class s2kb(base.Function):
+	"""
+	Conversion function: sectors to kbytes
+	"""
+	def __init__(self):
+		# 1 argument
+		super(self.__class__, self).__init__("s2kb", 1)
+
+	def execute(self, args):
+		if not super(self.__class__, self).execute(args):
+			return None
+		try:
+			return str(int(args[0]) / 2)
+		except ValueError:
+			return None
diff --git a/tuned/profiles/functions/functions.py b/tuned/profiles/functions/functions.py
new file mode 100644
index 0000000..335a0b9
--- /dev/null
+++ b/tuned/profiles/functions/functions.py
@@ -0,0 +1,48 @@
+import os
+import re
+import glob
+import repository
+import tuned.logs
+import tuned.consts as consts
+from tuned.utils.commands import commands
+
+log = tuned.logs.get()
+
+cmd = commands()
+
+class Functions():
+	"""
+	Built-in functions
+	"""
+
+	def __init__(self):
+		self._repository = repository.Repository()
+
+	def sub_func(self, mo):
+		sorig = mo.string[mo.start():mo.end()]
+		if mo.lastindex != 1:
+			return sorig
+		s = mo.string[mo.start(1):mo.end(1)]
+		if len(s) == 0:
+			return sorig
+		sl = re.split(r'(?<!\\):', s)
+		sl = map(lambda v: str(v).replace("\:", ":"), sl)
+		if not re.match(r'\w+$', sl[0]):
+			log.error("invalid function name '%s'" % sl[0])
+			return sorig
+		try:
+			f = self._repository.load_func(sl[0])
+		except ImportError:
+			log.error("function '%s' not implemented" % sl[0])
+			return sorig
+		s = f.execute(sl[1:])
+		if s is None:
+			return sorig
+		return s
+
+	def expand(self, s):
+		if s is None:
+			return s
+		r = re.compile(r'(?<!\\)\${f:([\w:\\]+)}')
+		# expand functions and convert all \${f:*} to ${f:*} (unescape)
+		return re.sub(r'\\(\${f:[\w:\\]+})', r'\1', r.sub(self.sub_func, s))
diff --git a/tuned/profiles/functions/repository.py b/tuned/profiles/functions/repository.py
new file mode 100644
index 0000000..0464a86
--- /dev/null
+++ b/tuned/profiles/functions/repository.py
@@ -0,0 +1,43 @@
+from tuned.utils.plugin_loader import PluginLoader
+import base
+import tuned.logs
+import tuned.consts as consts
+from tuned.utils.commands import commands
+
+log = tuned.logs.get()
+
+class Repository(PluginLoader):
+
+	def __init__(self):
+		super(self.__class__, self).__init__()
+		self._functions = {}
+
+	@property
+	def functions(self):
+		return self._functions
+
+	def _set_loader_parameters(self):
+		self._namespace = "tuned.profiles.functions"
+		self._prefix = consts.FUNCTION_PREFIX
+		self._interface = tuned.profiles.functions.base.Function
+
+	def create(self, function_name):
+		log.debug("creating function %s" % function_name)
+		function_cls = self.load_plugin(function_name)
+		function_instance = function_cls()
+		self._functions[function_name] = function_instance
+		return function_instance
+
+	# loads function from plugin file and return it
+	# if it is already loaded, just return it, it is not loaded again
+	def load_func(self, function_name):
+		if not function_name in self._functions:
+			return self.create(function_name)
+		return self._functions[function_name]
+
+	def delete(self, function):
+		assert isinstance(function, self._interface)
+		log.debug("removing function %s" % function)
+		for k, v in self._functions.items():
+			if v == function:
+				del self._functions[k]
diff --git a/tuned/profiles/loader.py b/tuned/profiles/loader.py
index 6991969..484c1fd 100644
--- a/tuned/profiles/loader.py
+++ b/tuned/profiles/loader.py
@@ -5,7 +5,6 @@ import os.path
 import collections
 import tuned.logs
 import re
-
 from tuned.profiles.exceptions import InvalidProfileException
 
 log = tuned.logs.get()
diff --git a/tuned/profiles/variables.py b/tuned/profiles/variables.py
index a1b6272..396e590 100644
--- a/tuned/profiles/variables.py
+++ b/tuned/profiles/variables.py
@@ -1,6 +1,7 @@
 import os
 import re
 import tuned.logs
+import functions.functions as functions
 import tuned.consts as consts
 from tuned.utils.commands import commands
 from configobj import ConfigObj
@@ -16,6 +17,7 @@ class Variables():
 		self._cmd = commands()
 		self._lookup_re = {}
 		self._lookup_env = {}
+		self._functions = functions.Functions()
 
 	def _add_env_prefix(self, s, prefix):
 		if s.find(prefix) == 0:
@@ -35,7 +37,7 @@ class Variables():
 		v = self.expand(value)
 		# variables referenced by ${VAR}, $ can be escaped by two $,
 		# i.e. the following will not expand: $${VAR}
-		self._lookup_re[r'(?<!\$)\${' + re.escape(s) + r'}'] = v
+		self._lookup_re[r'(?<!\\)\${' + re.escape(s) + r'}'] = v
 		self._lookup_env[self._add_env_prefix(s, consts.ENV_PREFIX)] = v
 
 	def add_dict(self, d):
@@ -64,11 +66,17 @@ class Variables():
 			else:
 				self.add_variable(item, cfg[item])
 
+	# expand static variables (no functions)
+	def expand_static(self, value):
+		return re.sub(r'\\(\${\w+})', r'\1', self._cmd.multiple_re_replace(self._lookup_re, value))
+
 	def expand(self, value):
 		if value is None:
 			return None
-		# expand variables and finally convert all $${VAR} to ${VAR} (unescape)
-		return re.sub(r'\$(\${\w+})', r'\1', self._cmd.multiple_re_replace(self._lookup_re, str(value)))
+		# expand variables and convert all \${VAR} to ${VAR} (unescape)
+		s = self.expand_static(str(value))
+		# expand built-in functions
+		return self._functions.expand(s)
 
 	def get_env(self):
 		return self._lookup_env