diff --git a/Makefile b/Makefile index 9d46479..056e53e 100644 --- a/Makefile +++ b/Makefile @@ -120,6 +120,10 @@ install: install-dirs cp -a profiles/* $(DESTDIR)$(TUNED_PROFILESDIR)/ mv $(DESTDIR)$(TUNED_PROFILESDIR)/realtime/realtime-variables.conf \ $(DESTDIR)/etc/tuned/realtime-variables.conf + mv $(DESTDIR)$(TUNED_PROFILESDIR)/realtime-virtual-guest/realtime-virtual-guest-variables.conf \ + $(DESTDIR)/etc/tuned/realtime-virtual-guest-variables.conf + mv $(DESTDIR)$(TUNED_PROFILESDIR)/realtime-virtual-host/realtime-virtual-host-variables.conf \ + $(DESTDIR)/etc/tuned/realtime-virtual-host-variables.conf install -pm 0644 recommend.conf $(DESTDIR)$(TUNED_PROFILESDIR)/recommend.conf # bash completion diff --git a/libexec/defirqaffinity.py b/libexec/defirqaffinity.py new file mode 100755 index 0000000..49f58bc --- /dev/null +++ b/libexec/defirqaffinity.py @@ -0,0 +1,127 @@ +# Helper script for realtime profiles provided by RT + +import os +import sys + +irqpath = "/proc/irq/" + +def bitmasklist(line): + fields = line.strip().split(",") + bitmasklist = [] + entry = 0 + for i in range(len(fields) - 1, -1, -1): + mask = int(fields[i], 16) + while mask != 0: + if mask & 1: + bitmasklist.append(entry) + mask >>= 1 + entry += 1 + return bitmasklist + +def get_cpumask(mask): + groups = [] + comma = 0 + while mask: + cpumaskstr = '' + m = mask & 0xffffffff + cpumaskstr += ('%x' % m) + if comma: + cpumaskstr += ',' + comma = 1 + mask >>= 32 + groups.append(cpumaskstr) + string = '' + for i in reversed(groups): + string += i + return string + +def parse_def_affinity(fname): + if os.getuid() != 0: + return + try: + f = file(fname) + line = f.readline() + f.close() + return bitmasklist(line) + except IOError: + return [ 0 ] + +def verify(shouldbemask): + inplacemask = 0 + fname = irqpath + "default_smp_affinity"; + cpulist = parse_def_affinity(fname) + for i in cpulist: + inplacemask = inplacemask | 1 << i; + if (inplacemask & ~shouldbemask): + sys.stderr.write("verify: failed: irqaffinity (%s) inplacemask=%x shouldbemask=%x\n" % (fname, inplacemask, shouldbemask)) + sys.exit(1) + + # now verify each /proc/irq/$num/smp_affinity + interruptdirs = [ f for f in listdir(irqpath) if isdir(join(irqpath,f)) ] + # IRQ 2 - cascaded signals from IRQs 8-15 (any devices configured to use IRQ 2 will actually be using IRQ 9) + interruptdirs.remove("2") + # IRQ 0 - system timer (cannot be changed) + interruptdirs.remove("0") + + for i in interruptdirs: + inplacemask = 0 + fname = irqpath + i + "/smp_affinity" + cpulist = parse_def_affinity(fname) + for i in cpulist: + inplacemask = inplacemask | 1 << i; + if (inplacemask & ~shouldbemask): + sys.stderr.write("verify: failed: irqaffinity (%s) inplacemask=%x shouldbemask=%x\n" % (fname, inplacemask, shouldbemask)) + sys.exit(1) + + sys.exit(0) + + + +# adjust default_smp_affinity +cpulist = parse_def_affinity(irqpath + "default_smp_affinity") +mask = 0 +for i in cpulist: + mask = mask | 1 << i; + +line = sys.argv[2] +fields = line.strip().split(",") + +for i in fields: + if sys.argv[1] == "add": + mask = mask | 1 << int(i); + elif sys.argv[1] == "remove" or sys.argv[1] == "verify": + mask = mask & ~(1 << int(i)); + +if sys.argv[1] == "verify": + verify(mask) + +string = get_cpumask(mask) + +fo = open(irqpath + "default_smp_affinity", "wb") +fo.write(string) +fo.close() + +# now adjust each /proc/irq/$num/smp_affinity + +interruptdirs = [ f for f in listdir(irqpath) if isdir(join(irqpath,f)) ] + +# IRQ 2 - cascaded signals from IRQs 8-15 (any devices configured to use IRQ 2 will actually be using IRQ 9) +interruptdirs.remove("2") +# IRQ 0 - system timer (cannot be changed) +interruptdirs.remove("0") + +for i in interruptdirs: + fname = irqpath + i + "/smp_affinity" + cpulist = parse_def_affinity(fname) + mask = 0 + for i in cpulist: + mask = mask | 1 << i; + for i in fields: + if sys.argv[1] == "add": + mask = mask | 1 << int(i); + elif sys.argv[1] == "remove": + mask = mask & ~(1 << int(i)); + string = get_cpumask(mask) + fo = open(fname, "wb") + fo.write(string) + fo.close() diff --git a/man/tuned-adm.8 b/man/tuned-adm.8 index efe5803..7757f35 100644 --- a/man/tuned-adm.8 +++ b/man/tuned-adm.8 @@ -69,7 +69,7 @@ performance or balanced profile if unsure. .TP .B off -Unload tunings.. +Unload tunings. .SH "FILES" .NF @@ -83,6 +83,9 @@ Unload tunings.. .BR tuned\-profiles\-atomic (7) .BR tuned\-profiles\-sap (7) .BR tuned\-profiles\-sap\-hana (7) +.BR tuned\-profiles\-oracle (7) +.BR tuned\-profiles\-realtime (7) +.BR tuned\-profiles\-nfv (7) .BR tuned\-profiles\-compat (7) .SH AUTHOR .NF diff --git a/man/tuned-profiles-atomic.7 b/man/tuned-profiles-atomic.7 index a9037fe..0de9435 100644 --- a/man/tuned-profiles-atomic.7 +++ b/man/tuned-profiles-atomic.7 @@ -54,6 +54,7 @@ netfilter connections tracking. .BR tuned\-profiles\-sap\-hana (7) .BR tuned\-profiles\-oracle (7) .BR tuned\-profiles\-realtime (7) +.BR tuned\-profiles\-nfv (7) .BR tuned\-profiles\-compat (7) .SH AUTHOR .NF diff --git a/man/tuned-profiles-compat.7 b/man/tuned-profiles-compat.7 index c45d3e2..599ad3a 100644 --- a/man/tuned-profiles-compat.7 +++ b/man/tuned-profiles-compat.7 @@ -88,6 +88,7 @@ set to performance. .BR tuned\-profiles\-sap\-hana (7) .BR tuned\-profiles\-oracle (7) .BR tuned\-profiles\-realtime (7) +.BR tuned\-profiles\-nfv (7) .SH AUTHOR .NF Jaroslav Škarvada diff --git a/man/tuned-profiles-nfv.7 b/man/tuned-profiles-nfv.7 new file mode 100644 index 0000000..1fefe6d --- /dev/null +++ b/man/tuned-profiles-nfv.7 @@ -0,0 +1,56 @@ +.\"/* +.\" * All rights reserved +.\" * Copyright (C) 2015 Red Hat, Inc. +.\" * Authors: Jaroslav Škarvada +.\" * +.\" * This program is free software; you can redistribute it and/or +.\" * modify it under the terms of the GNU General Public License +.\" * as published by the Free Software Foundation; either version 2 +.\" * of the License, or (at your option) any later version. +.\" * +.\" * This program is distributed in the hope that it will be useful, +.\" * but WITHOUT ANY WARRANTY; without even the implied warranty of +.\" * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +.\" * GNU General Public License for more details. +.\" * +.\" * You should have received a copy of the GNU General Public License +.\" * along with this program; if not, write to the Free Software +.\" * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +.\" */ +.\" +.TH TUNED_PROFILES_NFV "7" "1 Jul 2015" "Fedora Power Management SIG" "tuned" +.SH NAME +tuned\-profiles\-nfv - description of profiles provided for the NFV + +.SH DESCRIPTION +These profiles are provided for the Network Function Virtualization (NFV). + +.SH PROFILES +The following profiles are provided: + +.TP +.BI "realtime-virtual-guest" +Profile optimized for virtual guests based on realtime profile. + +.TP +.BI "realtime-virtual-host" +Profile optimized for virtual hosts based on realtime profile. + +.SH "FILES" +.NF +.I /etc/tuned/* +.I /usr/lib/tuned/* + +.SH "SEE ALSO" +.BR tuned (8) +.BR tuned\-adm (8) +.BR tuned\-profiles (7) +.BR tuned\-profiles\-atomic (7) +.BR tuned\-profiles\-sap (7) +.BR tuned\-profiles\-sap\-hana (7) +.BR tuned\-profiles\-oracle (7) +.BR tuned\-profiles\-realtime (7) +.BR tuned\-profiles\-compat (7) +.SH AUTHOR +.NF +Jaroslav Škarvada diff --git a/man/tuned-profiles-oracle.7 b/man/tuned-profiles-oracle.7 index 9256d62..d40ee3b 100644 --- a/man/tuned-profiles-oracle.7 +++ b/man/tuned-profiles-oracle.7 @@ -47,6 +47,7 @@ performance related kernel parameters. .BR tuned\-profiles\-sap\-hana (7) .BR tuned\-profiles\-atomic (7) .BR tuned\-profiles\-realtime (7) +.BR tuned\-profiles\-nfv (7) .BR tuned\-profiles\-compat (7) .SH AUTHOR .NF diff --git a/man/tuned-profiles-realtime.7 b/man/tuned-profiles-realtime.7 index 7363472..0d4c469 100644 --- a/man/tuned-profiles-realtime.7 +++ b/man/tuned-profiles-realtime.7 @@ -41,10 +41,11 @@ Profile optimized for realtime. .BR tuned (8) .BR tuned\-adm (8) .BR tuned\-profiles (7) +.BR tuned\-profiles\-atomic (7) .BR tuned\-profiles\-sap (7) .BR tuned\-profiles\-sap\-hana (7) .BR tuned\-profiles\-oracle (7) -.BR tuned\-profiles\-atomic (7) +.BR tuned\-profiles\-nfv (7) .BR tuned\-profiles\-compat (7) .SH AUTHOR .NF diff --git a/man/tuned-profiles-sap-hana.7 b/man/tuned-profiles-sap-hana.7 index 9c6160c..b6cfb93 100644 --- a/man/tuned-profiles-sap-hana.7 +++ b/man/tuned-profiles-sap-hana.7 @@ -56,6 +56,7 @@ recieve offload for the network adapter. .BR tuned\-profiles\-sap (7) .BR tuned\-profiles\-oracle (7) .BR tuned\-profiles\-realtime (7) +.BR tuned\-profiles\-nfv (7) .BR tuned\-profiles\-compat (7) .SH AUTHOR .NF diff --git a/man/tuned-profiles-sap.7 b/man/tuned-profiles-sap.7 index 190abbf..8da8a36 100644 --- a/man/tuned-profiles-sap.7 +++ b/man/tuned-profiles-sap.7 @@ -48,6 +48,7 @@ areas a process may have. .BR tuned\-profiles\-sap\-hana (7) .BR tuned\-profiles\-oracle (7) .BR tuned\-profiles\-realtime (7) +.BR tuned\-profiles\-nfv (7) .BR tuned\-profiles\-compat (7) .SH AUTHOR .NF diff --git a/man/tuned-profiles.7 b/man/tuned-profiles.7 index be7fb7c..f211763 100644 --- a/man/tuned-profiles.7 +++ b/man/tuned-profiles.7 @@ -124,6 +124,7 @@ It additionally enables more aggresive writeback of dirty pages. .BR tuned\-profiles\-sap-hana (7) .BR tuned\-profiles\-oracle (7) .BR tuned\-profiles\-realtime (7) +.BR tuned\-profiles\-nfv (7) .BR tuned\-profiles\-compat (7) .SH AUTHOR .NF diff --git a/profiles/realtime-virtual-guest/realtime-virtual-guest-variables.conf b/profiles/realtime-virtual-guest/realtime-virtual-guest-variables.conf new file mode 100644 index 0000000..b91e5f3 --- /dev/null +++ b/profiles/realtime-virtual-guest/realtime-virtual-guest-variables.conf @@ -0,0 +1,4 @@ +# Examples: +# isolated_cores=2,4-7 +# isolated_cores=2-23 +# diff --git a/profiles/realtime-virtual-guest/script.sh b/profiles/realtime-virtual-guest/script.sh new file mode 100755 index 0000000..46855d2 --- /dev/null +++ b/profiles/realtime-virtual-guest/script.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +. /usr/lib/tuned/functions + +start() { + python /usr/libexec/tuned/defirqaffinity.py "remove" "$TUNED_isolated_cores_expanded" && + tuna -c "$TUNED_isolated_cores_expanded" -i + return "$?" +} + +stop() { + tuna -c "$TUNED_isolated_cores_expanded" -I && + python /usr/libexec/tuned/defirqaffinity.py "add" "$TUNED_isolated_cores_expanded" + return "$?" +} + +verify() { + python /usr/libexec/tuned/defirqaffinity.py "verify" "$TUNED_isolated_cores_expanded" + return "$?" +} + +process $@ diff --git a/profiles/realtime-virtual-guest/tuned.conf b/profiles/realtime-virtual-guest/tuned.conf new file mode 100644 index 0000000..018713e --- /dev/null +++ b/profiles/realtime-virtual-guest/tuned.conf @@ -0,0 +1,14 @@ +# +# tuned configuration +# + +[main] +include=realtime + +[variables] +# User is responsible for adding isolated_cores=X-Y to realtime-virtual-guest-variables.conf +include=/etc/tuned/realtime-virtual-guest-variables.conf +isolated_cores_expanded=${f:cpulist_unpack:${isolated_cores}} + +[script] +script=script.sh diff --git a/profiles/realtime-virtual-host/find-lapictscdeadline-optimal.sh b/profiles/realtime-virtual-host/find-lapictscdeadline-optimal.sh new file mode 100755 index 0000000..d858eb6 --- /dev/null +++ b/profiles/realtime-virtual-host/find-lapictscdeadline-optimal.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +: ${1?"Usage: $0 latency-file"} + +lines=`wc -l $1 | cut -f 1 -d " "` +in_range=0 +prev_value=1 +for i in `seq 1 $lines`; do + a=`awk "NR==$i" $1 | cut -f 2 -d ":"` + value=$(($a*100/$prev_value)) + if [ $value -ge 98 -a $value -le 102 ]; then + in_range=$(($in_range + 1)) + else + in_range=0 + fi + if [ $in_range -ge 2 ]; then + echo -n "optimal value for lapic_timer_advance_ns is: " + awk "NR==$(($i - 1))" $1 | cut -f 1 -d ":" + exit 0 + fi + prev_value=$a +done +echo optimal not found +exit 1 diff --git a/profiles/realtime-virtual-host/realtime-virtual-host-variables.conf b/profiles/realtime-virtual-host/realtime-virtual-host-variables.conf new file mode 100644 index 0000000..b91e5f3 --- /dev/null +++ b/profiles/realtime-virtual-host/realtime-virtual-host-variables.conf @@ -0,0 +1,4 @@ +# Examples: +# isolated_cores=2,4-7 +# isolated_cores=2-23 +# diff --git a/profiles/realtime-virtual-host/run-tscdeadline-latency.sh b/profiles/realtime-virtual-host/run-tscdeadline-latency.sh new file mode 100755 index 0000000..7b54174 --- /dev/null +++ b/profiles/realtime-virtual-host/run-tscdeadline-latency.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +QEMU=/usr/libexec/qemu-kvm + +if [ ! -f /sys/module/kvm/parameters/lapic_timer_advance_ns ]; then + echo "/sys/module/kvm/parameters/lapic_timer_advance_ns not found" + exit 1 +fi + +dir=`mktemp -d` + +for i in `seq 1000 500 7000`; do + echo $i > /sys/module/kvm/parameters/lapic_timer_advance_ns + chrt -f 1 taskset -c $1 $QEMU -enable-kvm -device pc-testdev \ + -device isa-debug-exit,iobase=0xf4,iosize=0x4 \ + -display none -serial stdio -device pci-testdev \ + -kernel /usr/share/qemu-kvm/tscdeadline_latency.flat \ + -cpu host | grep latency | cut -f 2 -d ":" > $dir/out + + A=0 + while read l; do + A=$(($A+$l)) + done < $dir/out + + lines=`wc -l $dir/out | cut -f 1 -d " "`; ans=$(($A/$lines)); + echo $i: $ans +done diff --git a/profiles/realtime-virtual-host/script.sh b/profiles/realtime-virtual-host/script.sh new file mode 100755 index 0000000..dadadc8 --- /dev/null +++ b/profiles/realtime-virtual-host/script.sh @@ -0,0 +1,76 @@ +#!/bin/sh + +. /usr/lib/tuned/functions + +ltanfile=/sys/module/kvm/parameters/lapic_timer_advance_ns + +start() { + python /usr/libexec/tuned/defirqaffinity.py "remove" "$TUNED_isolated_cores_expanded" && + tuna -c "$TUNED_isolated_cores_expanded" -i + retval = "$?" + + if [ ! $retval -eq 0 ]; then + return $retval + fi + + modinfo -p kvm | grep -q kvmclock_periodic_sync + if [ "$?" -eq 0 ]; then + echo "options kvm kvmclock_periodic_sync=0" > /etc/modprobe.d/kvm.rt.tuned.conf + fi + + modinfo -p kvm_intel | grep -q ple_gap + if [ "$?" -eq 0 ]; then + echo "options kvm_intel ple_gap=0" >> /etc/modprobe.d/kvm.rt.tuned.conf + fi + + if [ -f lapic_timer_adv_ns.cpumodel ]; then + curmodel=`cat /proc/cpuinfo | grep "model name" | cut -f 2 -d ":" | uniq` + genmodel=`cat lapic_timer_adv_ns.cpumodel` + + if [ "$cpumodel" != "$genmodel" ]; then + rm -f lapic_timer_adv_ns + rm -f lapic_timer_adv_ns.cpumodel + fi + fi + + + if [ -f $ltanfile -a ! -f ./lapic_timer_adv_ns ]; then + if [ -f ./tscdeadline_latency.flat ]; then + tempdir=`mktemp -d` + isolatedcpu=`echo "$TUNED_isolated_cores_expanded" | cut -f 1 -d ","` + sh ./run-tscdeadline-latency.sh $isolatedcpu > $tempdir/lat.out + sh ./find-lapictscdeadline-optimal.sh $tempdir/lat.out > $tempdir/opt.out + if [ $? -eq 0 ]; then + echo `cat $tempdir/opt.out | cut -f 2 -d ":"` > ./lapic_timer_adv_ns + curmodel=`cat /proc/cpuinfo | grep "model name" | cut -f 2 -d ":" | uniq` + echo $curmodel > lapic_timer_adv_ns.cpumodel + fi + fi + fi + if [ -f $ltanfile -a -f ./lapic_timer_adv_ns ]; then + echo `cat ./lapic_timer_adv_ns` > $ltanfile + fi + + return $retval +} + +stop() { + rm -f /etc/modprobe.d/kvm.rt.tuned.conf + tuna -c "$TUNED_isolated_cores_expanded" -I && + python /usr/libexec/tuned/defirqaffinity.py "add" "$TUNED_isolated_cores_expanded" + return "$?" +} + +verify() { + python /usr/libexec/tuned/defirqaffinity.py "verify" "$TUNED_isolated_cores_expanded" + retval = "$?" + if [ $retval -eq 0 -a -f /sys/module/kvm/parameters/kvmclock_periodic_sync ]; then + retval = `cat /sys/module/kvm/parameters/kvmclock_periodic_sync` + fi + if [ $retval -eq 0 -a -f /sys/module/kvm_intel/parameters/ple_gap ]; then + retval = `cat /sys/module/kvm_intel/parameters/ple_gap` + fi + return $retval +} + +process $@ diff --git a/profiles/realtime-virtual-host/tuned.conf b/profiles/realtime-virtual-host/tuned.conf new file mode 100644 index 0000000..21fbfe2 --- /dev/null +++ b/profiles/realtime-virtual-host/tuned.conf @@ -0,0 +1,42 @@ +# +# tuned configuration +# +# Dependencies: +# +# - tuna +# - awk +# - wc +# +# Packaging: +# +# tscdeadline_latency.flat must be included in the profile directory +# so as to calculate optimal tsc deadline latency advancement value. +# +# How to build: +# +# git clone git://git.kernel.org/pub/scm/virt/kvm/kvm-unit-tests.git +# cd kvm-unit-tests/ +# ./configure && make + + +[main] +include=realtime + +[variables] +# User is responsible for adding isolated_cores=X-Y to realtime-virtual-host-variables.conf +include=/etc/tuned/realtime-virtual-host-variables.conf +isolated_cores_expanded=${f:cpulist_unpack:${isolated_cores}} + +[scheduler] +# group.group_name=rule_priority:scheduler_policy:scheduler_priority:core_affinity_in_hex:process_name_regex +# for i in `pgrep ksoftirqd` ; do grep Cpus_allowed_list /proc/$i/status ; done +group.ksoftirqd=0:f:2:*:ksoftirqd.* + +# for i in `pgrep rcuc` ; do grep Cpus_allowed_list /proc/$i/status ; done +group.rcuc=0:f:3:*:rcuc.* + +# for i in `pgrep rcub` ; do grep Cpus_allowed_list /proc/$i/status ; done +group.rcub=0:f:3:*:rcub.* + +[script] +script=script.sh diff --git a/tuned.spec b/tuned.spec index b16a64e..05fa6c3 100644 --- a/tuned.spec +++ b/tuned.spec @@ -101,6 +101,13 @@ Requires: %{name} = %{version}-%{release} %description profiles-realtime Additional tuned profile(s) targeted to realtime. +%package profiles-nfv +Summary: Additional tuned profile(s) targeted to Network Function Virtualization (NFV) +Requires: %{name} = %{version}-%{release}, %{name}-profiles-realtime = %{version}-%{release} + +%description profiles-nfv +Additional tuned profile(s) targeted to Network Function Virtualization (NFV). + %package profiles-compat Summary: Additional tuned profiles mainly for backward compatibility with tuned 1.0 Requires: %{name} = %{version}-%{release} @@ -197,6 +204,7 @@ fi %exclude %{_prefix}/lib/tuned/realtime %{_prefix}/lib/tuned %dir %{_sysconfdir}/tuned +%dir %{_libexecdir}/tuned %config(noreplace) %verify(not size mtime md5) %{_sysconfdir}/tuned/active_profile %config(noreplace) %{_sysconfdir}/tuned/tuned-main.conf %config(noreplace) %verify(not size mtime md5) %{_sysconfdir}/tuned/bootcmdline @@ -265,6 +273,15 @@ fi %{_prefix}/lib/tuned/realtime %{_mandir}/man7/tuned-profiles-realtime.7* +%files profiles-nfv +%defattr(-,root,root,-) +%config(noreplace) %{_sysconfdir}/tuned/realtime-virtual-guest-variables.conf +%config(noreplace) %{_sysconfdir}/tuned/realtime-virtual-host-variables.conf +%{_prefix}/lib/tuned/realtime-virtual-guest +%{_prefix}/lib/tuned/realtime-virtual-host +%{_libexecdir}/tuned/defirqaffinity* +%{_mandir}/man7/tuned-profiles-nfv.7* + %files profiles-compat %defattr(-,root,root,-) %{_prefix}/lib/tuned/default