Virtualization Guide #

# note cpuset mount point
cpuset_mnt=$(mount|grep cpuset|cut -d' ' -f3)

# create an isolated cpuset for your container
cset shield --userset=rtcpus --cpu=4-7 --kthread=on

# note path and id of your container
docker_path=$(docker run...)
docker_id=$(docker ps -q)

# move container content into the isolated cpuset
for i in $(cat ${cpuset_mnt}/system/docker/${docker_path}/tasks);
do
  echo $i > ${cpuset_mnt}/rtcpus/tasks;
done

# stop/destroy the container
docker stop ${docker_id}
docker rm ${docker_id}

# remove dir docker abandons in the shield system directory
rmdir ${cpuset_mnt}/system/docker

# tear down the shield, and you're done
cset shield --userset=rtcpus --cpu=4-7 --reset

#!/bin/sh

let START_CPU=4
let END_CPU=63
let ONLINE=1
let SHIELD_UP=0
GOVERNOR="performance"

DEFAULT_MASK=ffffffff,ffffffff
SHIELD_MASK=00000000,0000000f

if [ -f /proc/sys/kernel/sched_rt_runtime_us ]; then
  RT_RUNTIME=$(cat /proc/sys/kernel/sched_rt_runtime_us)
fi
if [ -f /proc/sys/kernel/nmi_watchdog ]; then
  NMI_WATCHDOG=$(cat /proc/sys/kernel/nmi_watchdog)
fi

CPUSET_ROOT=$(grep cpuset /proc/mounts|cut -d ' ' -f2)
if [ ! -z $CPUSET_ROOT ]; then
  if [ -d ${CPUSET_ROOT}/rtcpus ]; then
    let SHIELD_UP=1
  fi
  if [ -f ${CPUSET_ROOT}/cpuset.cpus ]; then
    CPUSET_PREFIX=cpuset.
  fi
fi

if [ $SHIELD_UP -eq 1 ]; then
  # take it down
  echo 1 > ${CPUSET_ROOT}/${CPUSET_PREFIX}sched_load_balance
  cset shield --userset=rtcpus --reset

  # restore default irq affinity
  echo ${DEFAULT_MASK} > /proc/irq/default_smp_affinity
  for irqlist in $(ls /proc/irq/*/smp_affinity); do
    echo ${DEFAULT_MASK} > $irqlist 2>/dev/null
  done

  if [ -f /proc/sys/kernel/timer_migration ]; then
    echo 1 > /proc/sys/kernel/timer_migration
  fi
  if [ -f /proc/sys/kernel/sched_rt_runtime_us ]; then
    echo ${RT_RUNTIME} > /proc/sys/kernel/sched_rt_runtime_us
  fi
  if [ -f /sys/kernel/debug/tracing/tracing_on ]; then
    echo 1 > /sys/kernel/debug/tracing/tracing_on
  fi
  if [ -f /sys/kernel/mm/transparent_hugepage/enabled ]; then
    echo always > /sys/kernel/mm/transparent_hugepage/enabled
  fi
  if [ -f /proc/sys/kernel/nmi_watchdog ]; then
   echo ${NMI_WATCHDOG} > /proc/sys/kernel/nmi_watchdog
  fi
  if [ -f /sys/devices/system/machinecheck/machinecheck0/check_interval ]; then
   echo 300 > /sys/devices/system/machinecheck/machinecheck0/check_interval
  fi
  if [ -f /sys/devices/virtual/workqueue/writeback/cpumask ]; then
   echo ${DEFAULT_MASK} > /sys/devices/virtual/workqueue/writeback/cpumask
  fi
  if [ -f /sys/devices/virtual/workqueue/cpumask ]; then
    echo ${DEFAULT_MASK} > /sys/devices/virtual/workqueue/cpumask
  fi
  if [ -f /proc/sys/vm/stat_interval ]; then
    echo 1 > /proc/sys/vm/stat_interval
  fi
  if [ -f /sys/module/processor/parameters/latency_factor ]; then
   echo 2 > /sys/module/processor/parameters/latency_factor
  fi
  if [ -f /sys/module/processor/parameters/ignore_ppc ]; then
   echo 0 > /sys/module/processor/parameters/ignore_ppc
  fi
  if [ -f /sys/module/processor/parameters/ignore_tpc ]; then
   echo 0 > /sys/module/processor/parameters/ignore_tpc
  fi
  if [ -f /etc/init.d/sgi_irqbalance ]; then
   /etc/init.d/sgi_irqbalance start
  fi
else
  # route irqs away from shielded cpus
  if [ -f /etc/init.d/sgi_irqbalance ]; then
    /etc/init.d/sgi_irqbalance stop
  fi
  echo $SHIELD_MASK > /proc/irq/default_smp_affinity
  for irqlist in $(ls /proc/irq/*/smp_affinity); do
    echo $SHIELD_MASK > $irqlist 2>/dev/null
  done

  # poke some buttons..
  if [ -f /proc/sys/kernel/sched_rt_runtime_us ]; then
    echo -1 > /proc/sys/kernel/sched_rt_runtime_us
  fi
  if [ -f /sys/kernel/debug/tracing/tracing_on ]; then
    echo 0 > /sys/kernel/debug/tracing/tracing_on
  fi
  if [ -f /sys/kernel/mm/transparent_hugepage/enabled ]; then
    echo never > /sys/kernel/mm/transparent_hugepage/enabled
  fi
  if [ -f /proc/sys/kernel/nmi_watchdog ]; then
    echo 0 > /proc/sys/kernel/nmi_watchdog
  fi
  if [ -f /sys/devices/system/machinecheck/machinecheck0/check_interval ]; then
    echo 0 > /sys/devices/system/machinecheck/machinecheck0/check_interval
  fi
  if [ -f /sys/devices/virtual/workqueue/writeback/cpumask ]; then
    echo ${SHIELD_MASK} > /sys/devices/virtual/workqueue/writeback/cpumask
  fi
  if [ -f /sys/devices/virtual/workqueue/cpumask ]; then
    echo ${SHIELD_MASK} > /sys/devices/virtual/workqueue/cpumask
  fi
  if [ -f /proc/sys/vm/stat_interval ]; then
    echo 999999 > /proc/sys/vm/stat_interval
  fi
  if [ -f /sys/module/processor/parameters/latency_factor ]; then
    echo 1 > /sys/module/processor/parameters/latency_factor
  fi
  if [ -f /sys/module/processor/parameters/ignore_ppc ]; then
    echo 1 > /sys/module/processor/parameters/ignore_ppc
  fi
  if [ -f /sys/module/processor/parameters/ignore_tpc ]; then
    echo 1 > /sys/module/processor/parameters/ignore_tpc
  fi

  # ...and fire up the shield
  cset shield --userset=rtcpus --cpu=${START_CPU}-${END_CPU} --kthread=on

  # If cpuset wasn't previously mounted (systemd will, like it or not),
  # it has now been mounted. Find the mount point.
  if [ -z $CPUSET_ROOT ]; then
   CPUSET_ROOT=$(grep cpuset /proc/mounts|cut -d ' ' -f2)
   if [ -z $CPUSET_ROOT ]; then
     # If it's not mounted now, bail.
     echo EEK, cupset is not mounted!
     exit
   else
     # ok, check for cgroup mount
     if [ -f ${CPUSET_ROOT}/cpuset.cpus ]; then
      CPUSET_PREFIX=cpuset.
     fi
   fi
  fi

  echo 0 > ${CPUSET_ROOT}/${CPUSET_PREFIX}sched_load_balance
  echo 1 > ${CPUSET_ROOT}/system/${CPUSET_PREFIX}sched_load_balance
  echo 0 > ${CPUSET_ROOT}/rtcpus/${CPUSET_PREFIX}sched_relax_domain_level
  # this ain't gonna happen in -rt kernels, but...
  if [ -f ${CPUSET_ROOT}/rtcpus/cpu.rt_runtime_us ]; then
    echo 300000 > ${CPUSET_ROOT}/system/cpu.rt_runtime_us
    echo 300000 > ${CPUSET_ROOT}/rtcpus/cpu.rt_runtime_us
  fi
  echo 0 > ${CPUSET_ROOT}/rtcpus/${CPUSET_PREFIX}sched_load_balance

  # wait a bit for sched_domain rebuild
  sleep 1

  # now go to hpc
  if [ -f ${CPUSET_ROOT}/rtcpus/${CPUSET_PREFIX}sched_hpc_rt ]; then
    echo 1 > ${CPUSET_ROOT}/rtcpus/${CPUSET_PREFIX}sched_hpc_rt
  fi

  # offline/online to migrate timers and whatnot
  if [ $ONLINE -eq 1 ]; then
    for i in `seq ${START_CPU} ${END_CPU}`; do
      echo 0 > /sys/devices/system/cpu/cpu$i/online
    done
    for i in `seq ${START_CPU} ${END_CPU}`; do
      echo 1 > /sys/devices/system/cpu/cpu$i/online
    done

    # re-add CPUs the kernel removed on offline
    echo ${START_CPU}-${END_CPU} > ${CPUSET_ROOT}/rtcpus/${CPUSET_PREFIX}cpus

    # and prioritize re-initialized kthreads
    systenctl restart set_kthread_prio
  fi
  if [ -f /proc/sys/kernel/timer_migration ]; then
    echo 0 > /proc/sys/kernel/timer_migration
  fi
  GOVERNOR="performance"
fi

if [ -f /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor ]; then
  CURRENT_GOVERNOR=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor)
  if ! [ $GOVERNOR = $CURRENT_GOVERNOR ]; then
    for i in $(ls /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor); do
     echo $GOVERNOR > $i;
    done
  fi
fi