Isolate Kernel Module to a Specific Core Using Cpuset

Isolate Kernel Module to a Specific Core Using Cpuset

So I want the module to get executed in an isolated core.

and

actually isolate a specific core in our system and execute just one
specific process to that core

This is a working source code compiled and tested on a Debian box using kernel 3.16. I'll describe how to load and unload first and what the parameter passed means.

All sources can be found on github here...

https://github.com/harryjackson/doc/tree/master/linux/kernel/toy/toy

Build and load the module...

make
insmod toy param_cpu_id=2

To unload the module use

rmmod toy

I'm not using modprobe because it expects some configuration etc. The parameter we're passing to the toy kernel module is the CPU we want to isolate. None of the device operations that get called will run unless they're executing on that CPU.

Once the module is loaded you can find it here

/dev/toy

Simple operations like

cat /dev/toy

create events that the kernel module catches and produces some output. You can see the output using dmesg.

Source code...

#include <linux/module.h>
#include <linux/fs.h>
#include <linux/miscdevice.h>
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Harry");
MODULE_DESCRIPTION("toy kernel module");
MODULE_VERSION("0.1"); 
#define  DEVICE_NAME "toy"
#define  CLASS_NAME  "toy"

static int    param_cpu_id;
module_param(param_cpu_id    , int, (S_IRUSR | S_IRGRP | S_IROTH));
MODULE_PARM_DESC(param_cpu_id, "CPU ID that operations run on");

//static void    bar(void *arg);
//static void    foo(void *cpu);
static int     toy_open(   struct inode *inodep, struct file *fp);
static ssize_t toy_read(   struct file *fp     , char *buffer, size_t len, loff_t * offset);
static ssize_t toy_write(  struct file *fp     , const char *buffer, size_t len, loff_t *);
static int     toy_release(struct inode *inodep, struct file *fp);

static struct file_operations toy_fops = {
  .owner = THIS_MODULE,
  .open = toy_open,
  .read = toy_read,
  .write = toy_write,
  .release = toy_release,
};

static struct miscdevice toy_device = {
  .minor = MISC_DYNAMIC_MINOR,
  .name = "toy",
  .fops = &toy_fops
};

//static int CPU_IDS[64] = {0};
static int toy_open(struct inode *inodep, struct file *filep) {
  int this_cpu = get_cpu();
  printk(KERN_INFO "open: called on CPU:%d\n", this_cpu);
  if(this_cpu == param_cpu_id) {
    printk(KERN_INFO "open: is on requested CPU: %d\n", smp_processor_id());
  }
  else {
    printk(KERN_INFO "open: not on requested CPU:%d\n", smp_processor_id());
  }
  put_cpu();
  return 0;
}
static ssize_t toy_read(struct file *filep, char *buffer, size_t len, loff_t *offset){
  int this_cpu = get_cpu();
  printk(KERN_INFO "read: called on CPU:%d\n", this_cpu);
  if(this_cpu == param_cpu_id) {
    printk(KERN_INFO "read: is on requested CPU: %d\n", smp_processor_id());
  }
  else {
    printk(KERN_INFO "read: not on requested CPU:%d\n", smp_processor_id());
  }
  put_cpu();
  return 0;
}
static ssize_t toy_write(struct file *filep, const char *buffer, size_t len, loff_t *offset){
  int this_cpu = get_cpu();
  printk(KERN_INFO "write called on CPU:%d\n", this_cpu);
  if(this_cpu == param_cpu_id) {
    printk(KERN_INFO "write: is on requested CPU: %d\n", smp_processor_id());
  }
  else {
    printk(KERN_INFO "write: not on requested CPU:%d\n", smp_processor_id());
  }
  put_cpu();
  return 0;
}
static int toy_release(struct inode *inodep, struct file *filep){
  int this_cpu = get_cpu();
  printk(KERN_INFO "release called on CPU:%d\n", this_cpu);
  if(this_cpu == param_cpu_id) {
    printk(KERN_INFO "release: is on requested CPU: %d\n", smp_processor_id());
  }
  else {
    printk(KERN_INFO "release: not on requested CPU:%d\n", smp_processor_id());
  }
  put_cpu();
  return 0;
}

static int __init toy_init(void) {
  int cpu_id;
  if(param_cpu_id < 0 || param_cpu_id > 4) {
    printk(KERN_INFO "toy: unable to load module without cpu parameter\n");
    return -1;
  }
  printk(KERN_INFO "toy: loading to device driver, param_cpu_id: %d\n", param_cpu_id);
  //preempt_disable(); // See notes below
  cpu_id = get_cpu();
  printk(KERN_INFO "toy init called and running on CPU: %d\n", cpu_id);
  misc_register(&toy_device);
  //preempt_enable(); // See notes below
  put_cpu();
  //smp_call_function_single(1,foo,(void *)(uintptr_t) 1,1);
  return 0;
}

static void __exit toy_exit(void) {
    misc_deregister(&toy_device);
    printk(KERN_INFO "toy exit called\n");
}

module_init(toy_init);
module_exit(toy_exit); 

The code above contains the two methods you asked for ie isolation of CPU and on init run on an isolated core.

On init get_cpu disables preemption ie anything that comes after it will not be preempted by the kernel and will run on one core. Note, this was done kernel using 3.16, your mileage may vary depending on your kernel version but I think these API's have been around a long time

This is the Makefile...

obj-m += toy.o

all:
    make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules

clean:
    make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean

Notes. get_cpu is declared in linux/smp.h as

#define get_cpu()   ({ preempt_disable(); smp_processor_id(); })
#define put_cpu()   preempt_enable()

so you don't actually need to call preempt_disable before calling get_cpu.
The get_cpu call is a wrapper around the following sequence of calls...

preempt_count_inc();
barrier();

and put_cpu is really doing this...

barrier();
if (unlikely(preempt_count_dec_and_test())) {
  __preempt_schedule();
}   

You can get as fancy as you like using the above. Almost all of this was taken from the following sources..

Google for... smp_call_function_single

Linux Kernel Development, book by Robert Love.

http://derekmolloy.ie/writing-a-linux-kernel-module-part-2-a-character-device/

https://github.com/vsinitsyn/reverse/blob/master/reverse.c

Run a kernel thread on an isolated cpu core

There is already one API schedule_work_on in the mainline kernel which you can use to run your workqueue thread on a specific core.

Few years ago I have used same API for the same purpose.
Have a look in the sample code.

static void
 myworkmod_work_handler(struct work_struct *w)
{
    printk(KERN_ERR "CPU is: %d\n", get_cpu());
    pr_info("work %u jiffies\n", (unsigned)onesec);
    put_cpu();
 }


 static int myworkmod_init(void)
 {
    onesec = msecs_to_jiffies(1000);
    pr_info("module loaded: %u jiffies\n", (unsigned)onesec);

    if (!wq)
            wq = create_singlethread_workqueue("myworkmod");
    if (wq)
            queue_delayed_work_on(2,wq, &myworkmod_work, onesec); //2 CPU no

    return 0;
  }

In your case I think you are using the schedule_work API which always hold the default CPU number. That is why you are getting the CPU 0. So you have to try the below one:

schedule_work_on(cpu_nr, &ctx->work);  //cpu_nr will the CPU no to be used.

How to bind certain kernel threads to a given core?

Several kernel threads are tied to a specific core, in order to effect capabilities needed by the SMP infrastructure, such as synchronization, interrupt handling and so on. The kworker, migration and ksoftirqd threads, for example, usually have one instance per virtual processor (e.g. 8 threads on a 4-core 8-thread CPU).

You cannot (and should not be able to) move those threads - without them that processor would not be fully usable by the system any more.

Why exactly do you want to move those threads anyway?

Could the affinity of thread created by kernel be set by cpuset ?

cpuset(7) is a manual page that describes a Linux userspace API in general. As the page states, you can use the sched_setaffinity(2) syscall to restrict a task to a specific set of CPUs.

The fact that sched_setaffinity(2) is a syscall should already make you notice that the functionality is intended for userspace usage. If you are writing kernel code, kernel threads have different internal APIs for this purpose (see kthread.h):

• kthread_bind(), which can be used to bind the kthread to a single CPU specified by its numerical ID.
• kthread_bind_mask(), which can be used to bind the kthread to one or more CPUs defined by a struct cpumask. You can initialize the right struct cpumask through cpumask_set_cpu(). This API is similar to the sched_setaffinity(2) syscall, but for kthreads.

How to make cpuset.cpu_exclusive function of cpuset work correctly

i believe it is a mis-understanding of cpu_exclusive flag, as i did. Here is the doc https://www.kernel.org/doc/Documentation/cgroup-v1/cpusets.txt, quoting:

If a cpuset is cpu or mem exclusive, no other cpuset, other than
a direct ancestor or descendant, may share any of the same CPUs or
Memory Nodes.

so one possible reason you have bash: echo: write error: Invalid argument, is that you have some other cgroup cpuset enabled, and it conflicts with your operations of echo 1 > my_cpuset/cpuset.cpu_exclusive

please run find . -name cpuset.cpus | xargs cat to list all your cgroup's target cpus.

assume you have 12 cpus, if you want to set cpu_exclusive of my_cpuset, you need to carefully modify all the other cgroups to use cpus, eg. 0-7, then set cpus of my_cpuset to be 8-11. After all these cpus configurations , you can set cpu_exclusive to be 1.

But still, other process can still use cpu 8-11. Only the tasks that belongs to the other cgroups will not use cpu 8-11

for me, i had some docker container running, which prevents me from setting my cpuset cpu_exclusive

with kernel doc, i do not think it is possible to use cpus exclusively by cgroup itself. One approach (i know this approach is running on production) is that we isolate cpus, and manage the cpu affinity/cpuset by ourselves

Pinning a thread to a core in a cpuset through C

Take a look at the pthread_setaffinity_np and pthread_getaffinity_np functions.

Example:

   #define _GNU_SOURCE
   #include <pthread.h>
   #include <stdio.h>
   #include <stdlib.h>
   #include <errno.h>

   #define handle_error_en(en, msg) \
           do { errno = en; perror(msg); exit(EXIT_FAILURE); } while (0)

   int
   main(int argc, char *argv[])
   {
       int s, j;
       cpu_set_t cpuset;
       pthread_t thread;

       thread = pthread_self();

       /* Set affinity mask to include CPUs 0 to 7 */

       CPU_ZERO(&cpuset);
       for (j = 0; j < 8; j++)
           CPU_SET(j, &cpuset);

       s = pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
       if (s != 0)
           handle_error_en(s, "pthread_setaffinity_np");

       /* Check the actual affinity mask assigned to the thread */

       s = pthread_getaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
       if (s != 0)
           handle_error_en(s, "pthread_getaffinity_np");

       printf("Set returned by pthread_getaffinity_np() contained:\n");
       for (j = 0; j < CPU_SETSIZE; j++)
           if (CPU_ISSET(j, &cpuset))
               printf("    CPU %d\n", j);

       exit(EXIT_SUCCESS);
   }

For more details, see the man page.

---

Related Topics