/* * Read-Copy Update module-based torture test facility * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * * Copyright (C) IBM Corporation, 2005, 2006 * * Authors: Paul E. McKenney <paulmck@us.ibm.com> *          Josh Triplett <josh@freedesktop.org> * * See also:  Documentation/RCU/torture.txt */#include <linux/types.h>#include <linux/kernel.h>#include <linux/init.h>#include <linux/module.h>#include <linux/kthread.h>#include <linux/err.h>#include <linux/spinlock.h>#include <linux/smp.h>#include <linux/rcupdate.h>#include <linux/interrupt.h>#include <linux/sched.h>#include <asm/atomic.h>#include <linux/bitops.h>#include <linux/module.h>#include <linux/completion.h>#include <linux/moduleparam.h>#include <linux/percpu.h>#include <linux/notifier.h>#include <linux/cpu.h>#include <linux/random.h>#include <linux/delay.h>#include <linux/byteorder/swabb.h>#include <linux/stat.h>#include <linux/srcu.h>MODULE_LICENSE("GPL");MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and "              "Josh Triplett <josh@freedesktop.org>");static int nreaders = -1;	/* # reader threads, defaults to 2*ncpus */static int nfakewriters = 4;	/* # fake writer threads */static int stat_interval;	/* Interval between stats, in seconds. */				/*  Defaults to "only at end of test". */static int verbose;		/* Print more debug info. */static int test_no_idle_hz;	/* Test RCU's support for tickless idle CPUs. */static int shuffle_interval = 5; /* Interval between shuffles (in sec)*/static char *torture_type = "rcu"; /* What RCU implementation to torture. */module_param(nreaders, int, 0444);MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");module_param(nfakewriters, int, 0444);MODULE_PARM_DESC(nfakewriters, "Number of RCU fake writer threads");module_param(stat_interval, int, 0444);MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");module_param(verbose, bool, 0444);MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");module_param(test_no_idle_hz, bool, 0444);MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");module_param(shuffle_interval, int, 0444);MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");module_param(torture_type, charp, 0444);MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)");#define TORTURE_FLAG "-torture:"#define PRINTK_STRING(s) \	do { printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)#define VERBOSE_PRINTK_STRING(s) \	do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)#define VERBOSE_PRINTK_ERRSTRING(s) \	do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG "!!! " s "\n", torture_type); } while (0)static char printk_buf[4096];static int nrealreaders;static struct task_struct *writer_task;static struct task_struct **fakewriter_tasks;static struct task_struct **reader_tasks;static struct task_struct *stats_task;static struct task_struct *shuffler_task;#define RCU_TORTURE_PIPE_LEN 10struct rcu_torture {	struct rcu_head rtort_rcu;	int rtort_pipe_count;	struct list_head rtort_free;	int rtort_mbtest;};static int fullstop = 0;	/* stop generating callbacks at test end. */static LIST_HEAD(rcu_torture_freelist);static struct rcu_torture *rcu_torture_current = NULL;static long rcu_torture_current_version = 0;static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];static DEFINE_SPINLOCK(rcu_torture_lock);static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) =	{ 0 };static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch) =	{ 0 };static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];static atomic_t n_rcu_torture_alloc;static atomic_t n_rcu_torture_alloc_fail;static atomic_t n_rcu_torture_free;static atomic_t n_rcu_torture_mberror;static atomic_t n_rcu_torture_error;static struct list_head rcu_torture_removed;/* * Allocate an element from the rcu_tortures pool. */static struct rcu_torture *rcu_torture_alloc(void){	struct list_head *p;	spin_lock_bh(&rcu_torture_lock);	if (list_empty(&rcu_torture_freelist)) {		atomic_inc(&n_rcu_torture_alloc_fail);		spin_unlock_bh(&rcu_torture_lock);		return NULL;	}	atomic_inc(&n_rcu_torture_alloc);	p = rcu_torture_freelist.next;	list_del_init(p);	spin_unlock_bh(&rcu_torture_lock);	return container_of(p, struct rcu_torture, rtort_free);}/* * Free an element to the rcu_tortures pool. */static voidrcu_torture_free(struct rcu_torture *p){	atomic_inc(&n_rcu_torture_free);	spin_lock_bh(&rcu_torture_lock);	list_add_tail(&p->rtort_free, &rcu_torture_freelist);	spin_unlock_bh(&rcu_torture_lock);}struct rcu_random_state {	unsigned long rrs_state;	long rrs_count;};#define RCU_RANDOM_MULT 39916801  /* prime */#define RCU_RANDOM_ADD	479001701 /* prime */#define RCU_RANDOM_REFRESH 10000#define DEFINE_RCU_RANDOM(name) struct rcu_random_state name = { 0, 0 }/* * Crude but fast random-number generator.  Uses a linear congruential * generator, with occasional help from get_random_bytes(). */static unsigned longrcu_random(struct rcu_random_state *rrsp){	long refresh;	if (--rrsp->rrs_count < 0) {		get_random_bytes(&refresh, sizeof(refresh));		rrsp->rrs_state += refresh;		rrsp->rrs_count = RCU_RANDOM_REFRESH;	}	rrsp->rrs_state = rrsp->rrs_state * RCU_RANDOM_MULT + RCU_RANDOM_ADD;	return swahw32(rrsp->rrs_state);}/* * Operations vector for selecting different types of tests. */struct rcu_torture_ops {	void (*init)(void);	void (*cleanup)(void);	int (*readlock)(void);	void (*readdelay)(struct rcu_random_state *rrsp);	void (*readunlock)(int idx);	int (*completed)(void);	void (*deferredfree)(struct rcu_torture *p);	void (*sync)(void);	int (*stats)(char *page);	char *name;};static struct rcu_torture_ops *cur_ops = NULL;/* * Definitions for rcu torture testing. */static int rcu_torture_read_lock(void) __acquires(RCU){	rcu_read_lock();	return 0;}static void rcu_read_delay(struct rcu_random_state *rrsp){	long delay;	const long longdelay = 200;	/* We want there to be long-running readers, but not all the time. */	delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay);	if (!delay)		udelay(longdelay);}static void rcu_torture_read_unlock(int idx) __releases(RCU){	rcu_read_unlock();}static int rcu_torture_completed(void){	return rcu_batches_completed();}static voidrcu_torture_cb(struct rcu_head *p){	int i;	struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);	if (fullstop) {		/* Test is ending, just drop callbacks on the floor. */		/* The next initialization will pick up the pieces. */		return;	}	i = rp->rtort_pipe_count;	if (i > RCU_TORTURE_PIPE_LEN)		i = RCU_TORTURE_PIPE_LEN;	atomic_inc(&rcu_torture_wcount[i]);	if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {		rp->rtort_mbtest = 0;		rcu_torture_free(rp);	} else		cur_ops->deferredfree(rp);}static void rcu_torture_deferred_free(struct rcu_torture *p){	call_rcu(&p->rtort_rcu, rcu_torture_cb);}static struct rcu_torture_ops rcu_ops = {	.init = NULL,	.cleanup = NULL,	.readlock = rcu_torture_read_lock,	.readdelay = rcu_read_delay,	.readunlock = rcu_torture_read_unlock,	.completed = rcu_torture_completed,	.deferredfree = rcu_torture_deferred_free,	.sync = synchronize_rcu,	.stats = NULL,	.name = "rcu"};static void rcu_sync_torture_deferred_free(struct rcu_torture *p){	int i;	struct rcu_torture *rp;	struct rcu_torture *rp1;	cur_ops->sync();	list_add(&p->rtort_free, &rcu_torture_removed);	list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {		i = rp->rtort_pipe_count;		if (i > RCU_TORTURE_PIPE_LEN)			i = RCU_TORTURE_PIPE_LEN;		atomic_inc(&rcu_torture_wcount[i]);		if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {			rp->rtort_mbtest = 0;			list_del(&rp->rtort_free);			rcu_torture_free(rp);		}	}}static void rcu_sync_torture_init(void){	INIT_LIST_HEAD(&rcu_torture_removed);}static struct rcu_torture_ops rcu_sync_ops = {	.init = rcu_sync_torture_init,	.cleanup = NULL,	.readlock = rcu_torture_read_lock,	.readdelay = rcu_read_delay,	.readunlock = rcu_torture_read_unlock,	.completed = rcu_torture_completed,	.deferredfree = rcu_sync_torture_deferred_free,	.sync = synchronize_rcu,	.stats = NULL,	.name = "rcu_sync"};/* * Definitions for rcu_bh torture testing. */static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH){	rcu_read_lock_bh();	return 0;}static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH){	rcu_read_unlock_bh();}static int rcu_bh_torture_completed(void){	return rcu_batches_completed_bh();}static void rcu_bh_torture_deferred_free(struct rcu_torture *p){	call_rcu_bh(&p->rtort_rcu, rcu_torture_cb);}struct rcu_bh_torture_synchronize {	struct rcu_head head;	struct completion completion;};static void rcu_bh_torture_wakeme_after_cb(struct rcu_head *head){	struct rcu_bh_torture_synchronize *rcu;	rcu = container_of(head, struct rcu_bh_torture_synchronize, head);	complete(&rcu->completion);}static void rcu_bh_torture_synchronize(void){	struct rcu_bh_torture_synchronize rcu;	init_completion(&rcu.completion);	call_rcu_bh(&rcu.head, rcu_bh_torture_wakeme_after_cb);	wait_for_completion(&rcu.completion);}static struct rcu_torture_ops rcu_bh_ops = {	.init = NULL,	.cleanup = NULL,	.readlock = rcu_bh_torture_read_lock,	.readdelay = rcu_read_delay,  /* just reuse rcu's version. */	.readunlock = rcu_bh_torture_read_unlock,	.completed = rcu_bh_torture_completed,	.deferredfree = rcu_bh_torture_deferred_free,	.sync = rcu_bh_torture_synchronize,	.stats = NULL,	.name = "rcu_bh"};static struct rcu_torture_ops rcu_bh_sync_ops = {	.init = rcu_sync_torture_init,	.cleanup = NULL,	.readlock = rcu_bh_torture_read_lock,	.readdelay = rcu_read_delay,  /* just reuse rcu's version. */	.readunlock = rcu_bh_torture_read_unlock,	.completed = rcu_bh_torture_completed,	.deferredfree = rcu_sync_torture_deferred_free,	.sync = rcu_bh_torture_synchronize,	.stats = NULL,	.name = "rcu_bh_sync"};/* * Definitions for srcu torture testing. */static struct srcu_struct srcu_ctl;static void srcu_torture_init(void){	init_srcu_struct(&srcu_ctl);	rcu_sync_torture_init();}static void srcu_torture_cleanup(void){	synchronize_srcu(&srcu_ctl);	cleanup_srcu_struct(&srcu_ctl);}static int srcu_torture_read_lock(void) __acquires(&srcu_ctl){	return srcu_read_lock(&srcu_ctl);}static void srcu_read_delay(struct rcu_random_state *rrsp){	long delay;	const long uspertick = 1000000 / HZ;	const long longdelay = 10;	/* We want there to be long-running readers, but not all the time. */	delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick);	if (!delay)		schedule_timeout_interruptible(longdelay);}static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl){	srcu_read_unlock(&srcu_ctl, idx);}static int srcu_torture_completed(void){	return srcu_batches_completed(&srcu_ctl);}static void srcu_torture_synchronize(void){	synchronize_srcu(&srcu_ctl);}static int srcu_torture_stats(char *page){	int cnt = 0;	int cpu;	int idx = srcu_ctl.completed & 0x1;	cnt += sprintf(&page[cnt], "%s%s per-CPU(idx=%d):",		       torture_type, TORTURE_FLAG, idx);	for_each_possible_cpu(cpu) {		cnt += sprintf(&page[cnt], " %d(%d,%d)", cpu,			       per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx],			       per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]);	}	cnt += sprintf(&page[cnt], "\n");	return cnt;}static struct rcu_torture_ops srcu_ops = {	.init = srcu_torture_init,	.cleanup = srcu_torture_cleanup,	.readlock = srcu_torture_read_lock,	.readdelay = srcu_read_delay,	.readunlock = srcu_torture_read_unlock,	.completed = srcu_torture_completed,	.deferredfree = rcu_sync_torture_deferred_free,	.sync = srcu_torture_synchronize,	.stats = srcu_torture_stats,	.name = "srcu"};/* * Definitions for sched torture testing. */static int sched_torture_read_lock(void){	preempt_disable();	return 0;}static void sched_torture_read_unlock(int idx){	preempt_enable();}static int sched_torture_completed(void){	return 0;}static void sched_torture_synchronize(void){	synchronize_sched();}static struct rcu_torture_ops sched_ops = {	.init = rcu_sync_torture_init,	.cleanup = NULL,	.readlock = sched_torture_read_lock,	.readdelay = rcu_read_delay,  /* just reuse rcu's version. */	.readunlock = sched_torture_read_unlock,	.completed = sched_torture_completed,	.deferredfree = rcu_sync_torture_deferred_free,	.sync = sched_torture_synchronize,	.stats = NULL,