/* * Implement CPU time clocks for the POSIX clock interface. */#include <linux/sched.h>#include <linux/posix-timers.h>#include <linux/errno.h>#include <linux/math64.h>#include <asm/uaccess.h>static int check_clock(const clockid_t which_clock){	int error = 0;	struct task_struct *p;	const pid_t pid = CPUCLOCK_PID(which_clock);	if (CPUCLOCK_WHICH(which_clock) >= CPUCLOCK_MAX)		return -EINVAL;	if (pid == 0)		return 0;	read_lock(&tasklist_lock);	p = find_task_by_vpid(pid);	if (!p || !(CPUCLOCK_PERTHREAD(which_clock) ?		   same_thread_group(p, current) : thread_group_leader(p))) {		error = -EINVAL;	}	read_unlock(&tasklist_lock);	return error;}static inline union cpu_time_counttimespec_to_sample(const clockid_t which_clock, const struct timespec *tp){	union cpu_time_count ret;	ret.sched = 0;		/* high half always zero when .cpu used */	if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) {		ret.sched = (unsigned long long)tp->tv_sec * NSEC_PER_SEC + tp->tv_nsec;	} else {		ret.cpu = timespec_to_cputime(tp);	}	return ret;}static void sample_to_timespec(const clockid_t which_clock,			       union cpu_time_count cpu,			       struct timespec *tp){	if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED)		*tp = ns_to_timespec(cpu.sched);	else		cputime_to_timespec(cpu.cpu, tp);}static inline int cpu_time_before(const clockid_t which_clock,				  union cpu_time_count now,				  union cpu_time_count then){	if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) {		return now.sched < then.sched;	}  else {		return cputime_lt(now.cpu, then.cpu);	}}static inline void cpu_time_add(const clockid_t which_clock,				union cpu_time_count *acc,			        union cpu_time_count val){	if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) {		acc->sched += val.sched;	}  else {		acc->cpu = cputime_add(acc->cpu, val.cpu);	}}static inline union cpu_time_count cpu_time_sub(const clockid_t which_clock,						union cpu_time_count a,						union cpu_time_count b){	if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) {		a.sched -= b.sched;	}  else {		a.cpu = cputime_sub(a.cpu, b.cpu);	}	return a;}/* * Divide and limit the result to res >= 1 * * This is necessary to prevent signal delivery starvation, when the result of * the division would be rounded down to 0. */static inline cputime_t cputime_div_non_zero(cputime_t time, unsigned long div){	cputime_t res = cputime_div(time, div);	return max_t(cputime_t, res, 1);}/* * Update expiry time from increment, and increase overrun count, * given the current clock sample. */static void bump_cpu_timer(struct k_itimer *timer,				  union cpu_time_count now){	int i;	if (timer->it.cpu.incr.sched == 0)		return;	if (CPUCLOCK_WHICH(timer->it_clock) == CPUCLOCK_SCHED) {		unsigned long long delta, incr;		if (now.sched < timer->it.cpu.expires.sched)			return;		incr = timer->it.cpu.incr.sched;		delta = now.sched + incr - timer->it.cpu.expires.sched;		/* Don't use (incr*2 < delta), incr*2 might overflow. */		for (i = 0; incr < delta - incr; i++)			incr = incr << 1;		for (; i >= 0; incr >>= 1, i--) {			if (delta < incr)				continue;			timer->it.cpu.expires.sched += incr;			timer->it_overrun += 1 << i;			delta -= incr;		}	} else {		cputime_t delta, incr;		if (cputime_lt(now.cpu, timer->it.cpu.expires.cpu))			return;		incr = timer->it.cpu.incr.cpu;		delta = cputime_sub(cputime_add(now.cpu, incr),				    timer->it.cpu.expires.cpu);		/* Don't use (incr*2 < delta), incr*2 might overflow. */		for (i = 0; cputime_lt(incr, cputime_sub(delta, incr)); i++)			     incr = cputime_add(incr, incr);		for (; i >= 0; incr = cputime_halve(incr), i--) {			if (cputime_lt(delta, incr))				continue;			timer->it.cpu.expires.cpu =				cputime_add(timer->it.cpu.expires.cpu, incr);			timer->it_overrun += 1 << i;			delta = cputime_sub(delta, incr);		}	}}static inline cputime_t prof_ticks(struct task_struct *p){	return cputime_add(p->utime, p->stime);}static inline cputime_t virt_ticks(struct task_struct *p){	return p->utime;}static inline unsigned long long sched_ns(struct task_struct *p){	return task_sched_runtime(p);}int posix_cpu_clock_getres(const clockid_t which_clock, struct timespec *tp){	int error = check_clock(which_clock);	if (!error) {		tp->tv_sec = 0;		tp->tv_nsec = ((NSEC_PER_SEC + HZ - 1) / HZ);		if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) {			/*			 * If sched_clock is using a cycle counter, we			 * don't have any idea of its true resolution			 * exported, but it is much more than 1s/HZ.			 */			tp->tv_nsec = 1;		}	}	return error;}int posix_cpu_clock_set(const clockid_t which_clock, const struct timespec *tp){	/*	 * You can never reset a CPU clock, but we check for other errors	 * in the call before failing with EPERM.	 */	int error = check_clock(which_clock);	if (error == 0) {		error = -EPERM;	}	return error;}/* * Sample a per-thread clock for the given task. */static int cpu_clock_sample(const clockid_t which_clock, struct task_struct *p,			    union cpu_time_count *cpu){	switch (CPUCLOCK_WHICH(which_clock)) {	default:		return -EINVAL;	case CPUCLOCK_PROF:		cpu->cpu = prof_ticks(p);		break;	case CPUCLOCK_VIRT:		cpu->cpu = virt_ticks(p);		break;	case CPUCLOCK_SCHED:		cpu->sched = sched_ns(p);		break;	}	return 0;}/* * Sample a process (thread group) clock for the given group_leader task. * Must be called with tasklist_lock held for reading. * Must be called with tasklist_lock held for reading, and p->sighand->siglock. */static int cpu_clock_sample_group_locked(unsigned int clock_idx,					 struct task_struct *p,					 union cpu_time_count *cpu){	struct task_struct *t = p; 	switch (clock_idx) {	default:		return -EINVAL;	case CPUCLOCK_PROF:		cpu->cpu = cputime_add(p->signal->utime, p->signal->stime);		do {			cpu->cpu = cputime_add(cpu->cpu, prof_ticks(t));			t = next_thread(t);		} while (t != p);		break;	case CPUCLOCK_VIRT:		cpu->cpu = p->signal->utime;		do {			cpu->cpu = cputime_add(cpu->cpu, virt_ticks(t));			t = next_thread(t);		} while (t != p);		break;	case CPUCLOCK_SCHED:		cpu->sched = p->signal->sum_sched_runtime;		/* Add in each other live thread.  */		while ((t = next_thread(t)) != p) {			cpu->sched += t->se.sum_exec_runtime;		}		cpu->sched += sched_ns(p);		break;	}	return 0;}/* * Sample a process (thread group) clock for the given group_leader task. * Must be called with tasklist_lock held for reading. */static int cpu_clock_sample_group(const clockid_t which_clock,				  struct task_struct *p,				  union cpu_time_count *cpu){	int ret;	unsigned long flags;	spin_lock_irqsave(&p->sighand->siglock, flags);	ret = cpu_clock_sample_group_locked(CPUCLOCK_WHICH(which_clock), p,					    cpu);	spin_unlock_irqrestore(&p->sighand->siglock, flags);	return ret;}int posix_cpu_clock_get(const clockid_t which_clock, struct timespec *tp){	const pid_t pid = CPUCLOCK_PID(which_clock);	int error = -EINVAL;	union cpu_time_count rtn;	if (pid == 0) {		/*		 * Special case constant value for our own clocks.		 * We don't have to do any lookup to find ourselves.		 */		if (CPUCLOCK_PERTHREAD(which_clock)) {			/*			 * Sampling just ourselves we can do with no locking.			 */			error = cpu_clock_sample(which_clock,						 current, &rtn);		} else {			read_lock(&tasklist_lock);			error = cpu_clock_sample_group(which_clock,						       current, &rtn);			read_unlock(&tasklist_lock);		}	} else {		/*		 * Find the given PID, and validate that the caller		 * should be able to see it.		 */		struct task_struct *p;		rcu_read_lock();		p = find_task_by_vpid(pid);		if (p) {			if (CPUCLOCK_PERTHREAD(which_clock)) {				if (same_thread_group(p, current)) {					error = cpu_clock_sample(which_clock,								 p, &rtn);				}			} else {				read_lock(&tasklist_lock);				if (thread_group_leader(p) && p->signal) {					error =					    cpu_clock_sample_group(which_clock,							           p, &rtn);				}				read_unlock(&tasklist_lock);			}		}		rcu_read_unlock();	}	if (error)		return error;	sample_to_timespec(which_clock, rtn, tp);	return 0;}/* * Validate the clockid_t for a new CPU-clock timer, and initialize the timer. * This is called from sys_timer_create with the new timer already locked. */int posix_cpu_timer_create(struct k_itimer *new_timer){	int ret = 0;	const pid_t pid = CPUCLOCK_PID(new_timer->it_clock);	struct task_struct *p;	if (CPUCLOCK_WHICH(new_timer->it_clock) >= CPUCLOCK_MAX)		return -EINVAL;	INIT_LIST_HEAD(&new_timer->it.cpu.entry);	new_timer->it.cpu.incr.sched = 0;	new_timer->it.cpu.expires.sched = 0;	read_lock(&tasklist_lock);	if (CPUCLOCK_PERTHREAD(new_timer->it_clock)) {		if (pid == 0) {			p = current;		} else {			p = find_task_by_vpid(pid);			if (p && !same_thread_group(p, current))				p = NULL;		}	} else {		if (pid == 0) {			p = current->group_leader;		} else {			p = find_task_by_vpid(pid);			if (p && !thread_group_leader(p))				p = NULL;		}	}	new_timer->it.cpu.task = p;	if (p) {		get_task_struct(p);	} else {		ret = -EINVAL;	}	read_unlock(&tasklist_lock);	return ret;}/* * Clean up a CPU-clock timer that is about to be destroyed. * This is called from timer deletion with the timer already locked. * If we return TIMER_RETRY, it's necessary to release the timer's lock * and try again.  (This happens when the timer is in the middle of firing.) */int posix_cpu_timer_del(struct k_itimer *timer){	struct task_struct *p = timer->it.cpu.task;	int ret = 0;	if (likely(p != NULL)) {		read_lock(&tasklist_lock);		if (unlikely(p->signal == NULL)) {			/*			 * We raced with the reaping of the task.			 * The deletion should have cleared us off the list.			 */			BUG_ON(!list_empty(&timer->it.cpu.entry));		} else {			spin_lock(&p->sighand->siglock);			if (timer->it.cpu.firing)				ret = TIMER_RETRY;			else				list_del(&timer->it.cpu.entry);			spin_unlock(&p->sighand->siglock);		}		read_unlock(&tasklist_lock);		if (!ret)			put_task_struct(p);	}	return ret;}/* * Clean out CPU timers still ticking when a thread exited.  The task * pointer is cleared, and the expiry time is replaced with the residual * time for later timer_gettime calls to return. * This must be called with the siglock held. */static void cleanup_timers(struct list_head *head,			   cputime_t utime, cputime_t stime,			   unsigned long long sum_exec_runtime){	struct cpu_timer_list *timer, *next;	cputime_t ptime = cputime_add(utime, stime);	list_for_each_entry_safe(timer, next, head, entry) {		list_del_init(&timer->entry);		if (cputime_lt(timer->expires.cpu, ptime)) {			timer->expires.cpu = cputime_zero;		} else {			timer->expires.cpu = cputime_sub(timer->expires.cpu,							 ptime);		}	}	++head;	list_for_each_entry_safe(timer, next, head, entry) {		list_del_init(&timer->entry);		if (cputime_lt(timer->expires.cpu, utime)) {			timer->expires.cpu = cputime_zero;		} else {			timer->expires.cpu = cputime_sub(timer->expires.cpu,							 utime);		}	}	++head;	list_for_each_entry_safe(timer, next, head, entry) {		list_del_init(&timer->entry);		if (timer->expires.sched < sum_exec_runtime) {			timer->expires.sched = 0;		} else {			timer->expires.sched -= sum_exec_runtime;		}	}}/* * These are both called with the siglock held, when the current thread * is being reaped.  When the final (leader) thread in the group is reaped, * posix_cpu_timers_exit_group will be called after posix_cpu_timers_exit. */void posix_cpu_timers_exit(struct task_struct *tsk){	cleanup_timers(tsk->cpu_timers,		       tsk->utime, tsk->stime, tsk->se.sum_exec_runtime);}void posix_cpu_timers_exit_group(struct task_struct *tsk){	cleanup_timers(tsk->signal->cpu_timers,		       cputime_add(tsk->utime, tsk->signal->utime),		       cputime_add(tsk->stime, tsk->signal->stime),		     tsk->se.sum_exec_runtime + tsk->signal->sum_sched_runtime);}/* * Set the expiry times of all the threads in the process so one of them * will go off before the process cumulative expiry total is reached. */static void process_timer_rebalance(struct task_struct *p,				    unsigned int clock_idx,				    union cpu_time_count expires,				    union cpu_time_count val){	cputime_t ticks, left;	unsigned long long ns, nsleft; 	struct task_struct *t = p;	unsigned int nthreads = atomic_read(&p->signal->live);	if (!nthreads)		return;	switch (clock_idx) {	default:		BUG();		break;	case CPUCLOCK_PROF:		left = cputime_div_non_zero(cputime_sub(expires.cpu, val.cpu),				       nthreads);		do {			if (likely(!(t->flags & PF_EXITING))) {				ticks = cputime_add(prof_ticks(t), left);				if (cputime_eq(t->it_prof_expires,					       cputime_zero) ||				    cputime_gt(t->it_prof_expires, ticks)) {					t->it_prof_expires = ticks;				}			}			t = next_thread(t);		} while (t != p);		break;	case CPUCLOCK_VIRT:		left = cputime_div_non_zero(cputime_sub(expires.cpu, val.cpu),				       nthreads);		do {			if (likely(!(t->flags & PF_EXITING))) {				ticks = cputime_add(virt_ticks(t), left);				if (cputime_eq(t->it_virt_expires,					       cputime_zero) ||				    cputime_gt(t->it_virt_expires, ticks)) {					t->it_virt_expires = ticks;				}			}			t = next_thread(t);		} while (t != p);		break;	case CPUCLOCK_SCHED:		nsleft = expires.sched - val.sched;		do_div(nsleft, nthreads);		nsleft = max_t(unsigned long long, nsleft, 1);		do {			if (likely(!(t->flags & PF_EXITING))) {				ns = t->se.sum_exec_runtime + nsleft;				if (t->it_sched_expires == 0 ||				    t->it_sched_expires > ns) {					t->it_sched_expires = ns;				}			}			t = next_thread(t);		} while (t != p);		break;	}}static void clear_dead_task(struct k_itimer *timer, union cpu_time_count now){	/*	 * That's all for this thread or process.	 * We leave our residual in expires to be reported.