/* *  linux/drivers/block/deadline-iosched.c * *  Deadline i/o scheduler. * *  Copyright (C) 2002 Jens Axboe <axboe@suse.de> */#include <linux/kernel.h>#include <linux/fs.h>#include <linux/blkdev.h>#include <linux/elevator.h>#include <linux/bio.h>#include <linux/config.h>#include <linux/module.h>#include <linux/slab.h>#include <linux/init.h>#include <linux/compiler.h>#include <linux/hash.h>#include <linux/rbtree.h>/* * See Documentation/deadline-iosched.txt */static int read_expire = HZ / 2;  /* max time before a read is submitted. */static int write_expire = 5 * HZ; /* ditto for writes, these limits are SOFT! */static int writes_starved = 2;    /* max times reads can starve a write */static int fifo_batch = 16;       /* # of sequential requests treated as one				     by the above parameters. For throughput. */static const int deadline_hash_shift = 5;#define DL_HASH_BLOCK(sec)	((sec) >> 3)#define DL_HASH_FN(sec)		(hash_long(DL_HASH_BLOCK((sec)), deadline_hash_shift))#define DL_HASH_ENTRIES		(1 << deadline_hash_shift)#define rq_hash_key(rq)		((rq)->sector + (rq)->nr_sectors)#define list_entry_hash(ptr)	list_entry((ptr), struct deadline_rq, hash)#define ON_HASH(drq)		(drq)->on_hashstruct deadline_data {	/*	 * run time data	 */	/*	 * requests (deadline_rq s) are present on both sort_list and fifo_list	 */	struct rb_root sort_list[2];		struct list_head fifo_list[2];		/*	 * next in sort order. read, write or both are NULL	 */	struct deadline_rq *next_drq[2];	struct list_head *dispatch;	/* driver dispatch queue */	struct list_head *hash;		/* request hash */	unsigned int batching;		/* number of sequential requests made */	sector_t last_sector;		/* head position */	unsigned int starved;		/* times reads have starved writes */	/*	 * settings that change how the i/o scheduler behaves	 */	int fifo_expire[2];	int fifo_batch;	int writes_starved;	int front_merges;	mempool_t *drq_pool;};/* * pre-request data. */struct deadline_rq {	/*	 * rbtree index, key is the starting offset	 */	struct rb_node rb_node;	sector_t rb_key;	struct request *request;	/*	 * request hash, key is the ending offset (for back merge lookup)	 */	struct list_head hash;	char on_hash;	/*	 * expire fifo	 */	struct list_head fifo;	unsigned long expires;};static void deadline_move_request(struct deadline_data *dd, struct deadline_rq *drq);static kmem_cache_t *drq_pool;#define RQ_DATA(rq)	((struct deadline_rq *) (rq)->elevator_private)/* * the back merge hash support functions */static inline void __deadline_del_drq_hash(struct deadline_rq *drq){	drq->on_hash = 0;	list_del_init(&drq->hash);}static inline void deadline_del_drq_hash(struct deadline_rq *drq){	if (ON_HASH(drq))		__deadline_del_drq_hash(drq);}static voiddeadline_remove_merge_hints(request_queue_t *q, struct deadline_rq *drq){	deadline_del_drq_hash(drq);	if (q->last_merge == drq->request)		q->last_merge = NULL;}static inline voiddeadline_add_drq_hash(struct deadline_data *dd, struct deadline_rq *drq){	struct request *rq = drq->request;	BUG_ON(ON_HASH(drq));	drq->on_hash = 1;	list_add(&drq->hash, &dd->hash[DL_HASH_FN(rq_hash_key(rq))]);}/* * move hot entry to front of chain */static inline voiddeadline_hot_drq_hash(struct deadline_data *dd, struct deadline_rq *drq){	struct request *rq = drq->request;	struct list_head *head = &dd->hash[DL_HASH_FN(rq_hash_key(rq))];	if (ON_HASH(drq) && drq->hash.prev != head) {		list_del(&drq->hash);		list_add(&drq->hash, head);	}}static struct request *deadline_find_drq_hash(struct deadline_data *dd, sector_t offset){	struct list_head *hash_list = &dd->hash[DL_HASH_FN(offset)];	struct list_head *entry, *next = hash_list->next;	while ((entry = next) != hash_list) {		struct deadline_rq *drq = list_entry_hash(entry);		struct request *__rq = drq->request;		next = entry->next;				BUG_ON(!ON_HASH(drq));		if (!rq_mergeable(__rq)) {			__deadline_del_drq_hash(drq);			continue;		}		if (rq_hash_key(__rq) == offset)			return __rq;	}	return NULL;}/* * rb tree support functions */#define RB_NONE		(2)#define RB_EMPTY(root)	((root)->rb_node == NULL)#define ON_RB(node)	((node)->rb_color != RB_NONE)#define RB_CLEAR(node)	((node)->rb_color = RB_NONE)#define rb_entry_drq(node)	rb_entry((node), struct deadline_rq, rb_node)#define DRQ_RB_ROOT(dd, drq)	(&(dd)->sort_list[rq_data_dir((drq)->request)])#define rq_rb_key(rq)		(rq)->sectorstatic struct deadline_rq *__deadline_add_drq_rb(struct deadline_data *dd, struct deadline_rq *drq){	struct rb_node **p = &DRQ_RB_ROOT(dd, drq)->rb_node;	struct rb_node *parent = NULL;	struct deadline_rq *__drq;	while (*p) {		parent = *p;		__drq = rb_entry_drq(parent);		if (drq->rb_key < __drq->rb_key)			p = &(*p)->rb_left;		else if (drq->rb_key > __drq->rb_key)			p = &(*p)->rb_right;		else			return __drq;	}	rb_link_node(&drq->rb_node, parent, p);	return NULL;}static voiddeadline_add_drq_rb(struct deadline_data *dd, struct deadline_rq *drq){	struct deadline_rq *__alias;	drq->rb_key = rq_rb_key(drq->request);retry:	__alias = __deadline_add_drq_rb(dd, drq);	if (!__alias) {		rb_insert_color(&drq->rb_node, DRQ_RB_ROOT(dd, drq));		return;	}	deadline_move_request(dd, __alias);	goto retry;}static inline voiddeadline_del_drq_rb(struct deadline_data *dd, struct deadline_rq *drq){	const int data_dir = rq_data_dir(drq->request);	if (dd->next_drq[data_dir] == drq) {		struct rb_node *rbnext = rb_next(&drq->rb_node);		dd->next_drq[data_dir] = NULL;		if (rbnext)			dd->next_drq[data_dir] = rb_entry_drq(rbnext);	}	if (ON_RB(&drq->rb_node)) {		rb_erase(&drq->rb_node, DRQ_RB_ROOT(dd, drq));		RB_CLEAR(&drq->rb_node);	}}static struct request *deadline_find_drq_rb(struct deadline_data *dd, sector_t sector, int data_dir){	struct rb_node *n = dd->sort_list[data_dir].rb_node;	struct deadline_rq *drq;	while (n) {		drq = rb_entry_drq(n);		if (sector < drq->rb_key)			n = n->rb_left;		else if (sector > drq->rb_key)			n = n->rb_right;		else			return drq->request;	}	return NULL;}/* * deadline_find_first_drq finds the first (lowest sector numbered) request * for the specified data_dir. Used to sweep back to the start of the disk * (1-way elevator) after we process the last (highest sector) request. */static struct deadline_rq *deadline_find_first_drq(struct deadline_data *dd, int data_dir){	struct rb_node *n = dd->sort_list[data_dir].rb_node;	for (;;) {		if (n->rb_left == NULL)			return rb_entry_drq(n);				n = n->rb_left;	}}/* * add drq to rbtree and fifo */static inline voiddeadline_add_request(struct request_queue *q, struct request *rq){	struct deadline_data *dd = q->elevator.elevator_data;	struct deadline_rq *drq = RQ_DATA(rq);	const int data_dir = rq_data_dir(drq->request);	deadline_add_drq_rb(dd, drq);	/*	 * set expire time (only used for reads) and add to fifo list	 */	drq->expires = jiffies + dd->fifo_expire[data_dir];	list_add_tail(&drq->fifo, &dd->fifo_list[data_dir]);	if (rq_mergeable(rq)) {		deadline_add_drq_hash(dd, drq);		if (!q->last_merge)			q->last_merge = rq;	}}/* * remove rq from rbtree, fifo, and hash */static void deadline_remove_request(request_queue_t *q, struct request *rq){	struct deadline_rq *drq = RQ_DATA(rq);	if (drq) {		struct deadline_data *dd = q->elevator.elevator_data;		list_del_init(&drq->fifo);		deadline_remove_merge_hints(q, drq);		deadline_del_drq_rb(dd, drq);	}}static intdeadline_merge(request_queue_t *q, struct request **req, struct bio *bio){	struct deadline_data *dd = q->elevator.elevator_data;	struct request *__rq;	int ret;	/*	 * try last_merge to avoid going to hash	 */	ret = elv_try_last_merge(q, bio);	if (ret != ELEVATOR_NO_MERGE) {		__rq = q->last_merge;		goto out_insert;	}	/*	 * see if the merge hash can satisfy a back merge	 */	__rq = deadline_find_drq_hash(dd, bio->bi_sector);	if (__rq) {		BUG_ON(__rq->sector + __rq->nr_sectors != bio->bi_sector);		if (elv_rq_merge_ok(__rq, bio)) {			ret = ELEVATOR_BACK_MERGE;			goto out;		}	}	/*	 * check for front merge	 */	if (dd->front_merges) {		sector_t rb_key = bio->bi_sector + bio_sectors(bio);		__rq = deadline_find_drq_rb(dd, rb_key, bio_data_dir(bio));		if (__rq) {			BUG_ON(rb_key != rq_rb_key(__rq));			if (elv_rq_merge_ok(__rq, bio)) {				ret = ELEVATOR_FRONT_MERGE;				goto out;			}		}	}	return ELEVATOR_NO_MERGE;out:	q->last_merge = __rq;out_insert:	if (ret)		deadline_hot_drq_hash(dd, RQ_DATA(__rq));	*req = __rq;	return ret;}static void deadline_merged_request(request_queue_t *q, struct request *req){	struct deadline_data *dd = q->elevator.elevator_data;	struct deadline_rq *drq = RQ_DATA(req);	/*	 * hash always needs to be repositioned, key is end sector	 */	deadline_del_drq_hash(drq);	deadline_add_drq_hash(dd, drq);	/*	 * if the merge was a front merge, we need to reposition request	 */	if (rq_rb_key(req) != drq->rb_key) {		deadline_del_drq_rb(dd, drq);		deadline_add_drq_rb(dd, drq);	}	q->last_merge = req;}static voiddeadline_merged_requests(request_queue_t *q, struct request *req,			 struct request *next){	struct deadline_data *dd = q->elevator.elevator_data;	struct deadline_rq *drq = RQ_DATA(req);	struct deadline_rq *dnext = RQ_DATA(next);	BUG_ON(!drq);	BUG_ON(!dnext);	/*	 * reposition drq (this is the merged request) in hash, and in rbtree	 * in case of a front merge	 */	deadline_del_drq_hash(drq);	deadline_add_drq_hash(dd, drq);	if (rq_rb_key(req) != drq->rb_key) {		deadline_del_drq_rb(dd, drq);		deadline_add_drq_rb(dd, drq);	}	/*	 * if dnext expires before drq, assign its expire time to drq	 * and move into dnext position (dnext will be deleted) in fifo	 */	if (!list_empty(&drq->fifo) && !list_empty(&dnext->fifo)) {		if (time_before(dnext->expires, drq->expires)) {			list_move(&drq->fifo, &dnext->fifo);			drq->expires = dnext->expires;		}	}	/*	 * kill knowledge of next, this one is a goner	 */	deadline_remove_request(q, next);}/* * move request from sort list to dispatch queue. */static inline voiddeadline_move_to_dispatch(struct deadline_data *dd, struct deadline_rq *drq){	request_queue_t *q = drq->request->q;	deadline_remove_request(q, drq->request);	list_add_tail(&drq->request->queuelist, dd->dispatch);}/* * move an entry to dispatch queue */static voiddeadline_move_request(struct deadline_data *dd, struct deadline_rq *drq){	const int data_dir = rq_data_dir(drq->request);	struct rb_node *rbnext = rb_next(&drq->rb_node);	dd->next_drq[READ] = NULL;	dd->next_drq[WRITE] = NULL;	if (rbnext)		dd->next_drq[data_dir] = rb_entry_drq(rbnext);