for(idx=0; idx < iocnt; idx++)                printf("iov[%2d] = 0x%8x :: %d\n", idx, iov[idx].iov_base,                       iov[idx].iov_len);            printf("error %s writing blocks %ld:%d (%d != %d)\n",                   strerror(errno), start_bnum, iocnt, ret, iocnt*bsize);            ret = EINVAL;            break;        }        ret = 0;        for(k=i; k < j; k++) {            if (ents[k]->func) {                ents[k]->func(ents[k]->logged_bnum, 1, ents[k]->arg);                ents[k]->func = NULL;            }            if (ents[k]->clone) {                free(ents[k]->clone);                ents[k]->clone = NULL;            } else {                ents[k]->flags &= ~CE_DIRTY;            }        }                    i = j - 1;  /* i gets incremented by the outer for loop */    }    /*       here we have to go back through and flush any blocks that are       still dirty.  with an arched brow you astutely ask, "but how       could this happen given the above loop?"  Ahhh young grasshopper       I say, the path through the cache is long and twisty and fraught       with peril.  The reason it can happen is that a block can be both       cloned and dirty.  The above loop would only flush the cloned half       of the data, not the main dirty block.  So we have to go back       through and see if there are any blocks that are still dirty.  If       there are we go back to the top of the function and do the whole       thing over.  Kind of grody but it is necessary to insure the       correctness of the log for the Be file system.    */       if (do_again == 0) {        for(i=0; i < n_ents; i++) {            if ((ents[i]->flags & CE_DIRTY) == 0 || ents[i]->lock)                continue;            do_again = 1;            break;        }        if (do_again)            goto restart;    }        release_iovec_array(iov);    return ret;}static voiddelete_cache_list(cache_ent_list *cel){    void      *junk;    cache_ent *ce, *next;        for(ce=cel->lru; ce; ce=next) {        next = ce->next;        if (ce->lock != 0) {            if (ce->func)                printf("*** shutdown_block_cache: block %ld, lock == %d "                       "(arg 0x%lx)!\n", ce->block_num, ce->lock,                       (ulong)ce->arg);            else                printf("*** shutdown_block_cache: block %ld, lock == %d!\n",                       ce->block_num, ce->lock);        }        if (ce->flags & CE_BUSY) {            printf("* shutdown block cache: bnum %ld is busy? ce 0x%lx\n",                   ce->block_num, (ulong)ce);        }        if ((ce->flags & CE_DIRTY) || ce->clone) {            flush_cache_ent(ce);        }        if (ce->clone)            free(ce->clone);        ce->clone = NULL;                if (ce->data)            free(ce->data);        ce->data = NULL;                if ((junk = hash_delete(&bc.ht, ce->dev, ce->block_num)) != ce) {            printf("*** free_device_cache: bad hash table entry %ld "                   "0x%lx != 0x%lx\n", ce->block_num, (ulong)junk, (ulong)ce);        }        memset(ce, 0xfd, sizeof(*ce));        free(ce);        bc.cur_blocks--;    }}voidshutdown_block_cache(void){    /* print_hash_stats(&bc.ht); */    if (bc.lock.s > 0)        LOCK(bc.lock);#ifndef USER    unregister_kernel_daemon(cache_flusher, NULL);#endif    delete_cache_list(&bc.normal);    delete_cache_list(&bc.locked);    bc.normal.lru = bc.normal.mru = NULL;    bc.locked.lru = bc.locked.mru = NULL;    shutdown_hash_table(&bc.ht);    if (bc.lock.s > 0)        free_lock(&bc.lock);    bc.lock.s = -1;    if (iovec_lock.s >= 0)        free_lock(&iovec_lock);}intinit_cache_for_device(int fd, fs_off_t max_blocks){    int ret = 0;        if (fd >= MAX_DEVICES)        return -1;    LOCK(bc.lock);    if (max_device_blocks[fd] != 0) {        printf("device %d is already initialized!\n", fd);        ret = -1;    } else {        max_device_blocks[fd] = max_blocks;    }    UNLOCK(bc.lock);    return ret;}/*   this routine assumes that bc.lock has been acquired*/   static cache_ent *block_lookup(int dev, fs_off_t bnum){    int        count = 0;    cache_ent *ce;    while (1) {        ce = hash_lookup(&bc.ht, dev, bnum);        if (ce == NULL)            return NULL;        if ((ce->flags & CE_BUSY) == 0) /* it's ok, break out and return it */            break;        /* else, it's busy and we need to retry our lookup */        UNLOCK(bc.lock);        snooze(5000);        if (count++ == 5000) {  /* then a lot of time has elapsed */            printf("block %ld isn't coming un-busy (ce @ 0x%lx)\n",                   ce->block_num, (ulong)ce);        }        LOCK(bc.lock);    }    if (ce->flags & CE_BUSY)        panic("block lookup: returning a busy block @ 0x%lx?!?\n",(ulong)ce);    return ce;}intset_blocks_info(int dev, fs_off_t *blocks, int nblocks,               void (*func)(fs_off_t bnum, size_t nblocks, void *arg), void *arg){    int        i, j, cur;    cache_ent *ce;    cache_ent *ents[NUM_FLUSH_BLOCKS];    LOCK(bc.lock);        for(i=0, cur=0; i < nblocks; i++) {        /* printf("sbi:   %ld (arg 0x%x)\n", blocks[i], arg); */        ce = block_lookup(dev, blocks[i]);        if (ce == NULL) {            panic("*** set_block_info can't find bnum %ld!\n", blocks[i]);            UNLOCK(bc.lock);            return ENOENT;   /* hopefully this doesn't happen... */        }        if (blocks[i] != ce->block_num || dev != ce->dev) {            UNLOCK(bc.lock);            panic("** error1: looked up dev %d block %ld but found dev %d "                    "bnum %ld\n", dev, blocks[i], ce->dev, ce->block_num);            return EBADF;        }        if (ce->lock == 0) {            panic("* set_block_info on bnum %ld (%d) but it's not locked!\n",                    blocks[i], nblocks);        }        if ((ce->flags & CE_DIRTY) == 0) {            panic("*** set_block_info on non-dirty block bnum %ld (%d)!\n",                    blocks[i], nblocks);        }        ce->flags |= CE_BUSY;     /* mark all blocks as busy till we're done */        /* if there is cloned data, it needs to be flushed now */        if (ce->clone && ce->func) {            ents[cur++] = ce;            if (cur >= NUM_FLUSH_BLOCKS) {                UNLOCK(bc.lock);                qsort(ents, cur, sizeof(cache_ent **), cache_ent_cmp);                flush_ents(ents, cur);                LOCK(bc.lock);                for(j=0; j < cur; j++)                    ents[j]->flags &= ~CE_BUSY;                cur = 0;            }        }    }        if (cur != 0) {        UNLOCK(bc.lock);        qsort(ents, cur, sizeof(cache_ent **), cache_ent_cmp);        flush_ents(ents, cur);        LOCK(bc.lock);        for(j=0; j < cur; j++)            ents[j]->flags &= ~CE_BUSY;        cur = 0;    }    /* now go through and set the info that we were asked to */    for(i=0; i < nblocks; i++) {        /* we can call hash_lookup() here because we know it's around */        ce = hash_lookup(&bc.ht, dev, blocks[i]);        if (ce == NULL) {            panic("*** set_block_info can't find bnum %ld!\n", blocks[i]);            UNLOCK(bc.lock);            return ENOENT;   /* hopefully this doesn't happen... */        }        ce->flags &= ~(CE_DIRTY | CE_BUSY);        if (ce->func != NULL) {            panic("*** set_block_info non-null callback on bnum %ld\n",                    ce->block_num);        }                if (ce->clone != NULL) {            panic("*** ce->clone == 0x%lx, not NULL in set_block_info\n",                    (ulong)ce->clone);        }                ce->clone = (void *)malloc(ce->bsize);        if (ce->clone == NULL)            panic("*** can't clone bnum %ld (bsize %d)\n",                    ce->block_num, ce->bsize);        memcpy(ce->clone, ce->data, ce->bsize);        ce->func   = func;        ce->arg    = arg;                ce->logged_bnum = blocks[i];        ce->lock--;        if (ce->lock < 0) {            printf("sbi: whoa nellie! ce @ 0x%lx (%ld) has lock == %d\n",                   (ulong)ce, ce->block_num, ce->lock);        }                if (ce->lock == 0) {            delete_from_list(&bc.locked, ce);            add_to_head(&bc.normal, ce);        }    }    UNLOCK(bc.lock);    return 0;}/* this function is only for use by flush_device() */static voiddo_flush(cache_ent **ents, int max){    int i;        for(i=0; i < max; i++) {        ents[i]->flags |= CE_BUSY;    }                    UNLOCK(bc.lock);                    qsort(ents, max, sizeof(cache_ent **), cache_ent_cmp);    flush_ents(ents, max);    LOCK(bc.lock);    for(i=0; i < max; i++) {        ents[i]->flags &= ~CE_BUSY;    }}intflush_device(int dev, int warn_locked){    int cur;    cache_ent *ce;    cache_ent *ents[NUM_FLUSH_BLOCKS];        LOCK(bc.lock);    cur = 0;    ce = bc.normal.lru;    while (ce) {        if (ce->dev != dev || (ce->flags & CE_BUSY)) {            ce = ce->next;            continue;        }                if ((ce->flags & CE_DIRTY) || ce->clone) {            ents[cur++] = ce;            if (cur >= NUM_FLUSH_BLOCKS) {                do_flush(ents, cur);                ce = bc.normal.lru;                cur = 0;                continue;            }        }        ce = ce->next;    }    if (cur != 0)        do_flush(ents, cur);    cur = 0;    ce = bc.locked.lru;    while (ce) {        if (ce->dev != dev || (ce->flags & CE_BUSY)) {            ce = ce->next;            continue;        }                if (ce->clone) {            ents[cur++] = ce;            if (cur >= NUM_FLUSH_BLOCKS) {                do_flush(ents, cur);                ce = bc.locked.lru;                cur = 0;                continue;            }        }        ce = ce->next;    }    if (cur != 0)        do_flush(ents, cur);    UNLOCK(bc.lock);    return 0;}static voidreal_remove_cached_blocks(int dev, int allow_writes, cache_ent_list *cel){    void      *junk;    cache_ent *ce, *next = NULL;        for(ce=cel->lru; ce; ce=next) {        next = ce->next;        if (ce->dev != dev) {            continue;        }        if (ce->lock != 0 || (ce->flags & CE_BUSY)) {            printf("*** remove_cached_dev: block %ld has lock = %d, flags "                   "0x%x! ce @ 0x%lx\n", ce->block_num, ce->lock, ce->flags,                   (ulong)ce);        }                if (allow_writes == ALLOW_WRITES &&            ((ce->flags & CE_DIRTY) || ce->clone)) {            ce->flags |= CE_BUSY;            flush_cache_ent(ce);            ce->flags &= ~CE_BUSY;        }        /* unlink this guy */        if (cel->lru == ce)            cel->lru = ce->next;        if (cel->mru == ce)            cel->mru = ce->prev;                            if (ce->prev)            ce->prev->next = ce->next;        if (ce->next)            ce->next->prev = ce->prev;        if (ce->clone)            free(ce->clone);        ce->clone = NULL;                if (ce->data)            free(ce->data);        ce->data = NULL;                if ((junk = hash_delete(&bc.ht, ce->dev, ce->block_num)) != ce) {            panic("*** remove_cached_device: bad hash table entry %ld "                   "0x%lx != 0x%lx\n", ce->block_num, (ulong)junk, (ulong)ce);        }        free(ce);        bc.cur_blocks--;    }}intremove_cached_device_blocks(int dev, int allow_writes){    LOCK(bc.lock);