mm->mmap_cache = prev;}/* * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that * is already present in an i_mmap tree without adjusting the tree. * The following helper function should be used when such adjustments * are necessary.  The "insert" vma (if any) is to be inserted * before we drop the necessary locks. */void vma_adjust(struct vm_area_struct *vma, unsigned long start,	unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert){	struct mm_struct *mm = vma->vm_mm;	struct vm_area_struct *next = vma->vm_next;	struct vm_area_struct *importer = NULL;	struct address_space *mapping = NULL;	struct prio_tree_root *root = NULL;	struct file *file = vma->vm_file;	struct anon_vma *anon_vma = NULL;	long adjust_next = 0;	int remove_next = 0;	if (next && !insert) {		if (end >= next->vm_end) {			/*			 * vma expands, overlapping all the next, and			 * perhaps the one after too (mprotect case 6).			 */again:			remove_next = 1 + (end > next->vm_end);			end = next->vm_end;			anon_vma = next->anon_vma;			importer = vma;		} else if (end > next->vm_start) {			/*			 * vma expands, overlapping part of the next:			 * mprotect case 5 shifting the boundary up.			 */			adjust_next = (end - next->vm_start) >> PAGE_SHIFT;			anon_vma = next->anon_vma;			importer = vma;		} else if (end < vma->vm_end) {			/*			 * vma shrinks, and !insert tells it's not			 * split_vma inserting another: so it must be			 * mprotect case 4 shifting the boundary down.			 */			adjust_next = - ((vma->vm_end - end) >> PAGE_SHIFT);			anon_vma = next->anon_vma;			importer = next;		}	}	if (file) {		mapping = file->f_mapping;		if (!(vma->vm_flags & VM_NONLINEAR))			root = &mapping->i_mmap;		spin_lock(&mapping->i_mmap_lock);		if (importer &&		    vma->vm_truncate_count != next->vm_truncate_count) {			/*			 * unmap_mapping_range might be in progress:			 * ensure that the expanding vma is rescanned.			 */			importer->vm_truncate_count = 0;		}		if (insert) {			insert->vm_truncate_count = vma->vm_truncate_count;			/*			 * Put into prio_tree now, so instantiated pages			 * are visible to arm/parisc __flush_dcache_page			 * throughout; but we cannot insert into address			 * space until vma start or end is updated.			 */			__vma_link_file(insert);		}	}	/*	 * When changing only vma->vm_end, we don't really need	 * anon_vma lock: but is that case worth optimizing out?	 */	if (vma->anon_vma)		anon_vma = vma->anon_vma;	if (anon_vma) {		spin_lock(&anon_vma->lock);		/*		 * Easily overlooked: when mprotect shifts the boundary,		 * make sure the expanding vma has anon_vma set if the		 * shrinking vma had, to cover any anon pages imported.		 */		if (importer && !importer->anon_vma) {			importer->anon_vma = anon_vma;			__anon_vma_link(importer);		}	}	if (root) {		flush_dcache_mmap_lock(mapping);		vma_prio_tree_remove(vma, root);		if (adjust_next)			vma_prio_tree_remove(next, root);	}	vma->vm_start = start;	vma->vm_end = end;	vma->vm_pgoff = pgoff;	if (adjust_next) {		next->vm_start += adjust_next << PAGE_SHIFT;		next->vm_pgoff += adjust_next;	}	if (root) {		if (adjust_next)			vma_prio_tree_insert(next, root);		vma_prio_tree_insert(vma, root);		flush_dcache_mmap_unlock(mapping);	}	if (remove_next) {		/*		 * vma_merge has merged next into vma, and needs		 * us to remove next before dropping the locks.		 */		__vma_unlink(mm, next, vma);		if (file)			__remove_shared_vm_struct(next, file, mapping);		if (next->anon_vma)			__anon_vma_merge(vma, next);	} else if (insert) {		/*		 * split_vma has split insert from vma, and needs		 * us to insert it before dropping the locks		 * (it may either follow vma or precede it).		 */		__insert_vm_struct(mm, insert);	}	if (anon_vma)		spin_unlock(&anon_vma->lock);	if (mapping)		spin_unlock(&mapping->i_mmap_lock);	if (remove_next) {		if (file) {			fput(file);			if (next->vm_flags & VM_EXECUTABLE)				removed_exe_file_vma(mm);		}		mm->map_count--;		mpol_put(vma_policy(next));		kmem_cache_free(vm_area_cachep, next);		/*		 * In mprotect's case 6 (see comments on vma_merge),		 * we must remove another next too. It would clutter		 * up the code too much to do both in one go.		 */		if (remove_next == 2) {			next = vma->vm_next;			goto again;		}	}	validate_mm(mm);}/* Flags that can be inherited from an existing mapping when merging */#define VM_MERGEABLE_FLAGS (VM_CAN_NONLINEAR)/* * If the vma has a ->close operation then the driver probably needs to release * per-vma resources, so we don't attempt to merge those. */static inline int is_mergeable_vma(struct vm_area_struct *vma,			struct file *file, unsigned long vm_flags){	if ((vma->vm_flags ^ vm_flags) & ~VM_MERGEABLE_FLAGS)		return 0;	if (vma->vm_file != file)		return 0;	if (vma->vm_ops && vma->vm_ops->close)		return 0;	return 1;}static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,					struct anon_vma *anon_vma2){	return !anon_vma1 || !anon_vma2 || (anon_vma1 == anon_vma2);}/* * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) * in front of (at a lower virtual address and file offset than) the vma. * * We cannot merge two vmas if they have differently assigned (non-NULL) * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. * * We don't check here for the merged mmap wrapping around the end of pagecache * indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which * wrap, nor mmaps which cover the final page at index -1UL. */static intcan_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,	struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff){	if (is_mergeable_vma(vma, file, vm_flags) &&	    is_mergeable_anon_vma(anon_vma, vma->anon_vma)) {		if (vma->vm_pgoff == vm_pgoff)			return 1;	}	return 0;}/* * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) * beyond (at a higher virtual address and file offset than) the vma. * * We cannot merge two vmas if they have differently assigned (non-NULL) * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. */static intcan_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,	struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff){	if (is_mergeable_vma(vma, file, vm_flags) &&	    is_mergeable_anon_vma(anon_vma, vma->anon_vma)) {		pgoff_t vm_pglen;		vm_pglen = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;		if (vma->vm_pgoff + vm_pglen == vm_pgoff)			return 1;	}	return 0;}/* * Given a mapping request (addr,end,vm_flags,file,pgoff), figure out * whether that can be merged with its predecessor or its successor. * Or both (it neatly fills a hole). * * In most cases - when called for mmap, brk or mremap - [addr,end) is * certain not to be mapped by the time vma_merge is called; but when * called for mprotect, it is certain to be already mapped (either at * an offset within prev, or at the start of next), and the flags of * this area are about to be changed to vm_flags - and the no-change * case has already been eliminated. * * The following mprotect cases have to be considered, where AAAA is * the area passed down from mprotect_fixup, never extending beyond one * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after: * *     AAAA             AAAA                AAAA          AAAA *    PPPPPPNNNNNN    PPPPPPNNNNNN    PPPPPPNNNNNN    PPPPNNNNXXXX *    cannot merge    might become    might become    might become *                    PPNNNNNNNNNN    PPPPPPPPPPNN    PPPPPPPPPPPP 6 or *    mmap, brk or    case 4 below    case 5 below    PPPPPPPPXXXX 7 or *    mremap move:                                    PPPPNNNNNNNN 8 *        AAAA *    PPPP    NNNN    PPPPPPPPPPPP    PPPPPPPPNNNN    PPPPNNNNNNNN *    might become    case 1 below    case 2 below    case 3 below * * Odd one out? Case 8, because it extends NNNN but needs flags of XXXX: * mprotect_fixup updates vm_flags & vm_page_prot on successful return. */struct vm_area_struct *vma_merge(struct mm_struct *mm,			struct vm_area_struct *prev, unsigned long addr,			unsigned long end, unsigned long vm_flags,		     	struct anon_vma *anon_vma, struct file *file,			pgoff_t pgoff, struct mempolicy *policy){	pgoff_t pglen = (end - addr) >> PAGE_SHIFT;	struct vm_area_struct *area, *next;	/*	 * We later require that vma->vm_flags == vm_flags,	 * so this tests vma->vm_flags & VM_SPECIAL, too.	 */	if (vm_flags & VM_SPECIAL)		return NULL;	if (prev)		next = prev->vm_next;	else		next = mm->mmap;	area = next;	if (next && next->vm_end == end)		/* cases 6, 7, 8 */		next = next->vm_next;	/*	 * Can it merge with the predecessor?	 */	if (prev && prev->vm_end == addr &&  			mpol_equal(vma_policy(prev), policy) &&			can_vma_merge_after(prev, vm_flags,						anon_vma, file, pgoff)) {		/*		 * OK, it can.  Can we now merge in the successor as well?		 */		if (next && end == next->vm_start &&				mpol_equal(policy, vma_policy(next)) &&				can_vma_merge_before(next, vm_flags,					anon_vma, file, pgoff+pglen) &&				is_mergeable_anon_vma(prev->anon_vma,						      next->anon_vma)) {							/* cases 1, 6 */			vma_adjust(prev, prev->vm_start,				next->vm_end, prev->vm_pgoff, NULL);		} else					/* cases 2, 5, 7 */			vma_adjust(prev, prev->vm_start,				end, prev->vm_pgoff, NULL);		return prev;	}	/*	 * Can this new request be merged in front of next?	 */	if (next && end == next->vm_start && 			mpol_equal(policy, vma_policy(next)) &&			can_vma_merge_before(next, vm_flags,					anon_vma, file, pgoff+pglen)) {		if (prev && addr < prev->vm_end)	/* case 4 */			vma_adjust(prev, prev->vm_start,				addr, prev->vm_pgoff, NULL);		else					/* cases 3, 8 */			vma_adjust(area, addr, next->vm_end,				next->vm_pgoff - pglen, NULL);		return area;	}	return NULL;}/* * find_mergeable_anon_vma is used by anon_vma_prepare, to check * neighbouring vmas for a suitable anon_vma, before it goes off * to allocate a new anon_vma.  It checks because a repetitive * sequence of mprotects and faults may otherwise lead to distinct * anon_vmas being allocated, preventing vma merge in subsequent * mprotect. */struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma){	struct vm_area_struct *near;	unsigned long vm_flags;	near = vma->vm_next;	if (!near)		goto try_prev;	/*	 * Since only mprotect tries to remerge vmas, match flags	 * which might be mprotected into each other later on.	 * Neither mlock nor madvise tries to remerge at present,	 * so leave their flags as obstructing a merge.	 */	vm_flags = vma->vm_flags & ~(VM_READ|VM_WRITE|VM_EXEC);	vm_flags |= near->vm_flags & (VM_READ|VM_WRITE|VM_EXEC);	if (near->anon_vma && vma->vm_end == near->vm_start && 			mpol_equal(vma_policy(vma), vma_policy(near)) &&			can_vma_merge_before(near, vm_flags,				NULL, vma->vm_file, vma->vm_pgoff +				((vma->vm_end - vma->vm_start) >> PAGE_SHIFT)))		return near->anon_vma;try_prev:	/*	 * It is potentially slow to have to call find_vma_prev here.	 * But it's only on the first write fault on the vma, not	 * every time, and we could devise a way to avoid it later	 * (e.g. stash info in next's anon_vma_node when assigning	 * an anon_vma, or when trying vma_merge).  Another time.	 */	BUG_ON(find_vma_prev(vma->vm_mm, vma->vm_start, &near) != vma);	if (!near)		goto none;	vm_flags = vma->vm_flags & ~(VM_READ|VM_WRITE|VM_EXEC);	vm_flags |= near->vm_flags & (VM_READ|VM_WRITE|VM_EXEC);	if (near->anon_vma && near->vm_end == vma->vm_start &&  			mpol_equal(vma_policy(near), vma_policy(vma)) &&			can_vma_merge_after(near, vm_flags,				NULL, vma->vm_file, vma->vm_pgoff))		return near->anon_vma;none:	/*	 * There's no absolute need to look only at touching neighbours:	 * we could search further afield for "compatible" anon_vmas.	 * But it would probably just be a waste of time searching,	 * or lead to too many vmas hanging off the same anon_vma.	 * We're trying to allow mprotect remerging later on,	 * not trying to minimize memory used for anon_vmas.	 */	return NULL;}#ifdef CONFIG_PROC_FSvoid vm_stat_account(struct mm_struct *mm, unsigned long flags,						struct file *file, long pages){	const unsigned long stack_flags		= VM_STACK_FLAGS & (VM_GROWSUP|VM_GROWSDOWN);	if (file) {		mm->shared_vm += pages;		if ((flags & (VM_EXEC|VM_WRITE)) == VM_EXEC)			mm->exec_vm += pages;	} else if (flags & stack_flags)		mm->stack_vm += pages;	if (flags & (VM_RESERVED|VM_IO))		mm->reserved_vm += pages;}#endif /* CONFIG_PROC_FS *//* * The caller must hold down_write(current->mm->mmap_sem). */unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,			unsigned long len, unsigned long prot,			unsigned long flags, unsigned long pgoff){	struct mm_struct * mm = current->mm;	struct inode *inode;	unsigned int vm_flags;	int error;	unsigned long reqprot = prot;	/*	 * Does the application expect PROT_READ to imply PROT_EXEC?	 *	 * (the exception is when the underlying filesystem is noexec	 *  mounted, in which case we dont add PROT_EXEC.)	 */	if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))		if (!(file && (file->f_path.mnt->mnt_flags & MNT_NOEXEC)))			prot |= PROT_EXEC;	if (!len)		return -EINVAL;	if (!(flags & MAP_FIXED))		addr = round_hint_to_min(addr);	error = arch_mmap_check(addr, len, flags);	if (error)		return error;	/* Careful about overflows.. */	len = PAGE_ALIGN(len);	if (!len || len > TASK_SIZE)		return -ENOMEM;	/* offset overflow? */	if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)               return -EOVERFLOW;	/* Too many mappings? */	if (mm->map_count > sysctl_max_map_count)		return -ENOMEM;	/* Obtain the address to map to. we verify (or select) it and ensure	 * that it represents a valid section of the address space.	 */	addr = get_unmapped_area(file, addr, len, pgoff, flags);	if (addr & ~PAGE_MASK)		return addr;	/* Do simple checking here so the lower-level routines won't have	 * to. we assume access permissions have been handled by the open	 * of the memory object, so we don't do any here.	 */	vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags) |			mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;	if (flags & MAP_LOCKED) {		if (!can_do_mlock())			return -EPERM;		vm_flags |= VM_LOCKED;	}	/* mlock MCL_FUTURE? */	if (vm_flags & VM_LOCKED) {		unsigned long locked, lock_limit;		locked = len >> PAGE_SHIFT;		locked += mm->locked_vm;		lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;		lock_limit >>= PAGE_SHIFT;		if (locked > lock_limit && !capable(CAP_IPC_LOCK))			return -EAGAIN;	}	inode = file ? file->f_path.dentry->d_inode : NULL;