The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/mm/mremap.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  *      linux/mm/remap.c
    3  *
    4  *      (C) Copyright 1996 Linus Torvalds
    5  */
    6 
    7 #include <linux/slab.h>
    8 #include <linux/smp_lock.h>
    9 #include <linux/shm.h>
   10 #include <linux/mman.h>
   11 #include <linux/swap.h>
   12 
   13 #include <asm/uaccess.h>
   14 #include <asm/pgalloc.h>
   15 
   16 extern int vm_enough_memory(long pages);
   17 
   18 static inline pte_t *get_one_pte(struct mm_struct *mm, unsigned long addr)
   19 {
   20         pgd_t * pgd;
   21         pmd_t * pmd;
   22         pte_t * pte = NULL;
   23 
   24         pgd = pgd_offset(mm, addr);
   25         if (pgd_none(*pgd))
   26                 goto end;
   27         if (pgd_bad(*pgd)) {
   28                 pgd_ERROR(*pgd);
   29                 pgd_clear(pgd);
   30                 goto end;
   31         }
   32 
   33         pmd = pmd_offset(pgd, addr);
   34         if (pmd_none(*pmd))
   35                 goto end;
   36         if (pmd_bad(*pmd)) {
   37                 pmd_ERROR(*pmd);
   38                 pmd_clear(pmd);
   39                 goto end;
   40         }
   41 
   42         pte = pte_offset(pmd, addr);
   43         if (pte_none(*pte))
   44                 pte = NULL;
   45 end:
   46         return pte;
   47 }
   48 
   49 static inline pte_t *alloc_one_pte(struct mm_struct *mm, unsigned long addr)
   50 {
   51         pmd_t * pmd;
   52         pte_t * pte = NULL;
   53 
   54         pmd = pmd_alloc(mm, pgd_offset(mm, addr), addr);
   55         if (pmd)
   56                 pte = pte_alloc(mm, pmd, addr);
   57         return pte;
   58 }
   59 
   60 static inline int copy_one_pte(struct mm_struct *mm, pte_t * src, pte_t * dst)
   61 {
   62         int error = 0;
   63         pte_t pte;
   64 
   65         if (!pte_none(*src)) {
   66                 pte = ptep_get_and_clear(src);
   67                 if (!dst) {
   68                         /* No dest?  We must put it back. */
   69                         dst = src;
   70                         error++;
   71                 }
   72                 set_pte(dst, pte);
   73         }
   74         return error;
   75 }
   76 
   77 static int move_one_page(struct mm_struct *mm, unsigned long old_addr, unsigned long new_addr)
   78 {
   79         int error = 0;
   80         pte_t * src;
   81 
   82         spin_lock(&mm->page_table_lock);
   83         src = get_one_pte(mm, old_addr);
   84         if (src)
   85                 error = copy_one_pte(mm, src, alloc_one_pte(mm, new_addr));
   86         spin_unlock(&mm->page_table_lock);
   87         return error;
   88 }
   89 
   90 static int move_page_tables(struct mm_struct * mm,
   91         unsigned long new_addr, unsigned long old_addr, unsigned long len)
   92 {
   93         unsigned long offset = len;
   94 
   95         flush_cache_range(mm, old_addr, old_addr + len);
   96 
   97         /*
   98          * This is not the clever way to do this, but we're taking the
   99          * easy way out on the assumption that most remappings will be
  100          * only a few pages.. This also makes error recovery easier.
  101          */
  102         while (offset) {
  103                 offset -= PAGE_SIZE;
  104                 if (move_one_page(mm, old_addr + offset, new_addr + offset))
  105                         goto oops_we_failed;
  106         }
  107         flush_tlb_range(mm, old_addr, old_addr + len);
  108         return 0;
  109 
  110         /*
  111          * Ok, the move failed because we didn't have enough pages for
  112          * the new page table tree. This is unlikely, but we have to
  113          * take the possibility into account. In that case we just move
  114          * all the pages back (this will work, because we still have
  115          * the old page tables)
  116          */
  117 oops_we_failed:
  118         flush_cache_range(mm, new_addr, new_addr + len);
  119         while ((offset += PAGE_SIZE) < len)
  120                 move_one_page(mm, new_addr + offset, old_addr + offset);
  121         zap_page_range(mm, new_addr, len);
  122         return -1;
  123 }
  124 
  125 static inline unsigned long move_vma(struct vm_area_struct * vma,
  126         unsigned long addr, unsigned long old_len, unsigned long new_len,
  127         unsigned long new_addr)
  128 {
  129         struct mm_struct * mm = vma->vm_mm;
  130         struct vm_area_struct * new_vma, * next, * prev;
  131         int allocated_vma;
  132 
  133         new_vma = NULL;
  134         next = find_vma_prev(mm, new_addr, &prev);
  135         if (next) {
  136                 if (prev && prev->vm_end == new_addr &&
  137                     can_vma_merge(prev, vma->vm_flags) && !vma->vm_file && !(vma->vm_flags & VM_SHARED)) {
  138                         spin_lock(&mm->page_table_lock);
  139                         prev->vm_end = new_addr + new_len;
  140                         spin_unlock(&mm->page_table_lock);
  141                         new_vma = prev;
  142                         if (next != prev->vm_next)
  143                                 BUG();
  144                         if (prev->vm_end == next->vm_start && can_vma_merge(next, prev->vm_flags)) {
  145                                 spin_lock(&mm->page_table_lock);
  146                                 prev->vm_end = next->vm_end;
  147                                 __vma_unlink(mm, next, prev);
  148                                 spin_unlock(&mm->page_table_lock);
  149 
  150                                 mm->map_count--;
  151                                 kmem_cache_free(vm_area_cachep, next);
  152                         }
  153                 } else if (next->vm_start == new_addr + new_len &&
  154                            can_vma_merge(next, vma->vm_flags) && !vma->vm_file && !(vma->vm_flags & VM_SHARED)) {
  155                         spin_lock(&mm->page_table_lock);
  156                         next->vm_start = new_addr;
  157                         spin_unlock(&mm->page_table_lock);
  158                         new_vma = next;
  159                 }
  160         } else {
  161                 prev = find_vma(mm, new_addr-1);
  162                 if (prev && prev->vm_end == new_addr &&
  163                     can_vma_merge(prev, vma->vm_flags) && !vma->vm_file && !(vma->vm_flags & VM_SHARED)) {
  164                         spin_lock(&mm->page_table_lock);
  165                         prev->vm_end = new_addr + new_len;
  166                         spin_unlock(&mm->page_table_lock);
  167                         new_vma = prev;
  168                 }
  169         }
  170 
  171         allocated_vma = 0;
  172         if (!new_vma) {
  173                 new_vma = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL);
  174                 if (!new_vma)
  175                         goto out;
  176                 allocated_vma = 1;
  177         }
  178 
  179         if (!move_page_tables(current->mm, new_addr, addr, old_len)) {
  180                 unsigned long vm_locked = vma->vm_flags & VM_LOCKED;
  181 
  182                 if (allocated_vma) {
  183                         *new_vma = *vma;
  184                         new_vma->vm_start = new_addr;
  185                         new_vma->vm_end = new_addr+new_len;
  186                         new_vma->vm_pgoff += (addr-vma->vm_start) >> PAGE_SHIFT;
  187                         new_vma->vm_raend = 0;
  188                         if (new_vma->vm_file)
  189                                 get_file(new_vma->vm_file);
  190                         if (new_vma->vm_ops && new_vma->vm_ops->open)
  191                                 new_vma->vm_ops->open(new_vma);
  192                         insert_vm_struct(current->mm, new_vma);
  193                 }
  194 
  195                 do_munmap(current->mm, addr, old_len);
  196 
  197                 current->mm->total_vm += new_len >> PAGE_SHIFT;
  198                 if (vm_locked) {
  199                         current->mm->locked_vm += new_len >> PAGE_SHIFT;
  200                         if (new_len > old_len)
  201                                 make_pages_present(new_addr + old_len,
  202                                                    new_addr + new_len);
  203                 }
  204                 return new_addr;
  205         }
  206         if (allocated_vma)
  207                 kmem_cache_free(vm_area_cachep, new_vma);
  208  out:
  209         return -ENOMEM;
  210 }
  211 
  212 /*
  213  * Expand (or shrink) an existing mapping, potentially moving it at the
  214  * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
  215  *
  216  * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
  217  * This option implies MREMAP_MAYMOVE.
  218  */
  219 unsigned long do_mremap(unsigned long addr,
  220         unsigned long old_len, unsigned long new_len,
  221         unsigned long flags, unsigned long new_addr)
  222 {
  223         struct vm_area_struct *vma;
  224         unsigned long ret = -EINVAL;
  225 
  226         if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE))
  227                 goto out;
  228 
  229         if (addr & ~PAGE_MASK)
  230                 goto out;
  231 
  232         old_len = PAGE_ALIGN(old_len);
  233         new_len = PAGE_ALIGN(new_len);
  234 
  235         /* new_addr is only valid if MREMAP_FIXED is specified */
  236         if (flags & MREMAP_FIXED) {
  237                 if (new_addr & ~PAGE_MASK)
  238                         goto out;
  239                 if (!(flags & MREMAP_MAYMOVE))
  240                         goto out;
  241 
  242                 if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len)
  243                         goto out;
  244 
  245                 /* Check if the location we're moving into overlaps the
  246                  * old location at all, and fail if it does.
  247                  */
  248                 if ((new_addr <= addr) && (new_addr+new_len) > addr)
  249                         goto out;
  250 
  251                 if ((addr <= new_addr) && (addr+old_len) > new_addr)
  252                         goto out;
  253 
  254                 do_munmap(current->mm, new_addr, new_len);
  255         }
  256 
  257         /*
  258          * Always allow a shrinking remap: that just unmaps
  259          * the unnecessary pages..
  260          */
  261         ret = addr;
  262         if (old_len >= new_len) {
  263                 do_munmap(current->mm, addr+new_len, old_len - new_len);
  264                 if (!(flags & MREMAP_FIXED) || (new_addr == addr))
  265                         goto out;
  266         }
  267 
  268         /*
  269          * Ok, we need to grow..  or relocate.
  270          */
  271         ret = -EFAULT;
  272         vma = find_vma(current->mm, addr);
  273         if (!vma || vma->vm_start > addr)
  274                 goto out;
  275         /* We can't remap across vm area boundaries */
  276         if (old_len > vma->vm_end - addr)
  277                 goto out;
  278         if (vma->vm_flags & VM_DONTEXPAND) {
  279                 if (new_len > old_len)
  280                         goto out;
  281         }
  282         if (vma->vm_flags & VM_LOCKED) {
  283                 unsigned long locked = current->mm->locked_vm << PAGE_SHIFT;
  284                 locked += new_len - old_len;
  285                 ret = -EAGAIN;
  286                 if (locked > current->rlim[RLIMIT_MEMLOCK].rlim_cur)
  287                         goto out;
  288         }
  289         ret = -ENOMEM;
  290         if ((current->mm->total_vm << PAGE_SHIFT) + (new_len - old_len)
  291             > current->rlim[RLIMIT_AS].rlim_cur)
  292                 goto out;
  293         /* Private writable mapping? Check memory availability.. */
  294         if ((vma->vm_flags & (VM_SHARED | VM_WRITE)) == VM_WRITE &&
  295             !(flags & MAP_NORESERVE)                             &&
  296             !vm_enough_memory((new_len - old_len) >> PAGE_SHIFT))
  297                 goto out;
  298 
  299         /* old_len exactly to the end of the area..
  300          * And we're not relocating the area.
  301          */
  302         if (old_len == vma->vm_end - addr &&
  303             !((flags & MREMAP_FIXED) && (addr != new_addr)) &&
  304             (old_len != new_len || !(flags & MREMAP_MAYMOVE))) {
  305                 unsigned long max_addr = TASK_SIZE;
  306                 if (vma->vm_next)
  307                         max_addr = vma->vm_next->vm_start;
  308                 /* can we just expand the current mapping? */
  309                 if (max_addr - addr >= new_len) {
  310                         int pages = (new_len - old_len) >> PAGE_SHIFT;
  311                         spin_lock(&vma->vm_mm->page_table_lock);
  312                         vma->vm_end = addr + new_len;
  313                         spin_unlock(&vma->vm_mm->page_table_lock);
  314                         current->mm->total_vm += pages;
  315                         if (vma->vm_flags & VM_LOCKED) {
  316                                 current->mm->locked_vm += pages;
  317                                 make_pages_present(addr + old_len,
  318                                                    addr + new_len);
  319                         }
  320                         ret = addr;
  321                         goto out;
  322                 }
  323         }
  324 
  325         /*
  326          * We weren't able to just expand or shrink the area,
  327          * we need to create a new one and move it..
  328          */
  329         ret = -ENOMEM;
  330         if (flags & MREMAP_MAYMOVE) {
  331                 if (!(flags & MREMAP_FIXED)) {
  332                         unsigned long map_flags = 0;
  333                         if (vma->vm_flags & VM_SHARED)
  334                                 map_flags |= MAP_SHARED;
  335 
  336                         new_addr = get_unmapped_area(vma->vm_file, 0, new_len, vma->vm_pgoff, map_flags);
  337                         ret = new_addr;
  338                         if (new_addr & ~PAGE_MASK)
  339                                 goto out;
  340                 }
  341                 ret = move_vma(vma, addr, old_len, new_len, new_addr);
  342         }
  343 out:
  344         return ret;
  345 }
  346 
  347 asmlinkage unsigned long sys_mremap(unsigned long addr,
  348         unsigned long old_len, unsigned long new_len,
  349         unsigned long flags, unsigned long new_addr)
  350 {
  351         unsigned long ret;
  352 
  353         down_write(&current->mm->mmap_sem);
  354         ret = do_mremap(addr, old_len, new_len, flags, new_addr);
  355         up_write(&current->mm->mmap_sem);
  356         return ret;
  357 }

Cache object: 5be368bcdf841b936330c3ef5a108fae


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.