[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/nptl/allocatestack.c

Version: -  FREEBSD  -  FREEBSD10  -  FREEBSD9  -  FREEBSD92  -  FREEBSD91  -  FREEBSD90  -  FREEBSD8  -  FREEBSD82  -  FREEBSD81  -  FREEBSD80  -  FREEBSD7  -  FREEBSD74  -  FREEBSD73  -  FREEBSD72  -  FREEBSD71  -  FREEBSD70  -  FREEBSD6  -  FREEBSD64  -  FREEBSD63  -  FREEBSD62  -  FREEBSD61  -  FREEBSD60  -  FREEBSD5  -  FREEBSD55  -  FREEBSD54  -  FREEBSD53  -  FREEBSD52  -  FREEBSD51  -  FREEBSD50  -  FREEBSD4  -  FREEBSD3  -  FREEBSD22  -  cheribsd  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1  -  FREEBSD-LIBC  -  FREEBSD8-LIBC  -  FREEBSD7-LIBC  -  FREEBSD6-LIBC  -  GLIBC27 
SearchContext: -  none  -  3  -  10 

    1 /* Copyright (C) 2002,2003,2004,2005,2006,2007 Free Software Foundation, Inc.
    2    This file is part of the GNU C Library.
    3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
    4 
    5    The GNU C Library is free software; you can redistribute it and/or
    6    modify it under the terms of the GNU Lesser General Public
    7    License as published by the Free Software Foundation; either
    8    version 2.1 of the License, or (at your option) any later version.
    9 
   10    The GNU C Library is distributed in the hope that it will be useful,
   11    but WITHOUT ANY WARRANTY; without even the implied warranty of
   12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   13    Lesser General Public License for more details.
   14 
   15    You should have received a copy of the GNU Lesser General Public
   16    License along with the GNU C Library; if not, write to the Free
   17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
   18    02111-1307 USA.  */
   19 
   20 #include <assert.h>
   21 #include <errno.h>
   22 #include <signal.h>
   23 #include <stdint.h>
   24 #include <string.h>
   25 #include <unistd.h>
   26 #include <sys/mman.h>
   27 #include <sys/param.h>
   28 #include <dl-sysdep.h>
   29 #include <tls.h>
   30 #include <lowlevellock.h>
   31 #include <kernel-features.h>
   32 
   33 
   34 #ifndef NEED_SEPARATE_REGISTER_STACK
   35 
   36 /* Most architectures have exactly one stack pointer.  Some have more.  */
   37 # define STACK_VARIABLES void *stackaddr = NULL
   38 
   39 /* How to pass the values to the 'create_thread' function.  */
   40 # define STACK_VARIABLES_ARGS stackaddr
   41 
   42 /* How to declare function which gets there parameters.  */
   43 # define STACK_VARIABLES_PARMS void *stackaddr
   44 
   45 /* How to declare allocate_stack.  */
   46 # define ALLOCATE_STACK_PARMS void **stack
   47 
   48 /* This is how the function is called.  We do it this way to allow
   49    other variants of the function to have more parameters.  */
   50 # define ALLOCATE_STACK(attr, pd) allocate_stack (attr, pd, &stackaddr)
   51 
   52 #else
   53 
   54 /* We need two stacks.  The kernel will place them but we have to tell
   55    the kernel about the size of the reserved address space.  */
   56 # define STACK_VARIABLES void *stackaddr = NULL; size_t stacksize = 0
   57 
   58 /* How to pass the values to the 'create_thread' function.  */
   59 # define STACK_VARIABLES_ARGS stackaddr, stacksize
   60 
   61 /* How to declare function which gets there parameters.  */
   62 # define STACK_VARIABLES_PARMS void *stackaddr, size_t stacksize
   63 
   64 /* How to declare allocate_stack.  */
   65 # define ALLOCATE_STACK_PARMS void **stack, size_t *stacksize
   66 
   67 /* This is how the function is called.  We do it this way to allow
   68    other variants of the function to have more parameters.  */
   69 # define ALLOCATE_STACK(attr, pd) \
   70   allocate_stack (attr, pd, &stackaddr, &stacksize)
   71 
   72 #endif
   73 
   74 
   75 /* Default alignment of stack.  */
   76 #ifndef STACK_ALIGN
   77 # define STACK_ALIGN __alignof__ (long double)
   78 #endif
   79 
   80 /* Default value for minimal stack size after allocating thread
   81    descriptor and guard.  */
   82 #ifndef MINIMAL_REST_STACK
   83 # define MINIMAL_REST_STACK     4096
   84 #endif
   85 
   86 
   87 /* Let the architecture add some flags to the mmap() call used to
   88    allocate stacks.  */
   89 #ifndef ARCH_MAP_FLAGS
   90 # define ARCH_MAP_FLAGS 0
   91 #endif
   92 
   93 /* This yields the pointer that TLS support code calls the thread pointer.  */
   94 #if TLS_TCB_AT_TP
   95 # define TLS_TPADJ(pd) (pd)
   96 #elif TLS_DTV_AT_TP
   97 # define TLS_TPADJ(pd) ((struct pthread *)((char *) (pd) + TLS_PRE_TCB_SIZE))
   98 #endif
   99 
  100 /* Cache handling for not-yet free stacks.  */
  101 
  102 /* Maximum size in kB of cache.  */
  103 static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default.  */
  104 static size_t stack_cache_actsize;
  105 
  106 /* Mutex protecting this variable.  */
  107 static int stack_cache_lock = LLL_LOCK_INITIALIZER;
  108 
  109 /* List of queued stack frames.  */
  110 static LIST_HEAD (stack_cache);
  111 
  112 /* List of the stacks in use.  */
  113 static LIST_HEAD (stack_used);
  114 
  115 /* List of the threads with user provided stacks in use.  No need to
  116    initialize this, since it's done in __pthread_initialize_minimal.  */
  117 list_t __stack_user __attribute__ ((nocommon));
  118 hidden_data_def (__stack_user)
  119 
  120 #if COLORING_INCREMENT != 0
  121 /* Number of threads created.  */
  122 static unsigned int nptl_ncreated;
  123 #endif
  124 
  125 
  126 /* Check whether the stack is still used or not.  */
  127 #define FREE_P(descr) ((descr)->tid <= 0)
  128 
  129 
  130 /* We create a double linked list of all cache entries.  Double linked
  131    because this allows removing entries from the end.  */
  132 
  133 
  134 /* Get a stack frame from the cache.  We have to match by size since
  135    some blocks might be too small or far too large.  */
  136 static struct pthread *
  137 get_cached_stack (size_t *sizep, void **memp)
  138 {
  139   size_t size = *sizep;
  140   struct pthread *result = NULL;
  141   list_t *entry;
  142 
  143   lll_lock (stack_cache_lock, LLL_PRIVATE);
  144 
  145   /* Search the cache for a matching entry.  We search for the
  146      smallest stack which has at least the required size.  Note that
  147      in normal situations the size of all allocated stacks is the
  148      same.  As the very least there are only a few different sizes.
  149      Therefore this loop will exit early most of the time with an
  150      exact match.  */
  151   list_for_each (entry, &stack_cache)
  152     {
  153       struct pthread *curr;
  154 
  155       curr = list_entry (entry, struct pthread, list);
  156       if (FREE_P (curr) && curr->stackblock_size >= size)
  157         {
  158           if (curr->stackblock_size == size)
  159             {
  160               result = curr;
  161               break;
  162             }
  163 
  164           if (result == NULL
  165               || result->stackblock_size > curr->stackblock_size)
  166             result = curr;
  167         }
  168     }
  169 
  170   if (__builtin_expect (result == NULL, 0)
  171       /* Make sure the size difference is not too excessive.  In that
  172          case we do not use the block.  */
  173       || __builtin_expect (result->stackblock_size > 4 * size, 0))
  174     {
  175       /* Release the lock.  */
  176       lll_unlock (stack_cache_lock, LLL_PRIVATE);
  177 
  178       return NULL;
  179     }
  180 
  181   /* Dequeue the entry.  */
  182   list_del (&result->list);
  183 
  184   /* And add to the list of stacks in use.  */
  185   list_add (&result->list, &stack_used);
  186 
  187   /* And decrease the cache size.  */
  188   stack_cache_actsize -= result->stackblock_size;
  189 
  190   /* Release the lock early.  */
  191   lll_unlock (stack_cache_lock, LLL_PRIVATE);
  192 
  193   /* Report size and location of the stack to the caller.  */
  194   *sizep = result->stackblock_size;
  195   *memp = result->stackblock;
  196 
  197   /* Cancellation handling is back to the default.  */
  198   result->cancelhandling = 0;
  199   result->cleanup = NULL;
  200 
  201   /* No pending event.  */
  202   result->nextevent = NULL;
  203 
  204   /* Clear the DTV.  */
  205   dtv_t *dtv = GET_DTV (TLS_TPADJ (result));
  206   memset (dtv, '\0', (dtv[-1].counter + 1) * sizeof (dtv_t));
  207 
  208   /* Re-initialize the TLS.  */
  209   _dl_allocate_tls_init (TLS_TPADJ (result));
  210 
  211   return result;
  212 }
  213 
  214 
  215 /* Free stacks until cache size is lower than LIMIT.  */
  216 static void
  217 free_stacks (size_t limit)
  218 {
  219   /* We reduce the size of the cache.  Remove the last entries until
  220      the size is below the limit.  */
  221   list_t *entry;
  222   list_t *prev;
  223 
  224   /* Search from the end of the list.  */
  225   list_for_each_prev_safe (entry, prev, &stack_cache)
  226     {
  227       struct pthread *curr;
  228 
  229       curr = list_entry (entry, struct pthread, list);
  230       if (FREE_P (curr))
  231         {
  232           /* Unlink the block.  */
  233           list_del (entry);
  234 
  235           /* Account for the freed memory.  */
  236           stack_cache_actsize -= curr->stackblock_size;
  237 
  238           /* Free the memory associated with the ELF TLS.  */
  239           _dl_deallocate_tls (TLS_TPADJ (curr), false);
  240 
  241           /* Remove this block.  This should never fail.  If it does
  242              something is really wrong.  */
  243           if (munmap (curr->stackblock, curr->stackblock_size) != 0)
  244             abort ();
  245 
  246           /* Maybe we have freed enough.  */
  247           if (stack_cache_actsize <= limit)
  248             break;
  249         }
  250     }
  251 }
  252 
  253 
  254 /* Add a stack frame which is not used anymore to the stack.  Must be
  255    called with the cache lock held.  */
  256 static inline void
  257 __attribute ((always_inline))
  258 queue_stack (struct pthread *stack)
  259 {
  260   /* We unconditionally add the stack to the list.  The memory may
  261      still be in use but it will not be reused until the kernel marks
  262      the stack as not used anymore.  */
  263   list_add (&stack->list, &stack_cache);
  264 
  265   stack_cache_actsize += stack->stackblock_size;
  266   if (__builtin_expect (stack_cache_actsize > stack_cache_maxsize, 0))
  267     free_stacks (stack_cache_maxsize);
  268 }
  269 
  270 
  271 /* This function is called indirectly from the freeres code in libc.  */
  272 void
  273 __free_stack_cache (void)
  274 {
  275   free_stacks (0);
  276 }
  277 
  278 
  279 static int
  280 internal_function
  281 change_stack_perm (struct pthread *pd
  282 #ifdef NEED_SEPARATE_REGISTER_STACK
  283                    , size_t pagemask
  284 #endif
  285                    )
  286 {
  287 #ifdef NEED_SEPARATE_REGISTER_STACK
  288   void *stack = (pd->stackblock
  289                  + (((((pd->stackblock_size - pd->guardsize) / 2)
  290                       & pagemask) + pd->guardsize) & pagemask));
  291   size_t len = pd->stackblock + pd->stackblock_size - stack;
  292 #elif _STACK_GROWS_DOWN
  293   void *stack = pd->stackblock + pd->guardsize;
  294   size_t len = pd->stackblock_size - pd->guardsize;
  295 #elif _STACK_GROWS_UP
  296   void *stack = pd->stackblock;
  297   size_t len = (uintptr_t) pd - pd->guardsize - (uintptr_t) pd->stackblock;
  298 #else
  299 # error "Define either _STACK_GROWS_DOWN or _STACK_GROWS_UP"
  300 #endif
  301   if (mprotect (stack, len, PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
  302     return errno;
  303 
  304   return 0;
  305 }
  306 
  307 
  308 static int
  309 allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
  310                 ALLOCATE_STACK_PARMS)
  311 {
  312   struct pthread *pd;
  313   size_t size;
  314   size_t pagesize_m1 = __getpagesize () - 1;
  315   void *stacktop;
  316 
  317   assert (attr != NULL);
  318   assert (powerof2 (pagesize_m1 + 1));
  319   assert (TCB_ALIGNMENT >= STACK_ALIGN);
  320 
  321   /* Get the stack size from the attribute if it is set.  Otherwise we
  322      use the default we determined at start time.  */
  323   size = attr->stacksize ?: __default_stacksize;
  324 
  325   /* Get memory for the stack.  */
  326   if (__builtin_expect (attr->flags & ATTR_FLAG_STACKADDR, 0))
  327     {
  328       uintptr_t adj;
  329 
  330       /* If the user also specified the size of the stack make sure it
  331          is large enough.  */
  332       if (attr->stacksize != 0
  333           && attr->stacksize < (__static_tls_size + MINIMAL_REST_STACK))
  334         return EINVAL;
  335 
  336       /* Adjust stack size for alignment of the TLS block.  */
  337 #if TLS_TCB_AT_TP
  338       adj = ((uintptr_t) attr->stackaddr - TLS_TCB_SIZE)
  339             & __static_tls_align_m1;
  340       assert (size > adj + TLS_TCB_SIZE);
  341 #elif TLS_DTV_AT_TP
  342       adj = ((uintptr_t) attr->stackaddr - __static_tls_size)
  343             & __static_tls_align_m1;
  344       assert (size > adj);
  345 #endif
  346 
  347       /* The user provided some memory.  Let's hope it matches the
  348          size...  We do not allocate guard pages if the user provided
  349          the stack.  It is the user's responsibility to do this if it
  350          is wanted.  */
  351 #if TLS_TCB_AT_TP
  352       pd = (struct pthread *) ((uintptr_t) attr->stackaddr
  353                                - TLS_TCB_SIZE - adj);
  354 #elif TLS_DTV_AT_TP
  355       pd = (struct pthread *) (((uintptr_t) attr->stackaddr
  356                                 - __static_tls_size - adj)
  357                                - TLS_PRE_TCB_SIZE);
  358 #endif
  359 
  360       /* The user provided stack memory needs to be cleared.  */
  361       memset (pd, '\0', sizeof (struct pthread));
  362 
  363       /* The first TSD block is included in the TCB.  */
  364       pd->specific[0] = pd->specific_1stblock;
  365 
  366       /* Remember the stack-related values.  */
  367       pd->stackblock = (char *) attr->stackaddr - size;
  368       pd->stackblock_size = size;
  369 
  370       /* This is a user-provided stack.  It will not be queued in the
  371          stack cache nor will the memory (except the TLS memory) be freed.  */
  372       pd->user_stack = true;
  373 
  374       /* This is at least the second thread.  */
  375       pd->header.multiple_threads = 1;
  376 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
  377       __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
  378 #endif
  379 
  380 #ifndef __ASSUME_PRIVATE_FUTEX
  381       /* The thread must know when private futexes are supported.  */
  382       pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
  383                                                 header.private_futex);
  384 #endif
  385 
  386 #ifdef NEED_DL_SYSINFO
  387       /* Copy the sysinfo value from the parent.  */
  388       THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
  389 #endif
  390 
  391       /* The process ID is also the same as that of the caller.  */
  392       pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
  393 
  394       /* Allocate the DTV for this thread.  */
  395       if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
  396         {
  397           /* Something went wrong.  */
  398           assert (errno == ENOMEM);
  399           return EAGAIN;
  400         }
  401 
  402 
  403       /* Prepare to modify global data.  */
  404       lll_lock (stack_cache_lock, LLL_PRIVATE);
  405 
  406       /* And add to the list of stacks in use.  */
  407       list_add (&pd->list, &__stack_user);
  408 
  409       lll_unlock (stack_cache_lock, LLL_PRIVATE);
  410     }
  411   else
  412     {
  413       /* Allocate some anonymous memory.  If possible use the cache.  */
  414       size_t guardsize;
  415       size_t reqsize;
  416       void *mem;
  417       const int prot = (PROT_READ | PROT_WRITE
  418                         | ((GL(dl_stack_flags) & PF_X) ? PROT_EXEC : 0));
  419 
  420 #if COLORING_INCREMENT != 0
  421       /* Add one more page for stack coloring.  Don't do it for stacks
  422          with 16 times pagesize or larger.  This might just cause
  423          unnecessary misalignment.  */
  424       if (size <= 16 * pagesize_m1)
  425         size += pagesize_m1 + 1;
  426 #endif
  427 
  428       /* Adjust the stack size for alignment.  */
  429       size &= ~__static_tls_align_m1;
  430       assert (size != 0);
  431 
  432       /* Make sure the size of the stack is enough for the guard and
  433          eventually the thread descriptor.  */
  434       guardsize = (attr->guardsize + pagesize_m1) & ~pagesize_m1;
  435       if (__builtin_expect (size < ((guardsize + __static_tls_size
  436                                      + MINIMAL_REST_STACK + pagesize_m1)
  437                                     & ~pagesize_m1),
  438                             0))
  439         /* The stack is too small (or the guard too large).  */
  440         return EINVAL;
  441 
  442       /* Try to get a stack from the cache.  */
  443       reqsize = size;
  444       pd = get_cached_stack (&size, &mem);
  445       if (pd == NULL)
  446         {
  447           /* To avoid aliasing effects on a larger scale than pages we
  448              adjust the allocated stack size if necessary.  This way
  449              allocations directly following each other will not have
  450              aliasing problems.  */
  451 #if MULTI_PAGE_ALIASING != 0
  452           if ((size % MULTI_PAGE_ALIASING) == 0)
  453             size += pagesize_m1 + 1;
  454 #endif
  455 
  456           mem = mmap (NULL, size, prot,
  457                       MAP_PRIVATE | MAP_ANONYMOUS | ARCH_MAP_FLAGS, -1, 0);
  458 
  459           if (__builtin_expect (mem == MAP_FAILED, 0))
  460             {
  461 #ifdef ARCH_RETRY_MMAP
  462               mem = ARCH_RETRY_MMAP (size);
  463               if (__builtin_expect (mem == MAP_FAILED, 0))
  464 #endif
  465                 return errno;
  466             }
  467 
  468           /* SIZE is guaranteed to be greater than zero.
  469              So we can never get a null pointer back from mmap.  */
  470           assert (mem != NULL);
  471 
  472 #if COLORING_INCREMENT != 0
  473           /* Atomically increment NCREATED.  */
  474           unsigned int ncreated = atomic_increment_val (&nptl_ncreated);
  475 
  476           /* We chose the offset for coloring by incrementing it for
  477              every new thread by a fixed amount.  The offset used
  478              module the page size.  Even if coloring would be better
  479              relative to higher alignment values it makes no sense to
  480              do it since the mmap() interface does not allow us to
  481              specify any alignment for the returned memory block.  */
  482           size_t coloring = (ncreated * COLORING_INCREMENT) & pagesize_m1;
  483 
  484           /* Make sure the coloring offsets does not disturb the alignment
  485              of the TCB and static TLS block.  */
  486           if (__builtin_expect ((coloring & __static_tls_align_m1) != 0, 0))
  487             coloring = (((coloring + __static_tls_align_m1)
  488                          & ~(__static_tls_align_m1))
  489                         & ~pagesize_m1);
  490 #else
  491           /* Unless specified we do not make any adjustments.  */
  492 # define coloring 0
  493 #endif
  494 
  495           /* Place the thread descriptor at the end of the stack.  */
  496 #if TLS_TCB_AT_TP
  497           pd = (struct pthread *) ((char *) mem + size - coloring) - 1;
  498 #elif TLS_DTV_AT_TP
  499           pd = (struct pthread *) ((((uintptr_t) mem + size - coloring
  500                                     - __static_tls_size)
  501                                     & ~__static_tls_align_m1)
  502                                    - TLS_PRE_TCB_SIZE);
  503 #endif
  504 
  505           /* Remember the stack-related values.  */
  506           pd->stackblock = mem;
  507           pd->stackblock_size = size;
  508 
  509           /* We allocated the first block thread-specific data array.
  510              This address will not change for the lifetime of this
  511              descriptor.  */
  512           pd->specific[0] = pd->specific_1stblock;
  513 
  514           /* This is at least the second thread.  */
  515           pd->header.multiple_threads = 1;
  516 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
  517           __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
  518 #endif
  519 
  520 #ifndef __ASSUME_PRIVATE_FUTEX
  521           /* The thread must know when private futexes are supported.  */
  522           pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
  523                                                     header.private_futex);
  524 #endif
  525 
  526 #ifdef NEED_DL_SYSINFO
  527           /* Copy the sysinfo value from the parent.  */
  528           THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
  529 #endif
  530 
  531           /* The process ID is also the same as that of the caller.  */
  532           pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
  533 
  534           /* Allocate the DTV for this thread.  */
  535           if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
  536             {
  537               /* Something went wrong.  */
  538               assert (errno == ENOMEM);
  539 
  540               /* Free the stack memory we just allocated.  */
  541               (void) munmap (mem, size);
  542 
  543               return EAGAIN;
  544             }
  545 
  546 
  547           /* Prepare to modify global data.  */
  548           lll_lock (stack_cache_lock, LLL_PRIVATE);
  549 
  550           /* And add to the list of stacks in use.  */
  551           list_add (&pd->list, &stack_used);
  552 
  553           lll_unlock (stack_cache_lock, LLL_PRIVATE);
  554 
  555 
  556           /* There might have been a race.  Another thread might have
  557              caused the stacks to get exec permission while this new
  558              stack was prepared.  Detect if this was possible and
  559              change the permission if necessary.  */
  560           if (__builtin_expect ((GL(dl_stack_flags) & PF_X) != 0
  561                                 && (prot & PROT_EXEC) == 0, 0))
  562             {
  563               int err = change_stack_perm (pd
  564 #ifdef NEED_SEPARATE_REGISTER_STACK
  565                                            , ~pagesize_m1
  566 #endif
  567                                            );
  568               if (err != 0)
  569                 {
  570                   /* Free the stack memory we just allocated.  */
  571                   (void) munmap (mem, size);
  572 
  573                   return err;
  574                 }
  575             }
  576 
  577 
  578           /* Note that all of the stack and the thread descriptor is
  579              zeroed.  This means we do not have to initialize fields
  580              with initial value zero.  This is specifically true for
  581              the 'tid' field which is always set back to zero once the
  582              stack is not used anymore and for the 'guardsize' field
  583              which will be read next.  */
  584         }
  585 
  586       /* Create or resize the guard area if necessary.  */
  587       if (__builtin_expect (guardsize > pd->guardsize, 0))
  588         {
  589 #ifdef NEED_SEPARATE_REGISTER_STACK
  590           char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
  591 #elif _STACK_GROWS_DOWN
  592           char *guard = mem;
  593 # elif _STACK_GROWS_UP
  594           char *guard = (char *) (((uintptr_t) pd - guardsize) & ~pagesize_m1);
  595 #endif
  596           if (mprotect (guard, guardsize, PROT_NONE) != 0)
  597             {
  598               int err;
  599             mprot_error:
  600               err = errno;
  601 
  602               lll_lock (stack_cache_lock, LLL_PRIVATE);
  603 
  604               /* Remove the thread from the list.  */
  605               list_del (&pd->list);
  606 
  607               lll_unlock (stack_cache_lock, LLL_PRIVATE);
  608 
  609               /* Get rid of the TLS block we allocated.  */
  610               _dl_deallocate_tls (TLS_TPADJ (pd), false);
  611 
  612               /* Free the stack memory regardless of whether the size
  613                  of the cache is over the limit or not.  If this piece
  614                  of memory caused problems we better do not use it
  615                  anymore.  Uh, and we ignore possible errors.  There
  616                  is nothing we could do.  */
  617               (void) munmap (mem, size);
  618 
  619               return err;
  620             }
  621 
  622           pd->guardsize = guardsize;
  623         }
  624       else if (__builtin_expect (pd->guardsize - guardsize > size - reqsize,
  625                                  0))
  626         {
  627           /* The old guard area is too large.  */
  628 
  629 #ifdef NEED_SEPARATE_REGISTER_STACK
  630           char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
  631           char *oldguard = mem + (((size - pd->guardsize) / 2) & ~pagesize_m1);
  632 
  633           if (oldguard < guard
  634               && mprotect (oldguard, guard - oldguard, prot) != 0)
  635             goto mprot_error;
  636 
  637           if (mprotect (guard + guardsize,
  638                         oldguard + pd->guardsize - guard - guardsize,
  639                         prot) != 0)
  640             goto mprot_error;
  641 #elif _STACK_GROWS_DOWN
  642           if (mprotect ((char *) mem + guardsize, pd->guardsize - guardsize,
  643                         prot) != 0)
  644             goto mprot_error;
  645 #elif _STACK_GROWS_UP
  646           if (mprotect ((char *) pd - pd->guardsize,
  647                         pd->guardsize - guardsize, prot) != 0)
  648             goto mprot_error;
  649 #endif
  650 
  651           pd->guardsize = guardsize;
  652         }
  653       /* The pthread_getattr_np() calls need to get passed the size
  654          requested in the attribute, regardless of how large the
  655          actually used guardsize is.  */
  656       pd->reported_guardsize = guardsize;
  657     }
  658 
  659   /* Initialize the lock.  We have to do this unconditionally since the
  660      stillborn thread could be canceled while the lock is taken.  */
  661   pd->lock = LLL_LOCK_INITIALIZER;
  662 
  663   /* The robust mutex lists also need to be initialized
  664      unconditionally because the cleanup for the previous stack owner
  665      might have happened in the kernel.  */
  666   pd->robust_head.futex_offset = (offsetof (pthread_mutex_t, __data.__lock)
  667                                   - offsetof (pthread_mutex_t,
  668                                               __data.__list.__next));
  669   pd->robust_head.list_op_pending = NULL;
  670 #ifdef __PTHREAD_MUTEX_HAVE_PREV
  671   pd->robust_prev = &pd->robust_head;
  672 #endif
  673   pd->robust_head.list = &pd->robust_head;
  674 
  675   /* We place the thread descriptor at the end of the stack.  */
  676   *pdp = pd;
  677 
  678 #if TLS_TCB_AT_TP
  679   /* The stack begins before the TCB and the static TLS block.  */
  680   stacktop = ((char *) (pd + 1) - __static_tls_size);
  681 #elif TLS_DTV_AT_TP
  682   stacktop = (char *) (pd - 1);
  683 #endif
  684 
  685 #ifdef NEED_SEPARATE_REGISTER_STACK
  686   *stack = pd->stackblock;
  687   *stacksize = stacktop - *stack;
  688 #elif _STACK_GROWS_DOWN
  689   *stack = stacktop;
  690 #elif _STACK_GROWS_UP
  691   *stack = pd->stackblock;
  692   assert (*stack > 0);
  693 #endif
  694 
  695   return 0;
  696 }
  697 
  698 
  699 void
  700 internal_function
  701 __deallocate_stack (struct pthread *pd)
  702 {
  703   lll_lock (stack_cache_lock, LLL_PRIVATE);
  704 
  705   /* Remove the thread from the list of threads with user defined
  706      stacks.  */
  707   list_del (&pd->list);
  708 
  709   /* Not much to do.  Just free the mmap()ed memory.  Note that we do
  710      not reset the 'used' flag in the 'tid' field.  This is done by
  711      the kernel.  If no thread has been created yet this field is
  712      still zero.  */
  713   if (__builtin_expect (! pd->user_stack, 1))
  714     (void) queue_stack (pd);
  715   else
  716     /* Free the memory associated with the ELF TLS.  */
  717     _dl_deallocate_tls (TLS_TPADJ (pd), false);
  718 
  719   lll_unlock (stack_cache_lock, LLL_PRIVATE);
  720 }
  721 
  722 
  723 int
  724 internal_function
  725 __make_stacks_executable (void **stack_endp)
  726 {
  727   /* First the main thread's stack.  */
  728   int err = _dl_make_stack_executable (stack_endp);
  729   if (err != 0)
  730     return err;
  731 
  732 #ifdef NEED_SEPARATE_REGISTER_STACK
  733   const size_t pagemask = ~(__getpagesize () - 1);
  734 #endif
  735 
  736   lll_lock (stack_cache_lock, LLL_PRIVATE);
  737 
  738   list_t *runp;
  739   list_for_each (runp, &stack_used)
  740     {
  741       err = change_stack_perm (list_entry (runp, struct pthread, list)
  742 #ifdef NEED_SEPARATE_REGISTER_STACK
  743                                , pagemask
  744 #endif
  745                                );
  746       if (err != 0)
  747         break;
  748     }
  749 
  750   /* Also change the permission for the currently unused stacks.  This
  751      might be wasted time but better spend it here than adding a check
  752      in the fast path.  */
  753   if (err == 0)
  754     list_for_each (runp, &stack_cache)
  755       {
  756         err = change_stack_perm (list_entry (runp, struct pthread, list)
  757 #ifdef NEED_SEPARATE_REGISTER_STACK
  758                                  , pagemask
  759 #endif
  760                                  );
  761         if (err != 0)
  762           break;
  763       }
  764 
  765   lll_unlock (stack_cache_lock, LLL_PRIVATE);
  766 
  767   return err;
  768 }
  769 
  770 
  771 /* In case of a fork() call the memory allocation in the child will be
  772    the same but only one thread is running.  All stacks except that of
  773    the one running thread are not used anymore.  We have to recycle
  774    them.  */
  775 void
  776 __reclaim_stacks (void)
  777 {
  778   struct pthread *self = (struct pthread *) THREAD_SELF;
  779 
  780   /* No locking necessary.  The caller is the only stack in use.  */
  781 
  782   /* Mark all stacks except the still running one as free.  */
  783   list_t *runp;
  784   list_for_each (runp, &stack_used)
  785     {
  786       struct pthread *curp = list_entry (runp, struct pthread, list);
  787       if (curp != self)
  788         {
  789           /* This marks the stack as free.  */
  790           curp->tid = 0;
  791 
  792           /* The PID field must be initialized for the new process.  */
  793           curp->pid = self->pid;
  794 
  795           /* Account for the size of the stack.  */
  796           stack_cache_actsize += curp->stackblock_size;
  797 
  798           if (curp->specific_used)
  799             {
  800               /* Clear the thread-specific data.  */
  801               memset (curp->specific_1stblock, '\0',
  802                       sizeof (curp->specific_1stblock));
  803 
  804               curp->specific_used = false;
  805 
  806               for (size_t cnt = 1; cnt < PTHREAD_KEY_1STLEVEL_SIZE; ++cnt)
  807                 if (curp->specific[cnt] != NULL)
  808                   {
  809                     memset (curp->specific[cnt], '\0',
  810                             sizeof (curp->specific_1stblock));
  811 
  812                     /* We have allocated the block which we do not
  813                        free here so re-set the bit.  */
  814                     curp->specific_used = true;
  815                   }
  816             }
  817         }
  818     }
  819 
  820   /* Reset the PIDs in any cached stacks.  */
  821   list_for_each (runp, &stack_cache)
  822     {
  823       struct pthread *curp = list_entry (runp, struct pthread, list);
  824       curp->pid = self->pid;
  825     }
  826 
  827   /* Add the stack of all running threads to the cache.  */
  828   list_splice (&stack_used, &stack_cache);
  829 
  830   /* Remove the entry for the current thread to from the cache list
  831      and add it to the list of running threads.  Which of the two
  832      lists is decided by the user_stack flag.  */
  833   list_del (&self->list);
  834 
  835   /* Re-initialize the lists for all the threads.  */
  836   INIT_LIST_HEAD (&stack_used);
  837   INIT_LIST_HEAD (&__stack_user);
  838 
  839   if (__builtin_expect (THREAD_GETMEM (self, user_stack), 0))
  840     list_add (&self->list, &__stack_user);
  841   else
  842     list_add (&self->list, &stack_used);
  843 
  844   /* There is one thread running.  */
  845   __nptl_nthreads = 1;
  846 
  847   /* Initialize the lock.  */
  848   stack_cache_lock = LLL_LOCK_INITIALIZER;
  849 }
  850 
  851 
  852 #if HP_TIMING_AVAIL
  853 # undef __find_thread_by_id
  854 /* Find a thread given the thread ID.  */
  855 attribute_hidden
  856 struct pthread *
  857 __find_thread_by_id (pid_t tid)
  858 {
  859   struct pthread *result = NULL;
  860 
  861   lll_lock (stack_cache_lock, LLL_PRIVATE);
  862 
  863   /* Iterate over the list with system-allocated threads first.  */
  864   list_t *runp;
  865   list_for_each (runp, &stack_used)
  866     {
  867       struct pthread *curp;
  868 
  869       curp = list_entry (runp, struct pthread, list);
  870 
  871       if (curp->tid == tid)
  872         {
  873           result = curp;
  874           goto out;
  875         }
  876     }
  877 
  878   /* Now the list with threads using user-allocated stacks.  */
  879   list_for_each (runp, &__stack_user)
  880     {
  881       struct pthread *curp;
  882 
  883       curp = list_entry (runp, struct pthread, list);
  884 
  885       if (curp->tid == tid)
  886         {
  887           result = curp;
  888           goto out;
  889         }
  890     }
  891 
  892  out:
  893   lll_unlock (stack_cache_lock, LLL_PRIVATE);
  894 
  895   return result;
  896 }
  897 #endif
  898 
  899 
  900 static void
  901 internal_function
  902 setxid_signal_thread (struct xid_command *cmdp, struct pthread *t)
  903 {
  904   if (! IS_DETACHED (t))
  905     {
  906       int ch;
  907       do
  908         {
  909           ch = t->cancelhandling;
  910 
  911           /* If the thread is exiting right now, ignore it.  */
  912           if ((ch & EXITING_BITMASK) != 0)
  913             return;
  914         }
  915       while (atomic_compare_and_exchange_bool_acq (&t->cancelhandling,
  916                                                    ch | SETXID_BITMASK, ch));
  917     }
  918 
  919   int val;
  920   INTERNAL_SYSCALL_DECL (err);
  921 #if __ASSUME_TGKILL
  922   val = INTERNAL_SYSCALL (tgkill, err, 3, THREAD_GETMEM (THREAD_SELF, pid),
  923                           t->tid, SIGSETXID);
  924 #else
  925 # ifdef __NR_tgkill
  926   val = INTERNAL_SYSCALL (tgkill, err, 3, THREAD_GETMEM (THREAD_SELF, pid),
  927                           t->tid, SIGSETXID);
  928   if (INTERNAL_SYSCALL_ERROR_P (val, err)
  929       && INTERNAL_SYSCALL_ERRNO (val, err) == ENOSYS)
  930 # endif
  931     val = INTERNAL_SYSCALL (tkill, err, 2, t->tid, SIGSETXID);
  932 #endif
  933 
  934   if (!INTERNAL_SYSCALL_ERROR_P (val, err))
  935     atomic_increment (&cmdp->cntr);
  936 }
  937 
  938 
  939 int
  940 attribute_hidden
  941 __nptl_setxid (struct xid_command *cmdp)
  942 {
  943   int result;
  944   lll_lock (stack_cache_lock, LLL_PRIVATE);
  945 
  946   __xidcmd = cmdp;
  947   cmdp->cntr = 0;
  948 
  949   struct pthread *self = THREAD_SELF;
  950 
  951   /* Iterate over the list with system-allocated threads first.  */
  952   list_t *runp;
  953   list_for_each (runp, &stack_used)
  954     {
  955       struct pthread *t = list_entry (runp, struct pthread, list);
  956       if (t == self)
  957         continue;
  958 
  959       setxid_signal_thread (cmdp, t);
  960     }
  961 
  962   /* Now the list with threads using user-allocated stacks.  */
  963   list_for_each (runp, &__stack_user)
  964     {
  965       struct pthread *t = list_entry (runp, struct pthread, list);
  966       if (t == self)
  967         continue;
  968 
  969       setxid_signal_thread (cmdp, t);
  970     }
  971 
  972   int cur = cmdp->cntr;
  973   while (cur != 0)
  974     {
  975       lll_futex_wait (&cmdp->cntr, cur, LLL_PRIVATE);
  976       cur = cmdp->cntr;
  977     }
  978 
  979   /* This must be last, otherwise the current thread might not have
  980      permissions to send SIGSETXID syscall to the other threads.  */
  981   INTERNAL_SYSCALL_DECL (err);
  982   result = INTERNAL_SYSCALL_NCS (cmdp->syscall_no, err, 3,
  983                                  cmdp->id[0], cmdp->id[1], cmdp->id[2]);
  984   if (INTERNAL_SYSCALL_ERROR_P (result, err))
  985     {
  986       __set_errno (INTERNAL_SYSCALL_ERRNO (result, err));
  987       result = -1;
  988     }
  989 
  990   lll_unlock (stack_cache_lock, LLL_PRIVATE);
  991   return result;
  992 }
  993 
  994 static inline void __attribute__((always_inline))
  995 init_one_static_tls (struct pthread *curp, struct link_map *map)
  996 {
  997   dtv_t *dtv = GET_DTV (TLS_TPADJ (curp));
  998 # if TLS_TCB_AT_TP
  999   void *dest = (char *) curp - map->l_tls_offset;
 1000 # elif TLS_DTV_AT_TP
 1001   void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
 1002 # else
 1003 #  error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
 1004 # endif
 1005 
 1006   /* Fill in the DTV slot so that a later LD/GD access will find it.  */
 1007   dtv[map->l_tls_modid].pointer.val = dest;
 1008   dtv[map->l_tls_modid].pointer.is_static = true;
 1009 
 1010   /* Initialize the memory.  */
 1011   memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
 1012           '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
 1013 }
 1014 
 1015 void
 1016 attribute_hidden
 1017 __pthread_init_static_tls (struct link_map *map)
 1018 {
 1019   lll_lock (stack_cache_lock, LLL_PRIVATE);
 1020 
 1021   /* Iterate over the list with system-allocated threads first.  */
 1022   list_t *runp;
 1023   list_for_each (runp, &stack_used)
 1024     init_one_static_tls (list_entry (runp, struct pthread, list), map);
 1025 
 1026   /* Now the list with threads using user-allocated stacks.  */
 1027   list_for_each (runp, &__stack_user)
 1028     init_one_static_tls (list_entry (runp, struct pthread, list), map);
 1029 
 1030   lll_unlock (stack_cache_lock, LLL_PRIVATE);
 1031 }
 1032 
 1033 
 1034 void
 1035 attribute_hidden
 1036 __wait_lookup_done (void)
 1037 {
 1038   lll_lock (stack_cache_lock, LLL_PRIVATE);
 1039 
 1040   struct pthread *self = THREAD_SELF;
 1041 
 1042   /* Iterate over the list with system-allocated threads first.  */
 1043   list_t *runp;
 1044   list_for_each (runp, &stack_used)
 1045     {
 1046       struct pthread *t = list_entry (runp, struct pthread, list);
 1047       if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
 1048         continue;
 1049 
 1050       int *const gscope_flagp = &t->header.gscope_flag;
 1051 
 1052       /* We have to wait until this thread is done with the global
 1053          scope.  First tell the thread that we are waiting and
 1054          possibly have to be woken.  */
 1055       if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
 1056                                                 THREAD_GSCOPE_FLAG_WAIT,
 1057                                                 THREAD_GSCOPE_FLAG_USED))
 1058         continue;
 1059 
 1060       do
 1061         lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE);
 1062       while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
 1063     }
 1064 
 1065   /* Now the list with threads using user-allocated stacks.  */
 1066   list_for_each (runp, &__stack_user)
 1067     {
 1068       struct pthread *t = list_entry (runp, struct pthread, list);
 1069       if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
 1070         continue;
 1071 
 1072       int *const gscope_flagp = &t->header.gscope_flag;
 1073 
 1074       /* We have to wait until this thread is done with the global
 1075          scope.  First tell the thread that we are waiting and
 1076          possibly have to be woken.  */
 1077       if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
 1078                                                 THREAD_GSCOPE_FLAG_WAIT,
 1079                                                 THREAD_GSCOPE_FLAG_USED))
 1080         continue;
 1081 
 1082       do
 1083         lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE);
 1084       while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
 1085     }
 1086 
 1087   lll_unlock (stack_cache_lock, LLL_PRIVATE);
 1088 }

Cache object: f32fb1ff7b67a28340debd1a28fc4df1


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.