The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kernel/wait.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  * Generic waiting primitives.
    3  *
    4  * (C) 2004 Nadia Yvette Chambers, Oracle
    5  */
    6 #include <linux/init.h>
    7 #include <linux/export.h>
    8 #include <linux/sched.h>
    9 #include <linux/mm.h>
   10 #include <linux/wait.h>
   11 #include <linux/hash.h>
   12 
   13 void __init_waitqueue_head(wait_queue_head_t *q, const char *name, struct lock_class_key *key)
   14 {
   15         spin_lock_init(&q->lock);
   16         lockdep_set_class_and_name(&q->lock, key, name);
   17         INIT_LIST_HEAD(&q->task_list);
   18 }
   19 
   20 EXPORT_SYMBOL(__init_waitqueue_head);
   21 
   22 void add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait)
   23 {
   24         unsigned long flags;
   25 
   26         wait->flags &= ~WQ_FLAG_EXCLUSIVE;
   27         spin_lock_irqsave(&q->lock, flags);
   28         __add_wait_queue(q, wait);
   29         spin_unlock_irqrestore(&q->lock, flags);
   30 }
   31 EXPORT_SYMBOL(add_wait_queue);
   32 
   33 void add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait)
   34 {
   35         unsigned long flags;
   36 
   37         wait->flags |= WQ_FLAG_EXCLUSIVE;
   38         spin_lock_irqsave(&q->lock, flags);
   39         __add_wait_queue_tail(q, wait);
   40         spin_unlock_irqrestore(&q->lock, flags);
   41 }
   42 EXPORT_SYMBOL(add_wait_queue_exclusive);
   43 
   44 void remove_wait_queue(wait_queue_head_t *q, wait_queue_t *wait)
   45 {
   46         unsigned long flags;
   47 
   48         spin_lock_irqsave(&q->lock, flags);
   49         __remove_wait_queue(q, wait);
   50         spin_unlock_irqrestore(&q->lock, flags);
   51 }
   52 EXPORT_SYMBOL(remove_wait_queue);
   53 
   54 
   55 /*
   56  * Note: we use "set_current_state()" _after_ the wait-queue add,
   57  * because we need a memory barrier there on SMP, so that any
   58  * wake-function that tests for the wait-queue being active
   59  * will be guaranteed to see waitqueue addition _or_ subsequent
   60  * tests in this thread will see the wakeup having taken place.
   61  *
   62  * The spin_unlock() itself is semi-permeable and only protects
   63  * one way (it only protects stuff inside the critical region and
   64  * stops them from bleeding out - it would still allow subsequent
   65  * loads to move into the critical region).
   66  */
   67 void
   68 prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state)
   69 {
   70         unsigned long flags;
   71 
   72         wait->flags &= ~WQ_FLAG_EXCLUSIVE;
   73         spin_lock_irqsave(&q->lock, flags);
   74         if (list_empty(&wait->task_list))
   75                 __add_wait_queue(q, wait);
   76         set_current_state(state);
   77         spin_unlock_irqrestore(&q->lock, flags);
   78 }
   79 EXPORT_SYMBOL(prepare_to_wait);
   80 
   81 void
   82 prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state)
   83 {
   84         unsigned long flags;
   85 
   86         wait->flags |= WQ_FLAG_EXCLUSIVE;
   87         spin_lock_irqsave(&q->lock, flags);
   88         if (list_empty(&wait->task_list))
   89                 __add_wait_queue_tail(q, wait);
   90         set_current_state(state);
   91         spin_unlock_irqrestore(&q->lock, flags);
   92 }
   93 EXPORT_SYMBOL(prepare_to_wait_exclusive);
   94 
   95 /**
   96  * finish_wait - clean up after waiting in a queue
   97  * @q: waitqueue waited on
   98  * @wait: wait descriptor
   99  *
  100  * Sets current thread back to running state and removes
  101  * the wait descriptor from the given waitqueue if still
  102  * queued.
  103  */
  104 void finish_wait(wait_queue_head_t *q, wait_queue_t *wait)
  105 {
  106         unsigned long flags;
  107 
  108         __set_current_state(TASK_RUNNING);
  109         /*
  110          * We can check for list emptiness outside the lock
  111          * IFF:
  112          *  - we use the "careful" check that verifies both
  113          *    the next and prev pointers, so that there cannot
  114          *    be any half-pending updates in progress on other
  115          *    CPU's that we haven't seen yet (and that might
  116          *    still change the stack area.
  117          * and
  118          *  - all other users take the lock (ie we can only
  119          *    have _one_ other CPU that looks at or modifies
  120          *    the list).
  121          */
  122         if (!list_empty_careful(&wait->task_list)) {
  123                 spin_lock_irqsave(&q->lock, flags);
  124                 list_del_init(&wait->task_list);
  125                 spin_unlock_irqrestore(&q->lock, flags);
  126         }
  127 }
  128 EXPORT_SYMBOL(finish_wait);
  129 
  130 /**
  131  * abort_exclusive_wait - abort exclusive waiting in a queue
  132  * @q: waitqueue waited on
  133  * @wait: wait descriptor
  134  * @mode: runstate of the waiter to be woken
  135  * @key: key to identify a wait bit queue or %NULL
  136  *
  137  * Sets current thread back to running state and removes
  138  * the wait descriptor from the given waitqueue if still
  139  * queued.
  140  *
  141  * Wakes up the next waiter if the caller is concurrently
  142  * woken up through the queue.
  143  *
  144  * This prevents waiter starvation where an exclusive waiter
  145  * aborts and is woken up concurrently and no one wakes up
  146  * the next waiter.
  147  */
  148 void abort_exclusive_wait(wait_queue_head_t *q, wait_queue_t *wait,
  149                         unsigned int mode, void *key)
  150 {
  151         unsigned long flags;
  152 
  153         __set_current_state(TASK_RUNNING);
  154         spin_lock_irqsave(&q->lock, flags);
  155         if (!list_empty(&wait->task_list))
  156                 list_del_init(&wait->task_list);
  157         else if (waitqueue_active(q))
  158                 __wake_up_locked_key(q, mode, key);
  159         spin_unlock_irqrestore(&q->lock, flags);
  160 }
  161 EXPORT_SYMBOL(abort_exclusive_wait);
  162 
  163 int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key)
  164 {
  165         int ret = default_wake_function(wait, mode, sync, key);
  166 
  167         if (ret)
  168                 list_del_init(&wait->task_list);
  169         return ret;
  170 }
  171 EXPORT_SYMBOL(autoremove_wake_function);
  172 
  173 int wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *arg)
  174 {
  175         struct wait_bit_key *key = arg;
  176         struct wait_bit_queue *wait_bit
  177                 = container_of(wait, struct wait_bit_queue, wait);
  178 
  179         if (wait_bit->key.flags != key->flags ||
  180                         wait_bit->key.bit_nr != key->bit_nr ||
  181                         test_bit(key->bit_nr, key->flags))
  182                 return 0;
  183         else
  184                 return autoremove_wake_function(wait, mode, sync, key);
  185 }
  186 EXPORT_SYMBOL(wake_bit_function);
  187 
  188 /*
  189  * To allow interruptible waiting and asynchronous (i.e. nonblocking)
  190  * waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are
  191  * permitted return codes. Nonzero return codes halt waiting and return.
  192  */
  193 int __sched
  194 __wait_on_bit(wait_queue_head_t *wq, struct wait_bit_queue *q,
  195                         int (*action)(void *), unsigned mode)
  196 {
  197         int ret = 0;
  198 
  199         do {
  200                 prepare_to_wait(wq, &q->wait, mode);
  201                 if (test_bit(q->key.bit_nr, q->key.flags))
  202                         ret = (*action)(q->key.flags);
  203         } while (test_bit(q->key.bit_nr, q->key.flags) && !ret);
  204         finish_wait(wq, &q->wait);
  205         return ret;
  206 }
  207 EXPORT_SYMBOL(__wait_on_bit);
  208 
  209 int __sched out_of_line_wait_on_bit(void *word, int bit,
  210                                         int (*action)(void *), unsigned mode)
  211 {
  212         wait_queue_head_t *wq = bit_waitqueue(word, bit);
  213         DEFINE_WAIT_BIT(wait, word, bit);
  214 
  215         return __wait_on_bit(wq, &wait, action, mode);
  216 }
  217 EXPORT_SYMBOL(out_of_line_wait_on_bit);
  218 
  219 int __sched
  220 __wait_on_bit_lock(wait_queue_head_t *wq, struct wait_bit_queue *q,
  221                         int (*action)(void *), unsigned mode)
  222 {
  223         do {
  224                 int ret;
  225 
  226                 prepare_to_wait_exclusive(wq, &q->wait, mode);
  227                 if (!test_bit(q->key.bit_nr, q->key.flags))
  228                         continue;
  229                 ret = action(q->key.flags);
  230                 if (!ret)
  231                         continue;
  232                 abort_exclusive_wait(wq, &q->wait, mode, &q->key);
  233                 return ret;
  234         } while (test_and_set_bit(q->key.bit_nr, q->key.flags));
  235         finish_wait(wq, &q->wait);
  236         return 0;
  237 }
  238 EXPORT_SYMBOL(__wait_on_bit_lock);
  239 
  240 int __sched out_of_line_wait_on_bit_lock(void *word, int bit,
  241                                         int (*action)(void *), unsigned mode)
  242 {
  243         wait_queue_head_t *wq = bit_waitqueue(word, bit);
  244         DEFINE_WAIT_BIT(wait, word, bit);
  245 
  246         return __wait_on_bit_lock(wq, &wait, action, mode);
  247 }
  248 EXPORT_SYMBOL(out_of_line_wait_on_bit_lock);
  249 
  250 void __wake_up_bit(wait_queue_head_t *wq, void *word, int bit)
  251 {
  252         struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit);
  253         if (waitqueue_active(wq))
  254                 __wake_up(wq, TASK_NORMAL, 1, &key);
  255 }
  256 EXPORT_SYMBOL(__wake_up_bit);
  257 
  258 /**
  259  * wake_up_bit - wake up a waiter on a bit
  260  * @word: the word being waited on, a kernel virtual address
  261  * @bit: the bit of the word being waited on
  262  *
  263  * There is a standard hashed waitqueue table for generic use. This
  264  * is the part of the hashtable's accessor API that wakes up waiters
  265  * on a bit. For instance, if one were to have waiters on a bitflag,
  266  * one would call wake_up_bit() after clearing the bit.
  267  *
  268  * In order for this to function properly, as it uses waitqueue_active()
  269  * internally, some kind of memory barrier must be done prior to calling
  270  * this. Typically, this will be smp_mb__after_clear_bit(), but in some
  271  * cases where bitflags are manipulated non-atomically under a lock, one
  272  * may need to use a less regular barrier, such fs/inode.c's smp_mb(),
  273  * because spin_unlock() does not guarantee a memory barrier.
  274  */
  275 void wake_up_bit(void *word, int bit)
  276 {
  277         __wake_up_bit(bit_waitqueue(word, bit), word, bit);
  278 }
  279 EXPORT_SYMBOL(wake_up_bit);
  280 
  281 wait_queue_head_t *bit_waitqueue(void *word, int bit)
  282 {
  283         const int shift = BITS_PER_LONG == 32 ? 5 : 6;
  284         const struct zone *zone = page_zone(virt_to_page(word));
  285         unsigned long val = (unsigned long)word << shift | bit;
  286 
  287         return &zone->wait_table[hash_long(val, zone->wait_table_bits)];
  288 }
  289 EXPORT_SYMBOL(bit_waitqueue);

Cache object: 534a615e7b0fc7a3e7606158a8f2ced9


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.