jfs_metapage.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*
    2  *   Copyright (c) International Business Machines Corp., 2000-2002
    3  *   Portions Copyright (c) Christoph Hellwig, 2001-2002
    4  *
    5  *   This program is free software;  you can redistribute it and/or modify
    6  *   it under the terms of the GNU General Public License as published by
    7  *   the Free Software Foundation; either version 2 of the License, or 
    8  *   (at your option) any later version.
    9  * 
   10  *   This program is distributed in the hope that it will be useful,
   11  *   but WITHOUT ANY WARRANTY;  without even the implied warranty of
   12  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See
   13  *   the GNU General Public License for more details.
   14  *
   15  *   You should have received a copy of the GNU General Public License
   16  *   along with this program;  if not, write to the Free Software 
   17  *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
   18  */
   19 
   20 #include <linux/fs.h>
   21 #include <linux/init.h>
   22 #include "jfs_incore.h"
   23 #include "jfs_filsys.h"
   24 #include "jfs_metapage.h"
   25 #include "jfs_txnmgr.h"
   26 #include "jfs_debug.h"
   27 
   28 extern struct task_struct *jfsCommitTask;
   29 static spinlock_t meta_lock = SPIN_LOCK_UNLOCKED;
   30 static wait_queue_head_t meta_wait;
   31 
   32 #ifdef CONFIG_JFS_STATISTICS
   33 struct {
   34         uint    pagealloc;      /* # of page allocations */
   35         uint    pagefree;       /* # of page frees */
   36         uint    lockwait;       /* # of sleeping lock_metapage() calls */
   37         uint    allocwait;      /* # of sleeping alloc_metapage() calls */
   38 } mpStat;
   39 #endif
   40 
   41 
   42 #define HASH_BITS 10            /* This makes hash_table 1 4K page */
   43 #define HASH_SIZE (1 << HASH_BITS)
   44 static struct metapage **hash_table = NULL;
   45 static unsigned long hash_order;
   46 
   47 
   48 static inline int metapage_locked(struct metapage *mp)
   49 {
   50         return test_bit(META_locked, &mp->flag);
   51 }
   52 
   53 static inline int trylock_metapage(struct metapage *mp)
   54 {
   55         return test_and_set_bit(META_locked, &mp->flag);
   56 }
   57 
   58 static inline void unlock_metapage(struct metapage *mp)
   59 {
   60         clear_bit(META_locked, &mp->flag);
   61         wake_up(&mp->wait);
   62 }
   63 
   64 static void __lock_metapage(struct metapage *mp)
   65 {
   66         DECLARE_WAITQUEUE(wait, current);
   67 
   68         INCREMENT(mpStat.lockwait);
   69 
   70         add_wait_queue_exclusive(&mp->wait, &wait);
   71         do {
   72                 set_current_state(TASK_UNINTERRUPTIBLE);
   73                 if (metapage_locked(mp)) {
   74                         spin_unlock(&meta_lock);
   75                         schedule();
   76                         spin_lock(&meta_lock);
   77                 }
   78         } while (trylock_metapage(mp));
   79         __set_current_state(TASK_RUNNING);
   80         remove_wait_queue(&mp->wait, &wait);
   81 }
   82 
   83 /* needs meta_lock */
   84 static inline void lock_metapage(struct metapage *mp)
   85 {
   86         if (trylock_metapage(mp))
   87                 __lock_metapage(mp);
   88 }
   89 
   90 /*
   91  * metapage pool is based on Linux 2.5's mempool
   92  *
   93  * Tap into reserved structures in critical paths where waiting on a
   94  * memory allocation could cause deadlock
   95  */
   96 #define METAPOOL_MIN_PAGES 32
   97 static struct metapage *reserved_metapages[METAPOOL_MIN_PAGES];
   98 static int num_reserved = 0;
   99 kmem_cache_t *metapage_cache;
  100 
  101 static void init_once(void *foo, kmem_cache_t *cachep, unsigned long flags)
  102 {
  103         struct metapage *mp = (struct metapage *)foo;
  104 
  105         if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
  106             SLAB_CTOR_CONSTRUCTOR) {
  107                 mp->lid = 0;
  108                 mp->lsn = 0;
  109                 mp->flag = 0;
  110                 mp->data = NULL;
  111                 mp->clsn = 0;
  112                 mp->log = NULL;
  113                 set_bit(META_free, &mp->flag);
  114                 init_waitqueue_head(&mp->wait);
  115         }
  116 }
  117 
  118 static void empty_reserved(void)
  119 {
  120         while (num_reserved--)
  121                 kmem_cache_free(metapage_cache,
  122                                 reserved_metapages[num_reserved]);
  123 }
  124 
  125 static struct metapage *alloc_metapage(int *dropped_lock, int no_wait)
  126 {
  127         struct metapage *new;
  128 
  129         *dropped_lock = 0;
  130 
  131         /*
  132          * Always try an atomic alloc first, to avoid dropping the
  133          * spinlock
  134          */
  135         new = kmem_cache_alloc(metapage_cache, GFP_ATOMIC);
  136         if (new)
  137                 return new;
  138 
  139         if (no_wait && num_reserved)
  140                 return reserved_metapages[--num_reserved];
  141 
  142         *dropped_lock = 1;
  143         spin_unlock(&meta_lock);
  144         new = kmem_cache_alloc(metapage_cache, GFP_NOFS);
  145         spin_lock(&meta_lock);
  146         return new;
  147 }
  148 
  149 static void __free_metapage(struct metapage *mp)
  150 {
  151         mp->flag = 0;
  152         set_bit(META_free, &mp->flag);
  153 
  154         if (num_reserved < METAPOOL_MIN_PAGES)
  155                 reserved_metapages[num_reserved++] = mp;
  156         else
  157                 kmem_cache_free(metapage_cache, mp);
  158 }
  159 
  160 static inline void free_metapage(struct metapage * mp)
  161 {
  162         spin_lock(&meta_lock);
  163         __free_metapage(mp);
  164         spin_unlock(&meta_lock);
  165 }
  166 
  167 int __init metapage_init(void)
  168 {
  169         struct metapage *mp;
  170 
  171         /*
  172          * Initialize wait queue
  173          */
  174         init_waitqueue_head(&meta_wait);
  175 
  176         /*
  177          * Allocate the metapage structures
  178          */
  179         metapage_cache = kmem_cache_create("jfs_mp", sizeof(struct metapage),
  180                                            0, 0, init_once, NULL);
  181         if (metapage_cache == NULL)
  182                 return -ENOMEM;
  183 
  184         while (num_reserved < METAPOOL_MIN_PAGES) {
  185                 mp = kmem_cache_alloc(metapage_cache, GFP_NOFS);
  186                 if (mp)
  187                         reserved_metapages[num_reserved++] = mp;
  188                 else {
  189                         empty_reserved();
  190                         kmem_cache_destroy(metapage_cache);
  191                         return -ENOMEM;
  192                 }
  193         }
  194         /*
  195          * Now the hash list
  196          */
  197         for (hash_order = 0;
  198              ((PAGE_SIZE << hash_order) / sizeof(void *)) < HASH_SIZE;
  199              hash_order++);
  200         hash_table =
  201             (struct metapage **) __get_free_pages(GFP_KERNEL, hash_order);
  202         assert(hash_table);
  203         memset(hash_table, 0, PAGE_SIZE << hash_order);
  204 
  205         return 0;
  206 }
  207 
  208 void metapage_exit(void)
  209 {
  210         empty_reserved();
  211         kmem_cache_destroy(metapage_cache);
  212 }
  213 
  214 /*
  215  * Basically same hash as in pagemap.h, but using our hash table
  216  */
  217 static struct metapage **meta_hash(struct address_space *mapping,
  218                                    unsigned long index)
  219 {
  220 #define i (((unsigned long)mapping)/ \
  221            (sizeof(struct inode) & ~(sizeof(struct inode) -1 )))
  222 #define s(x) ((x) + ((x) >> HASH_BITS))
  223         return hash_table + (s(i + index) & (HASH_SIZE - 1));
  224 #undef i
  225 #undef s
  226 }
  227 
  228 static struct metapage *search_hash(struct metapage ** hash_ptr,
  229                                     struct address_space *mapping,
  230                                unsigned long index)
  231 {
  232         struct metapage *ptr;
  233 
  234         for (ptr = *hash_ptr; ptr; ptr = ptr->hash_next) {
  235                 if ((ptr->mapping == mapping) && (ptr->index == index))
  236                         return ptr;
  237         }
  238 
  239         return NULL;
  240 }
  241 
  242 static void add_to_hash(struct metapage * mp, struct metapage ** hash_ptr)
  243 {
  244         if (*hash_ptr)
  245                 (*hash_ptr)->hash_prev = mp;
  246 
  247         mp->hash_prev = NULL;
  248         mp->hash_next = *hash_ptr;
  249         *hash_ptr = mp;
  250 }
  251 
  252 static void remove_from_hash(struct metapage * mp, struct metapage ** hash_ptr)
  253 {
  254         if (mp->hash_prev)
  255                 mp->hash_prev->hash_next = mp->hash_next;
  256         else {
  257                 assert(*hash_ptr == mp);
  258                 *hash_ptr = mp->hash_next;
  259         }
  260 
  261         if (mp->hash_next)
  262                 mp->hash_next->hash_prev = mp->hash_prev;
  263 }
  264 
  265 struct metapage *__get_metapage(struct inode *inode, unsigned long lblock,
  266                                 unsigned int size, int absolute,
  267                                 unsigned long new)
  268 {
  269         int dropped_lock;
  270         struct metapage **hash_ptr;
  271         int l2BlocksPerPage;
  272         int l2bsize;
  273         int no_wait;
  274         struct address_space *mapping;
  275         struct metapage *mp;
  276         unsigned long page_index;
  277         unsigned long page_offset;
  278 
  279         jfs_info("__get_metapage: inode = 0x%p, lblock = 0x%lx", inode, lblock);
  280 
  281         if (absolute)
  282                 mapping = inode->i_sb->s_bdev->bd_inode->i_mapping;
  283         else
  284                 mapping = inode->i_mapping;
  285 
  286         spin_lock(&meta_lock);
  287 
  288         hash_ptr = meta_hash(mapping, lblock);
  289 
  290         mp = search_hash(hash_ptr, mapping, lblock);
  291         if (mp) {
  292               page_found:
  293                 if (test_bit(META_discard, &mp->flag)) {
  294                         assert(new);    /* It's okay to reuse a discarded
  295                                          * if we expect it to be empty
  296                                          */
  297                         clear_bit(META_discard, &mp->flag);
  298                 }
  299                 mp->count++;
  300                 jfs_info("__get_metapage: found 0x%p, in hash", mp);
  301                 assert(mp->logical_size == size);
  302                 lock_metapage(mp);
  303                 spin_unlock(&meta_lock);
  304         } else {
  305                 l2bsize = inode->i_blkbits;
  306                 l2BlocksPerPage = PAGE_CACHE_SHIFT - l2bsize;
  307                 page_index = lblock >> l2BlocksPerPage;
  308                 page_offset = (lblock - (page_index << l2BlocksPerPage)) <<
  309                     l2bsize;
  310                 if ((page_offset + size) > PAGE_CACHE_SIZE) {
  311                         spin_unlock(&meta_lock);
  312                         jfs_err("MetaData crosses page boundary!!");
  313                         return NULL;
  314                 }
  315                 
  316                 /*
  317                  * Locks held on aggregate inode pages are usually
  318                  * not held long, and they are taken in critical code
  319                  * paths (committing dirty inodes, txCommit thread) 
  320                  * 
  321                  * Attempt to get metapage without blocking, tapping into
  322                  * reserves if necessary.
  323                  */
  324                 if (JFS_IP(inode)->fileset == AGGREGATE_I)
  325                         no_wait = 1;
  326                 else
  327                         no_wait = 0;
  328 
  329                 mp = alloc_metapage(&dropped_lock, no_wait);
  330                 if (dropped_lock) {
  331                         /* alloc_metapage blocked, we need to search the hash
  332                          * again.
  333                          */
  334                         struct metapage *mp2;
  335                         mp2 = search_hash(hash_ptr, mapping, lblock);
  336                         if (mp2) {
  337                                 __free_metapage(mp);
  338                                 mp = mp2;
  339                                 goto page_found;
  340                         }
  341                 }
  342                 mp->flag = 0;
  343                 lock_metapage(mp);
  344                 if (absolute)
  345                         set_bit(META_absolute, &mp->flag);
  346                 mp->xflag = COMMIT_PAGE;
  347                 mp->count = 1;
  348                 atomic_set(&mp->nohomeok,0);
  349                 mp->mapping = mapping;
  350                 mp->index = lblock;
  351                 mp->page = 0;
  352                 mp->logical_size = size;
  353                 add_to_hash(mp, hash_ptr);
  354                 spin_unlock(&meta_lock);
  355 
  356                 if (new) {
  357                         jfs_info("__get_metapage: Calling grab_cache_page");
  358                         mp->page = grab_cache_page(mapping, page_index);
  359                         if (!mp->page) {
  360                                 jfs_err("grab_cache_page failed!");
  361                                 goto freeit;
  362                         } else {
  363                                 INCREMENT(mpStat.pagealloc);
  364                                 UnlockPage(mp->page);
  365                         }
  366                 } else {
  367                         jfs_info("__get_metapage: Calling read_cache_page");
  368                         mp->page = read_cache_page(mapping, lblock,
  369                                     (filler_t *)mapping->a_ops->readpage, NULL);
  370                         if (IS_ERR(mp->page)) {
  371                                 jfs_err("read_cache_page failed!");
  372                                 goto freeit;
  373                         } else
  374                                 INCREMENT(mpStat.pagealloc);
  375                 }
  376                 mp->data = kmap(mp->page) + page_offset;
  377         }
  378         jfs_info("__get_metapage: returning = 0x%p", mp);
  379         return mp;
  380 
  381 freeit:
  382         spin_lock(&meta_lock);
  383         remove_from_hash(mp, hash_ptr);
  384         __free_metapage(mp);
  385         spin_unlock(&meta_lock);
  386         return NULL;
  387 }
  388 
  389 void hold_metapage(struct metapage * mp, int force)
  390 {
  391         spin_lock(&meta_lock);
  392 
  393         mp->count++;
  394 
  395         if (force) {
  396                 ASSERT (!(test_bit(META_forced, &mp->flag)));
  397                 if (trylock_metapage(mp))
  398                         set_bit(META_forced, &mp->flag);
  399         } else
  400                 lock_metapage(mp);
  401 
  402         spin_unlock(&meta_lock);
  403 }
  404 
  405 static void __write_metapage(struct metapage * mp)
  406 {
  407         int l2bsize = mp->mapping->host->i_blkbits;
  408         int l2BlocksPerPage = PAGE_CACHE_SHIFT - l2bsize;
  409         unsigned long page_index;
  410         unsigned long page_offset;
  411         int rc;
  412 
  413         jfs_info("__write_metapage: mp = 0x%p", mp);
  414 
  415         if (test_bit(META_discard, &mp->flag)) {
  416                 /*
  417                  * This metadata is no longer valid
  418                  */
  419                 clear_bit(META_dirty, &mp->flag);
  420                 return;
  421         }
  422 
  423         page_index = mp->page->index;
  424         page_offset =
  425             (mp->index - (page_index << l2BlocksPerPage)) << l2bsize;
  426 
  427         lock_page(mp->page);
  428         rc = mp->mapping->a_ops->prepare_write(NULL, mp->page, page_offset,
  429                                                page_offset +
  430                                                mp->logical_size);
  431         if (rc) {
  432                 jfs_err("prepare_write return %d!", rc);
  433                 ClearPageUptodate(mp->page);
  434                 UnlockPage(mp->page);
  435                 kunmap(mp->page);
  436                 clear_bit(META_dirty, &mp->flag);
  437                 return;
  438         }
  439         rc = mp->mapping->a_ops->commit_write(NULL, mp->page, page_offset,
  440                                               page_offset +
  441                                               mp->logical_size);
  442         if (rc) {
  443                 jfs_err("commit_write returned %d", rc);
  444         }
  445 
  446         UnlockPage(mp->page);
  447         clear_bit(META_dirty, &mp->flag);
  448 
  449         jfs_info("__write_metapage done");
  450 }
  451 
  452 static inline void sync_metapage(struct metapage *mp)
  453 {
  454         struct page *page = mp->page;
  455 
  456         page_cache_get(page);
  457         lock_page(page);
  458 
  459         /* we're done with this page - no need to check for errors */
  460         if (page->buffers) {
  461                 writeout_one_page(page);
  462                 waitfor_one_page(page);
  463         }
  464 
  465         UnlockPage(page);
  466         page_cache_release(page);
  467 }
  468 
  469 void release_metapage(struct metapage * mp)
  470 {
  471         struct jfs_log *log;
  472 
  473         jfs_info("release_metapage: mp = 0x%p, flag = 0x%lx", mp, mp->flag);
  474 
  475         spin_lock(&meta_lock);
  476         if (test_bit(META_forced, &mp->flag)) {
  477                 clear_bit(META_forced, &mp->flag);
  478                 mp->count--;
  479                 spin_unlock(&meta_lock);
  480                 return;
  481         }
  482 
  483         assert(mp->count);
  484         if (--mp->count || atomic_read(&mp->nohomeok)) {
  485                 unlock_metapage(mp);
  486                 spin_unlock(&meta_lock);
  487         } else {
  488                 remove_from_hash(mp, meta_hash(mp->mapping, mp->index));
  489                 spin_unlock(&meta_lock);
  490 
  491                 if (mp->page) {
  492                         kunmap(mp->page);
  493                         mp->data = 0;
  494                         if (test_bit(META_dirty, &mp->flag))
  495                                 __write_metapage(mp);
  496                         if (test_bit(META_sync, &mp->flag)) {
  497                                 sync_metapage(mp);
  498                                 clear_bit(META_sync, &mp->flag);
  499                         }
  500 
  501                         if (test_bit(META_discard, &mp->flag)) {
  502                                 lock_page(mp->page);
  503                                 block_flushpage(mp->page, 0);
  504                                 UnlockPage(mp->page);
  505                         }
  506 
  507                         page_cache_release(mp->page);
  508                         INCREMENT(mpStat.pagefree);
  509                 }
  510 
  511                 if (mp->lsn) {
  512                         /*
  513                          * Remove metapage from logsynclist.
  514                          */
  515                         log = mp->log;
  516                         LOGSYNC_LOCK(log);
  517                         mp->log = 0;
  518                         mp->lsn = 0;
  519                         mp->clsn = 0;
  520                         log->count--;
  521                         list_del(&mp->synclist);
  522                         LOGSYNC_UNLOCK(log);
  523                 }
  524 
  525                 free_metapage(mp);
  526         }
  527 }
  528 
  529 void __invalidate_metapages(struct inode *ip, s64 addr, int len)
  530 {
  531         struct metapage **hash_ptr;
  532         unsigned long lblock;
  533         int l2BlocksPerPage = PAGE_CACHE_SHIFT - ip->i_blkbits;
  534         /* All callers are interested in block device's mapping */
  535         struct address_space *mapping = ip->i_sb->s_bdev->bd_inode->i_mapping;
  536         struct metapage *mp;
  537         struct page *page;
  538 
  539         /*
  540          * First, mark metapages to discard.  They will eventually be
  541          * released, but should not be written.
  542          */
  543         for (lblock = addr; lblock < addr + len;
  544              lblock += 1 << l2BlocksPerPage) {
  545                 hash_ptr = meta_hash(mapping, lblock);
  546                 spin_lock(&meta_lock);
  547                 mp = search_hash(hash_ptr, mapping, lblock);
  548                 if (mp) {
  549                         set_bit(META_discard, &mp->flag);
  550                         spin_unlock(&meta_lock);
  551                         lock_page(mp->page);
  552                         block_flushpage(mp->page, 0);
  553                         UnlockPage(mp->page);
  554                 } else {
  555                         spin_unlock(&meta_lock);
  556                         page = find_lock_page(mapping, lblock>>l2BlocksPerPage);
  557                         if (page) {
  558                                 block_flushpage(page, 0);
  559                                 UnlockPage(page);
  560                         }
  561                 }
  562         }
  563 }
  564 
  565 #ifdef CONFIG_JFS_STATISTICS
  566 int jfs_mpstat_read(char *buffer, char **start, off_t offset, int length,
  567                     int *eof, void *data)
  568 {
  569         int len = 0;
  570         off_t begin;
  571 
  572         len += sprintf(buffer,
  573                        "JFS Metapage statistics\n"
  574                        "=======================\n"
  575                        "page allocations = %d\n"
  576                        "page frees = %d\n"
  577                        "lock waits = %d\n"
  578                        "allocation waits = %d\n",
  579                        mpStat.pagealloc,
  580                        mpStat.pagefree,
  581                        mpStat.lockwait,
  582                        mpStat.allocwait);
  583 
  584         begin = offset;
  585         *start = buffer + begin;
  586         len -= begin;
  587 
  588         if (len > length)
  589                 len = length;
  590         else
  591                 *eof = 1;
  592 
  593         if (len < 0)
  594                 len = 0;
  595 
  596         return len;
  597 }
  598 #endif
Cache object: 73542c14f897e82489d28f56beea6986
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/fs/jfs/jfs_metapage.c

FreeBSD/Linux Kernel Cross Reference
sys/fs/jfs/jfs_metapage.c
