The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/iokit/Kernel/IOHibernateRestoreKernel.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
    3  *
    4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
    5  * 
    6  * This file contains Original Code and/or Modifications of Original Code
    7  * as defined in and that are subject to the Apple Public Source License
    8  * Version 2.0 (the 'License'). You may not use this file except in
    9  * compliance with the License. The rights granted to you under the License
   10  * may not be used to create, or enable the creation or redistribution of,
   11  * unlawful or unlicensed copies of an Apple operating system, or to
   12  * circumvent, violate, or enable the circumvention or violation of, any
   13  * terms of an Apple operating system software license agreement.
   14  * 
   15  * Please obtain a copy of the License at
   16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
   17  * 
   18  * The Original Code and all software distributed under the License are
   19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
   20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
   21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
   22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
   23  * Please see the License for the specific language governing rights and
   24  * limitations under the License.
   25  * 
   26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
   27  */
   28 
   29 #include <stdint.h>
   30 #include <mach/mach_types.h>
   31 #include <mach/vm_param.h>
   32 #include <IOKit/IOHibernatePrivate.h>
   33 #include <IOKit/IOLib.h>
   34 #include <pexpert/boot.h>
   35 #include <crypto/aes.h>
   36 #include <libkern/libkern.h>
   37 
   38 #include "WKdm.h"
   39 #include "IOHibernateInternal.h"
   40 
   41 /*
   42 This code is linked into the kernel but part of the "__HIB" section, which means
   43 its used by code running in the special context of restoring the kernel text and data
   44 from the hibernation image read by the booter. hibernate_kernel_entrypoint() and everything
   45 it calls or references needs to be careful to only touch memory also in the "__HIB" section.
   46 */
   47 
   48 uint32_t gIOHibernateState;
   49 
   50 static IOHibernateImageHeader _hibernateHeader;
   51 IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader;
   52 
   53 static hibernate_graphics_t _hibernateGraphics;
   54 hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
   55 
   56 static hibernate_cryptwakevars_t _cryptWakeVars;
   57 hibernate_cryptwakevars_t * gIOHibernateCryptWakeVars = &_cryptWakeVars;
   58 
   59 vm_offset_t gIOHibernateWakeMap;            // ppnum
   60 vm_size_t   gIOHibernateWakeMapSize;
   61 
   62 #if __i386__
   63 extern void   acpi_wake_prot_entry(void);
   64 #endif
   65 
   66 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
   67 
   68 #define BASE 65521L /* largest prime smaller than 65536 */
   69 #define NMAX 5000  
   70 // NMAX (was 5521) the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
   71 
   72 #define DO1(buf,i)  {s1 += buf[i]; s2 += s1;}
   73 #define DO2(buf,i)  DO1(buf,i); DO1(buf,i+1);
   74 #define DO4(buf,i)  DO2(buf,i); DO2(buf,i+2);
   75 #define DO8(buf,i)  DO4(buf,i); DO4(buf,i+4);
   76 #define DO16(buf)   DO8(buf,0); DO8(buf,8);
   77 
   78 uint32_t
   79 hibernate_sum(uint8_t *buf, int32_t len)
   80 {
   81     unsigned long s1 = 1; // adler & 0xffff;
   82     unsigned long s2 = 0; // (adler >> 16) & 0xffff;
   83     int k;
   84 
   85     while (len > 0) {
   86         k = len < NMAX ? len : NMAX;
   87         len -= k;
   88         while (k >= 16) {
   89             DO16(buf);
   90             buf += 16;
   91             k -= 16;
   92         }
   93         if (k != 0) do {
   94             s1 += *buf++;
   95             s2 += s1;
   96         } while (--k);
   97         s1 %= BASE;
   98         s2 %= BASE;
   99     }
  100     return (s2 << 16) | s1;
  101 }
  102 
  103 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  104 
  105 static __inline__ unsigned int cntlzw(unsigned int num)
  106 {
  107         return clz(num);
  108 }
  109 
  110 static hibernate_bitmap_t *
  111 hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page)
  112 {
  113     uint32_t             bank;
  114     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
  115 
  116     for (bank = 0; bank < list->bank_count; bank++)
  117     {
  118         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
  119             break;
  120         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
  121     }
  122     if (bank == list->bank_count)
  123         bitmap = NULL;
  124         
  125     return (bitmap);
  126 }
  127 
  128 hibernate_bitmap_t *
  129 hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * pPage)
  130 {
  131     uint32_t             bank, page = *pPage;
  132     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
  133 
  134     for (bank = 0; bank < list->bank_count; bank++)
  135     {
  136         if (page <= bitmap->first_page)
  137         {
  138             *pPage = bitmap->first_page;
  139             break;
  140         }
  141         if (page <= bitmap->last_page)
  142             break;
  143         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
  144     }
  145     if (bank == list->bank_count)
  146         bitmap = NULL;
  147         
  148     return (bitmap);
  149 }
  150 
  151 void 
  152 hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
  153 {
  154     hibernate_bitmap_t * bitmap;
  155 
  156     bitmap = hibernate_page_bitmap(list, page);
  157     if (bitmap)
  158     {
  159         page -= bitmap->first_page;
  160         if (set)
  161             bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
  162             //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
  163         else
  164             bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
  165             //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
  166     }
  167 }
  168 
  169 boolean_t 
  170 hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
  171 {
  172     boolean_t            result = TRUE;
  173     hibernate_bitmap_t * bitmap;
  174 
  175     bitmap = hibernate_page_bitmap(list, page);
  176     if (bitmap)
  177     {
  178         page -= bitmap->first_page;
  179         result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
  180     }
  181     return (result);
  182 }
  183 
  184 // count bits clear or set (set == TRUE) starting at page.
  185 uint32_t
  186 hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t page)
  187 {
  188     uint32_t index, bit, bits;
  189     uint32_t count;
  190 
  191     count = 0;
  192 
  193     index = (page - bitmap->first_page) >> 5;
  194     bit = (page - bitmap->first_page) & 31;
  195 
  196     bits = bitmap->bitmap[index];
  197     if (set)
  198         bits = ~bits;
  199     bits = (bits << bit);
  200     if (bits)
  201         count += __builtin_clz(bits);
  202     else
  203     {
  204         count += 32 - bit;
  205         while (++index < bitmap->bitmapwords)
  206         {
  207             bits = bitmap->bitmap[index];
  208             if (set)
  209                 bits = ~bits;
  210             if (bits)
  211             {
  212                 count += __builtin_clz(bits);
  213                 break;
  214             }
  215             count += 32;
  216         }
  217     }
  218 
  219     return (count);
  220 }
  221 
  222 static vm_offset_t
  223 hibernate_page_list_grab(hibernate_page_list_t * list, uint32_t * pNextFree)
  224 {
  225     uint32_t             nextFree = *pNextFree;
  226     uint32_t             nextFreeInBank;
  227     hibernate_bitmap_t * bitmap;
  228 
  229     nextFreeInBank = nextFree + 1;
  230     while ((bitmap = hibernate_page_bitmap_pin(list, &nextFreeInBank)))
  231     {
  232         nextFreeInBank += hibernate_page_bitmap_count(bitmap, FALSE, nextFreeInBank);
  233         if (nextFreeInBank <= bitmap->last_page)
  234         {
  235             *pNextFree = nextFreeInBank;
  236             break;
  237         }
  238     }
  239 
  240     return (nextFree);
  241 }
  242 
  243 static uint32_t
  244 store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize, 
  245                 uint32_t * buffer, uint32_t ppnum)
  246 {
  247     uint64_t dst;
  248     uint32_t sum;
  249 
  250     dst = ptoa_64(ppnum);
  251     if (ppnum < 0x00100000)
  252         buffer = (uint32_t *) (uint32_t) dst;
  253 
  254     if (compressedSize != PAGE_SIZE)
  255     {
  256         WKdm_decompress((WK_word*) src, (WK_word*) buffer, PAGE_SIZE >> 2);
  257         src = buffer;
  258     }
  259 
  260     sum = hibernate_sum((uint8_t *) src, PAGE_SIZE);
  261 
  262     if (((uint64_t) (uint32_t) src) == dst)
  263         src = 0;
  264 
  265     hibernate_restore_phys_page((uint64_t) (uint32_t) src, dst, PAGE_SIZE, procFlags);
  266 
  267     return (sum);
  268 }
  269 
  270 static void 
  271 bcopy_internal(const void *src, void *dst, uint32_t len)
  272 {
  273     const char *s = src;
  274     char       *d = dst;
  275     uint32_t   idx = 0;
  276 
  277     while (idx < len)
  278     {
  279         d[idx] = s[idx];
  280         idx++;
  281     }
  282 }
  283 
  284 #define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
  285 
  286 long 
  287 hibernate_kernel_entrypoint(IOHibernateImageHeader * header, 
  288                             void * p2, void * p3, void * p4)
  289 {
  290     typedef void (*ResetProc)(void);
  291     uint32_t idx;
  292     uint32_t * src;
  293     uint32_t * buffer;
  294     uint32_t * pageIndexSource;
  295     hibernate_page_list_t * map;
  296     uint32_t count;
  297     uint32_t ppnum;
  298     uint32_t page;
  299     uint32_t conflictCount;
  300     uint32_t compressedSize;
  301     uint32_t uncompressedPages;
  302     uint32_t copyPageListHead;
  303     uint32_t * copyPageList;
  304     uint32_t copyPageIndex;
  305     uint32_t sum;
  306     uint32_t nextFree;
  307     uint32_t lastImagePage;
  308     uint32_t lastMapPage;
  309     uint32_t lastPageIndexPage;
  310     ResetProc proc;
  311 
  312     C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
  313 
  314     bcopy_internal(header, 
  315                 gIOHibernateCurrentHeader, 
  316                 sizeof(IOHibernateImageHeader));
  317 
  318     if (!p2)
  319     {
  320         count = header->graphicsInfoOffset;
  321         if (count)
  322             p2 = (void *)(((uintptr_t) header) - count);
  323     }
  324     if (p2) 
  325         bcopy_internal(p2, 
  326                 gIOHibernateGraphicsInfo, 
  327                 sizeof(hibernate_graphics_t));
  328     else
  329         gIOHibernateGraphicsInfo->physicalAddress = gIOHibernateGraphicsInfo->depth = 0;
  330 
  331     if (!p3)
  332     {
  333         count = header->cryptVarsOffset;
  334         if (count)
  335             p3 = (void *)(((uintptr_t) header) - count);
  336     }
  337     if (p3)
  338         bcopy_internal(p3, 
  339                 gIOHibernateCryptWakeVars, 
  340                 sizeof(hibernate_cryptvars_t));
  341 
  342     src = (uint32_t *)
  343                 (((uint32_t) &header->fileExtentMap[0]) 
  344                             + header->fileExtentMapSize 
  345                             + ptoa_32(header->restore1PageCount));
  346 
  347     if (header->previewSize)
  348     {
  349         pageIndexSource = src;
  350         map = (hibernate_page_list_t *)(((uint32_t) pageIndexSource) + header->previewSize);
  351         src = (uint32_t *) (((uint32_t) pageIndexSource) + header->previewPageListSize);
  352     }
  353     else
  354     {
  355         pageIndexSource = 0;
  356         map = (hibernate_page_list_t *) src;
  357         src = (uint32_t *) (((uint32_t) map) + header->bitmapSize);
  358     }
  359 
  360     lastPageIndexPage = atop_32(src);
  361 
  362     lastImagePage = atop_32(((uint32_t) header) + header->image1Size);
  363 
  364     lastMapPage = atop_32(((uint32_t) map) + header->bitmapSize);
  365 
  366     // knock all the image pages to be used out of free map
  367     for (ppnum = atop_32(header); ppnum <= lastImagePage; ppnum++)
  368     {
  369         hibernate_page_bitset(map, FALSE, ppnum);
  370     }
  371 
  372     nextFree = 0;
  373     hibernate_page_list_grab(map, &nextFree);
  374     buffer = (uint32_t *) ptoa_32(hibernate_page_list_grab(map, &nextFree));
  375 
  376     if (header->memoryMapSize && (count = header->memoryMapOffset))
  377     {
  378         p4 = (void *)(((uintptr_t) header) - count);
  379         gIOHibernateWakeMap     = hibernate_page_list_grab(map, &nextFree);
  380         gIOHibernateWakeMapSize = header->memoryMapSize;
  381         bcopy_internal(p4, (void  *) ptoa_32(gIOHibernateWakeMap), gIOHibernateWakeMapSize);
  382     }
  383     else
  384         gIOHibernateWakeMapSize = 0;
  385 
  386     sum = gIOHibernateCurrentHeader->actualRestore1Sum;
  387     gIOHibernateCurrentHeader->diag[0] = (uint32_t) header;
  388     gIOHibernateCurrentHeader->diag[1] = sum;
  389 
  390     uncompressedPages = 0;
  391     conflictCount     = 0;
  392     copyPageListHead  = 0;
  393     copyPageList      = 0;
  394     copyPageIndex     = PAGE_SIZE >> 2;
  395 
  396     compressedSize    = PAGE_SIZE;
  397 
  398     while (1)
  399     {
  400         if (pageIndexSource)
  401         {
  402             ppnum = pageIndexSource[0];
  403             count = pageIndexSource[1];
  404             pageIndexSource += 2;
  405             if (!count)
  406             {
  407                 pageIndexSource = 0;
  408                 src =  (uint32_t *) (((uint32_t) map) + gIOHibernateCurrentHeader->bitmapSize);
  409                 ppnum = src[0];
  410                 count = src[1];
  411                 src += 2;
  412             } 
  413         }
  414         else
  415         {
  416             ppnum = src[0];
  417             count = src[1];
  418             if (!count)
  419                 break;
  420             src += 2;
  421         }
  422 
  423         for (page = 0; page < count; page++, ppnum++)
  424         {
  425             uint32_t tag;
  426             int conflicts;
  427 
  428             if (!pageIndexSource)
  429             {
  430                 tag = *src++;
  431                 compressedSize = kIOHibernateTagLength & tag;
  432             }
  433 
  434 //    SINT(ppnum);
  435 
  436             conflicts = (((ppnum >= atop_32(map)) && (ppnum <= lastMapPage))
  437                       || ((ppnum >= atop_32(src)) && (ppnum <= lastImagePage)));
  438 
  439             if (pageIndexSource)
  440                 conflicts |= ((ppnum >= atop_32(pageIndexSource)) && (ppnum <= lastPageIndexPage));
  441 
  442             if (!conflicts)
  443             {
  444                 if (compressedSize)
  445                     sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
  446                                             src, compressedSize, buffer, ppnum);
  447                 uncompressedPages++;
  448             }
  449             else
  450             {
  451                 uint32_t   bufferPage;
  452                 uint32_t * dst;
  453 
  454                 conflictCount++;
  455 
  456                 // alloc new buffer page
  457                 bufferPage = hibernate_page_list_grab(map, &nextFree);
  458 
  459                 if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
  460                 {
  461                     // alloc new copy list page
  462                     uint32_t pageListPage = hibernate_page_list_grab(map, &nextFree);
  463                     // link to current
  464                     if (copyPageList)
  465                         copyPageList[1] = pageListPage;
  466                     else
  467                         copyPageListHead = pageListPage;
  468                     copyPageList = (uint32_t *) ptoa_32(pageListPage);
  469                     copyPageList[1] = 0;
  470                     copyPageIndex = 2;
  471                 }
  472 
  473                 copyPageList[copyPageIndex++] = ppnum;
  474                 copyPageList[copyPageIndex++] = bufferPage;
  475                 copyPageList[copyPageIndex++] = compressedSize;
  476                 copyPageList[0] = copyPageIndex;
  477 
  478                 dst = (uint32_t *) ptoa_32(bufferPage);
  479                 for (idx = 0; idx < ((compressedSize + 3) >> 2); idx++)
  480                     dst[idx] = src[idx];
  481             }
  482             src += ((compressedSize + 3) >> 2);
  483         }
  484     }
  485 
  486     // -- copy back conflicts
  487 
  488     copyPageList = (uint32_t *) ptoa_32(copyPageListHead);
  489     while (copyPageList)
  490     {
  491         for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3)
  492         {
  493             ppnum          =              copyPageList[copyPageIndex + 0];
  494             src            = (uint32_t *) ptoa_32(copyPageList[copyPageIndex + 1]);
  495             compressedSize =              copyPageList[copyPageIndex + 2];
  496 
  497             sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
  498                                     src, compressedSize, buffer, ppnum);
  499             uncompressedPages++;
  500         }
  501         copyPageList = (uint32_t *) ptoa_32(copyPageList[1]);
  502     }
  503 
  504     // -- image has been destroyed...
  505 
  506     gIOHibernateCurrentHeader->actualImage1Sum         = sum;
  507     gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages;
  508     gIOHibernateCurrentHeader->conflictCount           = conflictCount;
  509     gIOHibernateCurrentHeader->nextFree                = nextFree;
  510 
  511     gIOHibernateState = kIOHibernateStateWakingFromHibernate;
  512 
  513 #if __ppc__
  514     proc = (ResetProc) 0x100;
  515     __asm__ volatile("ori 0, 0, 0" : : );
  516     proc();
  517 #elif __i386__
  518     proc = (ResetProc) acpi_wake_prot_entry;
  519     // flush caches
  520     __asm__("wbinvd");
  521     proc();
  522 #elif __arm__
  523     proc = (ResetProc)0x00000000;
  524     proc();
  525 #endif
  526   
  527     return -1;
  528 }

Cache object: 2202e5d0e5bf7575587f416f53828bd3


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.