Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/contrib/openzfs/scripts/zimport.sh
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 #!/usr/bin/env bash 2 # 3 # Verify that an assortment of known good reference pools can be imported 4 # using different versions of OpenZFS code. 5 # 6 # By default references pools for the major ZFS implementation will be 7 # checked against the most recent OpenZFS tags and the master development branch. 8 # Alternate tags or branches may be verified with the '-s <src-tag> option. 9 # Passing the keyword "installed" will instruct the script to test whatever 10 # version is installed. 11 # 12 # Preferentially a reference pool is used for all tests. However, if one 13 # does not exist and the pool-tag matches one of the src-tags then a new 14 # reference pool will be created using binaries from that source build. 15 # This is particularly useful when you need to test your changes before 16 # opening a pull request. The keyword 'all' can be used as short hand 17 # refer to all available reference pools. 18 # 19 # New reference pools may be added by placing a bzip2 compressed tarball 20 # of the pool in the scripts/zfs-images directory and then passing 21 # the -p <pool-tag> option. To increase the test coverage reference pools 22 # should be collected for all the major ZFS implementations. Having these 23 # pools easily available is also helpful to the developers. 24 # 25 # Care should be taken to run these tests with a kernel supported by all 26 # the listed tags. Otherwise build failure will cause false positives. 27 # 28 # 29 # EXAMPLES: 30 # 31 # The following example will verify the zfs-0.6.2 tag, the master branch, 32 # and the installed zfs version can correctly import the listed pools. 33 # Note there is no reference pool available for master and installed but 34 # because binaries are available one is automatically constructed. The 35 # working directory is also preserved between runs (-k) preventing the 36 # need to rebuild from source for multiple runs. 37 # 38 # zimport.sh -k -f /var/tmp/zimport \ 39 # -s "zfs-0.6.2 master installed" \ 40 # -p "zevo-1.1.1 zol-0.6.2 zol-0.6.2-173 master installed" 41 # 42 # ------------------------ OpenZFS Source Versions ---------------- 43 # zfs-0.6.2 master 0.6.2-175_g36eb554 44 # ----------------------------------------------------------------- 45 # Clone ZFS Local Local Skip 46 # Build ZFS Pass Pass Skip 47 # ----------------------------------------------------------------- 48 # zevo-1.1.1 Pass Pass Pass 49 # zol-0.6.2 Pass Pass Pass 50 # zol-0.6.2-173 Fail Pass Pass 51 # master Pass Pass Pass 52 # installed Pass Pass Pass 53 # 54 55 BASE_DIR=$(dirname "$0") 56 SCRIPT_COMMON=common.sh 57 if [[ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]]; then 58 . "${BASE_DIR}/${SCRIPT_COMMON}" 59 else 60 echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 61 fi 62 63 PROG=zimport.sh 64 SRC_TAGS="zfs-0.6.5.11 master" 65 POOL_TAGS="all master" 66 POOL_CREATE_OPTIONS= 67 TEST_DIR=$(mktemp -u -d -p /var/tmp zimport.XXXXXXXX) 68 KEEP="no" 69 VERBOSE="no" 70 COLOR="yes" 71 REPO="https://github.com/openzfs" 72 IMAGES_DIR="${BASE_DIR}/zfs-images/" 73 IMAGES_TAR="https://github.com/openzfs/zfs-images/tarball/master" 74 ERROR=0 75 76 CONFIG_LOG="configure.log" 77 CONFIG_OPTIONS=${CONFIG_OPTIONS:-""} 78 MAKE_LOG="make.log" 79 MAKE_OPTIONS=${MAKE_OPTIONS:-"-s -j$(nproc)"} 80 81 COLOR_GREEN="\033[0;32m" 82 COLOR_RED="\033[0;31m" 83 COLOR_BROWN="\033[0;33m" 84 COLOR_RESET="\033[0m" 85 86 usage() { 87 cat << EOF 88 USAGE: 89 zimport.sh [hvl] [-r repo] [-s src-tag] [-i pool-dir] [-p pool-tag] 90 [-f path] [-o options] 91 92 DESCRIPTION: 93 ZPOOL import verification tests 94 95 OPTIONS: 96 -h Show this message 97 -v Verbose 98 -c No color 99 -k Keep temporary directory 100 -r <repo> Source repository ($REPO) 101 -s <src-tag>... Verify OpenZFS versions with the listed tags 102 -i <pool-dir> Pool image directory 103 -p <pool-tag>... Verify pools created with the listed tags 104 -f <path> Temporary directory to use 105 -o <options> Additional options to pass to 'zpool create' 106 107 EOF 108 } 109 110 while getopts 'hvckr:s:i:p:f:o:?' OPTION; do 111 case $OPTION in 112 h) 113 usage 114 exit 1 115 ;; 116 v) 117 VERBOSE="yes" 118 ;; 119 c) 120 COLOR="no" 121 ;; 122 k) 123 KEEP="yes" 124 ;; 125 r) 126 REPO="$OPTARG" 127 ;; 128 s) 129 SRC_TAGS="$OPTARG" 130 ;; 131 i) 132 IMAGES_DIR="$OPTARG" 133 ;; 134 p) 135 POOL_TAGS="$OPTARG" 136 ;; 137 f) 138 TEST_DIR="$OPTARG" 139 ;; 140 o) 141 POOL_CREATE_OPTIONS="$OPTARG" 142 ;; 143 *) 144 usage 145 exit 1 146 ;; 147 esac 148 done 149 150 # 151 # Verify the module start is not loaded 152 # 153 if lsmod | grep zfs >/dev/null; then 154 echo "ZFS modules must be unloaded" 155 exit 1 156 fi 157 158 # 159 # Create a random directory tree of files and sub-directories to 160 # to act as a copy source for the various regression tests. 161 # 162 populate() { 163 local ROOT=$1 164 local MAX_DIR_SIZE=$2 165 local MAX_FILE_SIZE=$3 166 167 mkdir -p "$ROOT"/{a,b,c,d,e,f,g}/{h,i} 168 DIRS=$(find "$ROOT") 169 170 for DIR in $DIRS; do 171 COUNT=$((RANDOM % MAX_DIR_SIZE)) 172 173 for _ in $(seq "$COUNT"); do 174 FILE=$(mktemp -p "$DIR") 175 SIZE=$((RANDOM % MAX_FILE_SIZE)) 176 dd if=/dev/urandom of="$FILE" bs=1k \ 177 count="$SIZE" &>/dev/null 178 done 179 done 180 181 return 0 182 } 183 184 SRC_DIR=$(mktemp -d -p /var/tmp/ zfs.src.XXXXXXXX) 185 trap 'rm -Rf "$SRC_DIR"' INT TERM EXIT 186 populate "$SRC_DIR" 10 100 187 188 SRC_DIR="$TEST_DIR/src" 189 SRC_DIR_ZFS="$SRC_DIR/zfs" 190 191 if [[ "$COLOR" = "no" ]]; then 192 COLOR_GREEN="" 193 COLOR_BROWN="" 194 COLOR_RED="" 195 COLOR_RESET="" 196 fi 197 198 pass_nonewline() { 199 echo -n -e "${COLOR_GREEN}Pass${COLOR_RESET}\t\t" 200 } 201 202 skip_nonewline() { 203 echo -n -e "${COLOR_BROWN}Skip${COLOR_RESET}\t\t" 204 } 205 206 fail_nonewline() { 207 echo -n -e "${COLOR_RED}Fail${COLOR_RESET}\t\t" 208 } 209 210 # 211 # Log a failure message, cleanup, and return an error. 212 # 213 fail() { 214 echo -e "$PROG: $1" >&2 215 $ZFS_SH -u >/dev/null 2>&1 216 exit 1 217 } 218 219 # 220 # Set several helper variables which are derived from a source tag. 221 # 222 # ZFS_TAG - The passed zfs-x.y.z tag 223 # ZFS_DIR - The zfs directory name 224 # ZFS_URL - The zfs github URL to fetch the tarball 225 # 226 src_set_vars() { 227 local TAG=$1 228 229 ZFS_TAG="$TAG" 230 ZFS_DIR="$SRC_DIR_ZFS/$ZFS_TAG" 231 ZFS_URL="$REPO/zfs/tarball/$ZFS_TAG" 232 233 if [[ "$TAG" = "installed" ]]; then 234 ZPOOL_CMD=$(command -v zpool) 235 ZFS_CMD=$(command -v zfs) 236 ZFS_SH="/usr/share/zfs/zfs.sh" 237 else 238 ZPOOL_CMD="./zpool" 239 ZFS_CMD="./zfs" 240 ZFS_SH="./scripts/zfs.sh" 241 fi 242 } 243 244 # 245 # Set several helper variables which are derived from a pool name such 246 # as zol-0.6.x, zevo-1.1.1, etc. These refer to example pools from various 247 # ZFS implementations which are used to verify compatibility. 248 # 249 # POOL_TAG - The example pools name in scripts/zfs-images/. 250 # POOL_BZIP - The full path to the example bzip2 compressed pool. 251 # POOL_DIR - The top level test path for this pool. 252 # POOL_DIR_PRISTINE - The directory containing a pristine version of the pool. 253 # POOL_DIR_COPY - The directory containing a working copy of the pool. 254 # POOL_DIR_SRC - Location of a source build if it exists for this pool. 255 # 256 pool_set_vars() { 257 local TAG=$1 258 259 POOL_TAG=$TAG 260 POOL_BZIP=$IMAGES_DIR/$POOL_TAG.tar.bz2 261 POOL_DIR=$TEST_DIR/pools/$POOL_TAG 262 POOL_DIR_PRISTINE=$POOL_DIR/pristine 263 POOL_DIR_COPY=$POOL_DIR/copy 264 POOL_DIR_SRC="$SRC_DIR_ZFS/${POOL_TAG//zol/zfs}" 265 } 266 267 # 268 # Construct a non-trivial pool given a specific version of the source. More 269 # interesting pools provide better test coverage so this function should 270 # extended as needed to create more realistic pools. 271 # 272 pool_create() { 273 pool_set_vars "$1" 274 src_set_vars "$1" 275 276 if [[ "$POOL_TAG" != "installed" ]]; then 277 cd "$POOL_DIR_SRC" || fail "Failed 'cd $POOL_DIR_SRC'" 278 fi 279 280 $ZFS_SH zfs="spa_config_path=$POOL_DIR_PRISTINE" || \ 281 fail "Failed to load kmods" 282 283 # Create a file vdev RAIDZ pool. 284 truncate -s 1G \ 285 "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \ 286 "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4" || \ 287 fail "Failed 'truncate -s 1G ...'" 288 # shellcheck disable=SC2086 289 $ZPOOL_CMD create $POOL_CREATE_OPTIONS "$POOL_TAG" raidz \ 290 "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \ 291 "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4" || \ 292 fail "Failed '$ZPOOL_CMD create $POOL_CREATE_OPTIONS $POOL_TAG ...'" 293 294 # Create a pool/fs filesystem with some random contents. 295 $ZFS_CMD create "$POOL_TAG/fs" || \ 296 fail "Failed '$ZFS_CMD create $POOL_TAG/fs'" 297 populate "/$POOL_TAG/fs/" 10 100 298 299 # Snapshot that filesystem, clone it, remove the files/dirs, 300 # replace them with new files/dirs. 301 $ZFS_CMD snap "$POOL_TAG/fs@snap" || \ 302 fail "Failed '$ZFS_CMD snap $POOL_TAG/fs@snap'" 303 $ZFS_CMD clone "$POOL_TAG/fs@snap" "$POOL_TAG/clone" || \ 304 fail "Failed '$ZFS_CMD clone $POOL_TAG/fs@snap $POOL_TAG/clone'" 305 # shellcheck disable=SC2086 306 rm -Rf /$POOL_TAG/clone/* 307 populate "/$POOL_TAG/clone/" 10 100 308 309 # Scrub the pool, delay slightly, then export it. It is now 310 # somewhat interesting for testing purposes. 311 $ZPOOL_CMD scrub "$POOL_TAG" || \ 312 fail "Failed '$ZPOOL_CMD scrub $POOL_TAG'" 313 sleep 10 314 $ZPOOL_CMD export "$POOL_TAG" || \ 315 fail "Failed '$ZPOOL_CMD export $POOL_TAG'" 316 317 $ZFS_SH -u || fail "Failed to unload kmods" 318 } 319 320 # If the zfs-images directory doesn't exist fetch a copy from Github then 321 # cache it in the $TEST_DIR and update $IMAGES_DIR. 322 if [[ ! -d "$IMAGES_DIR" ]]; then 323 IMAGES_DIR="$TEST_DIR/zfs-images" 324 mkdir -p "$IMAGES_DIR" 325 curl -sL "$IMAGES_TAR" | \ 326 tar -xz -C "$IMAGES_DIR" --strip-components=1 || \ 327 fail "Failed to download pool images" 328 fi 329 330 # Given the available images in the zfs-images directory substitute the 331 # list of available images for the reserved keyword 'all'. 332 for TAG in $POOL_TAGS; do 333 334 if [[ "$TAG" = "all" ]]; then 335 ALL_TAGS=$(echo "$IMAGES_DIR"/*.tar.bz2 | \ 336 sed "s|$IMAGES_DIR/||g;s|.tar.bz2||g") 337 NEW_TAGS="$NEW_TAGS $ALL_TAGS" 338 else 339 NEW_TAGS="$NEW_TAGS $TAG" 340 fi 341 done 342 POOL_TAGS="$NEW_TAGS" 343 344 if [[ "$VERBOSE" = "yes" ]]; then 345 echo "---------------------------- Options ----------------------------" 346 echo "VERBOSE=$VERBOSE" 347 echo "KEEP=$KEEP" 348 echo "REPO=$REPO" 349 echo "SRC_TAGS=$SRC_TAGS" 350 echo "POOL_TAGS=$POOL_TAGS" 351 echo "PATH=$TEST_DIR" 352 echo "POOL_CREATE_OPTIONS=$POOL_CREATE_OPTIONS" 353 echo 354 fi 355 356 if [[ ! -d "$TEST_DIR" ]]; then 357 mkdir -p "$TEST_DIR" 358 fi 359 360 if [[ ! -d "$SRC_DIR" ]]; then 361 mkdir -p "$SRC_DIR" 362 fi 363 364 # Print a header for all tags which are being tested. 365 echo "------------------------ OpenZFS Source Versions ----------------" 366 printf "%-16s" " " 367 for TAG in $SRC_TAGS; do 368 src_set_vars "$TAG" 369 370 if [[ "$TAG" = "installed" ]]; then 371 ZFS_VERSION=$(modinfo zfs | awk '/version:/ { print $2; exit }') 372 if [[ -n "$ZFS_VERSION" ]]; then 373 printf "%-16s" "$ZFS_VERSION" 374 else 375 fail "ZFS is not installed" 376 fi 377 else 378 printf "%-16s" "$TAG" 379 fi 380 done 381 echo -e "\n-----------------------------------------------------------------" 382 383 # 384 # Attempt to generate the tarball from your local git repository, if that 385 # fails then attempt to download the tarball from Github. 386 # 387 printf "%-16s" "Clone ZFS" 388 for TAG in $SRC_TAGS; do 389 src_set_vars "$TAG" 390 391 if [[ -d "$ZFS_DIR" ]]; then 392 skip_nonewline 393 elif [[ "$ZFS_TAG" = "installed" ]]; then 394 skip_nonewline 395 else 396 cd "$SRC_DIR" || fail "Failed 'cd $SRC_DIR'" 397 398 if [[ ! -d "$SRC_DIR_ZFS" ]]; then 399 mkdir -p "$SRC_DIR_ZFS" 400 fi 401 402 git archive --format=tar --prefix="$ZFS_TAG/ $ZFS_TAG" \ 403 -o "$SRC_DIR_ZFS/$ZFS_TAG.tar" &>/dev/null || \ 404 rm "$SRC_DIR_ZFS/$ZFS_TAG.tar" 405 if [[ -s "$SRC_DIR_ZFS/$ZFS_TAG.tar" ]]; then 406 tar -xf "$SRC_DIR_ZFS/$ZFS_TAG.tar" -C "$SRC_DIR_ZFS" 407 rm "$SRC_DIR_ZFS/$ZFS_TAG.tar" 408 echo -n -e "${COLOR_GREEN}Local${COLOR_RESET}\t\t" 409 else 410 mkdir -p "$ZFS_DIR" || fail "Failed to create $ZFS_DIR" 411 curl -sL "$ZFS_URL" | tar -xz -C "$ZFS_DIR" \ 412 --strip-components=1 || \ 413 fail "Failed to download $ZFS_URL" 414 echo -n -e "${COLOR_GREEN}Remote${COLOR_RESET}\t\t" 415 fi 416 fi 417 done 418 printf "\n" 419 420 # Build the listed tags 421 printf "%-16s" "Build ZFS" 422 for TAG in $SRC_TAGS; do 423 src_set_vars "$TAG" 424 425 if [[ -f "$ZFS_DIR/module/zfs/zfs.ko" ]]; then 426 skip_nonewline 427 elif [[ "$ZFS_TAG" = "installed" ]]; then 428 skip_nonewline 429 else 430 cd "$ZFS_DIR" || fail "Failed 'cd $ZFS_DIR'" 431 make distclean &>/dev/null 432 ./autogen.sh >>"$CONFIG_LOG" 2>&1 || \ 433 fail "Failed ZFS 'autogen.sh'" 434 # shellcheck disable=SC2086 435 ./configure $CONFIG_OPTIONS >>"$CONFIG_LOG" 2>&1 || \ 436 fail "Failed ZFS 'configure $CONFIG_OPTIONS'" 437 # shellcheck disable=SC2086 438 make $MAKE_OPTIONS >>"$MAKE_LOG" 2>&1 || \ 439 fail "Failed ZFS 'make $MAKE_OPTIONS'" 440 pass_nonewline 441 fi 442 done 443 printf "\n" 444 echo "-----------------------------------------------------------------" 445 446 # Either create a new pool using 'zpool create', or alternately restore an 447 # existing pool from another ZFS implementation for compatibility testing. 448 for TAG in $POOL_TAGS; do 449 pool_set_vars "$TAG" 450 SKIP=0 451 452 printf "%-16s" "$POOL_TAG" 453 rm -Rf "$POOL_DIR" 454 mkdir -p "$POOL_DIR_PRISTINE" 455 456 # Use the existing compressed image if available. 457 if [[ -f "$POOL_BZIP" ]]; then 458 tar -xjf "$POOL_BZIP" -C "$POOL_DIR_PRISTINE" \ 459 --strip-components=1 || \ 460 fail "Failed 'tar -xjf $POOL_BZIP" 461 # Use the installed version to create the pool. 462 elif [[ "$TAG" = "installed" ]]; then 463 pool_create "$TAG" 464 # A source build is available to create the pool. 465 elif [[ -d "$POOL_DIR_SRC" ]]; then 466 pool_create "$TAG" 467 else 468 SKIP=1 469 fi 470 471 # Verify 'zpool import' works for all listed source versions. 472 for SRC_TAG in $SRC_TAGS; do 473 474 if [[ "$SKIP" -eq 1 ]]; then 475 skip_nonewline 476 continue 477 fi 478 479 src_set_vars "$SRC_TAG" 480 if [[ "$SRC_TAG" != "installed" ]]; then 481 cd "$ZFS_DIR" || fail "Failed 'cd $ZFS_DIR'" 482 fi 483 $ZFS_SH zfs="spa_config_path=$POOL_DIR_COPY" 484 485 cp -a --sparse=always "$POOL_DIR_PRISTINE" \ 486 "$POOL_DIR_COPY" || \ 487 fail "Failed to copy $POOL_DIR_PRISTINE to $POOL_DIR_COPY" 488 POOL_NAME=$($ZPOOL_CMD import -d "$POOL_DIR_COPY" | \ 489 awk '/pool:/ { print $2; exit }') 490 491 if ! $ZPOOL_CMD import -N -d "$POOL_DIR_COPY" 492 "$POOL_NAME" &>/dev/null; then 493 fail_nonewline 494 ERROR=1 495 else 496 $ZPOOL_CMD export "$POOL_NAME" || \ 497 fail "Failed to export pool" 498 pass_nonewline 499 fi 500 501 rm -Rf "$POOL_DIR_COPY" 502 503 $ZFS_SH -u || fail "Failed to unload kmods" 504 done 505 printf "\n" 506 done 507 508 if [[ "$KEEP" = "no" ]]; then 509 rm -Rf "$TEST_DIR" 510 fi 511 512 exit "$ERROR"
Cache object: 58f72a00519bbbb7cc569720b849280f
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]