Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/contrib/openzfs/module/zfs/vdev_trim.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2016 by Delphix. All rights reserved. 24 * Copyright (c) 2019 by Lawrence Livermore National Security, LLC. 25 * Copyright (c) 2021 Hewlett Packard Enterprise Development LP 26 */ 27 28 #include <sys/spa.h> 29 #include <sys/spa_impl.h> 30 #include <sys/txg.h> 31 #include <sys/vdev_impl.h> 32 #include <sys/vdev_trim.h> 33 #include <sys/metaslab_impl.h> 34 #include <sys/dsl_synctask.h> 35 #include <sys/zap.h> 36 #include <sys/dmu_tx.h> 37 #include <sys/arc_impl.h> 38 39 /* 40 * TRIM is a feature which is used to notify a SSD that some previously 41 * written space is no longer allocated by the pool. This is useful because 42 * writes to a SSD must be performed to blocks which have first been erased. 43 * Ensuring the SSD always has a supply of erased blocks for new writes 44 * helps prevent the performance from deteriorating. 45 * 46 * There are two supported TRIM methods; manual and automatic. 47 * 48 * Manual TRIM: 49 * 50 * A manual TRIM is initiated by running the 'zpool trim' command. A single 51 * 'vdev_trim' thread is created for each leaf vdev, and it is responsible for 52 * managing that vdev TRIM process. This involves iterating over all the 53 * metaslabs, calculating the unallocated space ranges, and then issuing the 54 * required TRIM I/Os. 55 * 56 * While a metaslab is being actively trimmed it is not eligible to perform 57 * new allocations. After traversing all of the metaslabs the thread is 58 * terminated. Finally, both the requested options and current progress of 59 * the TRIM are regularly written to the pool. This allows the TRIM to be 60 * suspended and resumed as needed. 61 * 62 * Automatic TRIM: 63 * 64 * An automatic TRIM is enabled by setting the 'autotrim' pool property 65 * to 'on'. When enabled, a `vdev_autotrim' thread is created for each 66 * top-level (not leaf) vdev in the pool. These threads perform the same 67 * core TRIM process as a manual TRIM, but with a few key differences. 68 * 69 * 1) Automatic TRIM happens continuously in the background and operates 70 * solely on recently freed blocks (ms_trim not ms_allocatable). 71 * 72 * 2) Each thread is associated with a top-level (not leaf) vdev. This has 73 * the benefit of simplifying the threading model, it makes it easier 74 * to coordinate administrative commands, and it ensures only a single 75 * metaslab is disabled at a time. Unlike manual TRIM, this means each 76 * 'vdev_autotrim' thread is responsible for issuing TRIM I/Os for its 77 * children. 78 * 79 * 3) There is no automatic TRIM progress information stored on disk, nor 80 * is it reported by 'zpool status'. 81 * 82 * While the automatic TRIM process is highly effective it is more likely 83 * than a manual TRIM to encounter tiny ranges. Ranges less than or equal to 84 * 'zfs_trim_extent_bytes_min' (32k) are considered too small to efficiently 85 * TRIM and are skipped. This means small amounts of freed space may not 86 * be automatically trimmed. 87 * 88 * Furthermore, devices with attached hot spares and devices being actively 89 * replaced are skipped. This is done to avoid adding additional stress to 90 * a potentially unhealthy device and to minimize the required rebuild time. 91 * 92 * For this reason it may be beneficial to occasionally manually TRIM a pool 93 * even when automatic TRIM is enabled. 94 */ 95 96 /* 97 * Maximum size of TRIM I/O, ranges will be chunked in to 128MiB lengths. 98 */ 99 static unsigned int zfs_trim_extent_bytes_max = 128 * 1024 * 1024; 100 101 /* 102 * Minimum size of TRIM I/O, extents smaller than 32Kib will be skipped. 103 */ 104 static unsigned int zfs_trim_extent_bytes_min = 32 * 1024; 105 106 /* 107 * Skip uninitialized metaslabs during the TRIM process. This option is 108 * useful for pools constructed from large thinly-provisioned devices where 109 * TRIM operations are slow. As a pool ages an increasing fraction of 110 * the pools metaslabs will be initialized progressively degrading the 111 * usefulness of this option. This setting is stored when starting a 112 * manual TRIM and will persist for the duration of the requested TRIM. 113 */ 114 unsigned int zfs_trim_metaslab_skip = 0; 115 116 /* 117 * Maximum number of queued TRIM I/Os per leaf vdev. The number of 118 * concurrent TRIM I/Os issued to the device is controlled by the 119 * zfs_vdev_trim_min_active and zfs_vdev_trim_max_active module options. 120 */ 121 static unsigned int zfs_trim_queue_limit = 10; 122 123 /* 124 * The minimum number of transaction groups between automatic trims of a 125 * metaslab. This setting represents a trade-off between issuing more 126 * efficient TRIM operations, by allowing them to be aggregated longer, 127 * and issuing them promptly so the trimmed space is available. Note 128 * that this value is a minimum; metaslabs can be trimmed less frequently 129 * when there are a large number of ranges which need to be trimmed. 130 * 131 * Increasing this value will allow frees to be aggregated for a longer 132 * time. This can result is larger TRIM operations, and increased memory 133 * usage in order to track the ranges to be trimmed. Decreasing this value 134 * has the opposite effect. The default value of 32 was determined though 135 * testing to be a reasonable compromise. 136 */ 137 static unsigned int zfs_trim_txg_batch = 32; 138 139 /* 140 * The trim_args are a control structure which describe how a leaf vdev 141 * should be trimmed. The core elements are the vdev, the metaslab being 142 * trimmed and a range tree containing the extents to TRIM. All provided 143 * ranges must be within the metaslab. 144 */ 145 typedef struct trim_args { 146 /* 147 * These fields are set by the caller of vdev_trim_ranges(). 148 */ 149 vdev_t *trim_vdev; /* Leaf vdev to TRIM */ 150 metaslab_t *trim_msp; /* Disabled metaslab */ 151 range_tree_t *trim_tree; /* TRIM ranges (in metaslab) */ 152 trim_type_t trim_type; /* Manual or auto TRIM */ 153 uint64_t trim_extent_bytes_max; /* Maximum TRIM I/O size */ 154 uint64_t trim_extent_bytes_min; /* Minimum TRIM I/O size */ 155 enum trim_flag trim_flags; /* TRIM flags (secure) */ 156 157 /* 158 * These fields are updated by vdev_trim_ranges(). 159 */ 160 hrtime_t trim_start_time; /* Start time */ 161 uint64_t trim_bytes_done; /* Bytes trimmed */ 162 } trim_args_t; 163 164 /* 165 * Determines whether a vdev_trim_thread() should be stopped. 166 */ 167 static boolean_t 168 vdev_trim_should_stop(vdev_t *vd) 169 { 170 return (vd->vdev_trim_exit_wanted || !vdev_writeable(vd) || 171 vd->vdev_detached || vd->vdev_top->vdev_removing); 172 } 173 174 /* 175 * Determines whether a vdev_autotrim_thread() should be stopped. 176 */ 177 static boolean_t 178 vdev_autotrim_should_stop(vdev_t *tvd) 179 { 180 return (tvd->vdev_autotrim_exit_wanted || 181 !vdev_writeable(tvd) || tvd->vdev_removing || 182 spa_get_autotrim(tvd->vdev_spa) == SPA_AUTOTRIM_OFF); 183 } 184 185 /* 186 * The sync task for updating the on-disk state of a manual TRIM. This 187 * is scheduled by vdev_trim_change_state(). 188 */ 189 static void 190 vdev_trim_zap_update_sync(void *arg, dmu_tx_t *tx) 191 { 192 /* 193 * We pass in the guid instead of the vdev_t since the vdev may 194 * have been freed prior to the sync task being processed. This 195 * happens when a vdev is detached as we call spa_config_vdev_exit(), 196 * stop the trimming thread, schedule the sync task, and free 197 * the vdev. Later when the scheduled sync task is invoked, it would 198 * find that the vdev has been freed. 199 */ 200 uint64_t guid = *(uint64_t *)arg; 201 uint64_t txg = dmu_tx_get_txg(tx); 202 kmem_free(arg, sizeof (uint64_t)); 203 204 vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE); 205 if (vd == NULL || vd->vdev_top->vdev_removing || !vdev_is_concrete(vd)) 206 return; 207 208 uint64_t last_offset = vd->vdev_trim_offset[txg & TXG_MASK]; 209 vd->vdev_trim_offset[txg & TXG_MASK] = 0; 210 211 VERIFY3U(vd->vdev_leaf_zap, !=, 0); 212 213 objset_t *mos = vd->vdev_spa->spa_meta_objset; 214 215 if (last_offset > 0 || vd->vdev_trim_last_offset == UINT64_MAX) { 216 217 if (vd->vdev_trim_last_offset == UINT64_MAX) 218 last_offset = 0; 219 220 vd->vdev_trim_last_offset = last_offset; 221 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, 222 VDEV_LEAF_ZAP_TRIM_LAST_OFFSET, 223 sizeof (last_offset), 1, &last_offset, tx)); 224 } 225 226 if (vd->vdev_trim_action_time > 0) { 227 uint64_t val = (uint64_t)vd->vdev_trim_action_time; 228 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, 229 VDEV_LEAF_ZAP_TRIM_ACTION_TIME, sizeof (val), 230 1, &val, tx)); 231 } 232 233 if (vd->vdev_trim_rate > 0) { 234 uint64_t rate = (uint64_t)vd->vdev_trim_rate; 235 236 if (rate == UINT64_MAX) 237 rate = 0; 238 239 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, 240 VDEV_LEAF_ZAP_TRIM_RATE, sizeof (rate), 1, &rate, tx)); 241 } 242 243 uint64_t partial = vd->vdev_trim_partial; 244 if (partial == UINT64_MAX) 245 partial = 0; 246 247 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL, 248 sizeof (partial), 1, &partial, tx)); 249 250 uint64_t secure = vd->vdev_trim_secure; 251 if (secure == UINT64_MAX) 252 secure = 0; 253 254 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE, 255 sizeof (secure), 1, &secure, tx)); 256 257 258 uint64_t trim_state = vd->vdev_trim_state; 259 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE, 260 sizeof (trim_state), 1, &trim_state, tx)); 261 } 262 263 /* 264 * Update the on-disk state of a manual TRIM. This is called to request 265 * that a TRIM be started/suspended/canceled, or to change one of the 266 * TRIM options (partial, secure, rate). 267 */ 268 static void 269 vdev_trim_change_state(vdev_t *vd, vdev_trim_state_t new_state, 270 uint64_t rate, boolean_t partial, boolean_t secure) 271 { 272 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock)); 273 spa_t *spa = vd->vdev_spa; 274 275 if (new_state == vd->vdev_trim_state) 276 return; 277 278 /* 279 * Copy the vd's guid, this will be freed by the sync task. 280 */ 281 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP); 282 *guid = vd->vdev_guid; 283 284 /* 285 * If we're suspending, then preserve the original start time. 286 */ 287 if (vd->vdev_trim_state != VDEV_TRIM_SUSPENDED) { 288 vd->vdev_trim_action_time = gethrestime_sec(); 289 } 290 291 /* 292 * If we're activating, then preserve the requested rate and trim 293 * method. Setting the last offset and rate to UINT64_MAX is used 294 * as a sentinel to indicate they should be reset to default values. 295 */ 296 if (new_state == VDEV_TRIM_ACTIVE) { 297 if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE || 298 vd->vdev_trim_state == VDEV_TRIM_CANCELED) { 299 vd->vdev_trim_last_offset = UINT64_MAX; 300 vd->vdev_trim_rate = UINT64_MAX; 301 vd->vdev_trim_partial = UINT64_MAX; 302 vd->vdev_trim_secure = UINT64_MAX; 303 } 304 305 if (rate != 0) 306 vd->vdev_trim_rate = rate; 307 308 if (partial != 0) 309 vd->vdev_trim_partial = partial; 310 311 if (secure != 0) 312 vd->vdev_trim_secure = secure; 313 } 314 315 vdev_trim_state_t old_state = vd->vdev_trim_state; 316 boolean_t resumed = (old_state == VDEV_TRIM_SUSPENDED); 317 vd->vdev_trim_state = new_state; 318 319 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 320 VERIFY0(dmu_tx_assign(tx, TXG_WAIT)); 321 dsl_sync_task_nowait(spa_get_dsl(spa), vdev_trim_zap_update_sync, 322 guid, tx); 323 324 switch (new_state) { 325 case VDEV_TRIM_ACTIVE: 326 spa_event_notify(spa, vd, NULL, 327 resumed ? ESC_ZFS_TRIM_RESUME : ESC_ZFS_TRIM_START); 328 spa_history_log_internal(spa, "trim", tx, 329 "vdev=%s activated", vd->vdev_path); 330 break; 331 case VDEV_TRIM_SUSPENDED: 332 spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_SUSPEND); 333 spa_history_log_internal(spa, "trim", tx, 334 "vdev=%s suspended", vd->vdev_path); 335 break; 336 case VDEV_TRIM_CANCELED: 337 if (old_state == VDEV_TRIM_ACTIVE || 338 old_state == VDEV_TRIM_SUSPENDED) { 339 spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_CANCEL); 340 spa_history_log_internal(spa, "trim", tx, 341 "vdev=%s canceled", vd->vdev_path); 342 } 343 break; 344 case VDEV_TRIM_COMPLETE: 345 spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_FINISH); 346 spa_history_log_internal(spa, "trim", tx, 347 "vdev=%s complete", vd->vdev_path); 348 break; 349 default: 350 panic("invalid state %llu", (unsigned long long)new_state); 351 } 352 353 dmu_tx_commit(tx); 354 355 if (new_state != VDEV_TRIM_ACTIVE) 356 spa_notify_waiters(spa); 357 } 358 359 /* 360 * The zio_done_func_t done callback for each manual TRIM issued. It is 361 * responsible for updating the TRIM stats, reissuing failed TRIM I/Os, 362 * and limiting the number of in flight TRIM I/Os. 363 */ 364 static void 365 vdev_trim_cb(zio_t *zio) 366 { 367 vdev_t *vd = zio->io_vd; 368 369 mutex_enter(&vd->vdev_trim_io_lock); 370 if (zio->io_error == ENXIO && !vdev_writeable(vd)) { 371 /* 372 * The I/O failed because the vdev was unavailable; roll the 373 * last offset back. (This works because spa_sync waits on 374 * spa_txg_zio before it runs sync tasks.) 375 */ 376 uint64_t *offset = 377 &vd->vdev_trim_offset[zio->io_txg & TXG_MASK]; 378 *offset = MIN(*offset, zio->io_offset); 379 } else { 380 if (zio->io_error != 0) { 381 vd->vdev_stat.vs_trim_errors++; 382 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL, 383 0, 0, 0, 0, 1, zio->io_orig_size); 384 } else { 385 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL, 386 1, zio->io_orig_size, 0, 0, 0, 0); 387 } 388 389 vd->vdev_trim_bytes_done += zio->io_orig_size; 390 } 391 392 ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_MANUAL], >, 0); 393 vd->vdev_trim_inflight[TRIM_TYPE_MANUAL]--; 394 cv_broadcast(&vd->vdev_trim_io_cv); 395 mutex_exit(&vd->vdev_trim_io_lock); 396 397 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd); 398 } 399 400 /* 401 * The zio_done_func_t done callback for each automatic TRIM issued. It 402 * is responsible for updating the TRIM stats and limiting the number of 403 * in flight TRIM I/Os. Automatic TRIM I/Os are best effort and are 404 * never reissued on failure. 405 */ 406 static void 407 vdev_autotrim_cb(zio_t *zio) 408 { 409 vdev_t *vd = zio->io_vd; 410 411 mutex_enter(&vd->vdev_trim_io_lock); 412 413 if (zio->io_error != 0) { 414 vd->vdev_stat.vs_trim_errors++; 415 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO, 416 0, 0, 0, 0, 1, zio->io_orig_size); 417 } else { 418 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO, 419 1, zio->io_orig_size, 0, 0, 0, 0); 420 } 421 422 ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_AUTO], >, 0); 423 vd->vdev_trim_inflight[TRIM_TYPE_AUTO]--; 424 cv_broadcast(&vd->vdev_trim_io_cv); 425 mutex_exit(&vd->vdev_trim_io_lock); 426 427 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd); 428 } 429 430 /* 431 * The zio_done_func_t done callback for each TRIM issued via 432 * vdev_trim_simple(). It is responsible for updating the TRIM stats and 433 * limiting the number of in flight TRIM I/Os. Simple TRIM I/Os are best 434 * effort and are never reissued on failure. 435 */ 436 static void 437 vdev_trim_simple_cb(zio_t *zio) 438 { 439 vdev_t *vd = zio->io_vd; 440 441 mutex_enter(&vd->vdev_trim_io_lock); 442 443 if (zio->io_error != 0) { 444 vd->vdev_stat.vs_trim_errors++; 445 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_SIMPLE, 446 0, 0, 0, 0, 1, zio->io_orig_size); 447 } else { 448 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_SIMPLE, 449 1, zio->io_orig_size, 0, 0, 0, 0); 450 } 451 452 ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE], >, 0); 453 vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE]--; 454 cv_broadcast(&vd->vdev_trim_io_cv); 455 mutex_exit(&vd->vdev_trim_io_lock); 456 457 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd); 458 } 459 /* 460 * Returns the average trim rate in bytes/sec for the ta->trim_vdev. 461 */ 462 static uint64_t 463 vdev_trim_calculate_rate(trim_args_t *ta) 464 { 465 return (ta->trim_bytes_done * 1000 / 466 (NSEC2MSEC(gethrtime() - ta->trim_start_time) + 1)); 467 } 468 469 /* 470 * Issues a physical TRIM and takes care of rate limiting (bytes/sec) 471 * and number of concurrent TRIM I/Os. 472 */ 473 static int 474 vdev_trim_range(trim_args_t *ta, uint64_t start, uint64_t size) 475 { 476 vdev_t *vd = ta->trim_vdev; 477 spa_t *spa = vd->vdev_spa; 478 void *cb; 479 480 mutex_enter(&vd->vdev_trim_io_lock); 481 482 /* 483 * Limit manual TRIM I/Os to the requested rate. This does not 484 * apply to automatic TRIM since no per vdev rate can be specified. 485 */ 486 if (ta->trim_type == TRIM_TYPE_MANUAL) { 487 while (vd->vdev_trim_rate != 0 && !vdev_trim_should_stop(vd) && 488 vdev_trim_calculate_rate(ta) > vd->vdev_trim_rate) { 489 cv_timedwait_idle(&vd->vdev_trim_io_cv, 490 &vd->vdev_trim_io_lock, ddi_get_lbolt() + 491 MSEC_TO_TICK(10)); 492 } 493 } 494 ta->trim_bytes_done += size; 495 496 /* Limit in flight trimming I/Os */ 497 while (vd->vdev_trim_inflight[0] + vd->vdev_trim_inflight[1] + 498 vd->vdev_trim_inflight[2] >= zfs_trim_queue_limit) { 499 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock); 500 } 501 vd->vdev_trim_inflight[ta->trim_type]++; 502 mutex_exit(&vd->vdev_trim_io_lock); 503 504 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 505 VERIFY0(dmu_tx_assign(tx, TXG_WAIT)); 506 uint64_t txg = dmu_tx_get_txg(tx); 507 508 spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER); 509 mutex_enter(&vd->vdev_trim_lock); 510 511 if (ta->trim_type == TRIM_TYPE_MANUAL && 512 vd->vdev_trim_offset[txg & TXG_MASK] == 0) { 513 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP); 514 *guid = vd->vdev_guid; 515 516 /* This is the first write of this txg. */ 517 dsl_sync_task_nowait(spa_get_dsl(spa), 518 vdev_trim_zap_update_sync, guid, tx); 519 } 520 521 /* 522 * We know the vdev_t will still be around since all consumers of 523 * vdev_free must stop the trimming first. 524 */ 525 if ((ta->trim_type == TRIM_TYPE_MANUAL && 526 vdev_trim_should_stop(vd)) || 527 (ta->trim_type == TRIM_TYPE_AUTO && 528 vdev_autotrim_should_stop(vd->vdev_top))) { 529 mutex_enter(&vd->vdev_trim_io_lock); 530 vd->vdev_trim_inflight[ta->trim_type]--; 531 mutex_exit(&vd->vdev_trim_io_lock); 532 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd); 533 mutex_exit(&vd->vdev_trim_lock); 534 dmu_tx_commit(tx); 535 return (SET_ERROR(EINTR)); 536 } 537 mutex_exit(&vd->vdev_trim_lock); 538 539 if (ta->trim_type == TRIM_TYPE_MANUAL) 540 vd->vdev_trim_offset[txg & TXG_MASK] = start + size; 541 542 if (ta->trim_type == TRIM_TYPE_MANUAL) { 543 cb = vdev_trim_cb; 544 } else if (ta->trim_type == TRIM_TYPE_AUTO) { 545 cb = vdev_autotrim_cb; 546 } else { 547 cb = vdev_trim_simple_cb; 548 } 549 550 zio_nowait(zio_trim(spa->spa_txg_zio[txg & TXG_MASK], vd, 551 start, size, cb, NULL, ZIO_PRIORITY_TRIM, ZIO_FLAG_CANFAIL, 552 ta->trim_flags)); 553 /* vdev_trim_cb and vdev_autotrim_cb release SCL_STATE_ALL */ 554 555 dmu_tx_commit(tx); 556 557 return (0); 558 } 559 560 /* 561 * Issues TRIM I/Os for all ranges in the provided ta->trim_tree range tree. 562 * Additional parameters describing how the TRIM should be performed must 563 * be set in the trim_args structure. See the trim_args definition for 564 * additional information. 565 */ 566 static int 567 vdev_trim_ranges(trim_args_t *ta) 568 { 569 vdev_t *vd = ta->trim_vdev; 570 zfs_btree_t *t = &ta->trim_tree->rt_root; 571 zfs_btree_index_t idx; 572 uint64_t extent_bytes_max = ta->trim_extent_bytes_max; 573 uint64_t extent_bytes_min = ta->trim_extent_bytes_min; 574 spa_t *spa = vd->vdev_spa; 575 576 ta->trim_start_time = gethrtime(); 577 ta->trim_bytes_done = 0; 578 579 for (range_seg_t *rs = zfs_btree_first(t, &idx); rs != NULL; 580 rs = zfs_btree_next(t, &idx, &idx)) { 581 uint64_t size = rs_get_end(rs, ta->trim_tree) - rs_get_start(rs, 582 ta->trim_tree); 583 584 if (extent_bytes_min && size < extent_bytes_min) { 585 spa_iostats_trim_add(spa, ta->trim_type, 586 0, 0, 1, size, 0, 0); 587 continue; 588 } 589 590 /* Split range into legally-sized physical chunks */ 591 uint64_t writes_required = ((size - 1) / extent_bytes_max) + 1; 592 593 for (uint64_t w = 0; w < writes_required; w++) { 594 int error; 595 596 error = vdev_trim_range(ta, VDEV_LABEL_START_SIZE + 597 rs_get_start(rs, ta->trim_tree) + 598 (w *extent_bytes_max), MIN(size - 599 (w * extent_bytes_max), extent_bytes_max)); 600 if (error != 0) { 601 return (error); 602 } 603 } 604 } 605 606 return (0); 607 } 608 609 static void 610 vdev_trim_xlate_last_rs_end(void *arg, range_seg64_t *physical_rs) 611 { 612 uint64_t *last_rs_end = (uint64_t *)arg; 613 614 if (physical_rs->rs_end > *last_rs_end) 615 *last_rs_end = physical_rs->rs_end; 616 } 617 618 static void 619 vdev_trim_xlate_progress(void *arg, range_seg64_t *physical_rs) 620 { 621 vdev_t *vd = (vdev_t *)arg; 622 623 uint64_t size = physical_rs->rs_end - physical_rs->rs_start; 624 vd->vdev_trim_bytes_est += size; 625 626 if (vd->vdev_trim_last_offset >= physical_rs->rs_end) { 627 vd->vdev_trim_bytes_done += size; 628 } else if (vd->vdev_trim_last_offset > physical_rs->rs_start && 629 vd->vdev_trim_last_offset <= physical_rs->rs_end) { 630 vd->vdev_trim_bytes_done += 631 vd->vdev_trim_last_offset - physical_rs->rs_start; 632 } 633 } 634 635 /* 636 * Calculates the completion percentage of a manual TRIM. 637 */ 638 static void 639 vdev_trim_calculate_progress(vdev_t *vd) 640 { 641 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) || 642 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER)); 643 ASSERT(vd->vdev_leaf_zap != 0); 644 645 vd->vdev_trim_bytes_est = 0; 646 vd->vdev_trim_bytes_done = 0; 647 648 for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) { 649 metaslab_t *msp = vd->vdev_top->vdev_ms[i]; 650 mutex_enter(&msp->ms_lock); 651 652 uint64_t ms_free = (msp->ms_size - 653 metaslab_allocated_space(msp)) / 654 vdev_get_ndisks(vd->vdev_top); 655 656 /* 657 * Convert the metaslab range to a physical range 658 * on our vdev. We use this to determine if we are 659 * in the middle of this metaslab range. 660 */ 661 range_seg64_t logical_rs, physical_rs, remain_rs; 662 logical_rs.rs_start = msp->ms_start; 663 logical_rs.rs_end = msp->ms_start + msp->ms_size; 664 665 /* Metaslab space after this offset has not been trimmed. */ 666 vdev_xlate(vd, &logical_rs, &physical_rs, &remain_rs); 667 if (vd->vdev_trim_last_offset <= physical_rs.rs_start) { 668 vd->vdev_trim_bytes_est += ms_free; 669 mutex_exit(&msp->ms_lock); 670 continue; 671 } 672 673 /* Metaslab space before this offset has been trimmed */ 674 uint64_t last_rs_end = physical_rs.rs_end; 675 if (!vdev_xlate_is_empty(&remain_rs)) { 676 vdev_xlate_walk(vd, &remain_rs, 677 vdev_trim_xlate_last_rs_end, &last_rs_end); 678 } 679 680 if (vd->vdev_trim_last_offset > last_rs_end) { 681 vd->vdev_trim_bytes_done += ms_free; 682 vd->vdev_trim_bytes_est += ms_free; 683 mutex_exit(&msp->ms_lock); 684 continue; 685 } 686 687 /* 688 * If we get here, we're in the middle of trimming this 689 * metaslab. Load it and walk the free tree for more 690 * accurate progress estimation. 691 */ 692 VERIFY0(metaslab_load(msp)); 693 694 range_tree_t *rt = msp->ms_allocatable; 695 zfs_btree_t *bt = &rt->rt_root; 696 zfs_btree_index_t idx; 697 for (range_seg_t *rs = zfs_btree_first(bt, &idx); 698 rs != NULL; rs = zfs_btree_next(bt, &idx, &idx)) { 699 logical_rs.rs_start = rs_get_start(rs, rt); 700 logical_rs.rs_end = rs_get_end(rs, rt); 701 702 vdev_xlate_walk(vd, &logical_rs, 703 vdev_trim_xlate_progress, vd); 704 } 705 mutex_exit(&msp->ms_lock); 706 } 707 } 708 709 /* 710 * Load from disk the vdev's manual TRIM information. This includes the 711 * state, progress, and options provided when initiating the manual TRIM. 712 */ 713 static int 714 vdev_trim_load(vdev_t *vd) 715 { 716 int err = 0; 717 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) || 718 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER)); 719 ASSERT(vd->vdev_leaf_zap != 0); 720 721 if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE || 722 vd->vdev_trim_state == VDEV_TRIM_SUSPENDED) { 723 err = zap_lookup(vd->vdev_spa->spa_meta_objset, 724 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_LAST_OFFSET, 725 sizeof (vd->vdev_trim_last_offset), 1, 726 &vd->vdev_trim_last_offset); 727 if (err == ENOENT) { 728 vd->vdev_trim_last_offset = 0; 729 err = 0; 730 } 731 732 if (err == 0) { 733 err = zap_lookup(vd->vdev_spa->spa_meta_objset, 734 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_RATE, 735 sizeof (vd->vdev_trim_rate), 1, 736 &vd->vdev_trim_rate); 737 if (err == ENOENT) { 738 vd->vdev_trim_rate = 0; 739 err = 0; 740 } 741 } 742 743 if (err == 0) { 744 err = zap_lookup(vd->vdev_spa->spa_meta_objset, 745 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL, 746 sizeof (vd->vdev_trim_partial), 1, 747 &vd->vdev_trim_partial); 748 if (err == ENOENT) { 749 vd->vdev_trim_partial = 0; 750 err = 0; 751 } 752 } 753 754 if (err == 0) { 755 err = zap_lookup(vd->vdev_spa->spa_meta_objset, 756 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE, 757 sizeof (vd->vdev_trim_secure), 1, 758 &vd->vdev_trim_secure); 759 if (err == ENOENT) { 760 vd->vdev_trim_secure = 0; 761 err = 0; 762 } 763 } 764 } 765 766 vdev_trim_calculate_progress(vd); 767 768 return (err); 769 } 770 771 static void 772 vdev_trim_xlate_range_add(void *arg, range_seg64_t *physical_rs) 773 { 774 trim_args_t *ta = arg; 775 vdev_t *vd = ta->trim_vdev; 776 777 /* 778 * Only a manual trim will be traversing the vdev sequentially. 779 * For an auto trim all valid ranges should be added. 780 */ 781 if (ta->trim_type == TRIM_TYPE_MANUAL) { 782 783 /* Only add segments that we have not visited yet */ 784 if (physical_rs->rs_end <= vd->vdev_trim_last_offset) 785 return; 786 787 /* Pick up where we left off mid-range. */ 788 if (vd->vdev_trim_last_offset > physical_rs->rs_start) { 789 ASSERT3U(physical_rs->rs_end, >, 790 vd->vdev_trim_last_offset); 791 physical_rs->rs_start = vd->vdev_trim_last_offset; 792 } 793 } 794 795 ASSERT3U(physical_rs->rs_end, >, physical_rs->rs_start); 796 797 range_tree_add(ta->trim_tree, physical_rs->rs_start, 798 physical_rs->rs_end - physical_rs->rs_start); 799 } 800 801 /* 802 * Convert the logical range into physical ranges and add them to the 803 * range tree passed in the trim_args_t. 804 */ 805 static void 806 vdev_trim_range_add(void *arg, uint64_t start, uint64_t size) 807 { 808 trim_args_t *ta = arg; 809 vdev_t *vd = ta->trim_vdev; 810 range_seg64_t logical_rs; 811 logical_rs.rs_start = start; 812 logical_rs.rs_end = start + size; 813 814 /* 815 * Every range to be trimmed must be part of ms_allocatable. 816 * When ZFS_DEBUG_TRIM is set load the metaslab to verify this 817 * is always the case. 818 */ 819 if (zfs_flags & ZFS_DEBUG_TRIM) { 820 metaslab_t *msp = ta->trim_msp; 821 VERIFY0(metaslab_load(msp)); 822 VERIFY3B(msp->ms_loaded, ==, B_TRUE); 823 VERIFY(range_tree_contains(msp->ms_allocatable, start, size)); 824 } 825 826 ASSERT(vd->vdev_ops->vdev_op_leaf); 827 vdev_xlate_walk(vd, &logical_rs, vdev_trim_xlate_range_add, arg); 828 } 829 830 /* 831 * Each manual TRIM thread is responsible for trimming the unallocated 832 * space for each leaf vdev. This is accomplished by sequentially iterating 833 * over its top-level metaslabs and issuing TRIM I/O for the space described 834 * by its ms_allocatable. While a metaslab is undergoing trimming it is 835 * not eligible for new allocations. 836 */ 837 static __attribute__((noreturn)) void 838 vdev_trim_thread(void *arg) 839 { 840 vdev_t *vd = arg; 841 spa_t *spa = vd->vdev_spa; 842 trim_args_t ta; 843 int error = 0; 844 845 /* 846 * The VDEV_LEAF_ZAP_TRIM_* entries may have been updated by 847 * vdev_trim(). Wait for the updated values to be reflected 848 * in the zap in order to start with the requested settings. 849 */ 850 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0); 851 852 ASSERT(vdev_is_concrete(vd)); 853 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 854 855 vd->vdev_trim_last_offset = 0; 856 vd->vdev_trim_rate = 0; 857 vd->vdev_trim_partial = 0; 858 vd->vdev_trim_secure = 0; 859 860 VERIFY0(vdev_trim_load(vd)); 861 862 ta.trim_vdev = vd; 863 ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max; 864 ta.trim_extent_bytes_min = zfs_trim_extent_bytes_min; 865 ta.trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0); 866 ta.trim_type = TRIM_TYPE_MANUAL; 867 ta.trim_flags = 0; 868 869 /* 870 * When a secure TRIM has been requested infer that the intent 871 * is that everything must be trimmed. Override the default 872 * minimum TRIM size to prevent ranges from being skipped. 873 */ 874 if (vd->vdev_trim_secure) { 875 ta.trim_flags |= ZIO_TRIM_SECURE; 876 ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE; 877 } 878 879 uint64_t ms_count = 0; 880 for (uint64_t i = 0; !vd->vdev_detached && 881 i < vd->vdev_top->vdev_ms_count; i++) { 882 metaslab_t *msp = vd->vdev_top->vdev_ms[i]; 883 884 /* 885 * If we've expanded the top-level vdev or it's our 886 * first pass, calculate our progress. 887 */ 888 if (vd->vdev_top->vdev_ms_count != ms_count) { 889 vdev_trim_calculate_progress(vd); 890 ms_count = vd->vdev_top->vdev_ms_count; 891 } 892 893 spa_config_exit(spa, SCL_CONFIG, FTAG); 894 metaslab_disable(msp); 895 mutex_enter(&msp->ms_lock); 896 VERIFY0(metaslab_load(msp)); 897 898 /* 899 * If a partial TRIM was requested skip metaslabs which have 900 * never been initialized and thus have never been written. 901 */ 902 if (msp->ms_sm == NULL && vd->vdev_trim_partial) { 903 mutex_exit(&msp->ms_lock); 904 metaslab_enable(msp, B_FALSE, B_FALSE); 905 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 906 vdev_trim_calculate_progress(vd); 907 continue; 908 } 909 910 ta.trim_msp = msp; 911 range_tree_walk(msp->ms_allocatable, vdev_trim_range_add, &ta); 912 range_tree_vacate(msp->ms_trim, NULL, NULL); 913 mutex_exit(&msp->ms_lock); 914 915 error = vdev_trim_ranges(&ta); 916 metaslab_enable(msp, B_TRUE, B_FALSE); 917 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 918 919 range_tree_vacate(ta.trim_tree, NULL, NULL); 920 if (error != 0) 921 break; 922 } 923 924 spa_config_exit(spa, SCL_CONFIG, FTAG); 925 mutex_enter(&vd->vdev_trim_io_lock); 926 while (vd->vdev_trim_inflight[0] > 0) { 927 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock); 928 } 929 mutex_exit(&vd->vdev_trim_io_lock); 930 931 range_tree_destroy(ta.trim_tree); 932 933 mutex_enter(&vd->vdev_trim_lock); 934 if (!vd->vdev_trim_exit_wanted) { 935 if (vdev_writeable(vd)) { 936 vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE, 937 vd->vdev_trim_rate, vd->vdev_trim_partial, 938 vd->vdev_trim_secure); 939 } else if (vd->vdev_faulted) { 940 vdev_trim_change_state(vd, VDEV_TRIM_CANCELED, 941 vd->vdev_trim_rate, vd->vdev_trim_partial, 942 vd->vdev_trim_secure); 943 } 944 } 945 ASSERT(vd->vdev_trim_thread != NULL || vd->vdev_trim_inflight[0] == 0); 946 947 /* 948 * Drop the vdev_trim_lock while we sync out the txg since it's 949 * possible that a device might be trying to come online and must 950 * check to see if it needs to restart a trim. That thread will be 951 * holding the spa_config_lock which would prevent the txg_wait_synced 952 * from completing. 953 */ 954 mutex_exit(&vd->vdev_trim_lock); 955 txg_wait_synced(spa_get_dsl(spa), 0); 956 mutex_enter(&vd->vdev_trim_lock); 957 958 vd->vdev_trim_thread = NULL; 959 cv_broadcast(&vd->vdev_trim_cv); 960 mutex_exit(&vd->vdev_trim_lock); 961 962 thread_exit(); 963 } 964 965 /* 966 * Initiates a manual TRIM for the vdev_t. Callers must hold vdev_trim_lock, 967 * the vdev_t must be a leaf and cannot already be manually trimming. 968 */ 969 void 970 vdev_trim(vdev_t *vd, uint64_t rate, boolean_t partial, boolean_t secure) 971 { 972 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock)); 973 ASSERT(vd->vdev_ops->vdev_op_leaf); 974 ASSERT(vdev_is_concrete(vd)); 975 ASSERT3P(vd->vdev_trim_thread, ==, NULL); 976 ASSERT(!vd->vdev_detached); 977 ASSERT(!vd->vdev_trim_exit_wanted); 978 ASSERT(!vd->vdev_top->vdev_removing); 979 980 vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, rate, partial, secure); 981 vd->vdev_trim_thread = thread_create(NULL, 0, 982 vdev_trim_thread, vd, 0, &p0, TS_RUN, maxclsyspri); 983 } 984 985 /* 986 * Wait for the trimming thread to be terminated (canceled or stopped). 987 */ 988 static void 989 vdev_trim_stop_wait_impl(vdev_t *vd) 990 { 991 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock)); 992 993 while (vd->vdev_trim_thread != NULL) 994 cv_wait(&vd->vdev_trim_cv, &vd->vdev_trim_lock); 995 996 ASSERT3P(vd->vdev_trim_thread, ==, NULL); 997 vd->vdev_trim_exit_wanted = B_FALSE; 998 } 999 1000 /* 1001 * Wait for vdev trim threads which were listed to cleanly exit. 1002 */ 1003 void 1004 vdev_trim_stop_wait(spa_t *spa, list_t *vd_list) 1005 { 1006 (void) spa; 1007 vdev_t *vd; 1008 1009 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1010 1011 while ((vd = list_remove_head(vd_list)) != NULL) { 1012 mutex_enter(&vd->vdev_trim_lock); 1013 vdev_trim_stop_wait_impl(vd); 1014 mutex_exit(&vd->vdev_trim_lock); 1015 } 1016 } 1017 1018 /* 1019 * Stop trimming a device, with the resultant trimming state being tgt_state. 1020 * For blocking behavior pass NULL for vd_list. Otherwise, when a list_t is 1021 * provided the stopping vdev is inserted in to the list. Callers are then 1022 * required to call vdev_trim_stop_wait() to block for all the trim threads 1023 * to exit. The caller must hold vdev_trim_lock and must not be writing to 1024 * the spa config, as the trimming thread may try to enter the config as a 1025 * reader before exiting. 1026 */ 1027 void 1028 vdev_trim_stop(vdev_t *vd, vdev_trim_state_t tgt_state, list_t *vd_list) 1029 { 1030 ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER)); 1031 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock)); 1032 ASSERT(vd->vdev_ops->vdev_op_leaf); 1033 ASSERT(vdev_is_concrete(vd)); 1034 1035 /* 1036 * Allow cancel requests to proceed even if the trim thread has 1037 * stopped. 1038 */ 1039 if (vd->vdev_trim_thread == NULL && tgt_state != VDEV_TRIM_CANCELED) 1040 return; 1041 1042 vdev_trim_change_state(vd, tgt_state, 0, 0, 0); 1043 vd->vdev_trim_exit_wanted = B_TRUE; 1044 1045 if (vd_list == NULL) { 1046 vdev_trim_stop_wait_impl(vd); 1047 } else { 1048 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1049 list_insert_tail(vd_list, vd); 1050 } 1051 } 1052 1053 /* 1054 * Requests that all listed vdevs stop trimming. 1055 */ 1056 static void 1057 vdev_trim_stop_all_impl(vdev_t *vd, vdev_trim_state_t tgt_state, 1058 list_t *vd_list) 1059 { 1060 if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) { 1061 mutex_enter(&vd->vdev_trim_lock); 1062 vdev_trim_stop(vd, tgt_state, vd_list); 1063 mutex_exit(&vd->vdev_trim_lock); 1064 return; 1065 } 1066 1067 for (uint64_t i = 0; i < vd->vdev_children; i++) { 1068 vdev_trim_stop_all_impl(vd->vdev_child[i], tgt_state, 1069 vd_list); 1070 } 1071 } 1072 1073 /* 1074 * Convenience function to stop trimming of a vdev tree and set all trim 1075 * thread pointers to NULL. 1076 */ 1077 void 1078 vdev_trim_stop_all(vdev_t *vd, vdev_trim_state_t tgt_state) 1079 { 1080 spa_t *spa = vd->vdev_spa; 1081 list_t vd_list; 1082 vdev_t *vd_l2cache; 1083 1084 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1085 1086 list_create(&vd_list, sizeof (vdev_t), 1087 offsetof(vdev_t, vdev_trim_node)); 1088 1089 vdev_trim_stop_all_impl(vd, tgt_state, &vd_list); 1090 1091 /* 1092 * Iterate over cache devices and request stop trimming the 1093 * whole device in case we export the pool or remove the cache 1094 * device prematurely. 1095 */ 1096 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) { 1097 vd_l2cache = spa->spa_l2cache.sav_vdevs[i]; 1098 vdev_trim_stop_all_impl(vd_l2cache, tgt_state, &vd_list); 1099 } 1100 1101 vdev_trim_stop_wait(spa, &vd_list); 1102 1103 if (vd->vdev_spa->spa_sync_on) { 1104 /* Make sure that our state has been synced to disk */ 1105 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0); 1106 } 1107 1108 list_destroy(&vd_list); 1109 } 1110 1111 /* 1112 * Conditionally restarts a manual TRIM given its on-disk state. 1113 */ 1114 void 1115 vdev_trim_restart(vdev_t *vd) 1116 { 1117 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1118 ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER)); 1119 1120 if (vd->vdev_leaf_zap != 0) { 1121 mutex_enter(&vd->vdev_trim_lock); 1122 uint64_t trim_state = VDEV_TRIM_NONE; 1123 int err = zap_lookup(vd->vdev_spa->spa_meta_objset, 1124 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE, 1125 sizeof (trim_state), 1, &trim_state); 1126 ASSERT(err == 0 || err == ENOENT); 1127 vd->vdev_trim_state = trim_state; 1128 1129 uint64_t timestamp = 0; 1130 err = zap_lookup(vd->vdev_spa->spa_meta_objset, 1131 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_ACTION_TIME, 1132 sizeof (timestamp), 1, ×tamp); 1133 ASSERT(err == 0 || err == ENOENT); 1134 vd->vdev_trim_action_time = timestamp; 1135 1136 if (vd->vdev_trim_state == VDEV_TRIM_SUSPENDED || 1137 vd->vdev_offline) { 1138 /* load progress for reporting, but don't resume */ 1139 VERIFY0(vdev_trim_load(vd)); 1140 } else if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE && 1141 vdev_writeable(vd) && !vd->vdev_top->vdev_removing && 1142 vd->vdev_trim_thread == NULL) { 1143 VERIFY0(vdev_trim_load(vd)); 1144 vdev_trim(vd, vd->vdev_trim_rate, 1145 vd->vdev_trim_partial, vd->vdev_trim_secure); 1146 } 1147 1148 mutex_exit(&vd->vdev_trim_lock); 1149 } 1150 1151 for (uint64_t i = 0; i < vd->vdev_children; i++) { 1152 vdev_trim_restart(vd->vdev_child[i]); 1153 } 1154 } 1155 1156 /* 1157 * Used by the automatic TRIM when ZFS_DEBUG_TRIM is set to verify that 1158 * every TRIM range is contained within ms_allocatable. 1159 */ 1160 static void 1161 vdev_trim_range_verify(void *arg, uint64_t start, uint64_t size) 1162 { 1163 trim_args_t *ta = arg; 1164 metaslab_t *msp = ta->trim_msp; 1165 1166 VERIFY3B(msp->ms_loaded, ==, B_TRUE); 1167 VERIFY3U(msp->ms_disabled, >, 0); 1168 VERIFY(range_tree_contains(msp->ms_allocatable, start, size)); 1169 } 1170 1171 /* 1172 * Each automatic TRIM thread is responsible for managing the trimming of a 1173 * top-level vdev in the pool. No automatic TRIM state is maintained on-disk. 1174 * 1175 * N.B. This behavior is different from a manual TRIM where a thread 1176 * is created for each leaf vdev, instead of each top-level vdev. 1177 */ 1178 static __attribute__((noreturn)) void 1179 vdev_autotrim_thread(void *arg) 1180 { 1181 vdev_t *vd = arg; 1182 spa_t *spa = vd->vdev_spa; 1183 int shift = 0; 1184 1185 mutex_enter(&vd->vdev_autotrim_lock); 1186 ASSERT3P(vd->vdev_top, ==, vd); 1187 ASSERT3P(vd->vdev_autotrim_thread, !=, NULL); 1188 mutex_exit(&vd->vdev_autotrim_lock); 1189 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 1190 1191 while (!vdev_autotrim_should_stop(vd)) { 1192 int txgs_per_trim = MAX(zfs_trim_txg_batch, 1); 1193 boolean_t issued_trim = B_FALSE; 1194 uint64_t extent_bytes_max = zfs_trim_extent_bytes_max; 1195 uint64_t extent_bytes_min = zfs_trim_extent_bytes_min; 1196 1197 /* 1198 * All of the metaslabs are divided in to groups of size 1199 * num_metaslabs / zfs_trim_txg_batch. Each of these groups 1200 * is composed of metaslabs which are spread evenly over the 1201 * device. 1202 * 1203 * For example, when zfs_trim_txg_batch = 32 (default) then 1204 * group 0 will contain metaslabs 0, 32, 64, ...; 1205 * group 1 will contain metaslabs 1, 33, 65, ...; 1206 * group 2 will contain metaslabs 2, 34, 66, ...; and so on. 1207 * 1208 * On each pass through the while() loop one of these groups 1209 * is selected. This is accomplished by using a shift value 1210 * to select the starting metaslab, then striding over the 1211 * metaslabs using the zfs_trim_txg_batch size. This is 1212 * done to accomplish two things. 1213 * 1214 * 1) By dividing the metaslabs in to groups, and making sure 1215 * that each group takes a minimum of one txg to process. 1216 * Then zfs_trim_txg_batch controls the minimum number of 1217 * txgs which must occur before a metaslab is revisited. 1218 * 1219 * 2) Selecting non-consecutive metaslabs distributes the 1220 * TRIM commands for a group evenly over the entire device. 1221 * This can be advantageous for certain types of devices. 1222 */ 1223 for (uint64_t i = shift % txgs_per_trim; i < vd->vdev_ms_count; 1224 i += txgs_per_trim) { 1225 metaslab_t *msp = vd->vdev_ms[i]; 1226 range_tree_t *trim_tree; 1227 1228 spa_config_exit(spa, SCL_CONFIG, FTAG); 1229 metaslab_disable(msp); 1230 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 1231 1232 mutex_enter(&msp->ms_lock); 1233 1234 /* 1235 * Skip the metaslab when it has never been allocated 1236 * or when there are no recent frees to trim. 1237 */ 1238 if (msp->ms_sm == NULL || 1239 range_tree_is_empty(msp->ms_trim)) { 1240 mutex_exit(&msp->ms_lock); 1241 metaslab_enable(msp, B_FALSE, B_FALSE); 1242 continue; 1243 } 1244 1245 /* 1246 * Skip the metaslab when it has already been disabled. 1247 * This may happen when a manual TRIM or initialize 1248 * operation is running concurrently. In the case 1249 * of a manual TRIM, the ms_trim tree will have been 1250 * vacated. Only ranges added after the manual TRIM 1251 * disabled the metaslab will be included in the tree. 1252 * These will be processed when the automatic TRIM 1253 * next revisits this metaslab. 1254 */ 1255 if (msp->ms_disabled > 1) { 1256 mutex_exit(&msp->ms_lock); 1257 metaslab_enable(msp, B_FALSE, B_FALSE); 1258 continue; 1259 } 1260 1261 /* 1262 * Allocate an empty range tree which is swapped in 1263 * for the existing ms_trim tree while it is processed. 1264 */ 1265 trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL, 1266 0, 0); 1267 range_tree_swap(&msp->ms_trim, &trim_tree); 1268 ASSERT(range_tree_is_empty(msp->ms_trim)); 1269 1270 /* 1271 * There are two cases when constructing the per-vdev 1272 * trim trees for a metaslab. If the top-level vdev 1273 * has no children then it is also a leaf and should 1274 * be trimmed. Otherwise our children are the leaves 1275 * and a trim tree should be constructed for each. 1276 */ 1277 trim_args_t *tap; 1278 uint64_t children = vd->vdev_children; 1279 if (children == 0) { 1280 children = 1; 1281 tap = kmem_zalloc(sizeof (trim_args_t) * 1282 children, KM_SLEEP); 1283 tap[0].trim_vdev = vd; 1284 } else { 1285 tap = kmem_zalloc(sizeof (trim_args_t) * 1286 children, KM_SLEEP); 1287 1288 for (uint64_t c = 0; c < children; c++) { 1289 tap[c].trim_vdev = vd->vdev_child[c]; 1290 } 1291 } 1292 1293 for (uint64_t c = 0; c < children; c++) { 1294 trim_args_t *ta = &tap[c]; 1295 vdev_t *cvd = ta->trim_vdev; 1296 1297 ta->trim_msp = msp; 1298 ta->trim_extent_bytes_max = extent_bytes_max; 1299 ta->trim_extent_bytes_min = extent_bytes_min; 1300 ta->trim_type = TRIM_TYPE_AUTO; 1301 ta->trim_flags = 0; 1302 1303 if (cvd->vdev_detached || 1304 !vdev_writeable(cvd) || 1305 !cvd->vdev_has_trim || 1306 cvd->vdev_trim_thread != NULL) { 1307 continue; 1308 } 1309 1310 /* 1311 * When a device has an attached hot spare, or 1312 * is being replaced it will not be trimmed. 1313 * This is done to avoid adding additional 1314 * stress to a potentially unhealthy device, 1315 * and to minimize the required rebuild time. 1316 */ 1317 if (!cvd->vdev_ops->vdev_op_leaf) 1318 continue; 1319 1320 ta->trim_tree = range_tree_create(NULL, 1321 RANGE_SEG64, NULL, 0, 0); 1322 range_tree_walk(trim_tree, 1323 vdev_trim_range_add, ta); 1324 } 1325 1326 mutex_exit(&msp->ms_lock); 1327 spa_config_exit(spa, SCL_CONFIG, FTAG); 1328 1329 /* 1330 * Issue the TRIM I/Os for all ranges covered by the 1331 * TRIM trees. These ranges are safe to TRIM because 1332 * no new allocations will be performed until the call 1333 * to metaslab_enabled() below. 1334 */ 1335 for (uint64_t c = 0; c < children; c++) { 1336 trim_args_t *ta = &tap[c]; 1337 1338 /* 1339 * Always yield to a manual TRIM if one has 1340 * been started for the child vdev. 1341 */ 1342 if (ta->trim_tree == NULL || 1343 ta->trim_vdev->vdev_trim_thread != NULL) { 1344 continue; 1345 } 1346 1347 /* 1348 * After this point metaslab_enable() must be 1349 * called with the sync flag set. This is done 1350 * here because vdev_trim_ranges() is allowed 1351 * to be interrupted (EINTR) before issuing all 1352 * of the required TRIM I/Os. 1353 */ 1354 issued_trim = B_TRUE; 1355 1356 int error = vdev_trim_ranges(ta); 1357 if (error) 1358 break; 1359 } 1360 1361 /* 1362 * Verify every range which was trimmed is still 1363 * contained within the ms_allocatable tree. 1364 */ 1365 if (zfs_flags & ZFS_DEBUG_TRIM) { 1366 mutex_enter(&msp->ms_lock); 1367 VERIFY0(metaslab_load(msp)); 1368 VERIFY3P(tap[0].trim_msp, ==, msp); 1369 range_tree_walk(trim_tree, 1370 vdev_trim_range_verify, &tap[0]); 1371 mutex_exit(&msp->ms_lock); 1372 } 1373 1374 range_tree_vacate(trim_tree, NULL, NULL); 1375 range_tree_destroy(trim_tree); 1376 1377 metaslab_enable(msp, issued_trim, B_FALSE); 1378 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 1379 1380 for (uint64_t c = 0; c < children; c++) { 1381 trim_args_t *ta = &tap[c]; 1382 1383 if (ta->trim_tree == NULL) 1384 continue; 1385 1386 range_tree_vacate(ta->trim_tree, NULL, NULL); 1387 range_tree_destroy(ta->trim_tree); 1388 } 1389 1390 kmem_free(tap, sizeof (trim_args_t) * children); 1391 } 1392 1393 spa_config_exit(spa, SCL_CONFIG, FTAG); 1394 1395 /* 1396 * After completing the group of metaslabs wait for the next 1397 * open txg. This is done to make sure that a minimum of 1398 * zfs_trim_txg_batch txgs will occur before these metaslabs 1399 * are trimmed again. 1400 */ 1401 txg_wait_open(spa_get_dsl(spa), 0, issued_trim); 1402 1403 shift++; 1404 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 1405 } 1406 1407 for (uint64_t c = 0; c < vd->vdev_children; c++) { 1408 vdev_t *cvd = vd->vdev_child[c]; 1409 mutex_enter(&cvd->vdev_trim_io_lock); 1410 1411 while (cvd->vdev_trim_inflight[1] > 0) { 1412 cv_wait(&cvd->vdev_trim_io_cv, 1413 &cvd->vdev_trim_io_lock); 1414 } 1415 mutex_exit(&cvd->vdev_trim_io_lock); 1416 } 1417 1418 spa_config_exit(spa, SCL_CONFIG, FTAG); 1419 1420 /* 1421 * When exiting because the autotrim property was set to off, then 1422 * abandon any unprocessed ms_trim ranges to reclaim the memory. 1423 */ 1424 if (spa_get_autotrim(spa) == SPA_AUTOTRIM_OFF) { 1425 for (uint64_t i = 0; i < vd->vdev_ms_count; i++) { 1426 metaslab_t *msp = vd->vdev_ms[i]; 1427 1428 mutex_enter(&msp->ms_lock); 1429 range_tree_vacate(msp->ms_trim, NULL, NULL); 1430 mutex_exit(&msp->ms_lock); 1431 } 1432 } 1433 1434 mutex_enter(&vd->vdev_autotrim_lock); 1435 ASSERT(vd->vdev_autotrim_thread != NULL); 1436 vd->vdev_autotrim_thread = NULL; 1437 cv_broadcast(&vd->vdev_autotrim_cv); 1438 mutex_exit(&vd->vdev_autotrim_lock); 1439 1440 thread_exit(); 1441 } 1442 1443 /* 1444 * Starts an autotrim thread, if needed, for each top-level vdev which can be 1445 * trimmed. A top-level vdev which has been evacuated will never be trimmed. 1446 */ 1447 void 1448 vdev_autotrim(spa_t *spa) 1449 { 1450 vdev_t *root_vd = spa->spa_root_vdev; 1451 1452 for (uint64_t i = 0; i < root_vd->vdev_children; i++) { 1453 vdev_t *tvd = root_vd->vdev_child[i]; 1454 1455 mutex_enter(&tvd->vdev_autotrim_lock); 1456 if (vdev_writeable(tvd) && !tvd->vdev_removing && 1457 tvd->vdev_autotrim_thread == NULL) { 1458 ASSERT3P(tvd->vdev_top, ==, tvd); 1459 1460 tvd->vdev_autotrim_thread = thread_create(NULL, 0, 1461 vdev_autotrim_thread, tvd, 0, &p0, TS_RUN, 1462 maxclsyspri); 1463 ASSERT(tvd->vdev_autotrim_thread != NULL); 1464 } 1465 mutex_exit(&tvd->vdev_autotrim_lock); 1466 } 1467 } 1468 1469 /* 1470 * Wait for the vdev_autotrim_thread associated with the passed top-level 1471 * vdev to be terminated (canceled or stopped). 1472 */ 1473 void 1474 vdev_autotrim_stop_wait(vdev_t *tvd) 1475 { 1476 mutex_enter(&tvd->vdev_autotrim_lock); 1477 if (tvd->vdev_autotrim_thread != NULL) { 1478 tvd->vdev_autotrim_exit_wanted = B_TRUE; 1479 1480 while (tvd->vdev_autotrim_thread != NULL) { 1481 cv_wait(&tvd->vdev_autotrim_cv, 1482 &tvd->vdev_autotrim_lock); 1483 } 1484 1485 ASSERT3P(tvd->vdev_autotrim_thread, ==, NULL); 1486 tvd->vdev_autotrim_exit_wanted = B_FALSE; 1487 } 1488 mutex_exit(&tvd->vdev_autotrim_lock); 1489 } 1490 1491 /* 1492 * Wait for all of the vdev_autotrim_thread associated with the pool to 1493 * be terminated (canceled or stopped). 1494 */ 1495 void 1496 vdev_autotrim_stop_all(spa_t *spa) 1497 { 1498 vdev_t *root_vd = spa->spa_root_vdev; 1499 1500 for (uint64_t i = 0; i < root_vd->vdev_children; i++) 1501 vdev_autotrim_stop_wait(root_vd->vdev_child[i]); 1502 } 1503 1504 /* 1505 * Conditionally restart all of the vdev_autotrim_thread's for the pool. 1506 */ 1507 void 1508 vdev_autotrim_restart(spa_t *spa) 1509 { 1510 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1511 1512 if (spa->spa_autotrim) 1513 vdev_autotrim(spa); 1514 } 1515 1516 static __attribute__((noreturn)) void 1517 vdev_trim_l2arc_thread(void *arg) 1518 { 1519 vdev_t *vd = arg; 1520 spa_t *spa = vd->vdev_spa; 1521 l2arc_dev_t *dev = l2arc_vdev_get(vd); 1522 trim_args_t ta = {0}; 1523 range_seg64_t physical_rs; 1524 1525 ASSERT(vdev_is_concrete(vd)); 1526 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 1527 1528 vd->vdev_trim_last_offset = 0; 1529 vd->vdev_trim_rate = 0; 1530 vd->vdev_trim_partial = 0; 1531 vd->vdev_trim_secure = 0; 1532 1533 ta.trim_vdev = vd; 1534 ta.trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0); 1535 ta.trim_type = TRIM_TYPE_MANUAL; 1536 ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max; 1537 ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE; 1538 ta.trim_flags = 0; 1539 1540 physical_rs.rs_start = vd->vdev_trim_bytes_done = 0; 1541 physical_rs.rs_end = vd->vdev_trim_bytes_est = 1542 vdev_get_min_asize(vd); 1543 1544 range_tree_add(ta.trim_tree, physical_rs.rs_start, 1545 physical_rs.rs_end - physical_rs.rs_start); 1546 1547 mutex_enter(&vd->vdev_trim_lock); 1548 vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, 0, 0, 0); 1549 mutex_exit(&vd->vdev_trim_lock); 1550 1551 (void) vdev_trim_ranges(&ta); 1552 1553 spa_config_exit(spa, SCL_CONFIG, FTAG); 1554 mutex_enter(&vd->vdev_trim_io_lock); 1555 while (vd->vdev_trim_inflight[TRIM_TYPE_MANUAL] > 0) { 1556 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock); 1557 } 1558 mutex_exit(&vd->vdev_trim_io_lock); 1559 1560 range_tree_vacate(ta.trim_tree, NULL, NULL); 1561 range_tree_destroy(ta.trim_tree); 1562 1563 mutex_enter(&vd->vdev_trim_lock); 1564 if (!vd->vdev_trim_exit_wanted && vdev_writeable(vd)) { 1565 vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE, 1566 vd->vdev_trim_rate, vd->vdev_trim_partial, 1567 vd->vdev_trim_secure); 1568 } 1569 ASSERT(vd->vdev_trim_thread != NULL || 1570 vd->vdev_trim_inflight[TRIM_TYPE_MANUAL] == 0); 1571 1572 /* 1573 * Drop the vdev_trim_lock while we sync out the txg since it's 1574 * possible that a device might be trying to come online and 1575 * must check to see if it needs to restart a trim. That thread 1576 * will be holding the spa_config_lock which would prevent the 1577 * txg_wait_synced from completing. Same strategy as in 1578 * vdev_trim_thread(). 1579 */ 1580 mutex_exit(&vd->vdev_trim_lock); 1581 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0); 1582 mutex_enter(&vd->vdev_trim_lock); 1583 1584 /* 1585 * Update the header of the cache device here, before 1586 * broadcasting vdev_trim_cv which may lead to the removal 1587 * of the device. The same applies for setting l2ad_trim_all to 1588 * false. 1589 */ 1590 spa_config_enter(vd->vdev_spa, SCL_L2ARC, vd, 1591 RW_READER); 1592 memset(dev->l2ad_dev_hdr, 0, dev->l2ad_dev_hdr_asize); 1593 l2arc_dev_hdr_update(dev); 1594 spa_config_exit(vd->vdev_spa, SCL_L2ARC, vd); 1595 1596 vd->vdev_trim_thread = NULL; 1597 if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE) 1598 dev->l2ad_trim_all = B_FALSE; 1599 1600 cv_broadcast(&vd->vdev_trim_cv); 1601 mutex_exit(&vd->vdev_trim_lock); 1602 1603 thread_exit(); 1604 } 1605 1606 /* 1607 * Punches out TRIM threads for the L2ARC devices in a spa and assigns them 1608 * to vd->vdev_trim_thread variable. This facilitates the management of 1609 * trimming the whole cache device using TRIM_TYPE_MANUAL upon addition 1610 * to a pool or pool creation or when the header of the device is invalid. 1611 */ 1612 void 1613 vdev_trim_l2arc(spa_t *spa) 1614 { 1615 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1616 1617 /* 1618 * Locate the spa's l2arc devices and kick off TRIM threads. 1619 */ 1620 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) { 1621 vdev_t *vd = spa->spa_l2cache.sav_vdevs[i]; 1622 l2arc_dev_t *dev = l2arc_vdev_get(vd); 1623 1624 if (dev == NULL || !dev->l2ad_trim_all) { 1625 /* 1626 * Don't attempt TRIM if the vdev is UNAVAIL or if the 1627 * cache device was not marked for whole device TRIM 1628 * (ie l2arc_trim_ahead = 0, or the L2ARC device header 1629 * is valid with trim_state = VDEV_TRIM_COMPLETE and 1630 * l2ad_log_entries > 0). 1631 */ 1632 continue; 1633 } 1634 1635 mutex_enter(&vd->vdev_trim_lock); 1636 ASSERT(vd->vdev_ops->vdev_op_leaf); 1637 ASSERT(vdev_is_concrete(vd)); 1638 ASSERT3P(vd->vdev_trim_thread, ==, NULL); 1639 ASSERT(!vd->vdev_detached); 1640 ASSERT(!vd->vdev_trim_exit_wanted); 1641 ASSERT(!vd->vdev_top->vdev_removing); 1642 vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, 0, 0, 0); 1643 vd->vdev_trim_thread = thread_create(NULL, 0, 1644 vdev_trim_l2arc_thread, vd, 0, &p0, TS_RUN, maxclsyspri); 1645 mutex_exit(&vd->vdev_trim_lock); 1646 } 1647 } 1648 1649 /* 1650 * A wrapper which calls vdev_trim_ranges(). It is intended to be called 1651 * on leaf vdevs. 1652 */ 1653 int 1654 vdev_trim_simple(vdev_t *vd, uint64_t start, uint64_t size) 1655 { 1656 trim_args_t ta = {0}; 1657 range_seg64_t physical_rs; 1658 int error; 1659 physical_rs.rs_start = start; 1660 physical_rs.rs_end = start + size; 1661 1662 ASSERT(vdev_is_concrete(vd)); 1663 ASSERT(vd->vdev_ops->vdev_op_leaf); 1664 ASSERT(!vd->vdev_detached); 1665 ASSERT(!vd->vdev_top->vdev_removing); 1666 1667 ta.trim_vdev = vd; 1668 ta.trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0); 1669 ta.trim_type = TRIM_TYPE_SIMPLE; 1670 ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max; 1671 ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE; 1672 ta.trim_flags = 0; 1673 1674 ASSERT3U(physical_rs.rs_end, >=, physical_rs.rs_start); 1675 1676 if (physical_rs.rs_end > physical_rs.rs_start) { 1677 range_tree_add(ta.trim_tree, physical_rs.rs_start, 1678 physical_rs.rs_end - physical_rs.rs_start); 1679 } else { 1680 ASSERT3U(physical_rs.rs_end, ==, physical_rs.rs_start); 1681 } 1682 1683 error = vdev_trim_ranges(&ta); 1684 1685 mutex_enter(&vd->vdev_trim_io_lock); 1686 while (vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE] > 0) { 1687 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock); 1688 } 1689 mutex_exit(&vd->vdev_trim_io_lock); 1690 1691 range_tree_vacate(ta.trim_tree, NULL, NULL); 1692 range_tree_destroy(ta.trim_tree); 1693 1694 return (error); 1695 } 1696 1697 EXPORT_SYMBOL(vdev_trim); 1698 EXPORT_SYMBOL(vdev_trim_stop); 1699 EXPORT_SYMBOL(vdev_trim_stop_all); 1700 EXPORT_SYMBOL(vdev_trim_stop_wait); 1701 EXPORT_SYMBOL(vdev_trim_restart); 1702 EXPORT_SYMBOL(vdev_autotrim); 1703 EXPORT_SYMBOL(vdev_autotrim_stop_all); 1704 EXPORT_SYMBOL(vdev_autotrim_stop_wait); 1705 EXPORT_SYMBOL(vdev_autotrim_restart); 1706 EXPORT_SYMBOL(vdev_trim_l2arc); 1707 EXPORT_SYMBOL(vdev_trim_simple); 1708 1709 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, extent_bytes_max, UINT, ZMOD_RW, 1710 "Max size of TRIM commands, larger will be split"); 1711 1712 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, extent_bytes_min, UINT, ZMOD_RW, 1713 "Min size of TRIM commands, smaller will be skipped"); 1714 1715 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, metaslab_skip, UINT, ZMOD_RW, 1716 "Skip metaslabs which have never been initialized"); 1717 1718 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, txg_batch, UINT, ZMOD_RW, 1719 "Min number of txgs to aggregate frees before issuing TRIM"); 1720 1721 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, queue_limit, UINT, ZMOD_RW, 1722 "Max queued TRIMs outstanding per leaf vdev");
Cache object: d71151d8ee014442cec96e2a2aec224a
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]