mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import wiredtiger: dab483768edbfd660e54a8f753b664b7f7c86bdc from branch mongodb-8.2 (#49637)

Browse Source 
Co-authored-by: wt-vendoring-bot <wt-vendoring-bot@mongodb.com>
GitOrigin-RevId: ebc797b78f800667a81ee491425c9206d062a0b6
This commit is contained in:
wt-vendoring-bot[bot] 2026-03-16 10:55:25 +11:00 committed by MongoDB Bot
parent d77c2c1b4a
commit 8fe8ccc913
10 changed files with 774 additions and 620 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/third_party/wiredtiger/dist/stat_data.py vendored
View File

@ -72,6 +72,10 @@ class CapacityStat(Stat):
prefix = 'capacity'
def __init__(self, name, desc, flags=''):
Stat.__init__(self, name, CapacityStat.prefix, desc, flags)
class CheckpointCleanupStat(Stat):
prefix = 'checkpoint-cleanup'
def __init__(self, name, desc, flags=''):
Stat.__init__(self, name, CheckpointCleanupStat.prefix, desc, flags)
class CheckpointStat(Stat):
prefix = 'checkpoint'
def __init__(self, name, desc, flags=''):
@ -415,10 +419,17 @@ conn_stats = [
CapacityStat('fsync_all_fh_total', 'background fsync file handles considered'),
CapacityStat('fsync_all_time', 'background fsync time (msecs)', 'no_clear,no_scale'),
##########################################
# Checkpoint Cleanup statistics
##########################################
CheckpointCleanupStat('checkpoint_cleanup_duration', 'most recent duration on all eligible files (usecs)', 'no_clear,no_scale'),
CheckpointCleanupStat('checkpoint_cleanup_handle_processed', 'most recent handles processed'),
CheckpointCleanupStat('checkpoint_cleanup_inmem_pages_visited', 'most recent in-memory pages visited'),
CheckpointCleanupStat('checkpoint_cleanup_success', 'successful calls'),
##########################################
# Checkpoint statistics
##########################################
CheckpointStat('checkpoint_cleanup_success', 'checkpoint cleanup successful calls'),
CheckpointStat('checkpoint_fsync_post', 'fsync calls after allocating the transaction ID'),
CheckpointStat('checkpoint_fsync_post_duration', 'fsync duration after allocating the transaction ID (usecs)', 'no_clear,no_scale'),
CheckpointStat('checkpoint_generation', 'generation', 'no_clear,no_scale'),
@ -1123,16 +1134,20 @@ conn_dsrc_stats = [
CacheStat('cache_write_hs', 'page written requiring history store records'),
CacheStat('cache_write_restore', 'pages written requiring in-memory restoration'),
##########################################
# Checkpoint Cleanup statistics
##########################################
CheckpointCleanupStat('checkpoint_cleanup_pages_evict', 'pages added for eviction'),
CheckpointCleanupStat('checkpoint_cleanup_pages_obsolete_tw', 'pages dirtied due to obsolete time window'),
CheckpointCleanupStat('checkpoint_cleanup_pages_read_obsolete_tw', 'pages read into cache due to obsolete time window'),
CheckpointCleanupStat('checkpoint_cleanup_pages_read_reclaim_space', 'pages read into cache (reclaim_space)'),
CheckpointCleanupStat('checkpoint_cleanup_pages_removed', 'pages removed'),
CheckpointCleanupStat('checkpoint_cleanup_pages_visited', 'pages visited'),
CheckpointCleanupStat('checkpoint_cleanup_pages_walk_skipped', 'pages skipped during tree walk'),
##########################################
# Checkpoint statistics
##########################################
CheckpointStat('checkpoint_cleanup_pages_evict', 'pages added for eviction during checkpoint cleanup'),
CheckpointStat('checkpoint_cleanup_pages_obsolete_tw', 'pages dirtied due to obsolete time window by checkpoint cleanup'),
CheckpointStat('checkpoint_cleanup_pages_read_obsolete_tw', 'pages read into cache during checkpoint cleanup due to obsolete time window'),
CheckpointStat('checkpoint_cleanup_pages_read_reclaim_space', 'pages read into cache during checkpoint cleanup (reclaim_space)'),
CheckpointStat('checkpoint_cleanup_pages_removed', 'pages removed during checkpoint cleanup'),
CheckpointStat('checkpoint_cleanup_pages_visited', 'pages visited during checkpoint cleanup'),
CheckpointStat('checkpoint_cleanup_pages_walk_skipped', 'pages skipped during checkpoint cleanup tree walk'),
CheckpointStat('checkpoint_snapshot_acquired', 'checkpoint has acquired a snapshot for its transaction'),
##########################################

2
src/third_party/wiredtiger/import.data vendored
View File

@ -2,5 +2,5 @@
"vendor": "wiredtiger",
"github": "wiredtiger/wiredtiger",
"branch": "mongodb-8.2",
"commit": "ed5be22a19ee9e3cc5a8e3df1f4370782fd1b33b"
"commit": "dab483768edbfd660e54a8f753b664b7f7c86bdc"
}

31
src/third_party/wiredtiger/src/btree/bt_sync_obsolete.c vendored
View File

@ -484,11 +484,10 @@ __checkpoint_cleanup_walk_btree(WT_SESSION_IMPL *session, WT_ITEM *uri)
{
WT_BTREE *btree;
WT_DECL_RET;
WT_REF *ref;
uint32_t flags;
ref = NULL;
flags = WT_READ_NO_EVICT | WT_READ_VISIBLE_ALL;
WT_REF *ref = NULL;
uint32_t flags = WT_READ_NO_EVICT | WT_READ_VISIBLE_ALL;
uint32_t pages_visited = 0;
uint64_t elapsed_us, end_time, start_time = __wt_clock(session);
/* Open a handle for processing. */
ret = __wt_session_get_dhandle(session, uri->data, NULL, NULL, 0);
@ -518,6 +517,7 @@ __checkpoint_cleanup_walk_btree(WT_SESSION_IMPL *session, WT_ITEM *uri)
while ((ret = __wt_tree_walk_custom_skip(
session, &ref, __checkpoint_cleanup_page_skip, NULL, flags)) == 0 &&
ref != NULL) {
++pages_visited;
if (F_ISSET(ref, WT_REF_FLAG_INTERNAL)) {
WT_WITH_PAGE_INDEX(session, ret = __checkpoint_cleanup_obsolete_cleanup(session, ref));
} else {
@ -533,6 +533,14 @@ __checkpoint_cleanup_walk_btree(WT_SESSION_IMPL *session, WT_ITEM *uri)
}
err:
end_time = __wt_clock(session);
elapsed_us = WT_CLOCKDIFF_US(end_time, start_time);
__wt_verbose_debug1(session, WT_VERB_CHECKPOINT_CLEANUP,
"checkpoint cleanup btree walk completed (ret=%d), pages_visited=%" PRIu32
", elapsed_us=%" PRIu64,
ret, pages_visited, elapsed_us);
WT_STAT_CONN_SET(session, checkpoint_cleanup_inmem_pages_visited, pages_visited);
/* On error, clear any left-over tree walk. */
WT_TRET(__wt_page_release(session, ref, flags));
WT_TRET(__wt_session_release_dhandle(session));
@ -710,6 +718,9 @@ __checkpoint_cleanup_int(WT_SESSION_IMPL *session)
WT_DECL_ITEM(uri);
WT_DECL_RET;
uint32_t tables_processed = 0;
uint64_t elapsed_us, end_time, start_time = __wt_clock(session);
WT_RET(__wt_scr_alloc(session, 1024, &uri));
WT_ERR(__wt_buf_set(session, uri, WT_URI_FILE_PREFIX, strlen(WT_URI_FILE_PREFIX) + 1));
@ -722,6 +733,7 @@ __checkpoint_cleanup_int(WT_SESSION_IMPL *session)
continue;
}
WT_ERR(ret);
++tables_processed;
/* Check if we need to wait before continuing with the next file to minimize impact. */
if (S2C(session)->cc_cleanup.file_wait_ms > 0) {
@ -741,6 +753,15 @@ __checkpoint_cleanup_int(WT_SESSION_IMPL *session)
WT_ERR_NOTFOUND_OK(ret, false);
err:
end_time = __wt_clock(session);
elapsed_us = WT_CLOCKDIFF_US(end_time, start_time);
__wt_verbose_debug1(session, WT_VERB_CHECKPOINT_CLEANUP,
"checkpoint cleanup full iteration completed (ret=%d), tables_processed=%" PRIu32
" elapsed_us=%" PRIu64,
ret, tables_processed, elapsed_us);
WT_STAT_CONN_SET(session, checkpoint_cleanup_duration, elapsed_us);
WT_STAT_CONN_SET(session, checkpoint_cleanup_handle_processed, tables_processed);
__wt_scr_free(session, &uri);
return (ret);
}

23
src/third_party/wiredtiger/src/evict/evict_lru.c vendored
View File

@ -1407,9 +1407,7 @@ __evict_lru_walk(WT_SESSION_IMPL *session)
if (__evict_queue_full(queue) && !__evict_queue_full(other_queue))
queue = other_queue;
/*
* If both queues are full and haven't been empty on recent refills, we're done.
*/
/* If both queues are full and haven't been empty on recent refills, we're done. */
if (__evict_queue_full(queue) && evict->evict_empty_score < WT_EVICT_SCORE_CUTOFF) {
WT_STAT_CONN_INCR(session, eviction_queue_not_empty);
goto err;
@ -1528,24 +1526,22 @@ __evict_lru_walk(WT_SESSION_IMPL *session)
}
WT_STAT_CONN_INCRV(session, eviction_pages_queued_post_lru, queue->evict_candidates);
/*
* Add stats about pages that have been queued.
*/
/* Add stats about pages that have been queued. */
for (candidates = 0; candidates < queue->evict_candidates; ++candidates) {
WT_PAGE *page = queue->evict_queue[candidates].ref->page;
if (__wt_page_is_modified(page))
WT_STAT_CONN_DSRC_INCR(session, cache_eviction_pages_queued_dirty);
else if (page->modify != NULL)
WT_STAT_CONN_DSRC_INCR(session, cache_eviction_pages_queued_updates);
else
WT_STAT_CONN_DSRC_INCR(session, cache_eviction_pages_queued_clean);
if (page->modify != NULL)
WT_STAT_CONN_DSRC_INCR(session, cache_eviction_pages_queued_updates);
}
queue->evict_current = queue->evict_queue;
__wt_spin_unlock(session, &queue->evict_lock);
/*
* Signal any application or helper threads that may be waiting to help with eviction.
*/
/* Signal any application or helper threads that may be waiting to help with eviction. */
__wt_cond_signal(session, conn->evict_threads.wait_cond);
err:
@ -2549,11 +2545,12 @@ __evict_walk_tree(WT_SESSION_IMPL *session, WTI_EVICT_QUEUE *queue, u_int max_en
if (__wt_page_is_modified(page))
++pages_seen_dirty;
else if (page->modify != NULL)
++pages_seen_updates;
else
++pages_seen_clean;
if (page->modify != NULL)
++pages_seen_updates;
/* Count internal pages seen. */
if (F_ISSET(ref, WT_REF_FLAG_INTERNAL))
internal_pages_seen++;

19
src/third_party/wiredtiger/src/include/stat.h vendored
View File

@ -712,6 +712,16 @@ struct __wt_connection_stats {
int64_t capacity_time_log;
int64_t capacity_time_read;
int64_t capacity_time_chunkcache;
int64_t checkpoint_cleanup_duration;
int64_t checkpoint_cleanup_handle_processed;
int64_t checkpoint_cleanup_inmem_pages_visited;
int64_t checkpoint_cleanup_pages_evict;
int64_t checkpoint_cleanup_pages_obsolete_tw;
int64_t checkpoint_cleanup_pages_read_reclaim_space;
int64_t checkpoint_cleanup_pages_read_obsolete_tw;
int64_t checkpoint_cleanup_pages_removed;
int64_t checkpoint_cleanup_pages_walk_skipped;
int64_t checkpoint_cleanup_pages_visited;
int64_t checkpoint_cleanup_success;
int64_t checkpoint_snapshot_acquired;
int64_t checkpoint_skipped;
@ -741,13 +751,6 @@ struct __wt_connection_stats {
int64_t checkpoint_pages_visited_internal;
int64_t checkpoint_pages_visited_leaf;
int64_t checkpoint_pages_reconciled;
int64_t checkpoint_cleanup_pages_evict;
int64_t checkpoint_cleanup_pages_obsolete_tw;
int64_t checkpoint_cleanup_pages_read_reclaim_space;
int64_t checkpoint_cleanup_pages_read_obsolete_tw;
int64_t checkpoint_cleanup_pages_removed;
int64_t checkpoint_cleanup_pages_walk_skipped;
int64_t checkpoint_cleanup_pages_visited;
int64_t checkpoint_prep_running;
int64_t checkpoint_prep_max;
int64_t checkpoint_prep_min;
@ -1380,7 +1383,6 @@ struct __wt_dsrc_stats {
int64_t cache_state_refs_skipped;
int64_t cache_state_root_size;
int64_t cache_state_pages;
int64_t checkpoint_snapshot_acquired;
int64_t checkpoint_cleanup_pages_evict;
int64_t checkpoint_cleanup_pages_obsolete_tw;
int64_t checkpoint_cleanup_pages_read_reclaim_space;
@ -1388,6 +1390,7 @@ struct __wt_dsrc_stats {
int64_t checkpoint_cleanup_pages_removed;
int64_t checkpoint_cleanup_pages_walk_skipped;
int64_t checkpoint_cleanup_pages_visited;
int64_t checkpoint_snapshot_acquired;
int64_t compress_precomp_intl_max_page_size;
int64_t compress_precomp_leaf_max_page_size;
int64_t compress_write_fail;

1078
src/third_party/wiredtiger/src/include/wiredtiger.in vendored
View File

File diff suppressed because it is too large Load Diff

87
src/third_party/wiredtiger/src/support/stat.c vendored
View File

@ -173,14 +173,14 @@ static const char *const __stats_dsrc_desc[] = {
"cache_walk: Refs skipped during cache traversal",
"cache_walk: Size of the root page",
"cache_walk: Total number of pages currently in cache",
"checkpoint-cleanup: pages added for eviction",
"checkpoint-cleanup: pages dirtied due to obsolete time window",
"checkpoint-cleanup: pages read into cache (reclaim_space)",
"checkpoint-cleanup: pages read into cache due to obsolete time window",
"checkpoint-cleanup: pages removed",
"checkpoint-cleanup: pages skipped during tree walk",
"checkpoint-cleanup: pages visited",
"checkpoint: checkpoint has acquired a snapshot for its transaction",
"checkpoint: pages added for eviction during checkpoint cleanup",
"checkpoint: pages dirtied due to obsolete time window by checkpoint cleanup",
"checkpoint: pages read into cache during checkpoint cleanup (reclaim_space)",
"checkpoint: pages read into cache during checkpoint cleanup due to obsolete time window",
"checkpoint: pages removed during checkpoint cleanup",
"checkpoint: pages skipped during checkpoint cleanup tree walk",
"checkpoint: pages visited during checkpoint cleanup",
"compression: compressed page maximum internal page size prior to compression",
"compression: compressed page maximum leaf page size prior to compression ",
"compression: page written to disk failed to compress",
@ -532,7 +532,6 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats)
/* not clearing cache_state_refs_skipped */
/* not clearing cache_state_root_size */
/* not clearing cache_state_pages */
stats->checkpoint_snapshot_acquired = 0;
stats->checkpoint_cleanup_pages_evict = 0;
stats->checkpoint_cleanup_pages_obsolete_tw = 0;
stats->checkpoint_cleanup_pages_read_reclaim_space = 0;
@ -540,6 +539,7 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats)
stats->checkpoint_cleanup_pages_removed = 0;
stats->checkpoint_cleanup_pages_walk_skipped = 0;
stats->checkpoint_cleanup_pages_visited = 0;
stats->checkpoint_snapshot_acquired = 0;
/* not clearing compress_precomp_intl_max_page_size */
/* not clearing compress_precomp_leaf_max_page_size */
stats->compress_write_fail = 0;
@ -881,7 +881,6 @@ __wt_stat_dsrc_aggregate_single(WT_DSRC_STATS *from, WT_DSRC_STATS *to)
to->cache_state_refs_skipped += from->cache_state_refs_skipped;
to->cache_state_root_size += from->cache_state_root_size;
to->cache_state_pages += from->cache_state_pages;
to->checkpoint_snapshot_acquired += from->checkpoint_snapshot_acquired;
to->checkpoint_cleanup_pages_evict += from->checkpoint_cleanup_pages_evict;
to->checkpoint_cleanup_pages_obsolete_tw += from->checkpoint_cleanup_pages_obsolete_tw;
to->checkpoint_cleanup_pages_read_reclaim_space +=
@ -891,6 +890,7 @@ __wt_stat_dsrc_aggregate_single(WT_DSRC_STATS *from, WT_DSRC_STATS *to)
to->checkpoint_cleanup_pages_removed += from->checkpoint_cleanup_pages_removed;
to->checkpoint_cleanup_pages_walk_skipped += from->checkpoint_cleanup_pages_walk_skipped;
to->checkpoint_cleanup_pages_visited += from->checkpoint_cleanup_pages_visited;
to->checkpoint_snapshot_acquired += from->checkpoint_snapshot_acquired;
to->compress_precomp_intl_max_page_size += from->compress_precomp_intl_max_page_size;
to->compress_precomp_leaf_max_page_size += from->compress_precomp_leaf_max_page_size;
to->compress_write_fail += from->compress_write_fail;
@ -1251,7 +1251,6 @@ __wt_stat_dsrc_aggregate(WT_DSRC_STATS **from, WT_DSRC_STATS *to)
to->cache_state_refs_skipped += WT_STAT_DSRC_READ(from, cache_state_refs_skipped);
to->cache_state_root_size += WT_STAT_DSRC_READ(from, cache_state_root_size);
to->cache_state_pages += WT_STAT_DSRC_READ(from, cache_state_pages);
to->checkpoint_snapshot_acquired += WT_STAT_DSRC_READ(from, checkpoint_snapshot_acquired);
to->checkpoint_cleanup_pages_evict += WT_STAT_DSRC_READ(from, checkpoint_cleanup_pages_evict);
to->checkpoint_cleanup_pages_obsolete_tw +=
WT_STAT_DSRC_READ(from, checkpoint_cleanup_pages_obsolete_tw);
@ -1265,6 +1264,7 @@ __wt_stat_dsrc_aggregate(WT_DSRC_STATS **from, WT_DSRC_STATS *to)
WT_STAT_DSRC_READ(from, checkpoint_cleanup_pages_walk_skipped);
to->checkpoint_cleanup_pages_visited +=
WT_STAT_DSRC_READ(from, checkpoint_cleanup_pages_visited);
to->checkpoint_snapshot_acquired += WT_STAT_DSRC_READ(from, checkpoint_snapshot_acquired);
to->compress_precomp_intl_max_page_size +=
WT_STAT_DSRC_READ(from, compress_precomp_intl_max_page_size);
to->compress_precomp_leaf_max_page_size +=
@ -1747,7 +1747,17 @@ static const char *const __stats_connection_desc[] = {
"capacity: time waiting during logging (usecs)",
"capacity: time waiting during read (usecs)",
"capacity: time waiting for chunk cache IO bandwidth (usecs)",
"checkpoint: checkpoint cleanup successful calls",
"checkpoint-cleanup: most recent duration on all eligible files (usecs)",
"checkpoint-cleanup: most recent handles processed",
"checkpoint-cleanup: most recent in-memory pages visited",
"checkpoint-cleanup: pages added for eviction",
"checkpoint-cleanup: pages dirtied due to obsolete time window",
"checkpoint-cleanup: pages read into cache (reclaim_space)",
"checkpoint-cleanup: pages read into cache due to obsolete time window",
"checkpoint-cleanup: pages removed",
"checkpoint-cleanup: pages skipped during tree walk",
"checkpoint-cleanup: pages visited",
"checkpoint-cleanup: successful calls",
"checkpoint: checkpoint has acquired a snapshot for its transaction",
"checkpoint: checkpoints skipped because database was clean",
"checkpoint: fsync calls after allocating the transaction ID",
@ -1776,13 +1786,6 @@ static const char *const __stats_connection_desc[] = {
"checkpoint: number of internal pages visited",
"checkpoint: number of leaf pages visited",
"checkpoint: number of pages caused to be reconciled",
"checkpoint: pages added for eviction during checkpoint cleanup",
"checkpoint: pages dirtied due to obsolete time window by checkpoint cleanup",
"checkpoint: pages read into cache during checkpoint cleanup (reclaim_space)",
"checkpoint: pages read into cache during checkpoint cleanup due to obsolete time window",
"checkpoint: pages removed during checkpoint cleanup",
"checkpoint: pages skipped during checkpoint cleanup tree walk",
"checkpoint: pages visited during checkpoint cleanup",
"checkpoint: prepare currently running",
"checkpoint: prepare max time (msecs)",
"checkpoint: prepare min time (msecs)",
@ -2596,6 +2599,16 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats)
stats->capacity_time_log = 0;
stats->capacity_time_read = 0;
stats->capacity_time_chunkcache = 0;
/* not clearing checkpoint_cleanup_duration */
stats->checkpoint_cleanup_handle_processed = 0;
stats->checkpoint_cleanup_inmem_pages_visited = 0;
stats->checkpoint_cleanup_pages_evict = 0;
stats->checkpoint_cleanup_pages_obsolete_tw = 0;
stats->checkpoint_cleanup_pages_read_reclaim_space = 0;
stats->checkpoint_cleanup_pages_read_obsolete_tw = 0;
stats->checkpoint_cleanup_pages_removed = 0;
stats->checkpoint_cleanup_pages_walk_skipped = 0;
stats->checkpoint_cleanup_pages_visited = 0;
stats->checkpoint_cleanup_success = 0;
stats->checkpoint_snapshot_acquired = 0;
stats->checkpoint_skipped = 0;
@ -2625,13 +2638,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats)
stats->checkpoint_pages_visited_internal = 0;
stats->checkpoint_pages_visited_leaf = 0;
stats->checkpoint_pages_reconciled = 0;
stats->checkpoint_cleanup_pages_evict = 0;
stats->checkpoint_cleanup_pages_obsolete_tw = 0;
stats->checkpoint_cleanup_pages_read_reclaim_space = 0;
stats->checkpoint_cleanup_pages_read_obsolete_tw = 0;
stats->checkpoint_cleanup_pages_removed = 0;
stats->checkpoint_cleanup_pages_walk_skipped = 0;
stats->checkpoint_cleanup_pages_visited = 0;
/* not clearing checkpoint_prep_running */
/* not clearing checkpoint_prep_max */
/* not clearing checkpoint_prep_min */
@ -3484,6 +3490,24 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS *
to->capacity_time_log += WT_STAT_CONN_READ(from, capacity_time_log);
to->capacity_time_read += WT_STAT_CONN_READ(from, capacity_time_read);
to->capacity_time_chunkcache += WT_STAT_CONN_READ(from, capacity_time_chunkcache);
to->checkpoint_cleanup_duration += WT_STAT_CONN_READ(from, checkpoint_cleanup_duration);
to->checkpoint_cleanup_handle_processed +=
WT_STAT_CONN_READ(from, checkpoint_cleanup_handle_processed);
to->checkpoint_cleanup_inmem_pages_visited +=
WT_STAT_CONN_READ(from, checkpoint_cleanup_inmem_pages_visited);
to->checkpoint_cleanup_pages_evict += WT_STAT_CONN_READ(from, checkpoint_cleanup_pages_evict);
to->checkpoint_cleanup_pages_obsolete_tw +=
WT_STAT_CONN_READ(from, checkpoint_cleanup_pages_obsolete_tw);
to->checkpoint_cleanup_pages_read_reclaim_space +=
WT_STAT_CONN_READ(from, checkpoint_cleanup_pages_read_reclaim_space);
to->checkpoint_cleanup_pages_read_obsolete_tw +=
WT_STAT_CONN_READ(from, checkpoint_cleanup_pages_read_obsolete_tw);
to->checkpoint_cleanup_pages_removed +=
WT_STAT_CONN_READ(from, checkpoint_cleanup_pages_removed);
to->checkpoint_cleanup_pages_walk_skipped +=
WT_STAT_CONN_READ(from, checkpoint_cleanup_pages_walk_skipped);
to->checkpoint_cleanup_pages_visited +=
WT_STAT_CONN_READ(from, checkpoint_cleanup_pages_visited);
to->checkpoint_cleanup_success += WT_STAT_CONN_READ(from, checkpoint_cleanup_success);
to->checkpoint_snapshot_acquired += WT_STAT_CONN_READ(from, checkpoint_snapshot_acquired);
to->checkpoint_skipped += WT_STAT_CONN_READ(from, checkpoint_skipped);
@ -3516,19 +3540,6 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS *
WT_STAT_CONN_READ(from, checkpoint_pages_visited_internal);
to->checkpoint_pages_visited_leaf += WT_STAT_CONN_READ(from, checkpoint_pages_visited_leaf);
to->checkpoint_pages_reconciled += WT_STAT_CONN_READ(from, checkpoint_pages_reconciled);
to->checkpoint_cleanup_pages_evict += WT_STAT_CONN_READ(from, checkpoint_cleanup_pages_evict);
to->checkpoint_cleanup_pages_obsolete_tw +=
WT_STAT_CONN_READ(from, checkpoint_cleanup_pages_obsolete_tw);
to->checkpoint_cleanup_pages_read_reclaim_space +=
WT_STAT_CONN_READ(from, checkpoint_cleanup_pages_read_reclaim_space);
to->checkpoint_cleanup_pages_read_obsolete_tw +=
WT_STAT_CONN_READ(from, checkpoint_cleanup_pages_read_obsolete_tw);
to->checkpoint_cleanup_pages_removed +=
WT_STAT_CONN_READ(from, checkpoint_cleanup_pages_removed);
to->checkpoint_cleanup_pages_walk_skipped +=
WT_STAT_CONN_READ(from, checkpoint_cleanup_pages_walk_skipped);
to->checkpoint_cleanup_pages_visited +=
WT_STAT_CONN_READ(from, checkpoint_cleanup_pages_visited);
to->checkpoint_prep_running += WT_STAT_CONN_READ(from, checkpoint_prep_running);
to->checkpoint_prep_max += WT_STAT_CONN_READ(from, checkpoint_prep_max);
to->checkpoint_prep_min += WT_STAT_CONN_READ(from, checkpoint_prep_min);

31
src/third_party/wiredtiger/test/evergreen.yml vendored
View File

@ -16,6 +16,7 @@ post:
- func: "upload test/format config"
- func: "upload test/model workloads"
- func: "dump stderr/stdout"
- func: "upload stat files"
- func: "upload artifact"
vars:
postfix: -${execution}
@ -492,6 +493,36 @@ functions:
content_type: text/plain
remote_file: wiredtiger/${build_variant}/${revision}/artifacts/${task_name}_${build_id}/
preserve_path: true
"upload stat files":
- command: shell.exec
type: setup
params:
working_dir: "wiredtiger/cmake_build"
shell: bash
script: |
set -o errexit
set -o verbose
mkdir stat_files
${python_binary|python3} ../test/evergreen/collect_stat_files.py -d stat_files
- command: archive.targz_pack
type: setup
params:
target: wiredtiger/cmake_build/stat_files.tar.gz
source_dir: wiredtiger/cmake_build/stat_files
include:
- "./**"
- command: s3.put
type: setup
params:
aws_secret: ${aws_secret}
aws_key: ${aws_key}
local_file: wiredtiger/cmake_build/stat_files.tar.gz
bucket: build_external
permissions: public-read
content_type: application/tar
remote_file: wiredtiger/${build_variant}/${revision}/artifacts/${task_name}_${build_id}/stat_files.tar.gz
display_name: "Statistics"
"run data validation stress test checkpoint":
- *fetch_artifacts
- command: shell.exec

79
src/third_party/wiredtiger/test/evergreen/collect_stat_files.py vendored Normal file
View File

@ -0,0 +1,79 @@
#!/usr/bin/env python3
#
# Public Domain 2014-present MongoDB, Inc.
# Public Domain 2008-2014 WiredTiger, Inc.
#
# This is free and unencumbered software released into the public domain.
#
# Anyone is free to copy, modify, publish, use, compile, sell, or
# distribute this software, either in source code form or as a compiled
# binary, for any purpose, commercial or non-commercial, and by any
# means.
#
# In jurisdictions that recognize copyright laws, the author or authors
# of this software dedicate any and all copyright interest in the
# software to the public domain. We make this dedication for the benefit
# of the public at large and to the detriment of our heirs and
# successors. We intend this dedication to be an overt act of
# relinquishment in perpetuity of all present and future rights to this
# software under copyright law.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
import argparse
import os
import re
import shutil
def collect_stat_files(destination_dir, source_dir, regex):
"""
Collect all stat files from a source directory and copy them into a specified destination directory.
:param destination_dir: The destination directory where stat files are copied to.
:param source_dir: The source directory to search for stat files.
:param regex: A compiled regular expression to match stat file names.
"""
# Remove the leading "./" from the source directory name and replace all "/" with "-".
# This creates a unique directory name for the stat files being collected from this location.
destination_sub_dir = os.path.join(destination_dir, source_dir[2:].replace("/", "-"))
# Create the subdirectory that the files from this location will be copied to.
if not os.path.exists(destination_sub_dir):
os.makedirs(destination_sub_dir)
for item in os.listdir(source_dir):
path = os.path.join(source_dir, item)
if os.path.isfile(path) and regex.match(item):
file_path = os.path.join(source_dir, item)
shutil.copy(file_path, destination_sub_dir)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--destination-dir', required=True, help='Directory to collect stat files into for packing.')
args = parser.parse_args()
destination_dir = args.destination_dir
regex = re.compile(r'WiredTigerStat.*')
for walk_dir, _, files in os.walk("."):
if destination_dir in walk_dir:
# Skip searching for stat files in the destination directory, since that's where we're collecting them and we don't want to copy files that we've already collected.
continue
for file in files:
if regex.match(file):
# If current directory contains any stat files, collect them all into a single location for packing.
collect_stat_files(destination_dir, walk_dir, regex)
# Finished searching in this directory, move on to the next one.
break
if __name__ == "__main__":
main()

13
src/third_party/wiredtiger/test/suite/test_cc02.py vendored
View File

@ -30,6 +30,7 @@
# checkpoint:checkpoint_cleanup
# [END_TAGS]
import os
from test_cc01 import test_cc_base
from wiredtiger import stat
from wtscenario import make_scenarios
@ -93,16 +94,24 @@ class test_cc02(test_cc_base):
self.conn.set_timestamp(f'oldest_timestamp={new_ts}')
# Trigger obsolete cleanup.
# Depending whether the obsolete pages are on disk or in-memory, they should be flagged and
# discarded.
# Whether the obsolete pages are on disk or in-memory, they should be flagged and discarded.
self.wait_for_cc_to_run()
c = self.session.open_cursor('statistics:')
visited = c[stat.conn.checkpoint_cleanup_pages_visited][2]
obsolete_evicted = c[stat.conn.checkpoint_cleanup_pages_evict][2]
obsolete_on_disk = c[stat.conn.checkpoint_cleanup_pages_removed][2]
ckpt_cleanup_duration = c[stat.conn.checkpoint_cleanup_duration][2]
ckpt_cleanup_handle_processed = c[stat.conn.checkpoint_cleanup_handle_processed][2]
ckpt_cleanup_inmem_pages_visited = c[stat.conn.checkpoint_cleanup_inmem_pages_visited][2]
c.close()
# We should always visit pages for cleanup.
self.assertGreater(ckpt_cleanup_handle_processed, 0)
self.assertGreater(ckpt_cleanup_inmem_pages_visited, 0)
# Some Windows machines lack the time granularity to detect microseconds.
# Skip the time check on Windows.
if not os.name == "nt":
self.assertGreater(ckpt_cleanup_duration, 0)
self.assertGreater(visited, 0)
# Depending on the scenario, cleanup will be triggered differently.