From 64776d8d509a3c220d80fb3269a22b865d070b6b Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Thu, 1 Oct 2026 16:28:59 +0300 Subject: [PATCH 1/3] Measure VACUUM buffer usage in core Snapshot buffer activity and block I/O timing around heap, index and AO vacuum work. Assemble backend-local resource records and accumulate AO resources over active phases. Keep relation-local block counts, and subtract index resources from table-only totals to avoid double counting. Introduce optional instrumentation and its consumer hook. Allocate it only when a consumer is installed. Persistent storage and SQL access through the extension follow separately. Native timing and work counters are introduced in independent branches. --- src/backend/access/heap/vacuumlazy.c | 150 +++++++++++++++++++ src/backend/commands/vacuum_ao.c | 127 +++++++++++++++- src/backend/commands/vacuumparallel.c | 11 ++ src/backend/utils/activity/pgstat_relation.c | 6 + src/include/access/appendonly_compaction.h | 16 ++ src/include/commands/vacuum.h | 15 ++ src/include/pgstat.h | 35 +++++ 7 files changed, 357 insertions(+), 3 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index e3261b49a6c..02bbe3c529d 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -187,6 +187,8 @@ typedef struct LVRelState /* Error reporting state */ char *dbname; char *relnamespace; + Oid reloid; + Oid indoid; char *relname; char *indname; /* Current index name */ BlockNumber blkno; /* used only for heap operations */ @@ -225,6 +227,18 @@ typedef struct LVRelState int64 live_tuples; /* # live tuples remaining */ int64 recently_dead_tuples; /* # dead, but not yet removable */ int64 missed_dead_tuples; /* # removable, but not removed */ + + /* + * Resource usage of the index passes this process ran, subtracted from + * the heap report to avoid double-counting (see + * accumulate_heap_vacuum_statistics). + */ + PgStat_CommonCounts *extVacReportIdx; + + /* + * We need to accumulate index statistics for later subtraction from heap + * stats. + */ } LVRelState; /* @@ -312,6 +326,100 @@ static void update_vacuum_error_info(LVRelState *vacrel, static void restore_vacuum_error_info(LVRelState *vacrel, const LVSavedErrInfo *saved_vacrel); +/* Extended vacuum statistics functions */ + +/* + * extvac_stats_start - Allocate and snapshot extended instrumentation. + * + * Return NULL without allocating anything when there is no consumer. The + * caller frees a non-NULL result after reporting the phase, or its vacuum + * memory context cleans it up on error. + */ +LVExtStatCounters * +extvac_stats_start(Relation rel) +{ + LVExtStatCounters *counters; + + if (set_report_vacuum_hook == NULL) + return NULL; + + counters = palloc0(sizeof(LVExtStatCounters)); + counters->bufusage = pgBufferUsage; + + if (rel->pgstat_info && pgstat_track_counts) + { + counters->blocks_fetched = rel->pgstat_info->counts.blocks_fetched; + counters->blocks_hit = rel->pgstat_info->counts.blocks_hit; + } + return counters; +} + +/* + * extvac_stats_end - Diff resource usage since extvac_stats_start into report. + */ +void +extvac_stats_end(Relation rel, LVExtStatCounters * counters, + PgStat_CommonCounts * report) +{ + BufferUsage bufusage; + + memset(&bufusage, 0, sizeof(BufferUsage)); + BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &counters->bufusage); + + report->total_blks_read = bufusage.local_blks_read + bufusage.shared_blks_read; + report->total_blks_hit = bufusage.local_blks_hit + bufusage.shared_blks_hit; + report->total_blks_dirtied = bufusage.local_blks_dirtied + bufusage.shared_blks_dirtied; + report->total_blks_written = bufusage.shared_blks_written; + /* PostgreSQL 16 keeps a single timer for shared and local blocks */ + report->blk_read_time = INSTR_TIME_GET_MILLISEC(bufusage.blk_read_time); + report->blk_write_time = INSTR_TIME_GET_MILLISEC(bufusage.blk_write_time); + + if (rel->pgstat_info && pgstat_track_counts) + { + report->blks_fetched = rel->pgstat_info->counts.blocks_fetched - counters->blocks_fetched; + report->blks_hit = rel->pgstat_info->counts.blocks_hit - counters->blocks_hit; + } +} + +/* + * Build the heap-specific part of the extended vacuum report from the + * counters gathered in vacrel. + */ +static void +accumulate_heap_vacuum_statistics(LVRelState *vacrel, PgStat_VacuumRelationCounts * extVacStats) +{ + extVacStats->type = PGSTAT_EXTVAC_TABLE; + + /* + * Subtract the resource usage of the index passes this process ran: they + * are reported per index, and the database-wide aggregate would count + * them twice otherwise. Parallel workers report their own usage with + * their index passes, so it never enters the leader's counters. + */ + extVacStats->common.total_blks_read -= vacrel->extVacReportIdx->total_blks_read; + extVacStats->common.total_blks_hit -= vacrel->extVacReportIdx->total_blks_hit; + extVacStats->common.total_blks_dirtied -= vacrel->extVacReportIdx->total_blks_dirtied; + extVacStats->common.total_blks_written -= vacrel->extVacReportIdx->total_blks_written; + extVacStats->common.blk_read_time -= vacrel->extVacReportIdx->blk_read_time; + extVacStats->common.blk_write_time -= vacrel->extVacReportIdx->blk_write_time; +} + +/* + * Accumulate an index pass report into the running total that is later + * subtracted from the heap report. + */ +static void +accumulate_idxs_vacuum_statistics(LVRelState *vacrel, + PgStat_VacuumRelationCounts * extVacIdxStats) +{ + vacrel->extVacReportIdx->total_blks_read += extVacIdxStats->common.total_blks_read; + vacrel->extVacReportIdx->total_blks_hit += extVacIdxStats->common.total_blks_hit; + vacrel->extVacReportIdx->total_blks_dirtied += extVacIdxStats->common.total_blks_dirtied; + vacrel->extVacReportIdx->total_blks_written += extVacIdxStats->common.total_blks_written; + vacrel->extVacReportIdx->blk_read_time += extVacIdxStats->common.blk_read_time; + vacrel->extVacReportIdx->blk_write_time += extVacIdxStats->common.blk_write_time; +} + /* * lazy_vacuum_rel_heap() -- perform VACUUM for one heap relation * @@ -340,6 +448,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, TimestampTz starttime = 0; PgStat_Counter startreadtime = 0, startwritetime = 0; + LVExtStatCounters *extVacCounters; WalUsage startwalusage = pgWalUsage; BufferUsage startbufferusage = pgBufferUsage; ErrorContextCallback errcallback; @@ -359,6 +468,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, } } + extVacCounters = extvac_stats_start(rel); + pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); @@ -375,9 +486,12 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, * these temp copies. */ vacrel = (LVRelState *) palloc0(sizeof(LVRelState)); + if (extVacCounters != NULL) + vacrel->extVacReportIdx = palloc0(sizeof(PgStat_CommonCounts)); vacrel->dbname = get_database_name(MyDatabaseId); vacrel->relnamespace = get_namespace_name(RelationGetNamespace(rel)); vacrel->relname = pstrdup(RelationGetRelationName(rel)); + vacrel->reloid = RelationGetRelid(rel); vacrel->indname = NULL; vacrel->phase = VACUUM_ERRCB_PHASE_UNKNOWN; vacrel->verbose = verbose; @@ -615,6 +729,16 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, * soon in cases where the failsafe prevented significant amounts of heap * vacuuming. */ + if (extVacCounters != NULL) + { + PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; + + extvac_stats_end(rel, extVacCounters, &extVacReport->common); + accumulate_heap_vacuum_statistics(vacrel, extVacReport); + + pfree(extVacCounters); + pfree(vacrel->extVacReportIdx); + } pgstat_report_vacuum(RelationGetRelid(rel), rel->rd_rel->relisshared, Max(vacrel->new_live_tuples, 0), @@ -2734,7 +2858,9 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + LVExtStatCounters *extVacCounters; + extVacCounters = extvac_stats_start(indrel); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -2752,6 +2878,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, */ Assert(vacrel->indname == NULL); vacrel->indname = pstrdup(RelationGetRelationName(indrel)); + vacrel->indoid = RelationGetRelid(indrel); update_vacuum_error_info(vacrel, &saved_err_info, VACUUM_ERRCB_PHASE_VACUUM_INDEX, InvalidBlockNumber, InvalidOffsetNumber); @@ -2759,6 +2886,16 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, /* Do bulk deletion */ istat = vac_bulkdel_one_index(&ivinfo, istat, (void *) vacrel->dead_items); + if (extVacCounters != NULL) + { + PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; + + extvac_stats_end(indrel, extVacCounters, &extVacReport->common); + extVacReport->type = PGSTAT_EXTVAC_INDEX; + accumulate_idxs_vacuum_statistics(vacrel, extVacReport); + pfree(extVacCounters); + } + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); @@ -2783,7 +2920,9 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + LVExtStatCounters *extVacCounters; + extVacCounters = extvac_stats_start(indrel); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -2802,12 +2941,23 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, */ Assert(vacrel->indname == NULL); vacrel->indname = pstrdup(RelationGetRelationName(indrel)); + vacrel->indoid = RelationGetRelid(indrel); update_vacuum_error_info(vacrel, &saved_err_info, VACUUM_ERRCB_PHASE_INDEX_CLEANUP, InvalidBlockNumber, InvalidOffsetNumber); istat = vac_cleanup_one_index(&ivinfo, istat); + if (extVacCounters != NULL) + { + PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; + + extvac_stats_end(indrel, extVacCounters, &extVacReport->common); + extVacReport->type = PGSTAT_EXTVAC_INDEX; + accumulate_idxs_vacuum_statistics(vacrel, extVacReport); + pfree(extVacCounters); + } + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index 075c73fb1c1..f951390cf61 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -163,6 +163,13 @@ static void ao_vacuum_rel_recycle_dead_segments(Relation onerel, VacuumParams *p BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats); static AOVacuumRelStats *init_vacrelstats(void); static void cleanup_vacrelstats(AOVacuumRelStats **vacrelstatsp); +static void ao_accum_resources(PgStat_CommonCounts *dst, + const PgStat_CommonCounts *src, bool subtract); +static void ao_measure_index_resources(Relation indrel, + LVExtStatCounters *counters, + AOVacuumRelStats *vacrelstats); +static void ao_measure_table_resources(Relation rel, + AOVacuumRelStats *vacrelstats); static void ao_vacuum_rel_pre_cleanup(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats) @@ -414,6 +421,8 @@ init_vacrelstats() old_context = MemoryContextSwitchTo(TopMemoryContext); vacrelstats = (AOVacuumRelStats *) palloc0(sizeof(AOVacuumRelStats)); + if (set_report_vacuum_hook != NULL) + vacrelstats->extstats = palloc0(sizeof(AOVacuumExtStats)); MemoryContextSwitchTo(old_context); return vacrelstats; @@ -428,11 +437,26 @@ void ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy) { static AOVacuumRelStats *vacrelstats = NULL; + LVExtStatCounters *extcounters; + bool extstats = (set_report_vacuum_hook != NULL); Assert(RelationStorageIsAO(rel)); Assert(params != NULL); int ao_vacuum_phase = (params->options & VACUUM_AO_PHASE_MASK); + /* + * The phases of one vacuum share these stats and free them at the end of + * the last phase, so a vacuum that failed in between leaves them behind. + * Drop such leftovers: otherwise the counters of the failed vacuum are + * taken for the next one, whose progress is never reported because the + * progress command is only + * started along with the stats. + */ + if (vacrelstats != NULL && + (ao_vacuum_phase == VACOPT_AO_PRE_CLEANUP_PHASE || + vacrelstats->relid != RelationGetRelid(rel))) + cleanup_vacrelstats(&vacrelstats); + if (!vacrelstats) { if (ao_vacuum_phase != VACOPT_AO_PRE_CLEANUP_PHASE && Gp_role == GP_ROLE_EXECUTE) @@ -450,8 +474,12 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); vacrelstats = init_vacrelstats(); + vacrelstats->relid = RelationGetRelid(rel); } + /* Sample the resource usage of the phase for the extended statistics */ + extcounters = extvac_stats_start(rel); + /* * Do the actual work --- either FULL or "lazy" vacuum */ @@ -460,14 +488,93 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy else if (ao_vacuum_phase == VACOPT_AO_COMPACT_PHASE) ao_vacuum_rel_compact(rel, params, bstrategy, vacrelstats); else if (ao_vacuum_phase == VACOPT_AO_POST_CLEANUP_PHASE) - { ao_vacuum_rel_post_cleanup(rel, params, bstrategy, vacrelstats); + else + /* Do nothing here, we will launch the stages later */ + Assert(ao_vacuum_phase == 0); + + if (extstats && ao_vacuum_phase != 0) + { + extvac_stats_end(rel, extcounters, &extcounters->report.common); + ao_accum_resources(&vacrelstats->extstats->phases, + &extcounters->report.common, false); + } + + if (ao_vacuum_phase == VACOPT_AO_POST_CLEANUP_PHASE) + { + if (extstats) + ao_measure_table_resources(rel, vacrelstats); pgstat_progress_end_command(); cleanup_vacrelstats(&vacrelstats); } + + if (extcounters != NULL) + pfree(extcounters); +} + +/* + * Add the resource usage counters of src to dst, or subtract them from it. + */ +static void +ao_accum_resources(PgStat_CommonCounts *dst, const PgStat_CommonCounts *src, + bool subtract) +{ + if (subtract) + { + dst->total_blks_read -= src->total_blks_read; + dst->total_blks_hit -= src->total_blks_hit; + dst->total_blks_dirtied -= src->total_blks_dirtied; + dst->total_blks_written -= src->total_blks_written; + /* Per-relation block counts do not include the indexes. */ + dst->blk_read_time -= src->blk_read_time; + dst->blk_write_time -= src->blk_write_time; + } else - /* Do nothing here, we will launch the stages later */ - Assert(ao_vacuum_phase == 0); + { + dst->total_blks_read += src->total_blks_read; + dst->total_blks_hit += src->total_blks_hit; + dst->total_blks_dirtied += src->total_blks_dirtied; + dst->total_blks_written += src->total_blks_written; + dst->blks_fetched += src->blks_fetched; + dst->blks_hit += src->blks_hit; + dst->blk_read_time += src->blk_read_time; + dst->blk_write_time += src->blk_write_time; + } +} + +/* + * Report one pass over an index of an append-optimized table to + * set_report_vacuum_hook, as lazy_vacuum_one_index() does for the heap, and + * remember its resource usage, which the table's report must not count + * again. + */ +static void +ao_measure_index_resources(Relation indrel, LVExtStatCounters *counters, + AOVacuumRelStats *vacrelstats) +{ + PgStat_VacuumRelationCounts *report = &counters->report; + + extvac_stats_end(indrel, counters, &report->common); + report->type = PGSTAT_EXTVAC_INDEX; + + ao_accum_resources(&vacrelstats->extstats->indexes, &report->common, false); + pfree(counters); +} + +/* + * Report the vacuum of an append-optimized table, all of its phases, to + * set_report_vacuum_hook. + */ +static void +ao_measure_table_resources(Relation rel, AOVacuumRelStats *vacrelstats) +{ + PgStat_VacuumRelationCounts report; + + memset(&report, 0, sizeof(report)); + report.type = PGSTAT_EXTVAC_TABLE; + report.common = vacrelstats->extstats->phases; + ao_accum_resources(&report.common, &vacrelstats->extstats->indexes, true); + } /* @@ -574,22 +681,34 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs { for (i = 0; i < nindexes; i++) { + LVExtStatCounters *extcounters; + + extcounters = extvac_stats_start(Irel[i]); scan_index(Irel[i], aoRelation, elevel, bstrategy); + if (set_report_vacuum_hook) + ao_measure_index_resources(Irel[i], extcounters, + vacrelstats); } } else { for (i = 0; i < nindexes; i++) { + LVExtStatCounters *extcounters; + + extcounters = extvac_stats_start(Irel[i]); vacuum_appendonly_index(Irel[i], aoRelation, dead_segs, elevel, bstrategy, vacrelstats); + if (set_report_vacuum_hook) + ao_measure_index_resources(Irel[i], extcounters, + vacrelstats); } } } @@ -780,6 +899,8 @@ vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, int elevel, static void cleanup_vacrelstats(AOVacuumRelStats **vacrelstats) { + if ((*vacrelstats)->extstats != NULL) + pfree((*vacrelstats)->extstats); pfree(*vacrelstats); *vacrelstats = NULL; } diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index a79067fd461..9ec6d8fdd31 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -836,6 +836,7 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, IndexBulkDeleteResult *istat = NULL; IndexBulkDeleteResult *istat_res; IndexVacuumInfo ivinfo; + LVExtStatCounters *extVacCounters; /* * Update the pointer to the corresponding bulk-deletion result if someone @@ -844,6 +845,7 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, if (indstats->istat_updated) istat = &(indstats->istat); + extVacCounters = extvac_stats_start(indrel); ivinfo.index = indrel; ivinfo.heaprel = pvs->heaprel; ivinfo.analyze_only = false; @@ -871,6 +873,15 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, RelationGetRelationName(indrel)); } + if (extVacCounters != NULL) + { + PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; + + extvac_stats_end(indrel, extVacCounters, &extVacReport->common); + extVacReport->type = PGSTAT_EXTVAC_INDEX; + pfree(extVacCounters); + } + /* * Copy the index bulk-deletion result returned from ambulkdelete and * amvacuumcleanup to the DSM segment if it's the first cycle because they diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 45bbf9abcb7..41b528e6ccf 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -273,6 +273,12 @@ pgstat_report_vacuum(Oid tableoid, bool shared, pgstat_flush_io(false); } +/* + * Hook for extensions to receive extended vacuum statistics. + * NULL when no extension has registered. + */ +set_report_vacuum_hook_type set_report_vacuum_hook = NULL; + /* * Report that the table was just analyzed and flush IO statistics. * diff --git a/src/include/access/appendonly_compaction.h b/src/include/access/appendonly_compaction.h index 44aa78a39fb..9320a6131b2 100644 --- a/src/include/access/appendonly_compaction.h +++ b/src/include/access/appendonly_compaction.h @@ -13,6 +13,7 @@ #ifndef APPENDONLY_COMPACTION_H #define APPENDONLY_COMPACTION_H +#include "pgstat.h" #include "nodes/pg_list.h" #include "access/appendonly_visimap.h" #include "utils/rel.h" @@ -21,6 +22,13 @@ #define APPENDONLY_COMPACTION_SEGNO_INVALID (-1) +/* Optional resource counters for the extended-statistics hook. */ +typedef struct AOVacuumExtStats +{ + PgStat_CommonCounts phases; + PgStat_CommonCounts indexes; +} AOVacuumExtStats; + /* * Stats for progress reporting. * This is AO/AOCO counterpart of LVRelStats for Heap. It lives throughout @@ -31,6 +39,14 @@ typedef struct AOVacuumRelStats int nbytes_truncated; /* current # of bytes truncated from segment file */ int num_dead_tuples; /* current # of dead tuples */ int num_index_vacuumed; /* current # of indexes been vacuumed */ + /* the relation these stats were started for */ + Oid relid; + /* + * Resource usage for set_report_vacuum_hook, accumulated over the phases: + * of the phases as a whole, and of the index passes among them, which are + * reported per index and subtracted from the table's report. + */ + AOVacuumExtStats *extstats; /* allocated only when the hook is installed */ } AOVacuumRelStats; extern Bitmapset *AppendOptimizedCollectDeadSegments(Relation aorel); diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 431cd7221a8..a8c8a455e99 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -21,10 +21,12 @@ #include "catalog/pg_class.h" #include "catalog/pg_statistic.h" #include "catalog/pg_type.h" +#include "executor/instrument.h" #include "parser/parse_node.h" #include "storage/buf.h" #include "storage/lock.h" #include "utils/relcache.h" +#include "pgstat.h" #include "utils/snapshot.h" /* @@ -440,6 +442,19 @@ extern PGDLLIMPORT pg_atomic_uint32 *VacuumSharedCostBalance; extern PGDLLIMPORT pg_atomic_uint32 *VacuumActiveNWorkers; extern PGDLLIMPORT int VacuumCostBalanceLocal; +/* Allocated only when a hook consumes the extended statistics. */ +typedef struct LVExtStatCounters +{ + BufferUsage bufusage; + PgStat_Counter blocks_fetched; + PgStat_Counter blocks_hit; + PgStat_VacuumRelationCounts report; +} LVExtStatCounters; + +extern LVExtStatCounters *extvac_stats_start(Relation rel); +extern void extvac_stats_end(Relation rel, LVExtStatCounters *counters, + PgStat_CommonCounts *report); + extern PGDLLIMPORT bool VacuumFailsafeActive; extern PGDLLIMPORT double vacuum_cost_delay; extern PGDLLIMPORT int vacuum_cost_limit; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 2a85773c30c..3072794803c 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -117,6 +117,36 @@ typedef struct PgStat_FunctionCounts /* * Working state needed to accumulate per-function-call timing statistics. */ +/* + * Extended vacuum statistics - passed to extensions via set_report_vacuum_hook. + * Type of entry: table (heap), index, or database aggregate. + */ +typedef enum ExtVacReportType +{ + PGSTAT_EXTVAC_INVALID = 0, + PGSTAT_EXTVAC_TABLE = 1, + PGSTAT_EXTVAC_INDEX = 2, + PGSTAT_EXTVAC_DB = 3, +} ExtVacReportType; + +typedef struct PgStat_CommonCounts +{ + int64 total_blks_read; + int64 total_blks_hit; + int64 total_blks_dirtied; + int64 total_blks_written; + int64 blks_fetched; + int64 blks_hit; + double blk_read_time; + double blk_write_time; +} PgStat_CommonCounts; + +typedef struct PgStat_VacuumRelationCounts +{ + PgStat_CommonCounts common; + ExtVacReportType type; +} PgStat_VacuumRelationCounts; + typedef struct PgStat_FunctionCallUsage { /* Link to function's hashtable entry (must still be there at exit!) */ @@ -652,6 +682,11 @@ extern void pgstat_unlink_relation(Relation rel); extern void pgstat_report_vacuum(Oid tableoid, bool shared, PgStat_Counter livetuples, PgStat_Counter deadtuples); +/* Hook for extensions to receive extended vacuum statistics */ +typedef void (*set_report_vacuum_hook_type) (Oid tableoid, bool shared, + PgStat_VacuumRelationCounts * params); +extern PGDLLIMPORT set_report_vacuum_hook_type set_report_vacuum_hook; + extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, bool resetcounter); From b259261f5034cbcfb4989d6450cbfc1bd7663c49 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Thu, 1 Oct 2026 16:28:59 +0300 Subject: [PATCH 2/3] Measure VACUUM WAL usage in core Add WAL record, full-page-image and byte deltas to the backend-local VACUUM resource measurements. Snapshot pgWalUsage around the same heap, index and AO phases as buffer usage. Keep index WAL separate from table resources and accumulate only active AO phases. Reuse PostgreSQL's existing pgWalUsage instrumentation. Delivery, persistence and SQL exposure of these counters are added by the separate extension reporting commit. --- src/backend/access/heap/vacuumlazy.c | 13 +++++++++++++ src/backend/commands/vacuum_ao.c | 6 ++++++ src/include/commands/vacuum.h | 1 + src/include/pgstat.h | 3 +++ 4 files changed, 23 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 02bbe3c529d..5da380c2989 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -344,6 +344,7 @@ extvac_stats_start(Relation rel) return NULL; counters = palloc0(sizeof(LVExtStatCounters)); + counters->walusage = pgWalUsage; counters->bufusage = pgBufferUsage; if (rel->pgstat_info && pgstat_track_counts) @@ -361,8 +362,11 @@ void extvac_stats_end(Relation rel, LVExtStatCounters * counters, PgStat_CommonCounts * report) { + WalUsage walusage; BufferUsage bufusage; + memset(&walusage, 0, sizeof(WalUsage)); + WalUsageAccumDiff(&walusage, &pgWalUsage, &counters->walusage); memset(&bufusage, 0, sizeof(BufferUsage)); BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &counters->bufusage); @@ -373,6 +377,9 @@ extvac_stats_end(Relation rel, LVExtStatCounters * counters, /* PostgreSQL 16 keeps a single timer for shared and local blocks */ report->blk_read_time = INSTR_TIME_GET_MILLISEC(bufusage.blk_read_time); report->blk_write_time = INSTR_TIME_GET_MILLISEC(bufusage.blk_write_time); + report->wal_records = walusage.wal_records; + report->wal_fpi = walusage.wal_fpi; + report->wal_bytes = walusage.wal_bytes; if (rel->pgstat_info && pgstat_track_counts) { @@ -402,6 +409,9 @@ accumulate_heap_vacuum_statistics(LVRelState *vacrel, PgStat_VacuumRelationCount extVacStats->common.total_blks_written -= vacrel->extVacReportIdx->total_blks_written; extVacStats->common.blk_read_time -= vacrel->extVacReportIdx->blk_read_time; extVacStats->common.blk_write_time -= vacrel->extVacReportIdx->blk_write_time; + extVacStats->common.wal_records -= vacrel->extVacReportIdx->wal_records; + extVacStats->common.wal_fpi -= vacrel->extVacReportIdx->wal_fpi; + extVacStats->common.wal_bytes -= vacrel->extVacReportIdx->wal_bytes; } /* @@ -418,6 +428,9 @@ accumulate_idxs_vacuum_statistics(LVRelState *vacrel, vacrel->extVacReportIdx->total_blks_written += extVacIdxStats->common.total_blks_written; vacrel->extVacReportIdx->blk_read_time += extVacIdxStats->common.blk_read_time; vacrel->extVacReportIdx->blk_write_time += extVacIdxStats->common.blk_write_time; + vacrel->extVacReportIdx->wal_records += extVacIdxStats->common.wal_records; + vacrel->extVacReportIdx->wal_fpi += extVacIdxStats->common.wal_fpi; + vacrel->extVacReportIdx->wal_bytes += extVacIdxStats->common.wal_bytes; } /* diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index f951390cf61..38a83204700 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -528,6 +528,9 @@ ao_accum_resources(PgStat_CommonCounts *dst, const PgStat_CommonCounts *src, /* Per-relation block counts do not include the indexes. */ dst->blk_read_time -= src->blk_read_time; dst->blk_write_time -= src->blk_write_time; + dst->wal_records -= src->wal_records; + dst->wal_fpi -= src->wal_fpi; + dst->wal_bytes -= src->wal_bytes; } else { @@ -539,6 +542,9 @@ ao_accum_resources(PgStat_CommonCounts *dst, const PgStat_CommonCounts *src, dst->blks_hit += src->blks_hit; dst->blk_read_time += src->blk_read_time; dst->blk_write_time += src->blk_write_time; + dst->wal_records += src->wal_records; + dst->wal_fpi += src->wal_fpi; + dst->wal_bytes += src->wal_bytes; } } diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index a8c8a455e99..1dba80b2d84 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -445,6 +445,7 @@ extern PGDLLIMPORT int VacuumCostBalanceLocal; /* Allocated only when a hook consumes the extended statistics. */ typedef struct LVExtStatCounters { + WalUsage walusage; BufferUsage bufusage; PgStat_Counter blocks_fetched; PgStat_Counter blocks_hit; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 3072794803c..d0e43e30b8d 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -139,6 +139,9 @@ typedef struct PgStat_CommonCounts int64 blks_hit; double blk_read_time; double blk_write_time; + int64 wal_records; + int64 wal_fpi; + uint64 wal_bytes; } PgStat_CommonCounts; typedef struct PgStat_VacuumRelationCounts From 888e5c0c808bd9c161808c1ec0ddb82f6ccae2b7 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Fri, 9 Oct 2026 19:09:25 +0300 Subject: [PATCH 3/3] Report core VACUUM resources under VERBOSE without an extension hook Print buffer, WAL and I/O measurements for heap work, index passes, and each AO vacuum phase. Exclude index resources from the enclosing table or phase report, include local buffer writes, and keep ordinary VACUUM quiet. Remove the extension hook declaration, definition and collection gates from the instrumentation prerequisite. The extension patch introduces that interface together with its consumer. Drop unused AO aggregate reporting from the core-only patch. Add core TAP coverage without an extension and document VERBOSE output. --- doc/src/sgml/ref/vacuum.sgml | 7 ++ src/backend/access/heap/vacuumlazy.c | 60 +++++++++++--- src/backend/commands/vacuum_ao.c | 79 +++++++++---------- src/backend/commands/vacuumparallel.c | 7 +- src/backend/utils/activity/pgstat_relation.c | 6 -- src/include/access/appendonly_compaction.h | 11 +-- src/include/commands/vacuum.h | 7 +- src/include/pgstat.h | 6 +- src/test/modules/test_misc/meson.build | 1 + .../t/029_vacuum_resource_verbose.pl | 75 ++++++++++++++++++ 10 files changed, 187 insertions(+), 72 deletions(-) create mode 100644 src/test/modules/test_misc/t/029_vacuum_resource_verbose.pl diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index 2b85c3d385d..93d31af2543 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -140,6 +140,13 @@ VACUUM [ FULL ] [ FREEZE ] [ VERBOSE ] [ ANALYZE ] [ is enabled. Heap table resources + exclude the separately reported index passes. For append-optimized + tables, resources are reported separately for pre-cleanup, compaction + and post-cleanup, also excluding index work. These reports do not + require a statistics extension to be loaded. diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 5da380c2989..fcd77591fb6 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -331,16 +331,16 @@ static void restore_vacuum_error_info(LVRelState *vacrel, /* * extvac_stats_start - Allocate and snapshot extended instrumentation. * - * Return NULL without allocating anything when there is no consumer. The - * caller frees a non-NULL result after reporting the phase, or its vacuum - * memory context cleans it up on error. + * Return NULL without allocating anything unless VERBOSE is requested. + * The caller frees a non-NULL result after reporting the phase, or its + * vacuum memory context cleans it up on error. */ LVExtStatCounters * -extvac_stats_start(Relation rel) +extvac_stats_start(Relation rel, bool verbose) { LVExtStatCounters *counters; - if (set_report_vacuum_hook == NULL) + if (!verbose) return NULL; counters = palloc0(sizeof(LVExtStatCounters)); @@ -373,7 +373,7 @@ extvac_stats_end(Relation rel, LVExtStatCounters * counters, report->total_blks_read = bufusage.local_blks_read + bufusage.shared_blks_read; report->total_blks_hit = bufusage.local_blks_hit + bufusage.shared_blks_hit; report->total_blks_dirtied = bufusage.local_blks_dirtied + bufusage.shared_blks_dirtied; - report->total_blks_written = bufusage.shared_blks_written; + report->total_blks_written = bufusage.local_blks_written + bufusage.shared_blks_written; /* PostgreSQL 16 keeps a single timer for shared and local blocks */ report->blk_read_time = INSTR_TIME_GET_MILLISEC(bufusage.blk_read_time); report->blk_write_time = INSTR_TIME_GET_MILLISEC(bufusage.blk_write_time); @@ -388,6 +388,41 @@ extvac_stats_end(Relation rel, LVExtStatCounters * counters, } } +/* + * Print resource measurements for VACUUM VERBOSE. + */ +void +extvac_stats_log(Relation rel, const char *phase, const PgStat_CommonCounts *report) +{ + StringInfoData buf; + + initStringInfo(&buf); + appendStringInfo(&buf, _("vacuum resource usage for \"%s.%s\" (%s):\n"), + get_namespace_name(RelationGetNamespace(rel)), + RelationGetRelationName(rel), phase); + appendStringInfo(&buf, + _("buffer usage: %lld hits, %lld misses, %lld dirtied, %lld written\n"), + (long long) report->total_blks_hit, + (long long) report->total_blks_read, + (long long) report->total_blks_dirtied, + (long long) report->total_blks_written); + if (pgstat_track_counts) + appendStringInfo(&buf, _("relation buffer usage: %lld hits, %lld misses\n"), + (long long) report->blks_hit, + (long long) (report->blks_fetched - report->blks_hit)); + appendStringInfo(&buf, + _("WAL usage: %lld records, %lld full page images, %llu bytes\n"), + (long long) report->wal_records, + (long long) report->wal_fpi, + (unsigned long long) report->wal_bytes); + if (track_io_timing) + appendStringInfo(&buf, _("I/O timings: read: %.3f ms, write: %.3f ms\n"), + report->blk_read_time, report->blk_write_time); + + ereport(INFO, (errmsg_internal("%s", buf.data))); + pfree(buf.data); +} + /* * Build the heap-specific part of the extended vacuum report from the * counters gathered in vacrel. @@ -481,7 +516,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, } } - extVacCounters = extvac_stats_start(rel); + extVacCounters = extvac_stats_start(rel, verbose); pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); @@ -748,6 +783,9 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, extvac_stats_end(rel, extVacCounters, &extVacReport->common); accumulate_heap_vacuum_statistics(vacrel, extVacReport); + if (verbose) + extvac_stats_log(rel, _("heap (excluding indexes)"), + &extVacReport->common); pfree(extVacCounters); pfree(vacrel->extVacReportIdx); @@ -2873,7 +2911,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, LVSavedErrInfo saved_err_info; LVExtStatCounters *extVacCounters; - extVacCounters = extvac_stats_start(indrel); + extVacCounters = extvac_stats_start(indrel, vacrel->verbose); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -2904,6 +2942,8 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; extvac_stats_end(indrel, extVacCounters, &extVacReport->common); + if (vacrel->verbose) + extvac_stats_log(indrel, _("index bulk delete"), &extVacReport->common); extVacReport->type = PGSTAT_EXTVAC_INDEX; accumulate_idxs_vacuum_statistics(vacrel, extVacReport); pfree(extVacCounters); @@ -2935,7 +2975,7 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, LVSavedErrInfo saved_err_info; LVExtStatCounters *extVacCounters; - extVacCounters = extvac_stats_start(indrel); + extVacCounters = extvac_stats_start(indrel, vacrel->verbose); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -2966,6 +3006,8 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; extvac_stats_end(indrel, extVacCounters, &extVacReport->common); + if (vacrel->verbose) + extvac_stats_log(indrel, _("index cleanup"), &extVacReport->common); extVacReport->type = PGSTAT_EXTVAC_INDEX; accumulate_idxs_vacuum_statistics(vacrel, extVacReport); pfree(extVacCounters); diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index 38a83204700..fe61bd37760 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -161,15 +161,13 @@ static int vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats); static void ao_vacuum_rel_recycle_dead_segments(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats); -static AOVacuumRelStats *init_vacrelstats(void); +static AOVacuumRelStats *init_vacrelstats(bool instrument); static void cleanup_vacrelstats(AOVacuumRelStats **vacrelstatsp); static void ao_accum_resources(PgStat_CommonCounts *dst, const PgStat_CommonCounts *src, bool subtract); static void ao_measure_index_resources(Relation indrel, LVExtStatCounters *counters, - AOVacuumRelStats *vacrelstats); -static void ao_measure_table_resources(Relation rel, - AOVacuumRelStats *vacrelstats); + AOVacuumRelStats *vacrelstats, bool verbose); static void ao_vacuum_rel_pre_cleanup(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats) @@ -414,14 +412,14 @@ ao_vacuum_rel_compact(Relation onerel, VacuumParams *params, BufferAccessStrateg } static AOVacuumRelStats * -init_vacrelstats() +init_vacrelstats(bool instrument) { AOVacuumRelStats *vacrelstats; MemoryContext old_context; old_context = MemoryContextSwitchTo(TopMemoryContext); vacrelstats = (AOVacuumRelStats *) palloc0(sizeof(AOVacuumRelStats)); - if (set_report_vacuum_hook != NULL) + if (instrument) vacrelstats->extstats = palloc0(sizeof(AOVacuumExtStats)); MemoryContextSwitchTo(old_context); @@ -438,7 +436,8 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy { static AOVacuumRelStats *vacrelstats = NULL; LVExtStatCounters *extcounters; - bool extstats = (set_report_vacuum_hook != NULL); + bool verbose = (params->options & VACOPT_VERBOSE) != 0; + PgStat_CommonCounts index_start = {0}; Assert(RelationStorageIsAO(rel)); Assert(params != NULL); @@ -473,12 +472,14 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy } pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); - vacrelstats = init_vacrelstats(); + vacrelstats = init_vacrelstats(verbose); vacrelstats->relid = RelationGetRelid(rel); } /* Sample the resource usage of the phase for the extended statistics */ - extcounters = extvac_stats_start(rel); + extcounters = extvac_stats_start(rel, verbose); + if (verbose) + index_start = vacrelstats->extstats->indexes; /* * Do the actual work --- either FULL or "lazy" vacuum @@ -493,17 +494,27 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy /* Do nothing here, we will launch the stages later */ Assert(ao_vacuum_phase == 0); - if (extstats && ao_vacuum_phase != 0) + if (verbose && ao_vacuum_phase != 0) { extvac_stats_end(rel, extcounters, &extcounters->report.common); - ao_accum_resources(&vacrelstats->extstats->phases, - &extcounters->report.common, false); + if (Gp_role != GP_ROLE_DISPATCH) + { + PgStat_CommonCounts phase_usage = extcounters->report.common; + PgStat_CommonCounts index_usage = vacrelstats->extstats->indexes; + const char *phase_name = + ao_vacuum_phase == VACOPT_AO_PRE_CLEANUP_PHASE ? _("AO pre-cleanup (excluding indexes)") : + ao_vacuum_phase == VACOPT_AO_COMPACT_PHASE ? _("AO compaction (excluding indexes)") : + _("AO post-cleanup (excluding indexes)"); + + /* Only index passes executed during this phase are excluded. */ + ao_accum_resources(&index_usage, &index_start, true); + ao_accum_resources(&phase_usage, &index_usage, true); + extvac_stats_log(rel, phase_name, &phase_usage); + } } if (ao_vacuum_phase == VACOPT_AO_POST_CLEANUP_PHASE) { - if (extstats) - ao_measure_table_resources(rel, vacrelstats); pgstat_progress_end_command(); cleanup_vacrelstats(&vacrelstats); } @@ -549,40 +560,24 @@ ao_accum_resources(PgStat_CommonCounts *dst, const PgStat_CommonCounts *src, } /* - * Report one pass over an index of an append-optimized table to - * set_report_vacuum_hook, as lazy_vacuum_one_index() does for the heap, and - * remember its resource usage, which the table's report must not count - * again. + * Log one index pass and remember its resource usage so the enclosing AO + * phase's VERBOSE report does not count the same work again. */ static void ao_measure_index_resources(Relation indrel, LVExtStatCounters *counters, - AOVacuumRelStats *vacrelstats) + AOVacuumRelStats *vacrelstats, bool verbose) { PgStat_VacuumRelationCounts *report = &counters->report; extvac_stats_end(indrel, counters, &report->common); + if (verbose) + extvac_stats_log(indrel, _("AO index vacuum"), &report->common); report->type = PGSTAT_EXTVAC_INDEX; ao_accum_resources(&vacrelstats->extstats->indexes, &report->common, false); pfree(counters); } -/* - * Report the vacuum of an append-optimized table, all of its phases, to - * set_report_vacuum_hook. - */ -static void -ao_measure_table_resources(Relation rel, AOVacuumRelStats *vacrelstats) -{ - PgStat_VacuumRelationCounts report; - - memset(&report, 0, sizeof(report)); - report.type = PGSTAT_EXTVAC_TABLE; - report.common = vacrelstats->extstats->phases; - ao_accum_resources(&report.common, &vacrelstats->extstats->indexes, true); - -} - /* * Recycling AWAITING_DROP segments. */ @@ -689,14 +684,15 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs { LVExtStatCounters *extcounters; - extcounters = extvac_stats_start(Irel[i]); + extcounters = extvac_stats_start(Irel[i], elevel == INFO); scan_index(Irel[i], aoRelation, elevel, bstrategy); - if (set_report_vacuum_hook) + if (extcounters != NULL) ao_measure_index_resources(Irel[i], extcounters, - vacrelstats); + vacrelstats, + elevel == INFO && Gp_role != GP_ROLE_DISPATCH); } } else @@ -705,16 +701,17 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs { LVExtStatCounters *extcounters; - extcounters = extvac_stats_start(Irel[i]); + extcounters = extvac_stats_start(Irel[i], elevel == INFO); vacuum_appendonly_index(Irel[i], aoRelation, dead_segs, elevel, bstrategy, vacrelstats); - if (set_report_vacuum_hook) + if (extcounters != NULL) ao_measure_index_resources(Irel[i], extcounters, - vacrelstats); + vacrelstats, + elevel == INFO && Gp_role != GP_ROLE_DISPATCH); } } } diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index 9ec6d8fdd31..0f66ea6b825 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -845,7 +845,7 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, if (indstats->istat_updated) istat = &(indstats->istat); - extVacCounters = extvac_stats_start(indrel); + extVacCounters = extvac_stats_start(indrel, pvs->shared->elevel == INFO); ivinfo.index = indrel; ivinfo.heaprel = pvs->heaprel; ivinfo.analyze_only = false; @@ -878,6 +878,11 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; extvac_stats_end(indrel, extVacCounters, &extVacReport->common); + if (pvs->shared->elevel == INFO) + extvac_stats_log(indrel, + indstats->status == PARALLEL_INDVAC_STATUS_NEED_CLEANUP ? + _("index cleanup") : _("index bulk delete"), + &extVacReport->common); extVacReport->type = PGSTAT_EXTVAC_INDEX; pfree(extVacCounters); } diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 41b528e6ccf..45bbf9abcb7 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -273,12 +273,6 @@ pgstat_report_vacuum(Oid tableoid, bool shared, pgstat_flush_io(false); } -/* - * Hook for extensions to receive extended vacuum statistics. - * NULL when no extension has registered. - */ -set_report_vacuum_hook_type set_report_vacuum_hook = NULL; - /* * Report that the table was just analyzed and flush IO statistics. * diff --git a/src/include/access/appendonly_compaction.h b/src/include/access/appendonly_compaction.h index 9320a6131b2..92111c89005 100644 --- a/src/include/access/appendonly_compaction.h +++ b/src/include/access/appendonly_compaction.h @@ -22,10 +22,9 @@ #define APPENDONLY_COMPACTION_SEGNO_INVALID (-1) -/* Optional resource counters for the extended-statistics hook. */ +/* Optional resource counters for VACUUM VERBOSE. */ typedef struct AOVacuumExtStats { - PgStat_CommonCounts phases; PgStat_CommonCounts indexes; } AOVacuumExtStats; @@ -41,12 +40,8 @@ typedef struct AOVacuumRelStats int num_index_vacuumed; /* current # of indexes been vacuumed */ /* the relation these stats were started for */ Oid relid; - /* - * Resource usage for set_report_vacuum_hook, accumulated over the phases: - * of the phases as a whole, and of the index passes among them, which are - * reported per index and subtracted from the table's report. - */ - AOVacuumExtStats *extstats; /* allocated only when the hook is installed */ + /* Index resources are excluded from each phase's VERBOSE report. */ + AOVacuumExtStats *extstats; /* allocated only for VERBOSE */ } AOVacuumRelStats; extern Bitmapset *AppendOptimizedCollectDeadSegments(Relation aorel); diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 1dba80b2d84..f19205c584a 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -442,7 +442,7 @@ extern PGDLLIMPORT pg_atomic_uint32 *VacuumSharedCostBalance; extern PGDLLIMPORT pg_atomic_uint32 *VacuumActiveNWorkers; extern PGDLLIMPORT int VacuumCostBalanceLocal; -/* Allocated only when a hook consumes the extended statistics. */ +/* Allocated only for VERBOSE. */ typedef struct LVExtStatCounters { WalUsage walusage; @@ -452,10 +452,13 @@ typedef struct LVExtStatCounters PgStat_VacuumRelationCounts report; } LVExtStatCounters; -extern LVExtStatCounters *extvac_stats_start(Relation rel); +extern LVExtStatCounters *extvac_stats_start(Relation rel, bool verbose); extern void extvac_stats_end(Relation rel, LVExtStatCounters *counters, PgStat_CommonCounts *report); +extern void extvac_stats_log(Relation rel, const char *phase, + const PgStat_CommonCounts *report); + extern PGDLLIMPORT bool VacuumFailsafeActive; extern PGDLLIMPORT double vacuum_cost_delay; extern PGDLLIMPORT int vacuum_cost_limit; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index d0e43e30b8d..a23eaf95d6a 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -118,7 +118,7 @@ typedef struct PgStat_FunctionCounts * Working state needed to accumulate per-function-call timing statistics. */ /* - * Extended vacuum statistics - passed to extensions via set_report_vacuum_hook. + * Resource measurements collected during VACUUM. * Type of entry: table (heap), index, or database aggregate. */ typedef enum ExtVacReportType @@ -685,10 +685,6 @@ extern void pgstat_unlink_relation(Relation rel); extern void pgstat_report_vacuum(Oid tableoid, bool shared, PgStat_Counter livetuples, PgStat_Counter deadtuples); -/* Hook for extensions to receive extended vacuum statistics */ -typedef void (*set_report_vacuum_hook_type) (Oid tableoid, bool shared, - PgStat_VacuumRelationCounts * params); -extern PGDLLIMPORT set_report_vacuum_hook_type set_report_vacuum_hook; extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, diff --git a/src/test/modules/test_misc/meson.build b/src/test/modules/test_misc/meson.build index 911084ac0fe..51c8a091119 100644 --- a/src/test/modules/test_misc/meson.build +++ b/src/test/modules/test_misc/meson.build @@ -10,6 +10,7 @@ tests += { 't/002_tablespace.pl', 't/003_check_guc.pl', 't/004_io_direct.pl', + 't/029_vacuum_resource_verbose.pl', ], }, } diff --git a/src/test/modules/test_misc/t/029_vacuum_resource_verbose.pl b/src/test/modules/test_misc/t/029_vacuum_resource_verbose.pl new file mode 100644 index 00000000000..94e5a413553 --- /dev/null +++ b/src/test/modules/test_misc/t/029_vacuum_resource_verbose.pl @@ -0,0 +1,75 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Resource instrumentation has a core consumer even without an extension. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('vacuum_resource_verbose'); +$node->init; +$node->append_conf('postgresql.conf', q{ +autovacuum = off +track_io_timing = on +gp_appendonly_compaction_threshold = 1 +}); +$node->start; +is($node->safe_psql('postgres', 'SHOW shared_preload_libraries'), '', + 'VERBOSE resource reporting requires no preloaded extension'); + +for my $storage ('heap', 'row', 'column') +{ + my $table = "verbose_$storage"; + my $options = $storage eq 'heap' ? '' : + "WITH (appendonly = true, orientation = $storage)"; + $node->safe_psql('postgres', qq{ +CREATE TABLE $table (id int, padding text) $options; +CREATE INDEX ${table}_idx ON $table (id); +INSERT INTO $table SELECT g, repeat('x', 100) FROM generate_series(1, 4000) g; +DELETE FROM $table WHERE id <= 1000; +}); + my ($stdout, $stderr); + is($node->psql('postgres', "VACUUM $table", + stdout => \$stdout, stderr => \$stderr), 0, + "$storage plain vacuum succeeds"); + unlike($stderr, qr/vacuum resource usage for/, + "$storage plain vacuum does not emit resource reports"); + $node->safe_psql('postgres', "DELETE FROM $table WHERE id <= 2000"); + is($node->psql('postgres', "VACUUM (VERBOSE, INDEX_CLEANUP ON) $table", + stdout => \$stdout, stderr => \$stderr), 0, + "$storage verbose vacuum succeeds"); + my @phases = $storage eq 'heap' ? ('heap (excluding indexes)') : + ('AO pre-cleanup (excluding indexes)', 'AO compaction (excluding indexes)', + 'AO post-cleanup (excluding indexes)'); + for my $phase (@phases) + { + my ($report) = $stderr =~ + /vacuum resource usage for "public\.\Q$table\E" \(\Q$phase\E\):\n(.*?)(?=\n(?:INFO|CONTEXT|WARNING):|\z)/s; + ok(defined $report, "$storage reports $phase without an extension"); + $report //= ''; + like($report, qr/buffer usage: \d+ hits, \d+ misses, \d+ dirtied, \d+ written/, + "$phase reports nonnegative buffer counters"); + like($report, qr/WAL usage: \d+ records, \d+ full page images, \d+ bytes/, + "$phase reports WAL counters"); + like($report, qr/I\/O timings: read: \d+\.\d+ ms, write: \d+\.\d+ ms/, + "$phase reports I/O times in milliseconds"); + like($report, qr/WAL usage: [1-9]\d* records/, + 'compaction produces WAL') if $phase eq 'AO compaction (excluding indexes)'; + } + my $index_phase = $storage eq 'heap' ? 'index bulk delete' : 'AO index vacuum'; + like($stderr, qr/vacuum resource usage for "public\.\Q${table}_idx\E" \(\Q$index_phase\E\):/, + "$storage reports index resources separately"); +} + +# Timing output is conditional, while collection of buffer/WAL counters is not. +my ($stdout, $stderr); +is($node->psql('postgres', + 'SET track_io_timing = off; VACUUM (VERBOSE) verbose_heap', + stdout => \$stdout, stderr => \$stderr), 0, 'verbose vacuum with I/O timing disabled'); +like($stderr, qr/vacuum resource usage for "public\.verbose_heap"/, + 'resource report remains available without I/O timing'); +unlike($stderr, qr/I\/O timings:/, 'disabled I/O timing is not printed'); + +$node->stop; +done_testing();