diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index 2b85c3d385d..93d31af2543 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -140,6 +140,13 @@ VACUUM [ FULL ] [ FREEZE ] [ VERBOSE ] [ ANALYZE ] [ is enabled. Heap table resources + exclude the separately reported index passes. For append-optimized + tables, resources are reported separately for pre-cleanup, compaction + and post-cleanup, also excluding index work. These reports do not + require a statistics extension to be loaded. diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index e3261b49a6c..fcd77591fb6 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -187,6 +187,8 @@ typedef struct LVRelState /* Error reporting state */ char *dbname; char *relnamespace; + Oid reloid; + Oid indoid; char *relname; char *indname; /* Current index name */ BlockNumber blkno; /* used only for heap operations */ @@ -225,6 +227,18 @@ typedef struct LVRelState int64 live_tuples; /* # live tuples remaining */ int64 recently_dead_tuples; /* # dead, but not yet removable */ int64 missed_dead_tuples; /* # removable, but not removed */ + + /* + * Resource usage of the index passes this process ran, subtracted from + * the heap report to avoid double-counting (see + * accumulate_heap_vacuum_statistics). + */ + PgStat_CommonCounts *extVacReportIdx; + + /* + * We need to accumulate index statistics for later subtraction from heap + * stats. + */ } LVRelState; /* @@ -312,6 +326,148 @@ static void update_vacuum_error_info(LVRelState *vacrel, static void restore_vacuum_error_info(LVRelState *vacrel, const LVSavedErrInfo *saved_vacrel); +/* Extended vacuum statistics functions */ + +/* + * extvac_stats_start - Allocate and snapshot extended instrumentation. + * + * Return NULL without allocating anything unless VERBOSE is requested. + * The caller frees a non-NULL result after reporting the phase, or its + * vacuum memory context cleans it up on error. + */ +LVExtStatCounters * +extvac_stats_start(Relation rel, bool verbose) +{ + LVExtStatCounters *counters; + + if (!verbose) + return NULL; + + counters = palloc0(sizeof(LVExtStatCounters)); + counters->walusage = pgWalUsage; + counters->bufusage = pgBufferUsage; + + if (rel->pgstat_info && pgstat_track_counts) + { + counters->blocks_fetched = rel->pgstat_info->counts.blocks_fetched; + counters->blocks_hit = rel->pgstat_info->counts.blocks_hit; + } + return counters; +} + +/* + * extvac_stats_end - Diff resource usage since extvac_stats_start into report. + */ +void +extvac_stats_end(Relation rel, LVExtStatCounters * counters, + PgStat_CommonCounts * report) +{ + WalUsage walusage; + BufferUsage bufusage; + + memset(&walusage, 0, sizeof(WalUsage)); + WalUsageAccumDiff(&walusage, &pgWalUsage, &counters->walusage); + memset(&bufusage, 0, sizeof(BufferUsage)); + BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &counters->bufusage); + + report->total_blks_read = bufusage.local_blks_read + bufusage.shared_blks_read; + report->total_blks_hit = bufusage.local_blks_hit + bufusage.shared_blks_hit; + report->total_blks_dirtied = bufusage.local_blks_dirtied + bufusage.shared_blks_dirtied; + report->total_blks_written = bufusage.local_blks_written + bufusage.shared_blks_written; + /* PostgreSQL 16 keeps a single timer for shared and local blocks */ + report->blk_read_time = INSTR_TIME_GET_MILLISEC(bufusage.blk_read_time); + report->blk_write_time = INSTR_TIME_GET_MILLISEC(bufusage.blk_write_time); + report->wal_records = walusage.wal_records; + report->wal_fpi = walusage.wal_fpi; + report->wal_bytes = walusage.wal_bytes; + + if (rel->pgstat_info && pgstat_track_counts) + { + report->blks_fetched = rel->pgstat_info->counts.blocks_fetched - counters->blocks_fetched; + report->blks_hit = rel->pgstat_info->counts.blocks_hit - counters->blocks_hit; + } +} + +/* + * Print resource measurements for VACUUM VERBOSE. + */ +void +extvac_stats_log(Relation rel, const char *phase, const PgStat_CommonCounts *report) +{ + StringInfoData buf; + + initStringInfo(&buf); + appendStringInfo(&buf, _("vacuum resource usage for \"%s.%s\" (%s):\n"), + get_namespace_name(RelationGetNamespace(rel)), + RelationGetRelationName(rel), phase); + appendStringInfo(&buf, + _("buffer usage: %lld hits, %lld misses, %lld dirtied, %lld written\n"), + (long long) report->total_blks_hit, + (long long) report->total_blks_read, + (long long) report->total_blks_dirtied, + (long long) report->total_blks_written); + if (pgstat_track_counts) + appendStringInfo(&buf, _("relation buffer usage: %lld hits, %lld misses\n"), + (long long) report->blks_hit, + (long long) (report->blks_fetched - report->blks_hit)); + appendStringInfo(&buf, + _("WAL usage: %lld records, %lld full page images, %llu bytes\n"), + (long long) report->wal_records, + (long long) report->wal_fpi, + (unsigned long long) report->wal_bytes); + if (track_io_timing) + appendStringInfo(&buf, _("I/O timings: read: %.3f ms, write: %.3f ms\n"), + report->blk_read_time, report->blk_write_time); + + ereport(INFO, (errmsg_internal("%s", buf.data))); + pfree(buf.data); +} + +/* + * Build the heap-specific part of the extended vacuum report from the + * counters gathered in vacrel. + */ +static void +accumulate_heap_vacuum_statistics(LVRelState *vacrel, PgStat_VacuumRelationCounts * extVacStats) +{ + extVacStats->type = PGSTAT_EXTVAC_TABLE; + + /* + * Subtract the resource usage of the index passes this process ran: they + * are reported per index, and the database-wide aggregate would count + * them twice otherwise. Parallel workers report their own usage with + * their index passes, so it never enters the leader's counters. + */ + extVacStats->common.total_blks_read -= vacrel->extVacReportIdx->total_blks_read; + extVacStats->common.total_blks_hit -= vacrel->extVacReportIdx->total_blks_hit; + extVacStats->common.total_blks_dirtied -= vacrel->extVacReportIdx->total_blks_dirtied; + extVacStats->common.total_blks_written -= vacrel->extVacReportIdx->total_blks_written; + extVacStats->common.blk_read_time -= vacrel->extVacReportIdx->blk_read_time; + extVacStats->common.blk_write_time -= vacrel->extVacReportIdx->blk_write_time; + extVacStats->common.wal_records -= vacrel->extVacReportIdx->wal_records; + extVacStats->common.wal_fpi -= vacrel->extVacReportIdx->wal_fpi; + extVacStats->common.wal_bytes -= vacrel->extVacReportIdx->wal_bytes; +} + +/* + * Accumulate an index pass report into the running total that is later + * subtracted from the heap report. + */ +static void +accumulate_idxs_vacuum_statistics(LVRelState *vacrel, + PgStat_VacuumRelationCounts * extVacIdxStats) +{ + vacrel->extVacReportIdx->total_blks_read += extVacIdxStats->common.total_blks_read; + vacrel->extVacReportIdx->total_blks_hit += extVacIdxStats->common.total_blks_hit; + vacrel->extVacReportIdx->total_blks_dirtied += extVacIdxStats->common.total_blks_dirtied; + vacrel->extVacReportIdx->total_blks_written += extVacIdxStats->common.total_blks_written; + vacrel->extVacReportIdx->blk_read_time += extVacIdxStats->common.blk_read_time; + vacrel->extVacReportIdx->blk_write_time += extVacIdxStats->common.blk_write_time; + vacrel->extVacReportIdx->wal_records += extVacIdxStats->common.wal_records; + vacrel->extVacReportIdx->wal_fpi += extVacIdxStats->common.wal_fpi; + vacrel->extVacReportIdx->wal_bytes += extVacIdxStats->common.wal_bytes; +} + /* * lazy_vacuum_rel_heap() -- perform VACUUM for one heap relation * @@ -340,6 +496,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, TimestampTz starttime = 0; PgStat_Counter startreadtime = 0, startwritetime = 0; + LVExtStatCounters *extVacCounters; WalUsage startwalusage = pgWalUsage; BufferUsage startbufferusage = pgBufferUsage; ErrorContextCallback errcallback; @@ -359,6 +516,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, } } + extVacCounters = extvac_stats_start(rel, verbose); + pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); @@ -375,9 +534,12 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, * these temp copies. */ vacrel = (LVRelState *) palloc0(sizeof(LVRelState)); + if (extVacCounters != NULL) + vacrel->extVacReportIdx = palloc0(sizeof(PgStat_CommonCounts)); vacrel->dbname = get_database_name(MyDatabaseId); vacrel->relnamespace = get_namespace_name(RelationGetNamespace(rel)); vacrel->relname = pstrdup(RelationGetRelationName(rel)); + vacrel->reloid = RelationGetRelid(rel); vacrel->indname = NULL; vacrel->phase = VACUUM_ERRCB_PHASE_UNKNOWN; vacrel->verbose = verbose; @@ -615,6 +777,19 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, * soon in cases where the failsafe prevented significant amounts of heap * vacuuming. */ + if (extVacCounters != NULL) + { + PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; + + extvac_stats_end(rel, extVacCounters, &extVacReport->common); + accumulate_heap_vacuum_statistics(vacrel, extVacReport); + if (verbose) + extvac_stats_log(rel, _("heap (excluding indexes)"), + &extVacReport->common); + + pfree(extVacCounters); + pfree(vacrel->extVacReportIdx); + } pgstat_report_vacuum(RelationGetRelid(rel), rel->rd_rel->relisshared, Max(vacrel->new_live_tuples, 0), @@ -2734,7 +2909,9 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + LVExtStatCounters *extVacCounters; + extVacCounters = extvac_stats_start(indrel, vacrel->verbose); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -2752,6 +2929,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, */ Assert(vacrel->indname == NULL); vacrel->indname = pstrdup(RelationGetRelationName(indrel)); + vacrel->indoid = RelationGetRelid(indrel); update_vacuum_error_info(vacrel, &saved_err_info, VACUUM_ERRCB_PHASE_VACUUM_INDEX, InvalidBlockNumber, InvalidOffsetNumber); @@ -2759,6 +2937,18 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, /* Do bulk deletion */ istat = vac_bulkdel_one_index(&ivinfo, istat, (void *) vacrel->dead_items); + if (extVacCounters != NULL) + { + PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; + + extvac_stats_end(indrel, extVacCounters, &extVacReport->common); + if (vacrel->verbose) + extvac_stats_log(indrel, _("index bulk delete"), &extVacReport->common); + extVacReport->type = PGSTAT_EXTVAC_INDEX; + accumulate_idxs_vacuum_statistics(vacrel, extVacReport); + pfree(extVacCounters); + } + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); @@ -2783,7 +2973,9 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + LVExtStatCounters *extVacCounters; + extVacCounters = extvac_stats_start(indrel, vacrel->verbose); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -2802,12 +2994,25 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, */ Assert(vacrel->indname == NULL); vacrel->indname = pstrdup(RelationGetRelationName(indrel)); + vacrel->indoid = RelationGetRelid(indrel); update_vacuum_error_info(vacrel, &saved_err_info, VACUUM_ERRCB_PHASE_INDEX_CLEANUP, InvalidBlockNumber, InvalidOffsetNumber); istat = vac_cleanup_one_index(&ivinfo, istat); + if (extVacCounters != NULL) + { + PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; + + extvac_stats_end(indrel, extVacCounters, &extVacReport->common); + if (vacrel->verbose) + extvac_stats_log(indrel, _("index cleanup"), &extVacReport->common); + extVacReport->type = PGSTAT_EXTVAC_INDEX; + accumulate_idxs_vacuum_statistics(vacrel, extVacReport); + pfree(extVacCounters); + } + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index 075c73fb1c1..fe61bd37760 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -161,8 +161,13 @@ static int vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats); static void ao_vacuum_rel_recycle_dead_segments(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats); -static AOVacuumRelStats *init_vacrelstats(void); +static AOVacuumRelStats *init_vacrelstats(bool instrument); static void cleanup_vacrelstats(AOVacuumRelStats **vacrelstatsp); +static void ao_accum_resources(PgStat_CommonCounts *dst, + const PgStat_CommonCounts *src, bool subtract); +static void ao_measure_index_resources(Relation indrel, + LVExtStatCounters *counters, + AOVacuumRelStats *vacrelstats, bool verbose); static void ao_vacuum_rel_pre_cleanup(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats) @@ -407,13 +412,15 @@ ao_vacuum_rel_compact(Relation onerel, VacuumParams *params, BufferAccessStrateg } static AOVacuumRelStats * -init_vacrelstats() +init_vacrelstats(bool instrument) { AOVacuumRelStats *vacrelstats; MemoryContext old_context; old_context = MemoryContextSwitchTo(TopMemoryContext); vacrelstats = (AOVacuumRelStats *) palloc0(sizeof(AOVacuumRelStats)); + if (instrument) + vacrelstats->extstats = palloc0(sizeof(AOVacuumExtStats)); MemoryContextSwitchTo(old_context); return vacrelstats; @@ -428,11 +435,27 @@ void ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy) { static AOVacuumRelStats *vacrelstats = NULL; + LVExtStatCounters *extcounters; + bool verbose = (params->options & VACOPT_VERBOSE) != 0; + PgStat_CommonCounts index_start = {0}; Assert(RelationStorageIsAO(rel)); Assert(params != NULL); int ao_vacuum_phase = (params->options & VACUUM_AO_PHASE_MASK); + /* + * The phases of one vacuum share these stats and free them at the end of + * the last phase, so a vacuum that failed in between leaves them behind. + * Drop such leftovers: otherwise the counters of the failed vacuum are + * taken for the next one, whose progress is never reported because the + * progress command is only + * started along with the stats. + */ + if (vacrelstats != NULL && + (ao_vacuum_phase == VACOPT_AO_PRE_CLEANUP_PHASE || + vacrelstats->relid != RelationGetRelid(rel))) + cleanup_vacrelstats(&vacrelstats); + if (!vacrelstats) { if (ao_vacuum_phase != VACOPT_AO_PRE_CLEANUP_PHASE && Gp_role == GP_ROLE_EXECUTE) @@ -449,9 +472,15 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy } pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); - vacrelstats = init_vacrelstats(); + vacrelstats = init_vacrelstats(verbose); + vacrelstats->relid = RelationGetRelid(rel); } + /* Sample the resource usage of the phase for the extended statistics */ + extcounters = extvac_stats_start(rel, verbose); + if (verbose) + index_start = vacrelstats->extstats->indexes; + /* * Do the actual work --- either FULL or "lazy" vacuum */ @@ -460,14 +489,93 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy else if (ao_vacuum_phase == VACOPT_AO_COMPACT_PHASE) ao_vacuum_rel_compact(rel, params, bstrategy, vacrelstats); else if (ao_vacuum_phase == VACOPT_AO_POST_CLEANUP_PHASE) - { ao_vacuum_rel_post_cleanup(rel, params, bstrategy, vacrelstats); + else + /* Do nothing here, we will launch the stages later */ + Assert(ao_vacuum_phase == 0); + + if (verbose && ao_vacuum_phase != 0) + { + extvac_stats_end(rel, extcounters, &extcounters->report.common); + if (Gp_role != GP_ROLE_DISPATCH) + { + PgStat_CommonCounts phase_usage = extcounters->report.common; + PgStat_CommonCounts index_usage = vacrelstats->extstats->indexes; + const char *phase_name = + ao_vacuum_phase == VACOPT_AO_PRE_CLEANUP_PHASE ? _("AO pre-cleanup (excluding indexes)") : + ao_vacuum_phase == VACOPT_AO_COMPACT_PHASE ? _("AO compaction (excluding indexes)") : + _("AO post-cleanup (excluding indexes)"); + + /* Only index passes executed during this phase are excluded. */ + ao_accum_resources(&index_usage, &index_start, true); + ao_accum_resources(&phase_usage, &index_usage, true); + extvac_stats_log(rel, phase_name, &phase_usage); + } + } + + if (ao_vacuum_phase == VACOPT_AO_POST_CLEANUP_PHASE) + { pgstat_progress_end_command(); cleanup_vacrelstats(&vacrelstats); } + + if (extcounters != NULL) + pfree(extcounters); +} + +/* + * Add the resource usage counters of src to dst, or subtract them from it. + */ +static void +ao_accum_resources(PgStat_CommonCounts *dst, const PgStat_CommonCounts *src, + bool subtract) +{ + if (subtract) + { + dst->total_blks_read -= src->total_blks_read; + dst->total_blks_hit -= src->total_blks_hit; + dst->total_blks_dirtied -= src->total_blks_dirtied; + dst->total_blks_written -= src->total_blks_written; + /* Per-relation block counts do not include the indexes. */ + dst->blk_read_time -= src->blk_read_time; + dst->blk_write_time -= src->blk_write_time; + dst->wal_records -= src->wal_records; + dst->wal_fpi -= src->wal_fpi; + dst->wal_bytes -= src->wal_bytes; + } else - /* Do nothing here, we will launch the stages later */ - Assert(ao_vacuum_phase == 0); + { + dst->total_blks_read += src->total_blks_read; + dst->total_blks_hit += src->total_blks_hit; + dst->total_blks_dirtied += src->total_blks_dirtied; + dst->total_blks_written += src->total_blks_written; + dst->blks_fetched += src->blks_fetched; + dst->blks_hit += src->blks_hit; + dst->blk_read_time += src->blk_read_time; + dst->blk_write_time += src->blk_write_time; + dst->wal_records += src->wal_records; + dst->wal_fpi += src->wal_fpi; + dst->wal_bytes += src->wal_bytes; + } +} + +/* + * Log one index pass and remember its resource usage so the enclosing AO + * phase's VERBOSE report does not count the same work again. + */ +static void +ao_measure_index_resources(Relation indrel, LVExtStatCounters *counters, + AOVacuumRelStats *vacrelstats, bool verbose) +{ + PgStat_VacuumRelationCounts *report = &counters->report; + + extvac_stats_end(indrel, counters, &report->common); + if (verbose) + extvac_stats_log(indrel, _("AO index vacuum"), &report->common); + report->type = PGSTAT_EXTVAC_INDEX; + + ao_accum_resources(&vacrelstats->extstats->indexes, &report->common, false); + pfree(counters); } /* @@ -574,22 +682,36 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs { for (i = 0; i < nindexes; i++) { + LVExtStatCounters *extcounters; + + extcounters = extvac_stats_start(Irel[i], elevel == INFO); scan_index(Irel[i], aoRelation, elevel, bstrategy); + if (extcounters != NULL) + ao_measure_index_resources(Irel[i], extcounters, + vacrelstats, + elevel == INFO && Gp_role != GP_ROLE_DISPATCH); } } else { for (i = 0; i < nindexes; i++) { + LVExtStatCounters *extcounters; + + extcounters = extvac_stats_start(Irel[i], elevel == INFO); vacuum_appendonly_index(Irel[i], aoRelation, dead_segs, elevel, bstrategy, vacrelstats); + if (extcounters != NULL) + ao_measure_index_resources(Irel[i], extcounters, + vacrelstats, + elevel == INFO && Gp_role != GP_ROLE_DISPATCH); } } } @@ -780,6 +902,8 @@ vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, int elevel, static void cleanup_vacrelstats(AOVacuumRelStats **vacrelstats) { + if ((*vacrelstats)->extstats != NULL) + pfree((*vacrelstats)->extstats); pfree(*vacrelstats); *vacrelstats = NULL; } diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index a79067fd461..0f66ea6b825 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -836,6 +836,7 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, IndexBulkDeleteResult *istat = NULL; IndexBulkDeleteResult *istat_res; IndexVacuumInfo ivinfo; + LVExtStatCounters *extVacCounters; /* * Update the pointer to the corresponding bulk-deletion result if someone @@ -844,6 +845,7 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, if (indstats->istat_updated) istat = &(indstats->istat); + extVacCounters = extvac_stats_start(indrel, pvs->shared->elevel == INFO); ivinfo.index = indrel; ivinfo.heaprel = pvs->heaprel; ivinfo.analyze_only = false; @@ -871,6 +873,20 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, RelationGetRelationName(indrel)); } + if (extVacCounters != NULL) + { + PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; + + extvac_stats_end(indrel, extVacCounters, &extVacReport->common); + if (pvs->shared->elevel == INFO) + extvac_stats_log(indrel, + indstats->status == PARALLEL_INDVAC_STATUS_NEED_CLEANUP ? + _("index cleanup") : _("index bulk delete"), + &extVacReport->common); + extVacReport->type = PGSTAT_EXTVAC_INDEX; + pfree(extVacCounters); + } + /* * Copy the index bulk-deletion result returned from ambulkdelete and * amvacuumcleanup to the DSM segment if it's the first cycle because they diff --git a/src/include/access/appendonly_compaction.h b/src/include/access/appendonly_compaction.h index 44aa78a39fb..92111c89005 100644 --- a/src/include/access/appendonly_compaction.h +++ b/src/include/access/appendonly_compaction.h @@ -13,6 +13,7 @@ #ifndef APPENDONLY_COMPACTION_H #define APPENDONLY_COMPACTION_H +#include "pgstat.h" #include "nodes/pg_list.h" #include "access/appendonly_visimap.h" #include "utils/rel.h" @@ -21,6 +22,12 @@ #define APPENDONLY_COMPACTION_SEGNO_INVALID (-1) +/* Optional resource counters for VACUUM VERBOSE. */ +typedef struct AOVacuumExtStats +{ + PgStat_CommonCounts indexes; +} AOVacuumExtStats; + /* * Stats for progress reporting. * This is AO/AOCO counterpart of LVRelStats for Heap. It lives throughout @@ -31,6 +38,10 @@ typedef struct AOVacuumRelStats int nbytes_truncated; /* current # of bytes truncated from segment file */ int num_dead_tuples; /* current # of dead tuples */ int num_index_vacuumed; /* current # of indexes been vacuumed */ + /* the relation these stats were started for */ + Oid relid; + /* Index resources are excluded from each phase's VERBOSE report. */ + AOVacuumExtStats *extstats; /* allocated only for VERBOSE */ } AOVacuumRelStats; extern Bitmapset *AppendOptimizedCollectDeadSegments(Relation aorel); diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 431cd7221a8..f19205c584a 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -21,10 +21,12 @@ #include "catalog/pg_class.h" #include "catalog/pg_statistic.h" #include "catalog/pg_type.h" +#include "executor/instrument.h" #include "parser/parse_node.h" #include "storage/buf.h" #include "storage/lock.h" #include "utils/relcache.h" +#include "pgstat.h" #include "utils/snapshot.h" /* @@ -440,6 +442,23 @@ extern PGDLLIMPORT pg_atomic_uint32 *VacuumSharedCostBalance; extern PGDLLIMPORT pg_atomic_uint32 *VacuumActiveNWorkers; extern PGDLLIMPORT int VacuumCostBalanceLocal; +/* Allocated only for VERBOSE. */ +typedef struct LVExtStatCounters +{ + WalUsage walusage; + BufferUsage bufusage; + PgStat_Counter blocks_fetched; + PgStat_Counter blocks_hit; + PgStat_VacuumRelationCounts report; +} LVExtStatCounters; + +extern LVExtStatCounters *extvac_stats_start(Relation rel, bool verbose); +extern void extvac_stats_end(Relation rel, LVExtStatCounters *counters, + PgStat_CommonCounts *report); + +extern void extvac_stats_log(Relation rel, const char *phase, + const PgStat_CommonCounts *report); + extern PGDLLIMPORT bool VacuumFailsafeActive; extern PGDLLIMPORT double vacuum_cost_delay; extern PGDLLIMPORT int vacuum_cost_limit; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 2a85773c30c..a23eaf95d6a 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -117,6 +117,39 @@ typedef struct PgStat_FunctionCounts /* * Working state needed to accumulate per-function-call timing statistics. */ +/* + * Resource measurements collected during VACUUM. + * Type of entry: table (heap), index, or database aggregate. + */ +typedef enum ExtVacReportType +{ + PGSTAT_EXTVAC_INVALID = 0, + PGSTAT_EXTVAC_TABLE = 1, + PGSTAT_EXTVAC_INDEX = 2, + PGSTAT_EXTVAC_DB = 3, +} ExtVacReportType; + +typedef struct PgStat_CommonCounts +{ + int64 total_blks_read; + int64 total_blks_hit; + int64 total_blks_dirtied; + int64 total_blks_written; + int64 blks_fetched; + int64 blks_hit; + double blk_read_time; + double blk_write_time; + int64 wal_records; + int64 wal_fpi; + uint64 wal_bytes; +} PgStat_CommonCounts; + +typedef struct PgStat_VacuumRelationCounts +{ + PgStat_CommonCounts common; + ExtVacReportType type; +} PgStat_VacuumRelationCounts; + typedef struct PgStat_FunctionCallUsage { /* Link to function's hashtable entry (must still be there at exit!) */ @@ -652,6 +685,7 @@ extern void pgstat_unlink_relation(Relation rel); extern void pgstat_report_vacuum(Oid tableoid, bool shared, PgStat_Counter livetuples, PgStat_Counter deadtuples); + extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, bool resetcounter); diff --git a/src/test/modules/test_misc/meson.build b/src/test/modules/test_misc/meson.build index 911084ac0fe..51c8a091119 100644 --- a/src/test/modules/test_misc/meson.build +++ b/src/test/modules/test_misc/meson.build @@ -10,6 +10,7 @@ tests += { 't/002_tablespace.pl', 't/003_check_guc.pl', 't/004_io_direct.pl', + 't/029_vacuum_resource_verbose.pl', ], }, } diff --git a/src/test/modules/test_misc/t/029_vacuum_resource_verbose.pl b/src/test/modules/test_misc/t/029_vacuum_resource_verbose.pl new file mode 100644 index 00000000000..94e5a413553 --- /dev/null +++ b/src/test/modules/test_misc/t/029_vacuum_resource_verbose.pl @@ -0,0 +1,75 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Resource instrumentation has a core consumer even without an extension. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('vacuum_resource_verbose'); +$node->init; +$node->append_conf('postgresql.conf', q{ +autovacuum = off +track_io_timing = on +gp_appendonly_compaction_threshold = 1 +}); +$node->start; +is($node->safe_psql('postgres', 'SHOW shared_preload_libraries'), '', + 'VERBOSE resource reporting requires no preloaded extension'); + +for my $storage ('heap', 'row', 'column') +{ + my $table = "verbose_$storage"; + my $options = $storage eq 'heap' ? '' : + "WITH (appendonly = true, orientation = $storage)"; + $node->safe_psql('postgres', qq{ +CREATE TABLE $table (id int, padding text) $options; +CREATE INDEX ${table}_idx ON $table (id); +INSERT INTO $table SELECT g, repeat('x', 100) FROM generate_series(1, 4000) g; +DELETE FROM $table WHERE id <= 1000; +}); + my ($stdout, $stderr); + is($node->psql('postgres', "VACUUM $table", + stdout => \$stdout, stderr => \$stderr), 0, + "$storage plain vacuum succeeds"); + unlike($stderr, qr/vacuum resource usage for/, + "$storage plain vacuum does not emit resource reports"); + $node->safe_psql('postgres', "DELETE FROM $table WHERE id <= 2000"); + is($node->psql('postgres', "VACUUM (VERBOSE, INDEX_CLEANUP ON) $table", + stdout => \$stdout, stderr => \$stderr), 0, + "$storage verbose vacuum succeeds"); + my @phases = $storage eq 'heap' ? ('heap (excluding indexes)') : + ('AO pre-cleanup (excluding indexes)', 'AO compaction (excluding indexes)', + 'AO post-cleanup (excluding indexes)'); + for my $phase (@phases) + { + my ($report) = $stderr =~ + /vacuum resource usage for "public\.\Q$table\E" \(\Q$phase\E\):\n(.*?)(?=\n(?:INFO|CONTEXT|WARNING):|\z)/s; + ok(defined $report, "$storage reports $phase without an extension"); + $report //= ''; + like($report, qr/buffer usage: \d+ hits, \d+ misses, \d+ dirtied, \d+ written/, + "$phase reports nonnegative buffer counters"); + like($report, qr/WAL usage: \d+ records, \d+ full page images, \d+ bytes/, + "$phase reports WAL counters"); + like($report, qr/I\/O timings: read: \d+\.\d+ ms, write: \d+\.\d+ ms/, + "$phase reports I/O times in milliseconds"); + like($report, qr/WAL usage: [1-9]\d* records/, + 'compaction produces WAL') if $phase eq 'AO compaction (excluding indexes)'; + } + my $index_phase = $storage eq 'heap' ? 'index bulk delete' : 'AO index vacuum'; + like($stderr, qr/vacuum resource usage for "public\.\Q${table}_idx\E" \(\Q$index_phase\E\):/, + "$storage reports index resources separately"); +} + +# Timing output is conditional, while collection of buffer/WAL counters is not. +my ($stdout, $stderr); +is($node->psql('postgres', + 'SET track_io_timing = off; VACUUM (VERBOSE) verbose_heap', + stdout => \$stdout, stderr => \$stderr), 0, 'verbose vacuum with I/O timing disabled'); +like($stderr, qr/vacuum resource usage for "public\.verbose_heap"/, + 'resource report remains available without I/O timing'); +unlike($stderr, qr/I\/O timings:/, 'disabled I/O timing is not printed'); + +$node->stop; +done_testing();