diff --git a/CHANGELOG.md b/CHANGELOG.md index bd26c1a..0023ff6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased] +### Fixed + +- **Opening the Overview on a large database was one of the most expensive things that database did.** On a 160 GB PostgreSQL with ~500 tables, `pg_stat_statements` ranked the Overview's two size queries first and second across *every* role on the cluster: the Largest tables chart averaged 62 seconds a call and the Database size card 26, together over seven hours of server time in three weeks, on a database the application was meanwhile trying to serve. Both are `pg_total_relation_size()` over every row of `pg_stat_user_tables`, which has the server stat the files of every table, TOAST table and index one relation at a time — and both go through Eloquent rather than the read-only executor, so no statement timeout ever stopped them. Three changes; the page shows the same thing, one figure aside by a rounding error: + + - **Database size now comes from `pg_database_size()`.** One walk of the database directory instead of one per relation: on the database above, 1 second against 8 on a quiet server. It also counts the system catalogs and the schemas the panel leaves out — which is what "database size" means to the person reading the card, and a 0.03% difference there — and it runs through the read-only executor, so it is bounded by the package's statement timeout like every other statistics read. + - **The Largest tables chart no longer polls.** Filament re-renders a widget every five seconds by default, and every render of this one sized the whole database again. Where the disk has gone does not change between two polls; reloading the page redraws it. + - **The vitals card reads each figure once per render.** The cache hit ratio was queried three times — value, description, colour — and the active sessions twice, on a card that polls. + + The Tables resource still orders by `pg_total_relation_size()` across the whole catalog, which is the same cost on each list load; it is left for a separate change because making it cheap without losing exact ordering is a design question rather than a fix. + ## [1.2.2] - 2026-09-21 ### Fixed diff --git a/src/Filament/Widgets/DatabaseVitals.php b/src/Filament/Widgets/DatabaseVitals.php index c552f84..25e8e59 100644 --- a/src/Filament/Widgets/DatabaseVitals.php +++ b/src/Filament/Widgets/DatabaseVitals.php @@ -6,6 +6,7 @@ use Filament\Widgets\StatsOverviewWidget; use Filament\Widgets\StatsOverviewWidget\Stat; +use Heyosseus\Vacuum\Database\ReadOnlyExecutor; use Heyosseus\Vacuum\Filament\Concerns\GatedWidget; use Heyosseus\Vacuum\Filament\Models\Session as SessionModel; use Heyosseus\Vacuum\Filament\Models\Table as TableModel; @@ -32,31 +33,49 @@ final class DatabaseVitals extends StatsOverviewWidget #[Override] protected function getStats(): array { + // Each figure is read once per render. The cache hit ratio used to be asked for + // three times -- the value, its description, its colour -- and the active sessions + // twice, each ask a query of its own, on a card that polls. + $cacheHitRatio = $this->cacheHitRatio(); + $readingFromMemory = $cacheHitRatio >= $this->cacheThreshold(); + $activeSessions = $this->activeSessions(); + return [ Stat::make('Database size', Bytes::human($this->totalBytes())) ->description(number_format($this->tables()).' tables') ->color('gray'), - Stat::make('Cache hit ratio', number_format($this->cacheHitRatio() * 100, 2).'%') - ->description($this->cacheHitRatio() >= $this->cacheThreshold() ? 'Reading from memory' : 'Going to disk') - ->color($this->cacheHitRatio() >= $this->cacheThreshold() ? 'success' : 'warning'), + Stat::make('Cache hit ratio', number_format($cacheHitRatio * 100, 2).'%') + ->description($readingFromMemory ? 'Reading from memory' : 'Going to disk') + ->color($readingFromMemory ? 'success' : 'warning'), Stat::make('Sessions', (string) $this->sessions()) - ->description($this->activeSessions().' active') - ->color($this->activeSessions() > 0 ? 'info' : 'gray'), + ->description($activeSessions.' active') + ->color($activeSessions > 0 ? 'info' : 'gray'), ]; } + /** + * The database's size, as PostgreSQL itself reports it. + * + * This used to sum pg_total_relation_size() over every row of pg_stat_user_tables, + * which has the server stat the files of every table, TOAST table and index one + * relation at a time, outside any statement timeout. On a 160 GB database with ~500 + * tables that measured 8 seconds on a quiet server and 26 on average under load, + * on every render of a card that polls. pg_database_size() walks the database + * directory once and returns the same figure in about a second there. + * + * The two are not quite the same number: pg_database_size() also counts the system + * catalogs and the schemas the panel leaves out. That is what "database size" means + * to the person reading the card, and on the database above the difference was + * 0.03%. It runs through the read-only executor, so it is bounded by the same + * statement timeout as the package's other statistics queries. + */ private function totalBytes(): int { - // first() rather than value(): value() would try to read back an attribute named - // after the whole raw expression, which is not a column and never resolves. The - // aggregate is aliased and read by that alias instead. - $row = TableModel::query() - ->selectRaw('coalesce(sum(pg_total_relation_size(pg_stat_user_tables.relid)), 0) AS bytes') - ->first(); - - $bytes = $row?->getAttribute('bytes'); + $row = app(ReadOnlyExecutor::class)->select('SELECT pg_database_size(current_database()) AS bytes')[0] ?? []; + + $bytes = $row['bytes'] ?? null; return is_numeric($bytes) ? (int) $bytes : 0; } diff --git a/src/Filament/Widgets/LargestTables.php b/src/Filament/Widgets/LargestTables.php index 5aecd69..9a26f04 100644 --- a/src/Filament/Widgets/LargestTables.php +++ b/src/Filament/Widgets/LargestTables.php @@ -25,6 +25,15 @@ final class LargestTables extends ChartWidget protected int|string|array $columnSpan = 1; + /** + * No polling. Filament's default re-renders the chart every five seconds, and every + * render sizes every relation in the database -- pg_total_relation_size() across + * pg_stat_user_tables, which is seconds on a large database and far longer under + * load. Where the disk has gone does not change between two polls; reloading the + * page redraws it. + */ + protected ?string $pollingInterval = null; + #[Override] protected function getType(): string { diff --git a/tests/Filament/OverviewTest.php b/tests/Filament/OverviewTest.php index 137c71f..4e814cf 100644 --- a/tests/Filament/OverviewTest.php +++ b/tests/Filament/OverviewTest.php @@ -11,6 +11,7 @@ use Heyosseus\Vacuum\Filament\Widgets\IndexFootprint; use Heyosseus\Vacuum\Filament\Widgets\LargestTables; use Heyosseus\Vacuum\Filament\Widgets\RunningVacuums; +use Heyosseus\Vacuum\Support\Bytes; use Heyosseus\Vacuum\Vacuum; use Illuminate\Contracts\View\View; use Illuminate\Http\Request; @@ -47,6 +48,33 @@ ->and($vitals)->toHaveCount(3); }); +it('reports the database size PostgreSQL itself reports', function (): void { + $bytes = DB::selectOne('SELECT pg_database_size(current_database()) AS bytes')->bytes; + + $vitals = invokeProtected(app(DatabaseVitals::class), 'getStats'); + + expect($vitals[0]->getValue())->toBe(Bytes::human((int) $bytes)); +}); + +it('reads each vital once per render, however many places the card shows it', function (): void { + DB::flushQueryLog(); + DB::enableQueryLog(); + + invokeProtected(app(DatabaseVitals::class), 'getStats'); + + $queries = collect(DB::getQueryLog())->pluck('query'); + + DB::disableQueryLog(); + + expect($queries->filter(static fn (string $sql): bool => str_contains($sql, 'pg_stat_database')))->toHaveCount(1) + ->and($queries->filter(static fn (string $sql): bool => str_contains($sql, "state = 'active'")))->toHaveCount(1) + ->and($queries->filter(static fn (string $sql): bool => str_contains($sql, 'pg_total_relation_size')))->toBeEmpty(); +}); + +it('does not poll the largest tables, which sizes every relation on each render', function (): void { + expect(invokeProtected(app(LargestTables::class), 'getPollingInterval'))->toBeNull(); +}); + it('shapes each chart from real numbers', function (): void { $severity = app(FindingsBySeverity::class); $largest = app(LargestTables::class);