diff --git a/Darling/Darling.Tests/Fixtures/OriginPlans/memory_grant_wait_plan.sqlplan b/Darling/Darling.Tests/Fixtures/OriginPlans/memory_grant_wait_plan.sqlplan new file mode 100644 index 000000000..b27fd82de --- /dev/null +++ b/Darling/Darling.Tests/Fixtures/OriginPlans/memory_grant_wait_plan.sqlplan @@ -0,0 +1,572 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Darling/Darling.Tests/Fixtures/OriginPlans/serially-parallel.sqlplan b/Darling/Darling.Tests/Fixtures/OriginPlans/serially-parallel.sqlplan new file mode 100644 index 000000000..8a1f9d32d Binary files /dev/null and b/Darling/Darling.Tests/Fixtures/OriginPlans/serially-parallel.sqlplan differ diff --git a/Darling/Darling.Tests/Fixtures/OriginPlans/spill_plan.sqlplan b/Darling/Darling.Tests/Fixtures/OriginPlans/spill_plan.sqlplan new file mode 100644 index 000000000..9d54d66e5 --- /dev/null +++ b/Darling/Darling.Tests/Fixtures/OriginPlans/spill_plan.sqlplan @@ -0,0 +1,572 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Darling/Darling.Tests/PlanSync4531Tests.cs b/Darling/Darling.Tests/PlanSync4531Tests.cs index a82e62d90..08a59dfbe 100644 --- a/Darling/Darling.Tests/PlanSync4531Tests.cs +++ b/Darling/Darling.Tests/PlanSync4531Tests.cs @@ -34,7 +34,8 @@ private static ParsedPlan Analyze(PlanStatement stmt) MemoryGrant = new MemoryGrantInfo { GrantedMemoryKB = 2_097_152, // 2 GB - MaxUsedMemoryKB = 1024 // 1 MB used — well past the 10x/1GB thresholds + MaxUsedMemoryKB = 1024, // 1 MB used — well past the 10x/1GB thresholds + HasMaxUsedMemory = true // an actual plan reports MaxUsedMemory } }; diff --git a/Darling/Darling.Tests/PlanSync4535StatementRulesTests.cs b/Darling/Darling.Tests/PlanSync4535StatementRulesTests.cs index dda9bc9c5..1e7c0dbb4 100644 --- a/Darling/Darling.Tests/PlanSync4535StatementRulesTests.cs +++ b/Darling/Darling.Tests/PlanSync4535StatementRulesTests.cs @@ -45,7 +45,7 @@ private static AnalyzerConfig Disabling(int ruleNumber) => private static PlanStatement Rule9_MemoryGrant() => new() { - MemoryGrant = new MemoryGrantInfo { GrantedMemoryKB = 2097152, MaxUsedMemoryKB = 10240 } + MemoryGrant = new MemoryGrantInfo { GrantedMemoryKB = 2097152, MaxUsedMemoryKB = 10240, HasMaxUsedMemory = true } }; private static PlanStatement Rule18_CompileMemoryExceeded() => new() diff --git a/Darling/Darling.Tests/PlanSync4686ExcessiveMemoryGrantTests.cs b/Darling/Darling.Tests/PlanSync4686ExcessiveMemoryGrantTests.cs new file mode 100644 index 000000000..ba20cfc3b --- /dev/null +++ b/Darling/Darling.Tests/PlanSync4686ExcessiveMemoryGrantTests.cs @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Linq; +using PerformanceMonitor.PlanAnalysis; +using Xunit; + +namespace Darling.Tests; + +/// +/// #4686 (erikdarlingdata/PerformanceStudio#608) - Rule 9's excessive-grant check. A grant of 1 GB or +/// more that used none of it is the worst waste there is, and the check used to skip it because it only +/// looked at plans where MaxUsedMemory was above 0. The other half of the fix is just as important: a +/// plan that has no MaxUsedMemory at all (an estimated plan, or one with no runtime grant info) must not +/// read as "used nothing". +/// +/// memory_grant_wait_plan.sqlplan (copied verbatim from PerformanceStudio's public repo) has a +/// 10,851,312 KB grant, so it clears the 1 GB floor. Each test edits the MemoryGrantInfo attributes in +/// the XML text, so the parser and the analyzer are both in the path. +/// +public sealed class PlanSync4686ExcessiveMemoryGrantTests +{ + private const string Fixture = "memory_grant_wait_plan.sqlplan"; + private const string Granted = "GrantedMemory=\"10851312\""; + private const string MaxUsed = "MaxUsedMemory=\"10232840\""; + + private static ParsedPlan Analyze(Func editXml) + { + var path = Path.Combine(AppContext.BaseDirectory, "Fixtures", "OriginPlans", Fixture); + var xml = File.ReadAllText(path); + Assert.Contains(Granted, xml); + Assert.Contains(MaxUsed, xml); + + var plan = ShowPlanParser.Parse(editXml(xml)); + PlanAnalyzer.Analyze(plan); + return plan; + } + + private static PlanWarning[] Excessive(ParsedPlan plan) => + PlanStatements.EnumerateAll(plan) + .SelectMany(s => s.PlanWarnings) + .Where(w => w.WarningType == "Excessive Memory Grant") + .ToArray(); + + private static PlanStatement FirstStatement(ParsedPlan plan) => + PlanStatements.EnumerateAll(plan).First(s => s.MemoryGrant != null); + + [Fact] + public void GrantThatUsedNothing_Fires() + { + var plan = Analyze(xml => xml.Replace(MaxUsed, "MaxUsedMemory=\"0\"")); + + var warning = Assert.Single(Excessive(plan)); + Assert.Equal(9, warning.RuleNumber); + Assert.Equal(PlanWarningSeverity.Warning, warning.Severity); + Assert.Contains("used none of it", warning.Message); + } + + [Fact] + public void GrantThatUsedNothing_MessageHasNoRatio() + { + var plan = Analyze(xml => xml.Replace(MaxUsed, "MaxUsedMemory=\"0\"")); + + var message = Assert.Single(Excessive(plan)).Message; + Assert.DoesNotContain("overestimate", message); + Assert.DoesNotContain("only used", message); + Assert.DoesNotContain("Infinity", message); + Assert.DoesNotContain("NaN", message); + } + + [Fact] + public void MaxUsedMemoryMissingFromXml_DoesNotFire() + { + // No MaxUsedMemory attribute, as in an estimated plan or a plan with no runtime grant info. + // The parser reports 0 for it, and 0 there means "not reported", not "used nothing". + var plan = Analyze(xml => xml.Replace(" " + MaxUsed, "")); + + Assert.Empty(Excessive(plan)); + } + + [Fact] + public void MaxUsedMemoryMissingFromXml_ParserSaysItWasNotReported() + { + var absent = Analyze(xml => xml.Replace(" " + MaxUsed, "")); + var zero = Analyze(xml => xml.Replace(MaxUsed, "MaxUsedMemory=\"0\"")); + var present = Analyze(xml => xml); + + Assert.False(FirstStatement(absent).MemoryGrant!.HasMaxUsedMemory); + Assert.True(FirstStatement(zero).MemoryGrant!.HasMaxUsedMemory); + Assert.True(FirstStatement(present).MemoryGrant!.HasMaxUsedMemory); + } + + [Fact] + public void GrantThatUsedNothing_JustUnderOneGb_DoesNotFire() + { + var plan = Analyze(xml => xml + .Replace(Granted, "GrantedMemory=\"1048575\"") + .Replace(MaxUsed, "MaxUsedMemory=\"0\"")); + + Assert.Empty(Excessive(plan)); + } + + [Fact] + public void GrantThatUsedNothing_AtOneGbExactly_Fires() + { + var plan = Analyze(xml => xml + .Replace(Granted, "GrantedMemory=\"1048576\"") + .Replace(MaxUsed, "MaxUsedMemory=\"0\"")); + + Assert.Single(Excessive(plan)); + } + + [Fact] + public void GrantThatUsedALittle_StillReportsTheRatio() + { + // 10,851,312 KB granted, 1,000 KB used: the old ratio message, unchanged. + var plan = Analyze(xml => xml.Replace(MaxUsed, "MaxUsedMemory=\"1000\"")); + + var message = Assert.Single(Excessive(plan)).Message; + Assert.Contains("only used", message); + Assert.Contains("x overestimate", message); + Assert.DoesNotContain("used none of it", message); + } + + [Fact] + public void GrantThatUsedMostOfIt_DoesNotFire() + { + // The fixture as shipped: 10,851,312 KB granted, 10,232,840 KB used. + var plan = Analyze(xml => xml); + + Assert.Empty(Excessive(plan)); + } + + [Fact] + public void GrantUnderTenTimesTheUse_DoesNotFire() + { + // 5.4x, under the 10x floor. + var plan = Analyze(xml => xml.Replace(MaxUsed, "MaxUsedMemory=\"2000000\"")); + + Assert.Empty(Excessive(plan)); + } + + [Fact] + public void GrantThatUsedNothing_OnAnAdaptiveJoinThatRanNestedLoops_KeepsTheAdaptiveJoinNote() + { + // The #4531 note is appended to both messages, so a hand-built statement (no XML) with a used-none + // grant and an adaptive join that ran Nested Loops carries it after the used-none sentence. + var stmt = new PlanStatement + { + RootNode = new PlanNode + { + PhysicalOp = "Adaptive Join", + LogicalOp = "Adaptive Join", + IsAdaptive = true, + ActualJoinType = "Nested Loops" + }, + MemoryGrant = new MemoryGrantInfo { GrantedMemoryKB = 2_097_152, MaxUsedMemoryKB = 0, HasMaxUsedMemory = true } + }; + PlanAnalyzer.Analyze(new ParsedPlan { Batches = [new PlanBatch { Statements = [stmt] }] }); + + var message = Assert.Single(stmt.PlanWarnings, w => w.WarningType == "Excessive Memory Grant").Message; + Assert.Contains("used none of it", message); + Assert.Contains("adaptive join", message); + } +} diff --git a/Darling/Darling.Tests/PlanSync4687CeGuessDetectionTests.cs b/Darling/Darling.Tests/PlanSync4687CeGuessDetectionTests.cs new file mode 100644 index 000000000..37bb00daa --- /dev/null +++ b/Darling/Darling.Tests/PlanSync4687CeGuessDetectionTests.cs @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Collections.Generic; +using System.Linq; +using PerformanceMonitor.PlanAnalysis; +using Xunit; + +namespace Darling.Tests; + +/// +/// #4687 (erikdarlingdata/PerformanceStudio#608) - Rule 33 names the default guess an estimate matches. +/// The labels used to be wrong: 30% was called the equality guess (it is the inequality guess), and 10% +/// and 1% were called inequality guesses that no predicate produces. The numbers below were measured on +/// SQL Server 2025 against a 100,000-row heap with no statistics, using estimated plans +/// (SET SHOWPLAN_XML), CE 70 through FORCE_LEGACY_CARDINALITY_ESTIMATION and CE 120 to 170 through the +/// compatibility level. +/// +/// Each row is what the optimizer wrote for EstimateRows on the scan, so every test feeds the rule a +/// value the engine produced, not one worked out by hand. The code-built plans follow the pattern in +/// PlanSync4535NodeRulesATests. +/// +public sealed class PlanSync4687CeGuessDetectionTests +{ + private const double HundredThousand = 100_000; + + /// + /// Runs the whole analyzer over one estimated table scan and returns the text of its + /// "Estimated Plan CE Guess" warning, or null when the rule said nothing. + /// + private static string? Label(int ceModelVersion, double estimateRows, double tableCardinality = HundredThousand) + { + var node = new PlanNode + { + NodeId = 0, + PhysicalOp = "Table Scan", + LogicalOp = "Table Scan", + EstimateRows = estimateRows, + TableCardinality = tableCardinality, + EstimatedTotalSubtreeCost = 1.0, + Predicate = "[t].[a]>(5)" + }; + var stmt = new PlanStatement + { + RootNode = node, + StatementSubTreeCost = 1.0, + CardinalityEstimationModelVersion = ceModelVersion + }; + var plan = new ParsedPlan { Batches = [new PlanBatch { Statements = [stmt] }] }; + PlanAnalyzer.Analyze(plan); + + return node.Warnings.FirstOrDefault(w => w.WarningType == "Estimated Plan CE Guess")?.Message; + } + + // ---- what each measured guess is called ---------------------------------------------------- + + [Theory] + // a > 5, a < 5, a >= 5: 30% in every estimator + [InlineData(170, 30000, "the 30% guess for an inequality such as > or <")] + [InlineData(70, 30000, "the 30% guess for an inequality such as > or <")] + [InlineData(0, 30000, "the 30% guess for an inequality such as > or <")] + // a BETWEEN 5 AND 10, a >= 5 AND a <= 10, and LIKE (CE 120+): 9% + [InlineData(170, 9000, "the 9% guess for BETWEEN or a two-sided range on a column with known values, or for LIKE (9.0%)")] + [InlineData(120, 9000, "the 9% guess for BETWEEN or a two-sided range on a column with known values, or for LIKE (9.0%)")] + // a > 5 AND b > 5 and BETWEEN on variables under CE 70: 30% * 30% = 9% + [InlineData(70, 9000, "the 9% guess for BETWEEN, a two-sided range, or two inequalities on different columns (9.0%)")] + // no estimator named: only what both estimators agree on + [InlineData(0, 9000, "the 9% guess for BETWEEN or a two-sided range (9.0%)")] + // a > 5 AND b > 5, and BETWEEN on variables, under CE 120+: 30% * sqrt(30%) = 16.43% + [InlineData(170, 16431.7, "the 16.4% guess for two inequalities on different columns, or for a BETWEEN or range on variables or on an expression")] + [InlineData(120, 16431.7, "the 16.4% guess for two inequalities on different columns")] + [InlineData(0, 16431.7, "the 16.4% guess for two inequalities on different columns")] + // a = b, and from CE 130 an equality on an expression such as ABS(a) = 5: 10% + [InlineData(170, 10000, "the 10% guess for comparing one column with another, or for an equality on an expression such as a function of a column")] + [InlineData(130, 10000, "the 10% guess for comparing one column with another, or for an equality on an expression")] + [InlineData(0, 10000, "the 10% guess for comparing one column with another, or for an equality on an expression")] + [InlineData(120, 10000, "the 10% guess for comparing one column with another (10.0%)")] + [InlineData(70, 10000, "the 10% guess for comparing one column with another (10.0%)")] + // a = b AND c = d under CE 70: 10% * 10% = 1% + [InlineData(70, 1000, "the 1% guess that CE 70 gets from multiplying two 10% guesses")] + [InlineData(0, 1000, "the 1% guess that CE 70 gets from multiplying two 10% guesses")] + // a = 5 and a IS NULL with no statistics: CE 120+ rows^0.5 (316.228), CE 70 rows^0.75 (5623.41) + [InlineData(170, 316.228, "the square root of the row count (0.3%)")] + [InlineData(120, 316.228, "the square root of the row count (0.3%)")] + [InlineData(0, 316.228, "the square root of the row count (0.3%)")] + [InlineData(70, 5623.41, "the row count to the power 0.75 (5.6%)")] + [InlineData(0, 5623.41, "the row count to the power 0.75 (5.6%)")] + public void MeasuredGuess_IsNamedForWhatProducedIt(int ceModelVersion, double estimateRows, string expected) + { + var label = Label(ceModelVersion, estimateRows); + + Assert.NotNull(label); + Assert.Contains(expected, label); + } + + [Fact] + public void ThirtyPercent_IsTheInequalityGuess_NotTheEqualityGuess() + { + var label = Label(170, 30000); + + Assert.NotNull(label); + Assert.DoesNotContain("30% equality", label); + Assert.Contains("inequality", label); + } + + [Fact] + public void EqualityGuess_MessageSaysItIsAnEqualityGuess() + { + var label = Label(170, 316.228); + + Assert.NotNull(label); + Assert.Contains("equality guess", label); + } + + // ---- estimates that are not a guess in that estimator ---------------------------------------- + + [Theory] + [InlineData(70, 16431.7)] // CE 70 has no 16.43% guess: it gets 9% for the same predicate + [InlineData(170, 1000)] // CE 120+ has no 1% guess: a = b AND c = d is 3.16% there + [InlineData(120, 1000)] + [InlineData(170, 5623.41)] // rows^0.75 is CE 70's equality guess only + [InlineData(70, 316.228)] // the square root is CE 120+'s equality guess only + [InlineData(170, 25000)] // ordinary estimates + [InlineData(170, 50000)] + [InlineData(70, 12345)] + [InlineData(170, 322.6)] // 2% above the equality guess, outside its 1% tolerance + [InlineData(170, 309.9)] // 2% below + public void NotAGuess_GetsNoLabel(int ceModelVersion, double estimateRows) + { + Assert.Null(Label(ceModelVersion, estimateRows)); + } + + [Fact] + public void EqualityGuess_ToleratesHalfAPercent() + { + Assert.NotNull(Label(170, 316.228 * 1.005)); + Assert.NotNull(Label(170, 316.228 * 0.995)); + } + + // ---- the equality guess follows the table's row count ---------------------------------------- + + [Theory] + [InlineData(400_000, 170, 632.456)] // measured: sqrt(400,000) + [InlineData(400_000, 70, 15905.4)] // measured: 400,000^0.75 + [InlineData(1_000_000, 170, 1000)] + [InlineData(1_000_000, 70, 31622.8)] + // 100,000,000^0.75 is 1,000,000, exactly 1% of the table. It is still the equality guess. + [InlineData(100_000_000, 70, 1_000_000)] + public void EqualityGuess_ScalesWithTheTable(double tableRows, int ceModelVersion, double estimateRows) + { + var label = Label(ceModelVersion, estimateRows, tableRows); + + Assert.NotNull(label); + Assert.Contains("equality guess", label); + Assert.DoesNotContain("1% guess", label); + } + + [Fact] + public void EqualityGuess_IsNotLookedForBelowTheTableSizeFloor() + { + // The rule only looks at tables of 100,000 rows or more. The same estimate on a table just + // under the floor gets no label: there the equality guess could sit on a fixed guess. + Assert.Null(Label(170, System.Math.Sqrt(99_999), 99_999)); + } + + // ---- the estimator version reaches the rule from the plan XML -------------------------------- + + private const string PlanTemplate = """ + + """; + + private static IEnumerable Nodes(PlanNode node) + { + yield return node; + foreach (var child in node.Children) + foreach (var descendant in Nodes(child)) + yield return descendant; + } + + /// + /// The shape of a real estimated plan for one of the probe queries, with the estimate, the + /// estimator version and the predicate filled in from what SQL Server 2025 produced. + /// + private static string? LabelFromXml(string sql, string predicate, int ce, string estimateRows) + { + var xml = PlanTemplate + .Replace("{SQL}", sql) + .Replace("{PREDICATE}", predicate) + .Replace("{CE}", ce.ToString()) + .Replace("{ROWS}", estimateRows); + + var plan = ShowPlanParser.Parse(xml); + PlanAnalyzer.Analyze(plan); + + return PlanStatements.EnumerateAll(plan) + .Where(s => s.RootNode != null) + .SelectMany(s => Nodes(s.RootNode!)) + .SelectMany(n => n.Warnings) + .FirstOrDefault(w => w.WarningType == "Estimated Plan CE Guess")?.Message; + } + + private const string TwoInequalitiesSql = "a > 5 AND b > 5"; + private const string TwoInequalitiesPredicate = "[t].[a]>(5) AND [t].[b]>(5)"; + + [Fact] + public void TwoInequalitiesOnDifferentColumns_Ce170_IsSixteenPointFourPercent() + { + // WHERE a > 5 AND b > 5 under CE 170: EstimateRows="16431.7" + var label = LabelFromXml(TwoInequalitiesSql, TwoInequalitiesPredicate, 170, "16431.7"); + + Assert.NotNull(label); + Assert.Contains("the 16.4% guess for two inequalities on different columns", label); + } + + [Fact] + public void TwoInequalitiesOnDifferentColumns_Ce70_IsNinePercent() + { + // The same query under CE 70: EstimateRows="9000", and the label says why. + var label = LabelFromXml(TwoInequalitiesSql, TwoInequalitiesPredicate, 70, "9000"); + + Assert.NotNull(label); + Assert.Contains("the 9% guess for BETWEEN, a two-sided range, or two inequalities on different columns", label); + } + + [Fact] + public void Equality_Ce170_IsTheSquareRootOfTheRowCount() + { + // WHERE a = 5 under CE 170: EstimateRows="316.228" + var label = LabelFromXml("a = 5", "[t].[a]=(5)", 170, "316.228"); + + Assert.NotNull(label); + Assert.Contains("the square root of the row count", label); + } + + [Fact] + public void Equality_Ce70_IsTheRowCountToTheThreeQuarters() + { + // WHERE a = 5 under CE 70: EstimateRows="5623.41" + var label = LabelFromXml("a = 5", "[t].[a]=(5)", 70, "5623.41"); + + Assert.NotNull(label); + Assert.Contains("the row count to the power 0.75", label); + } +} diff --git a/Darling/Darling.Tests/PlanSync4690ExpensiveOperatorExchangeTests.cs b/Darling/Darling.Tests/PlanSync4690ExpensiveOperatorExchangeTests.cs new file mode 100644 index 000000000..adbe15482 --- /dev/null +++ b/Darling/Darling.Tests/PlanSync4690ExpensiveOperatorExchangeTests.cs @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using PerformanceMonitor.PlanAnalysis; +using Xunit; + +namespace Darling.Tests; + +/// +/// #4690 (erikdarlingdata/PerformanceStudio#608) - Rule 35 (Expensive Operator) used to rank exchanges +/// by their self-time. An exchange's elapsed time is mostly spent waiting on the operators that feed it +/// and drain it, so it could be named the expensive operator when the operator next to it was the real +/// problem. serially-parallel.sqlplan shows it plainly: the Sort took 17,111 ms and the +/// Repartition Streams below it, which feeds the Sort, was given the same 17,111 ms as if it had done +/// the same work. The three fixtures are copied verbatim from PerformanceStudio's public repo. +/// +public sealed class PlanSync4690ExpensiveOperatorExchangeTests +{ + private static ParsedPlan LoadAndAnalyze(string fileName) + { + var xml = File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "Fixtures", "OriginPlans", fileName)); + var plan = ShowPlanParser.Parse(xml); + PlanAnalyzer.Analyze(plan); + BenefitScorer.Score(plan); + return plan; + } + + private static IEnumerable Nodes(PlanNode node) + { + yield return node; + foreach (var child in node.Children) + foreach (var descendant in Nodes(child)) + yield return descendant; + } + + private static List AllNodes(ParsedPlan plan) => + PlanStatements.EnumerateAll(plan) + .Where(s => s.RootNode != null) + .SelectMany(s => Nodes(s.RootNode!)) + .ToList(); + + private static List ExchangesIn(ParsedPlan plan) => + AllNodes(plan).Where(n => n.PhysicalOp == "Parallelism").ToList(); + + [Theory] + [InlineData("serially-parallel.sqlplan")] + [InlineData("memory_grant_wait_plan.sqlplan")] + [InlineData("spill_plan.sqlplan")] + public void Exchange_IsNeverNamedTheExpensiveOperator(string fixture) + { + var plan = LoadAndAnalyze(fixture); + + var exchanges = ExchangesIn(plan); + Assert.NotEmpty(exchanges); // so the assertion below can't pass on a plan with no exchange + Assert.All(exchanges, e => + Assert.DoesNotContain(e.Warnings, w => w.WarningType == "Expensive Operator")); + } + + [Fact] + public void SeriallyParallelPlan_StillNamesTheSort() + { + // The Sort is the operator that did the work, and it keeps its Rule 35 warning. Only the + // exchange that mirrored its time drops out. + var plan = LoadAndAnalyze("serially-parallel.sqlplan"); + + var warning = Assert.Single(AllNodes(plan).SelectMany(n => n.Warnings), w => w.WarningType == "Expensive Operator"); + Assert.StartsWith("Sort took", warning.Message); + } + + // ---- built in code: the same timing on an exchange and on an ordinary operator --------------- + + private static PlanNode Analyze(string physicalOp, string logicalOp) + { + var node = new PlanNode + { + NodeId = 1, + PhysicalOp = physicalOp, + LogicalOp = logicalOp, + HasActualStats = true, + ActualExecutions = 1, + ActualRows = 100, + EstimateRows = 100, + ActualElapsedMs = 6000 + }; + var stmt = new PlanStatement + { + RootNode = node, + QueryTimeStats = new QueryTimeInfo { ElapsedTimeMs = 10000, CpuTimeMs = 10000 } + }; + PlanAnalyzer.Analyze(new ParsedPlan { Batches = [new PlanBatch { Statements = [stmt] }] }); + return node; + } + + [Fact] + public void ControlOperator_With60PercentOfTheStatement_IsFlagged() + { + var node = Analyze("Sort", "Sort"); + + var warning = Assert.Single(node.Warnings, w => w.WarningType == "Expensive Operator"); + Assert.Equal(35, warning.RuleNumber); + } + + [Theory] + [InlineData("Repartition Streams")] + [InlineData("Gather Streams")] + [InlineData("Distribute Streams")] + public void Exchange_With60PercentOfTheStatement_IsNotFlagged(string logicalOp) + { + var node = Analyze("Parallelism", logicalOp); + + Assert.DoesNotContain(node.Warnings, w => w.WarningType == "Expensive Operator"); + } +} diff --git a/PerformanceMonitor.PlanAnalysis/PlanAnalyzer.cs b/PerformanceMonitor.PlanAnalysis/PlanAnalyzer.cs index 941b35889..06f9a7e23 100644 --- a/PerformanceMonitor.PlanAnalysis/PlanAnalyzer.cs +++ b/PerformanceMonitor.PlanAnalysis/PlanAnalyzer.cs @@ -365,15 +365,19 @@ private static void AnalyzeStatement(PlanStatement stmt, AnalyzerConfig cfg, Ser { var grant = stmt.MemoryGrant; - // Excessive grant — granted far more than actually used - if (grant.GrantedMemoryKB > 0 && grant.MaxUsedMemoryKB > 0) - { - var wasteRatio = (double)grant.GrantedMemoryKB / grant.MaxUsedMemoryKB; - if (wasteRatio >= 10 && grant.GrantedMemoryKB >= 1048576) + // Excessive grant — granted far more than actually used. A grant that used nothing is the + // worst case, so MaxUsedMemory="0" counts, but only when the plan reported it: a plan with + // no MaxUsedMemory attribute (estimated, or no runtime grant info) says nothing about use (#4686). + if (grant.GrantedMemoryKB >= 1048576 && grant.HasMaxUsedMemory) + { + var usedNothing = grant.MaxUsedMemoryKB <= 0; + var wasteRatio = usedNothing ? 0 : (double)grant.GrantedMemoryKB / grant.MaxUsedMemoryKB; + if (usedNothing || wasteRatio >= 10) { var grantMB = grant.GrantedMemoryKB / 1024.0; - var usedMB = grant.MaxUsedMemoryKB / 1024.0; - var message = $"Granted {grantMB:N0} MB but only used {usedMB:N0} MB ({wasteRatio:F0}x overestimate). The unused memory is reserved and unavailable to other queries."; + var message = usedNothing + ? $"Granted {grantMB:N0} MB but the query used none of it. The unused memory is reserved and unavailable to other queries." + : $"Granted {grantMB:N0} MB but only used {grant.MaxUsedMemoryKB / 1024.0:N0} MB ({wasteRatio:F0}x overestimate). The unused memory is reserved and unavailable to other queries."; // Note adaptive joins that chose Nested Loops at runtime — the grant // was sized for a hash join that never happened. @@ -1220,17 +1224,20 @@ _ when nonSargableReason.StartsWith("Function call", StringComparison.OrdinalIgn } // Rule 33: Estimated plan CE guess detection — scans with telltale default selectivity - // When the optimizer uses a local variable or can't sniff, it falls back to density-based - // guesses: 30% (equality), 10% (inequality), 9% (LIKE/between), ~16.43% (sqrt(30%)), - // 1% (multi-inequality). On large tables, these guesses can hide the need for an index. + // When the optimizer has no statistics to use (a local variable it can't sniff, a column with + // no statistics, an expression), it falls back on fixed guesses: 30% for an inequality, 9% + // for BETWEEN or LIKE, ~16.4% for two inequalities, 10% for comparing two columns, and an + // equality guess that grows with the table. DetectCeGuess has the measured details and + // which estimator (CE 70 or 120+) each one belongs to. On large tables, these guesses can + // hide the need for an index (#4687). if (!cfg.IsRuleDisabled(33) && !node.HasActualStats && IsRowstoreScan(node) - && node.TableCardinality >= 100_000 && node.EstimateRows > 0 + && node.TableCardinality >= CeGuessMinTableRows && node.EstimateRows > 0 && !string.IsNullOrEmpty(node.Predicate)) { var impact = BuildScanImpactDetails(node, stmt); if (impact.CostPct >= 50) { - var guessDesc = DetectCeGuess(node.EstimateRows, node.TableCardinality); + var guessDesc = DetectCeGuess(node.EstimateRows, node.TableCardinality, stmt.CardinalityEstimationModelVersion); if (guessDesc != null) { node.Warnings.Add(new PlanWarning @@ -1570,7 +1577,12 @@ _ when nonSargableReason.StartsWith("Function call", StringComparison.OrdinalIgn // one or two operators always take most of the time just because there's almost // nothing else to divide it among, so the share points at nothing. The benefit % is // just the self-time share. + // Exchanges (Parallelism) are skipped: their self-time is mostly time spent waiting on the + // operators that feed them and drain them, not work of their own. On a live plan an exchange + // feeding a spilling sort showed 21 s of elapsed time on 2.4 s of CPU per thread, and was + // named as the expensive operator while the sort beside it was the real problem (#4690). if (!cfg.IsRuleDisabled(35) && node.HasActualStats && node.Warnings.Count == 0 + && !IsExchangeOperator(node) && stmt.QueryTimeStats != null && stmt.QueryTimeStats.ElapsedTimeMs >= 1000) { var selfMs = GetOperatorOwnElapsedMs(node); @@ -1601,6 +1613,15 @@ _ when nonSargableReason.StartsWith("Function call", StringComparison.OrdinalIgn } } + /// + /// True for parallelism exchange operators (Gather/Distribute/Repartition Streams), whose + /// timings reflect time spent waiting on the operators around them rather than the operator's + /// own work. + /// + private static bool IsExchangeOperator(PlanNode node) => + node.PhysicalOp == "Parallelism" + || node.LogicalOp is "Gather Streams" or "Distribute Streams" or "Repartition Streams"; + /// /// Detects the NOT IN with nullable column pattern: statement has NOT IN, /// and a nearby Nested Loops Anti Semi Join has an IS NULL residual predicate. @@ -3002,23 +3023,72 @@ private static string StripBrackets(string bracketPart) => return null; } + // Rule 33 only looks at tables with at least this many rows, and DetectCeGuess relies on that: + // the equality guess is a power of the row count, so on a small table it lands on the fixed + // guesses below, and from this size up it stays clear of them (0.3% and 5.6% at this size, + // falling as the table grows). + private const double CeGuessMinTableRows = 100_000; + /// /// Detects well-known CE default selectivity guesses by comparing EstimateRows to TableCardinality. /// Returns a description of the guess pattern, or null if no known pattern matches. + /// + /// Where each guess comes from was measured on SQL Server 2025: a 100,000-row heap with no + /// statistics, estimated plans through SET SHOWPLAN_XML, CE 70 through + /// FORCE_LEGACY_CARDINALITY_ESTIMATION and CE 120 to 170 through the compatibility level. The + /// versions from 120 to 170 agree except where a row says otherwise. + /// equality, a = 5 or a IS NULL CE 120+: rows^0.5 CE 70: rows^0.75 + /// inequality, a > 5 30%, every CE + /// BETWEEN or a two-sided range CE 70: 9% CE 120+: 9% on a column with known values + /// LIKE CE 120+: 9% CE 70: not a fixed guess + /// two inequalities, two columns CE 120+: 16.43% CE 70: 9% + /// range on variables, or on an expression, ABS(a) BETWEEN 5 AND 10 CE 120+: 16.43% CE 70: 9% + /// one column compared with another 10%, every CE + /// equality on an expression, ABS(a) = 5 CE 130+: 10% CE 120: rows^0.5 CE 70: rows^0.75 + /// two 10% guesses, a = b AND c = d CE 120+: 3.16% CE 70: 1% + /// The 16.43% is 30% times the square root of 30%, how CE 120+ combines two 30% guesses. /// - private static string? DetectCeGuess(double estimateRows, double tableCardinality) + /// The scan's estimated row count. + /// The table's row count. + /// + /// The statement's CardinalityEstimationModelVersion: 70 is the legacy estimator, 120 and later + /// the current one, and 0 means the plan did not say, so both stay possible. + /// + private static string? DetectCeGuess(double estimateRows, double tableCardinality, int ceModelVersion = 0) { if (tableCardinality <= 0) return null; var selectivity = estimateRows / tableCardinality; + var pct = $"{selectivity * 100:N1}%"; + var legacy = ceModelVersion == 70; + var current = ceModelVersion >= 120; + + // Equality is not a fixed share of the table, so it is checked on its own, to 1%. The two + // estimators use different powers of the row count, so a plan that names its estimator only + // gets the one that estimator uses. + static bool Near(double rows, double guess) => Math.Abs(rows - guess) <= guess * 0.01; + + if (!legacy && Near(estimateRows, Math.Sqrt(tableCardinality))) + return $"matches the equality guess (an equality or IS NULL with no statistics to use), the square root of the row count ({pct})"; + if (!current && Near(estimateRows, Math.Pow(tableCardinality, 0.75))) + return $"matches the equality guess (an equality or IS NULL with no statistics to use), the row count to the power 0.75 ({pct})"; - // Known CE guess selectivities with a 2% tolerance band + // The fixed guesses, with a 2% tolerance band return selectivity switch { - >= 0.29 and <= 0.31 => $"matches the 30% equality guess ({selectivity * 100:N1}%)", - >= 0.098 and <= 0.102 => $"matches the 10% inequality guess ({selectivity * 100:N1}%)", - >= 0.088 and <= 0.092 => $"matches the 9% LIKE/BETWEEN guess ({selectivity * 100:N1}%)", - >= 0.155 and <= 0.175 => $"matches the ~16.4% compound predicate guess ({selectivity * 100:N1}%)", - >= 0.009 and <= 0.011 => $"matches the 1% multi-inequality guess ({selectivity * 100:N1}%)", + >= 0.29 and <= 0.31 => + $"matches the 30% guess for an inequality such as > or < ({pct})", + >= 0.088 and <= 0.092 => + current ? $"matches the 9% guess for BETWEEN or a two-sided range on a column with known values, or for LIKE ({pct})" + : legacy ? $"matches the 9% guess for BETWEEN, a two-sided range, or two inequalities on different columns ({pct})" + : $"matches the 9% guess for BETWEEN or a two-sided range ({pct})", + >= 0.098 and <= 0.102 => + ceModelVersion is 0 or >= 130 + ? $"matches the 10% guess for comparing one column with another, or for an equality on an expression such as a function of a column ({pct})" + : $"matches the 10% guess for comparing one column with another ({pct})", + >= 0.155 and <= 0.175 when !legacy => + $"matches the 16.4% guess for two inequalities on different columns, or for a BETWEEN or range on variables or on an expression ({pct})", + >= 0.009 and <= 0.011 when !current => + $"matches the 1% guess that CE 70 gets from multiplying two 10% guesses, as when two predicates each compare one column with another ({pct})", _ => null }; } diff --git a/PerformanceMonitor.PlanAnalysis/PlanModels.cs b/PerformanceMonitor.PlanAnalysis/PlanModels.cs index b793e1cf1..d12d8659b 100644 --- a/PerformanceMonitor.PlanAnalysis/PlanModels.cs +++ b/PerformanceMonitor.PlanAnalysis/PlanModels.cs @@ -500,6 +500,13 @@ public class MemoryGrantInfo public long RequestedMemoryKB { get; set; } public long GrantedMemoryKB { get; set; } public long MaxUsedMemoryKB { get; set; } + /// + /// True when the plan XML carried a MaxUsedMemory attribute. An actual plan does, and there 0 + /// means the query used none of its grant. An estimated plan, or a plan with no runtime grant + /// info, does not, and there is 0 only because nothing was + /// reported. A rule that acts on "used nothing" must check this first. + /// + public bool HasMaxUsedMemory { get; set; } public long GrantWaitTimeMs { get; set; } public long LastRequestedMemoryKB { get; set; } public string? IsMemoryGrantFeedbackAdjusted { get; set; } diff --git a/PerformanceMonitor.PlanAnalysis/ShowPlanParser.cs b/PerformanceMonitor.PlanAnalysis/ShowPlanParser.cs index cc5d63053..3ea1c52b6 100644 --- a/PerformanceMonitor.PlanAnalysis/ShowPlanParser.cs +++ b/PerformanceMonitor.PlanAnalysis/ShowPlanParser.cs @@ -648,6 +648,8 @@ private static void ParseQueryPlanElements(PlanStatement stmt, XElement stmtEl, RequestedMemoryKB = ParseLong(memEl.Attribute("RequestedMemory")?.Value), GrantedMemoryKB = ParseLong(memEl.Attribute("GrantedMemory")?.Value), MaxUsedMemoryKB = ParseLong(memEl.Attribute("MaxUsedMemory")?.Value), + HasMaxUsedMemory = long.TryParse(memEl.Attribute("MaxUsedMemory")?.Value, + System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out _), GrantWaitTimeMs = ParseLong(memEl.Attribute("GrantWaitTime")?.Value), LastRequestedMemoryKB = ParseLong(memEl.Attribute("LastRequestedMemory")?.Value), IsMemoryGrantFeedbackAdjusted = memEl.Attribute("IsMemoryGrantFeedbackAdjusted")?.Value