feat(query): Support extended expression queries

.gitignore

@@ -63,6 +63,7 @@ BenchmarkDotNet.Artifacts/
 # Local-only directories (development artifacts, not for upstream)
 bench/
 docs/
+scripts/query-oracle/
 
 # .NET
 project.lock.json

Directory.Packages.props

@@ -6,7 +6,7 @@
 
   <ItemGroup>
     <!-- Infrastructure -->
-    <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="10.0.201" />
+    <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="10.0.303" />
     <PackageVersion Include="Meziantou.Analyzer" Version="3.0.27" />
     <!-- Benchmarks -->
     <PackageVersion Include="BenchmarkDotNet" Version="0.15.8" />

benchmarks/XLSight.Benchmarks/ExpressionQueryBenchmarks.cs

@@ -0,0 +1,223 @@
+using System.Globalization;
+using System.IO.Compression;
+using System.Text;
+using System.Runtime.InteropServices;
+using BenchmarkDotNet.Attributes;
+using XLSight;
+using XLSight.Query;
+
+namespace XLSight.Benchmarks;
+
+/// <summary>
+/// Measures the expression query path against a direct reader loop for realistic workbook sizes.
+/// The group parameter intentionally includes both low and high cardinality: row count is usually
+/// the dominant cost, while high cardinality exercises the bounded group-state allocation path.
+/// </summary>
+[MemoryDiagnoser]
+public class ExpressionQueryBenchmarks
+{
+    [Params(1_000, 10_000)]
+    public int Rows { get; set; }
+
+    [Params(5, 10_000)]
+    public int Groups { get; set; }
+
+    [Params("ShortInline", "LongInline", "LongShared")]
+    public string TextFormat { get; set; } = "ShortInline";
+
+    private byte[] _workbook = null!;
+    private string _query = null!;
+    private SheetQuerySpec _spec = null!;
+
+    [GlobalSetup]
+    public void Setup()
+    {
+        _workbook = WorkbookBytes(Rows, Groups, TextFormat);
+        _query = $"""
+            FROM Data!A1:D{Rows + 1} HEADER ROW 1
+            SELECT SUM(Value * 1.15) FILTER (WHERE Flag = 'Y') AS Adjusted
+            GROUP BY Category, Flag
+            """;
+        _spec = SheetQuerySpec.Parse(_query);
+        ValidateEquivalentResults();
+    }
+
+    [Benchmark]
+    public QueryResult DslExpression()
+    {
+        using var stream = new MemoryStream(_workbook, writable: false);
+        using var workbook = ExcelWorkbook.Open(stream);
+        return workbook.ExecuteQuery(_spec);
+    }
+
+    [Benchmark(Baseline = true)]
+    public QueryResult DirectReaderEquivalent()
+    {
+        using var stream = new MemoryStream(_workbook, writable: false);
+        using var workbook = ExcelWorkbook.Open(stream);
+        return ReadDirectResult(workbook);
+    }
+
+    private QueryResult ReadDirectResult(ExcelWorkbook workbook)
+    {
+        using var reader = workbook.GetRangeReader("Data", $"A1:D{Rows + 1}");
+
+        var totals = new Dictionary<(string Category, string Flag), AggregateState>(Math.Min(Rows, Groups * 2));
+        bool header = true;
+        while (reader.Read())
+        {
+            if (header)
+            {
+                header = false;
+                continue;
+            }
+
+            var row = reader.Current;
+            if (!row.GetCell(2).TryGetText(out string? category) || !row.GetCell(4).TryGetText(out string? flag))
+            {
+                continue;
+            }
+
+            var key = (category, flag);
+            AggregateState state = totals.GetValueOrDefault(key);
+            if (row.GetCell(3).TryGetNumber(out double value)
+                && string.Equals(flag, "Y", StringComparison.Ordinal))
+            {
+                state = new AggregateState(state.Sum + (value * 1.15), HasValue: true);
+            }
+
+            totals[key] = state;
+        }
+
+        var rows = new List<QueryResultRow>(totals.Count);
+        foreach (((string category, string flag), AggregateState state) in totals)
+        {
+            rows.Add(new QueryResultRow
+            {
+                Values = new[]
+                {
+                    ExcelCellValue.FromText(category),
+                    ExcelCellValue.FromText(flag),
+                    state.HasValue ? ExcelCellValue.FromNumber(state.Sum) : ExcelCellValue.Empty,
+                },
+            });
+        }
+
+        return new QueryResult
+        {
+            Columns = ["Category", "Flag", "Adjusted"],
+            Rows = rows,
+            RowsScanned = Rows,
+            RowsMatched = Rows,
+            Unaggregatable = [],
+        };
+    }
+
+    private void ValidateEquivalentResults()
+    {
+        using var stream = new MemoryStream(_workbook, writable: false);
+        using var workbook = ExcelWorkbook.Open(stream);
+        QueryResult actual = workbook.ExecuteQuery(_spec);
+        QueryResult expected = ReadDirectResult(workbook);
+
+        if (!actual.Columns.SequenceEqual(expected.Columns, StringComparer.Ordinal)
+            || actual.Rows.Count != expected.Rows.Count
+            || actual.RowsScanned != expected.RowsScanned
+            || actual.RowsMatched != expected.RowsMatched)
+        {
+            throw new InvalidOperationException("Expression and direct benchmark results have different shapes or scan statistics.");
+        }
+
+        for (int i = 0; i < actual.Rows.Count; i++)
+        {
+            if (!actual.Rows[i].Values.Span.SequenceEqual(expected.Rows[i].Values.Span))
+            {
+                throw new InvalidOperationException($"Expression and direct benchmark results differ at group row {i}.");
+            }
+        }
+    }
+
+    [StructLayout(LayoutKind.Auto)]
+    private readonly record struct AggregateState(double Sum, bool HasValue);
+
+    private static byte[] WorkbookBytes(int rows, int groups, string textFormat)
+    {
+        using var stream = new MemoryStream();
+        using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true))
+        {
+            WriteEntry(archive, "xl/workbook.xml", """
+                <workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"
+                          xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
+                  <sheets><sheet name="Data" sheetId="1" r:id="rId1" /></sheets>
+                </workbook>
+                """);
+            WriteEntry(archive, "xl/_rels/workbook.xml.rels", """
+                <Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
+                  <Relationship Id="rId1" Target="worksheets/sheet1.xml" />
+                </Relationships>
+                """);
+
+            string prefix = textFormat is "ShortInline" ? "G" : new string('x', 120) + "-Category-";
+            int categories = Math.Min(rows, groups);
+            bool shared = textFormat is "LongShared";
+            if (shared) { WriteSharedStrings(archive, prefix, categories); }
+
+            var sheet = new StringBuilder(rows * 120);
+            sheet.Append("<worksheet xmlns=\"http://schemas.openxmlformats.org/spreadsheetml/2006/main\"><sheetData>");
+            sheet.Append("<row r=\"1\"><c r=\"A1\" t=\"inlineStr\"><is><t>Id</t></is></c>");
+            sheet.Append("<c r=\"B1\" t=\"inlineStr\"><is><t>Category</t></is></c>");
+            sheet.Append("<c r=\"C1\" t=\"inlineStr\"><is><t>Value</t></is></c>");
+            sheet.Append("<c r=\"D1\" t=\"inlineStr\"><is><t>Flag</t></is></c></row>");
+
+            for (int row = 1; row <= rows; row++)
+            {
+                int sheetRow = row + 1;
+                int category = (row - 1) % groups;
+                int value = ((row - 1) % 1_000) + 1;
+                string flag = row % 3 == 0 ? "Y" : "N";
+                sheet.Append(CultureInfo.InvariantCulture, $"<row r=\"{sheetRow}\">");
+                sheet.Append(CultureInfo.InvariantCulture, $"<c r=\"A{sheetRow}\"><v>{row}</v></c>");
+                if (shared)
+                {
+                    sheet.Append(CultureInfo.InvariantCulture, $"<c r=\"B{sheetRow}\" t=\"s\"><v>{category}</v></c>");
+                }
+                else
+                {
+                    sheet.Append(CultureInfo.InvariantCulture, $"<c r=\"B{sheetRow}\" t=\"inlineStr\"><is><t>{prefix}{category}</t></is></c>");
+                }
+                sheet.Append(CultureInfo.InvariantCulture, $"<c r=\"C{sheetRow}\"><v>{value}</v></c>");
+                if (shared)
+                {
+                    sheet.Append(CultureInfo.InvariantCulture, $"<c r=\"D{sheetRow}\" t=\"s\"><v>{categories + (row % 3 == 0 ? 1 : 0)}</v></c></row>");
+                }
+                else
+                {
+                    sheet.Append(CultureInfo.InvariantCulture, $"<c r=\"D{sheetRow}\" t=\"inlineStr\"><is><t>{flag}</t></is></c></row>");
+                }
+            }
+
+            sheet.Append("</sheetData></worksheet>");
+            WriteEntry(archive, "xl/worksheets/sheet1.xml", sheet.ToString());
+        }
+
+        return stream.ToArray();
+    }
+
+    private static void WriteSharedStrings(ZipArchive archive, string prefix, int categories)
+    {
+        var strings = new StringBuilder("<sst xmlns=\"http://schemas.openxmlformats.org/spreadsheetml/2006/main\">");
+        for (int i = 0; i < categories; i++)
+        {
+            strings.Append(CultureInfo.InvariantCulture, $"<si><t>{prefix}{i}</t></si>");
+        }
+        strings.Append("<si><t>N</t></si><si><t>Y</t></si></sst>");
+        WriteEntry(archive, "xl/sharedStrings.xml", strings.ToString());
+    }
+
+    private static void WriteEntry(ZipArchive archive, string path, string content)
+    {
+        var entry = archive.CreateEntry(path);
+        using var writer = new StreamWriter(entry.Open(), Encoding.UTF8);
+        writer.Write(content);
+    }
+}

benchmarks/XLSight.Benchmarks/packages.lock.json

@@ -34,13 +34,13 @@
       },
       "Microsoft.SourceLink.GitHub": {
         "type": "Direct",
-        "requested": "[10.0.201, )",
-        "resolved": "10.0.201",
-        "contentHash": "qxYAmO4ktzd9L+HMdnqWucxpu7bI9undPyACXOMqPyhaiMtbpbYL/n0ACyWIJlbyEJrXFwxiOaBOSasLtDvsCg==",
+        "requested": "[10.0.303, )",
+        "resolved": "10.0.303",
+        "contentHash": "fxqLxLWx4yBAabjJgSIehVITPysG30YwAMWI6LkWyrgxDlnzFKITVj9SpBAGDeFlM2QR91T4X7Z3MFkzb4kzEw==",
         "dependencies": {
-          "Microsoft.Build.Tasks.Git": "10.0.201",
-          "Microsoft.SourceLink.Common": "10.0.201",
-          "System.IO.Hashing": "10.0.5"
+          "Microsoft.Build.Tasks.Git": "10.0.303",
+          "Microsoft.SourceLink.Common": "10.0.303",
+          "System.IO.Hashing": "10.0.11"
         }
       },
       "MiniExcel": {
@@ -71,10 +71,10 @@
       },
       "Microsoft.Build.Tasks.Git": {
         "type": "Transitive",
-        "resolved": "10.0.201",
-        "contentHash": "DMYBnrFZvLnBKn14VavEuuIr31CY6YY2i2L9P8DorS/Qp6ifRR8ZPLdJCFLFfjikNq8DykbYyLd/RP6lSqHcWw==",
+        "resolved": "10.0.303",
+        "contentHash": "tUblJMHhGAg4WKq3fzfJqXpV8o01WmQnWvgopNBnbhVQB6qwW2c8guCTCFMILD9Dekhe8MROBJ4lEk8AtXKisQ==",
         "dependencies": {
-          "System.IO.Hashing": "10.0.5"
+          "System.IO.Hashing": "10.0.11"
         }
       },
       "Microsoft.CodeAnalysis.Analyzers": {
@@ -174,8 +174,8 @@
       },
       "Microsoft.SourceLink.Common": {
         "type": "Transitive",
-        "resolved": "10.0.201",
-        "contentHash": "QbBYhkjgL6rCnBfDbzsAJLlsad13TlBHqYCFDIw56OO2g6ix+9RsmY8uxiQGdWwFKbZXaXyAA6jDCzFYVGCZDw=="
+        "resolved": "10.0.303",
+        "contentHash": "rEJw+kxSZHFRnSR5rIT2I4qYDnEhoOOl0SGS7wCeYXG88u008LwIb0oEZtj/auGvaQBMvy7jvPu6VFli8zUAZg=="
       },
       "Perfolizer": {
         "type": "Transitive",
@@ -197,8 +197,8 @@
       },
       "System.IO.Hashing": {
         "type": "Transitive",
-        "resolved": "10.0.5",
-        "contentHash": "8IBJWcCT9+e4Bmevm4T7+fQEiAh133KGiz4oiVTgJckd3Q76OFdR1falgn9lpz7+C4HJvogCDJeAa2QmvbeVtg=="
+        "resolved": "10.0.11",
+        "contentHash": "OzKDcIRkeNJeC8qAsbn8yJXnfTLP1dtkWILe+T56Gf/z+IkAASi7sMqLqJQat08j5z/mRN5xVtoAwbkMNMoBUQ=="
       },
       "System.Management": {
         "type": "Transitive",

src/XLSight.Query/ExcelWorkbookQueryExtensions.cs

@@ -188,6 +188,8 @@ private static void ValidateSupportedHeader(SheetQuerySpec spec)
     private static SheetQuery ConfigureSheetQuery(ExcelWorkbook workbook, SheetQuerySpec spec, int headerRow)
     {
         SheetQuery query = workbook.QueryRange(spec.Sheet, spec.Range, headerRow);
+        query.WithGroupLimit(spec.GroupLimit);
+        if (spec.ExpressionPlan is { } plan) { return query.WithExpressions(plan); }
 
         foreach (SheetQueryPredicate predicate in spec.Predicates)
         {

src/XLSight.Query/Internal/AggregateAccumulator.cs

@@ -22,7 +22,7 @@ public bool TryAccumulate(AggregateKind kind, in ExcelCellValue cell)
     {
         if (kind is AggregateKind.Sum or AggregateKind.Average)
         {
-            if (!cell.TryGetNumber(out double number))
+            if (!cell.TryGetNumber(out double number) || !double.IsFinite(number))
             {
                 return false;
             }
@@ -38,6 +38,7 @@ public bool TryAccumulate(AggregateKind kind, in ExcelCellValue cell)
         CellType valueKind;
         if (cell.TryGetNumber(out double n))
         {
+            if (!double.IsFinite(n)) { return false; }
             value = n;
             valueKind = CellType.Number;
         }
@@ -75,8 +76,8 @@ public bool TryAccumulate(AggregateKind kind, in ExcelCellValue cell)
     {
         AggregateKind.Count => ExcelCellValue.FromNumber(Count),
         _ when !HasValue => ExcelCellValue.Empty,
-        AggregateKind.Sum => ExcelCellValue.FromNumber(Sum),
-        AggregateKind.Average => ExcelCellValue.FromNumber(Sum / Count),
+        AggregateKind.Sum => double.IsFinite(Sum) ? ExcelCellValue.FromNumber(Sum) : ExcelCellValue.Empty,
+        AggregateKind.Average => double.IsFinite(Sum) ? ExcelCellValue.FromNumber(Sum / Count) : ExcelCellValue.Empty,
         AggregateKind.Min => ToCell(Min),
         AggregateKind.Max => ToCell(Max),
         _ => ExcelCellValue.Empty,

src/XLSight.Query/Internal/AggregateExpression.cs

@@ -0,0 +1,6 @@
+namespace XLSight.Query.Internal;
+
+internal sealed record AggregateExpression(
+    AggregateKind Kind,
+    QueryExpression? Argument,
+    QueryExpression? Filter) : QueryExpression;

src/XLSight.Query/Internal/BinaryExpression.cs

@@ -0,0 +1,3 @@
+namespace XLSight.Query.Internal;
+
+internal sealed record BinaryExpression(string Operator, QueryExpression Left, QueryExpression Right) : QueryExpression;

src/XLSight.Query/Internal/ColumnExpression.cs

@@ -0,0 +1,3 @@
+namespace XLSight.Query.Internal;
+
+internal sealed record ColumnExpression(string Name) : QueryExpression;

src/XLSight.Query/Internal/EmptyExpression.cs

@@ -0,0 +1,3 @@
+namespace XLSight.Query.Internal;
+
+internal sealed record EmptyExpression(QueryExpression Operand, bool Negated) : QueryExpression;