diff --git a/.github/workflows/validate-and-package.yml b/.github/workflows/validate-and-package.yml index d195b60..d421574 100644 --- a/.github/workflows/validate-and-package.yml +++ b/.github/workflows/validate-and-package.yml @@ -73,6 +73,7 @@ jobs: **/*.csproj - name: Restore GitVersion tool + if: github.event_name != 'pull_request' shell: pwsh run: | $ErrorActionPreference = 'Stop' @@ -82,6 +83,7 @@ jobs: } - name: Compute version properties + if: github.event_name != 'pull_request' id: version shell: pwsh run: | @@ -140,9 +142,9 @@ jobs: dotnet build CosmosDBShell.sln --configuration $env:BUILD_CONFIGURATION --no-restore - /p:Version=${{ steps.version.outputs.assembly_version }} - /p:FileVersion=${{ steps.version.outputs.file_version }} - /p:InformationalVersion=${{ steps.version.outputs.informational_version }} + $(if ('${{ steps.version.outputs.assembly_version }}') { "/p:Version=${{ steps.version.outputs.assembly_version }}" }) + $(if ('${{ steps.version.outputs.file_version }}') { "/p:FileVersion=${{ steps.version.outputs.file_version }}" }) + $(if ('${{ steps.version.outputs.informational_version }}') { "/p:InformationalVersion=${{ steps.version.outputs.informational_version }}" }) shell: pwsh - name: Test solution @@ -187,6 +189,7 @@ jobs: if-no-files-found: ignore - name: Publish runtime artifacts + if: github.event_name != 'pull_request' shell: pwsh run: | $ErrorActionPreference = 'Stop' @@ -214,6 +217,7 @@ jobs: } - name: Pack NuGet artifacts + if: github.event_name != 'pull_request' shell: pwsh run: | New-Item -ItemType Directory -Path out/nupkg -Force | Out-Null @@ -232,6 +236,7 @@ jobs: /p:ContinuousIntegrationBuild=true - name: Validate NuGet package set + if: github.event_name != 'pull_request' shell: pwsh run: | $pkgDir = Join-Path $pwd 'out/nupkg' @@ -257,9 +262,9 @@ jobs: $_.Name -notmatch '^CosmosDBShell\.(win-x64|win-arm64|linux-x64|linux-arm64|osx-x64|osx-arm64)\..+\.nupkg$' } - if (-not $pointerPackages -or $pointerPackages.Count -ne 1) { + if ($pointerPackages -and $pointerPackages.Count -gt 1) { $names = @($pointerPackages | ForEach-Object { $_.Name }) - Write-Error "Expected exactly one pointer package (non-RID). Found: $($names -join ', ')" + Write-Error "Expected at most one pointer package (non-RID). Found: $($names -join ', ')" exit 1 } @@ -270,7 +275,22 @@ jobs: exit 1 } + - name: Validate pointer package is available for upload + if: github.event_name != 'pull_request' + shell: pwsh + run: | + $pkgDir = Join-Path $pwd 'out/nupkg' + $pointerPackages = Get-ChildItem -Path (Join-Path $pkgDir 'CosmosDBShell.*.nupkg') -ErrorAction SilentlyContinue | Where-Object { + $_.Name -notmatch '^CosmosDBShell\.(win-x64|win-arm64|linux-x64|linux-arm64|osx-x64|osx-arm64)\..+\.nupkg$' + } + + if (-not $pointerPackages) { + Write-Error 'Expected a pointer package in out/nupkg, but none was found.' + exit 1 + } + - name: List packaged NuGet files + if: github.event_name != 'pull_request' shell: pwsh run: | Get-ChildItem -Path out/nupkg -Filter *.nupkg -File | Sort-Object Name | ForEach-Object { @@ -278,6 +298,7 @@ jobs: } - name: Write package install summary + if: github.event_name != 'pull_request' shell: pwsh run: | $summary = $env:GITHUB_STEP_SUMMARY @@ -308,6 +329,7 @@ jobs: $lines -join "`n" | Out-File -FilePath $summary -Encoding utf8 -Append - name: Upload pointer package + if: github.event_name != 'pull_request' uses: actions/upload-artifact@v4 with: name: CosmosDBShell-pointer-${{ steps.version.outputs.artifact_suffix }} @@ -322,6 +344,7 @@ jobs: if-no-files-found: error - name: Upload win-x64 package + if: github.event_name != 'pull_request' uses: actions/upload-artifact@v4 with: name: CosmosDBShell-win-x64-${{ steps.version.outputs.artifact_suffix }} @@ -336,6 +359,7 @@ jobs: if-no-files-found: error - name: Upload linux-x64 package + if: github.event_name != 'pull_request' uses: actions/upload-artifact@v4 with: name: CosmosDBShell-linux-x64-${{ steps.version.outputs.artifact_suffix }} @@ -343,6 +367,7 @@ jobs: if-no-files-found: error - name: Upload linux-arm64 package + if: github.event_name != 'pull_request' uses: actions/upload-artifact@v4 with: name: CosmosDBShell-linux-arm64-${{ steps.version.outputs.artifact_suffix }} @@ -350,6 +375,7 @@ jobs: if-no-files-found: error - name: Upload osx-x64 package + if: github.event_name != 'pull_request' uses: actions/upload-artifact@v4 with: name: CosmosDBShell-osx-x64-${{ steps.version.outputs.artifact_suffix }} @@ -357,6 +383,7 @@ jobs: if-no-files-found: error - name: Upload osx-arm64 package + if: github.event_name != 'pull_request' uses: actions/upload-artifact@v4 with: name: CosmosDBShell-osx-arm64-${{ steps.version.outputs.artifact_suffix }} diff --git a/.pipelines/CosmosDB-Shell-Official.yml b/.pipelines/CosmosDB-Shell-Official.yml index 25e9f75..d19aa2b 100644 --- a/.pipelines/CosmosDB-Shell-Official.yml +++ b/.pipelines/CosmosDB-Shell-Official.yml @@ -244,6 +244,45 @@ extends: files_to_sign: "**/*.exe;**/*.dll" search_root: '$(Build.SourcesDirectory)\out' + - task: PowerShell@2 + displayName: "Zip signed RID publish folders" + condition: succeeded() + inputs: + targetType: inline + pwsh: true + script: | + $ErrorActionPreference = 'Stop' + $version = "$(CosmosDBShell_PackageVersion)" + $outDir = "$(Build.SourcesDirectory)\out" + $zipDir = Join-Path $outDir 'zip' + New-Item -ItemType Directory -Path $zipDir -Force | Out-Null + + $rids = @('win-x64', 'win-arm64', 'linux-x64', 'linux-arm64', 'osx-x64', 'osx-arm64') + foreach ($rid in $rids) { + $ridDir = Join-Path $outDir $rid + if (-not (Test-Path $ridDir)) { + Write-Warning "Skipping $rid — publish folder not found at $ridDir." + continue + } + + $zipPath = Join-Path $zipDir ("cosmosdbshell_{0}_{1}.zip" -f $rid, $version) + if (Test-Path $zipPath) { + Remove-Item -Path $zipPath -Force + } + + Write-Host "Creating $zipPath" + [System.IO.Compression.ZipFile]::CreateFromDirectory( + $ridDir, + $zipPath, + [System.IO.Compression.CompressionLevel]::Optimal, + $false) + } + + Write-Host "Generated archives:" + Get-ChildItem -Path $zipDir -Filter *.zip | Sort-Object Name | ForEach-Object { + Write-Host " - $($_.Name) [$($_.Length) bytes]" + } + - task: DotNetCoreCLI@2 displayName: "Build Fuzzer" inputs: diff --git a/CosmosDBShell.Tests/CommandTests/FilterCommandTests.cs b/CosmosDBShell.Tests/CommandTests/FilterCommandTests.cs new file mode 100644 index 0000000..29c580b --- /dev/null +++ b/CosmosDBShell.Tests/CommandTests/FilterCommandTests.cs @@ -0,0 +1,171 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace CosmosShell.Tests.CommandTests; + +using System.Text.Json; +using Azure.Data.Cosmos.Shell.Commands; +using Azure.Data.Cosmos.Shell.Core; +using Azure.Data.Cosmos.Shell.Parser; + +public class FilterCommandTests +{ + [Fact] + public async Task ExecuteAsync_AppliesPathExpression_AndPreservesStructuredResult() + { + var shell = ShellInterpreter.CreateInstance(); + var state = new CommandState + { + Result = new ShellJson(JsonSerializer.SerializeToElement(new + { + items = new[] + { + new { id = "1", status = "active" }, + new { id = "2", status = "inactive" }, + }, + })), + }; + + var command = new FilterCommand + { + ExpressionText = ".items[0].id", + }; + + var result = await command.ExecuteAsync(shell, state, string.Empty, CancellationToken.None); + + Assert.Same(state, result); + Assert.False(result.IsPrinted); + var json = Assert.IsType(result.Result); + Assert.Equal("1", json.Value.GetString()); + } + + [Fact] + public async Task ExecuteAsync_AppliesMapProjection() + { + var shell = ShellInterpreter.CreateInstance(); + var state = new CommandState + { + Result = new ShellJson(JsonSerializer.SerializeToElement(new + { + items = new[] + { + new { id = "1", status = "active" }, + new { id = "2", status = "inactive" }, + }, + })), + }; + + var command = new FilterCommand + { + ExpressionText = ".items | map({id, status})", + }; + + var result = await command.ExecuteAsync(shell, state, string.Empty, CancellationToken.None); + + var json = Assert.IsType(result.Result); + Assert.Equal(JsonValueKind.Array, json.Value.ValueKind); + Assert.Equal("1", json.Value[0].GetProperty("id").GetString()); + Assert.Equal("active", json.Value[0].GetProperty("status").GetString()); + Assert.Equal("2", json.Value[1].GetProperty("id").GetString()); + } + + [Fact] + public async Task ExecuteAsync_NormalizesSequenceResult_ToJsonArray() + { + var shell = ShellInterpreter.CreateInstance(); + var state = new CommandState + { + Result = new ShellJson(JsonSerializer.SerializeToElement(new + { + items = new[] + { + new { id = "1" }, + new { id = "2" }, + }, + })), + }; + + var command = new FilterCommand + { + ExpressionText = ".items[] | .id", + }; + + var result = await command.ExecuteAsync(shell, state, string.Empty, CancellationToken.None); + + var json = Assert.IsType(result.Result); + Assert.Equal(JsonValueKind.Array, json.Value.ValueKind); + Assert.Equal("1", json.Value[0].GetString()); + Assert.Equal("2", json.Value[1].GetString()); + } + + [Fact] + public async Task ExecuteAsync_NormalizesTextResult_ToJsonString() + { + var shell = ShellInterpreter.CreateInstance(); + var state = new CommandState + { + Result = new ShellJson(JsonSerializer.SerializeToElement(new { id = "1" })), + }; + + var command = new FilterCommand + { + ExpressionText = "type", + }; + + var result = await command.ExecuteAsync(shell, state, string.Empty, CancellationToken.None); + + var json = Assert.IsType(result.Result); + Assert.Equal(JsonValueKind.String, json.Value.ValueKind); + Assert.Equal("object", json.Value.GetString()); + } + + [Fact] + public async Task ExecuteAsync_NormalizesBooleanResult_ToJsonBoolean() + { + var shell = ShellInterpreter.CreateInstance(); + var state = new CommandState + { + Result = new ShellJson(JsonSerializer.SerializeToElement(new { id = "1" })), + }; + + var command = new FilterCommand + { + ExpressionText = "true", + }; + + var result = await command.ExecuteAsync(shell, state, string.Empty, CancellationToken.None); + + var json = Assert.IsType(result.Result); + Assert.Equal(JsonValueKind.True, json.Value.ValueKind); + } + + [Fact] + public async Task ExecuteAsync_ThrowsWhenInputMissing() + { + var shell = ShellInterpreter.CreateInstance(); + var command = new FilterCommand + { + ExpressionText = ".items", + }; + + var ex = await Assert.ThrowsAsync(() => command.ExecuteAsync(shell, new CommandState(), string.Empty, CancellationToken.None)); + Assert.Contains("requires piped JSON input", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task ExecuteAsync_ThrowsWhenExpressionInvalid() + { + var shell = ShellInterpreter.CreateInstance(); + var state = new CommandState + { + Result = new ShellJson(JsonSerializer.SerializeToElement(new { items = new[] { 1, 2 } })), + }; + var command = new FilterCommand + { + ExpressionText = ".items[", + }; + + await Assert.ThrowsAsync(() => command.ExecuteAsync(shell, state, string.Empty, CancellationToken.None)); + } +} \ No newline at end of file diff --git a/CosmosDBShell.Tests/Integration/FilterIntegrationTests.cs b/CosmosDBShell.Tests/Integration/FilterIntegrationTests.cs new file mode 100644 index 0000000..488c3c6 --- /dev/null +++ b/CosmosDBShell.Tests/Integration/FilterIntegrationTests.cs @@ -0,0 +1,73 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace CosmosShell.Tests.Integration; + +using System.Text.Json; + +using Azure.Data.Cosmos.Shell.Parser; + +public class FilterIntegrationTests : IntegrationTestBase +{ + [Fact] + public async Task Filter_PipelineProjectsQuotedProperties() + { + Shell.SetVariable("data", new ShellJson(JsonSerializer.SerializeToElement(new + { + items = new[] + { + new Dictionary { ["Volcano Name"] = "Abu", ["Country"] = "Japan", ["Region"] = "Honshu-Japan" }, + new Dictionary { ["Volcano Name"] = "Acamarachi", ["Country"] = "Chile", ["Region"] = "Chile-N" }, + }, + }))); + + var outputFile = CaptureOutputFile(); + var state = await RunScriptAsync("echo $data | filter '.items | map({\"Volcano Name\": .[\"Volcano Name\"], Country})'"); + + Assert.False(state.IsError, FormatError(state)); + var output = await ReadRedirectAsync(outputFile); + var json = JsonDocument.Parse(output).RootElement; + Assert.Equal(JsonValueKind.Array, json.ValueKind); + Assert.Equal("Abu", json[0].GetProperty("Volcano Name").GetString()); + Assert.Equal("Japan", json[0].GetProperty("Country").GetString()); + Assert.False(json[0].TryGetProperty("Region", out _)); + Assert.Equal("Acamarachi", json[1].GetProperty("Volcano Name").GetString()); + Assert.Equal("Chile", json[1].GetProperty("Country").GetString()); + } + + [Fact] + public async Task Filter_PipelineContainsStringLiteralReturnsJsonBoolean() + { + Shell.SetVariable("data", new ShellJson(JsonSerializer.SerializeToElement(new + { + tags = new[] { "dev", "prod" }, + }))); + + var outputFile = CaptureOutputFile(); + var state = await RunScriptAsync("echo $data | filter '.tags | contains(\"prod\")'"); + + Assert.False(state.IsError, FormatError(state)); + var output = await ReadRedirectAsync(outputFile); + var json = JsonDocument.Parse(output).RootElement; + Assert.Equal(JsonValueKind.True, json.ValueKind); + } + + [Fact] + public async Task Filter_PipelineTypeReturnsJsonString() + { + Shell.SetVariable("data", new ShellJson(JsonSerializer.SerializeToElement(new + { + id = "item-1", + }))); + + var outputFile = CaptureOutputFile(); + var state = await RunScriptAsync("echo $data | filter 'type'"); + + Assert.False(state.IsError, FormatError(state)); + var output = await ReadRedirectAsync(outputFile); + var json = JsonDocument.Parse(output).RootElement; + Assert.Equal(JsonValueKind.String, json.ValueKind); + Assert.Equal("object", json.GetString()); + } +} diff --git a/CosmosDBShell.Tests/Parser/ExpressionTests.cs b/CosmosDBShell.Tests/Parser/ExpressionTests.cs index fb34b86..4f675f9 100644 --- a/CosmosDBShell.Tests/Parser/ExpressionTests.cs +++ b/CosmosDBShell.Tests/Parser/ExpressionTests.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using System.Text.Json; using Azure.Data.Cosmos.Shell.Core; using Azure.Data.Cosmos.Shell.Parser; @@ -17,7 +18,7 @@ private Expression ParseExpression(string input) { var lexer = new Lexer(input); var parser = new ExpressionParser(lexer); - return parser.ParseExpression(); + return parser.ParseFilterExpression(); } private async Task EvaluateExpressionAsync(string input) @@ -26,6 +27,17 @@ private async Task EvaluateExpressionAsync(string input) return await expression.EvaluateAsync(ShellInterpreter.Instance, new CommandState(), CancellationToken.None); } + private async Task EvaluateExpressionWithJsonAsync(string input, object? value) + { + var expression = ParseExpression(input); + var state = new CommandState + { + Result = new ShellJson(JsonSerializer.SerializeToElement(value)), + }; + + return await expression.EvaluateAsync(ShellInterpreter.Instance, state, CancellationToken.None); + } + #pragma warning disable CS0618, VSTHRD002 // Type or member is obsolete, Synchronously waiting on tasks private ShellObject EvaluateExpression(string input) { @@ -535,6 +547,191 @@ public void ParseExpression_VariableArrayAccess_ReturnsJsonPathExpression() Assert.Equal("items[0]", path.JSonPath); } + [Fact] + public void ParseExpression_FilterRootPath_ReturnsFilterPathExpression() + { + var expr = ParseExpression(".items[0].id"); + var path = Assert.IsType(expr); + Assert.Equal(3, path.Segments.Count); + Assert.IsType(path.Segments[0]); + Assert.IsType(path.Segments[1]); + Assert.IsType(path.Segments[2]); + } + + [Fact] + public void ParseExpression_FilterQuotedPropertyPath_ReturnsFilterPathExpression() + { + var expr = ParseExpression(".[\"Volcano Name\"]"); + var path = Assert.IsType(expr); + var segment = Assert.IsType(Assert.Single(path.Segments)); + Assert.Equal("Volcano Name", segment.Name); + } + + [Fact] + public void ParseExpression_FilterBuiltinCall_ReturnsFilterCallExpression() + { + var expr = ParseExpression("map(.id)"); + var call = Assert.IsType(expr); + Assert.Equal("map", call.Name); + Assert.Single(call.Arguments); + } + + [Fact] + public void ParseExpression_FilterPipe_ReturnsFilterPipeExpression() + { + var expr = ParseExpression(".items | length"); + var pipe = Assert.IsType(expr); + Assert.IsType(pipe.Left); + Assert.IsType(pipe.Right); + } + + [Fact] + public void ParseExpression_ObjectShorthand_ReturnsJsonExpression() + { + var expr = ParseExpression("{id, status}"); + var json = Assert.IsType(expr); + Assert.Equal(2, json.Properties.Count); + Assert.All(json.Properties.Values, value => Assert.IsType(value)); + } + + [Fact] + public async Task EvaluateExpression_FilterRootPath_ReturnsPropertyValue() + { + var result = await EvaluateExpressionWithJsonAsync(".items[0].id", new { items = new[] { new { id = "1" } } }); + var json = Assert.IsType(result); + Assert.Equal("1", json.Value.GetString()); + } + + [Fact] + public async Task EvaluateExpression_FilterQuotedPropertyPath_ReturnsPropertyValue() + { + var result = await EvaluateExpressionWithJsonAsync(".[\"Volcano Name\"]", new Dictionary { ["Volcano Name"] = "Abu" }); + var json = Assert.IsType(result); + Assert.Equal("Abu", json.Value.GetString()); + } + + [Fact] + public async Task EvaluateExpression_FilterDotQuotedPropertyPath_ReturnsPropertyValue() + { + var result = await EvaluateExpressionWithJsonAsync(".\"Volcano Name\"", new Dictionary { ["Volcano Name"] = "Abu" }); + var json = Assert.IsType(result); + Assert.Equal("Abu", json.Value.GetString()); + } + + [Fact] + public async Task EvaluateExpression_FilterMapProjection_WithQuotedPropertyPath_ReturnsProjectedArray() + { + var result = await EvaluateExpressionWithJsonAsync( + ".items | map({\"Volcano Name\": .[\"Volcano Name\"], Country})", + new + { + items = new[] + { + new Dictionary { ["Volcano Name"] = "Abu", ["Country"] = "Japan", ["Region"] = "Honshu-Japan" }, + new Dictionary { ["Volcano Name"] = "Acamarachi", ["Country"] = "Chile", ["Region"] = "Chile-N" }, + }, + }); + + var json = Assert.IsType(result); + Assert.Equal(JsonValueKind.Array, json.Value.ValueKind); + Assert.Equal("Abu", json.Value[0].GetProperty("Volcano Name").GetString()); + Assert.Equal("Japan", json.Value[0].GetProperty("Country").GetString()); + Assert.False(json.Value[0].TryGetProperty("Region", out _)); + Assert.Equal("Acamarachi", json.Value[1].GetProperty("Volcano Name").GetString()); + Assert.Equal("Chile", json.Value[1].GetProperty("Country").GetString()); + } + + [Fact] + public async Task EvaluateExpression_FilterPipeLength_ReturnsCount() + { + var result = await EvaluateExpressionWithJsonAsync(".items | length", new { items = new[] { 1, 2, 3 } }); + var number = Assert.IsType(result); + Assert.Equal(3, number.Value); + } + + [Fact] + public async Task EvaluateExpression_FilterMap_ReturnsProjectedArray() + { + var result = await EvaluateExpressionWithJsonAsync(".items | map(.id)", new { items = new[] { new { id = "b" }, new { id = "a" } } }); + var json = Assert.IsType(result); + Assert.Equal(JsonValueKind.Array, json.Value.ValueKind); + Assert.Equal("b", json.Value[0].GetString()); + Assert.Equal("a", json.Value[1].GetString()); + } + + [Fact] + public async Task EvaluateExpression_FilterMapType_ReturnsStringArray() + { + var result = await EvaluateExpressionWithJsonAsync(".items | map(type)", new { items = new object?[] { "active", 42, true } }); + var json = Assert.IsType(result); + Assert.Equal(JsonValueKind.Array, json.Value.ValueKind); + Assert.Equal("string", json.Value[0].GetString()); + Assert.Equal("number", json.Value[1].GetString()); + Assert.Equal("boolean", json.Value[2].GetString()); + } + + [Fact] + public async Task EvaluateExpression_FilterContains_WithStringLiteral_ReturnsTrue() + { + var result = await EvaluateExpressionWithJsonAsync(".tags | contains(\"prod\")", new { tags = new[] { "dev", "prod" } }); + var shellBool = Assert.IsType(result); + Assert.True(shellBool.Value); + } + + [Fact] + public async Task EvaluateExpression_FilterContains_WithBooleanLiteral_ReturnsTrue() + { + var result = await EvaluateExpressionWithJsonAsync(".flags | contains(true)", new { flags = new[] { false, true } }); + var shellBool = Assert.IsType(result); + Assert.True(shellBool.Value); + } + + [Theory] + [InlineData("length(.items)")] + [InlineData("keys(.items)")] + [InlineData("type(.items)")] + public async Task EvaluateExpression_FilterZeroArgumentBuiltin_WithArgument_Throws(string expression) + { + await Assert.ThrowsAsync(() => EvaluateExpressionWithJsonAsync(expression, new { items = new[] { 1, 2, 3 } })); + } + + [Fact] + public async Task EvaluateExpression_FilterSelect_ReturnsFilteredArray() + { + var result = await EvaluateExpressionWithJsonAsync( + ".items | select(.status == \"active\")", + new { items = new[] { new { id = "1", status = "active" }, new { id = "2", status = "inactive" } } }); + + var json = Assert.IsType(result); + Assert.Equal(1, json.Value.GetArrayLength()); + Assert.Equal("1", json.Value[0].GetProperty("id").GetString()); + } + + [Fact] + public async Task EvaluateExpression_FilterSortBy_ReturnsSortedArray() + { + var result = await EvaluateExpressionWithJsonAsync( + ".items | sort_by(.id)", + new { items = new[] { new { id = "b" }, new { id = "a" } } }); + + var json = Assert.IsType(result); + Assert.Equal("a", json.Value[0].GetProperty("id").GetString()); + Assert.Equal("b", json.Value[1].GetProperty("id").GetString()); + } + + [Fact] + public async Task EvaluateExpression_FilterIterationPipe_ReturnsSequenceAsJsonArray() + { + var result = await EvaluateExpressionWithJsonAsync( + ".items[] | .id", + new { items = new[] { new { id = "1" }, new { id = "2" } } }); + + var sequence = Assert.IsType(result); + Assert.Equal(2, sequence.Elements.Count); + Assert.Equal("1", sequence.Elements[0].GetString()); + Assert.Equal("2", sequence.Elements[1].GetString()); + } + #endregion #region Decimal Expression Tests diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Commands/FilterCommand.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Commands/FilterCommand.cs new file mode 100644 index 0000000..324ceb3 --- /dev/null +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Commands/FilterCommand.cs @@ -0,0 +1,65 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Azure.Data.Cosmos.Shell.Commands; + +using Azure.Data.Cosmos.Shell.Core; +using Azure.Data.Cosmos.Shell.Parser; +using Azure.Data.Cosmos.Shell.Util; + +[CosmosCommand("filter")] +[CosmosExample("query \"SELECT * FROM c\" | filter '.items[0]'", Description = "Extract the first query result")] +[CosmosExample("query \"SELECT * FROM c\" | filter '.items | map({id, status})'", Description = "Project selected fields from query results")] +[CosmosExample("ls | filter '.items | length'", Description = "Count listed items")] +[CosmosExample("query \"SELECT * FROM c\" | filter '.items[] | .id'", Description = "Extract ids from each item")] +internal class FilterCommand : CosmosCommand +{ + [CosmosParameter("expression")] + public string? ExpressionText { get; init; } + + public override async Task ExecuteAsync(ShellInterpreter shell, CommandState commandState, string commandText, CancellationToken token) + { + if (string.IsNullOrWhiteSpace(this.ExpressionText)) + { + throw new CommandException("filter", MessageService.GetString("command-filter-error-no_expression")); + } + + if (commandState.Result == null) + { + throw new CommandException("filter", MessageService.GetString("command-filter-error-no_input")); + } + + var evaluatedInput = commandState.Result.ConvertShellObject(DataType.Json); + if (evaluatedInput is not System.Text.Json.JsonElement) + { + throw new CommandException("filter", MessageService.GetString("command-filter-error-invalid_input")); + } + + var lexer = new Lexer(this.ExpressionText); + var parser = new ExpressionParser(lexer); + var expression = parser.ParseFilterExpression(); + + if (lexer.Errors.HasErrors) + { + throw new CommandException("filter", lexer.Errors[0].Message); + } + + var result = await expression.EvaluateAsync(shell, commandState, token); + if (result is ShellSequence sequence) + { + commandState.Result = new ShellJson(FilterExpressionUtilities.ToJsonArray(sequence.Elements)); + } + else if (result is ShellJson) + { + commandState.Result = result; + } + else + { + commandState.Result = new ShellJson(FilterExpressionUtilities.ToJsonElement(result)); + } + + commandState.IsPrinted = false; + return commandState; + } +} \ No newline at end of file diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Core/ShellInterpreter.Highlighter.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Core/ShellInterpreter.Highlighter.cs index de6db4b..b189dd5 100644 --- a/CosmosDBShell/Azure.Data.Cosmos.Shell.Core/ShellInterpreter.Highlighter.cs +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Core/ShellInterpreter.Highlighter.cs @@ -292,6 +292,15 @@ public void Visit(BinaryOperatorExpression binaryOperatorExpression) binaryOperatorExpression.Right.Accept(this); } + public void Visit(FilterPipeExpression filterPipeExpression) + { + filterPipeExpression.Left.Accept(this); + this.AppendUpTo(filterPipeExpression.PipeToken.Start); + this.result.Append(Theme.FormatJsonBracket(filterPipeExpression.PipeToken.Value)); + this.currentPosition = filterPipeExpression.PipeToken.End; + filterPipeExpression.Right.Accept(this); + } + public void Visit(ParensExpression parensExpression) { parensExpression.InnerExpression.Accept(this); @@ -454,6 +463,11 @@ public void Visit(JSonPathExpression jSonPathExpression) this.AppendUpTo(jSonPathExpression.Start + jSonPathExpression.Length); } + public void Visit(FilterPathExpression filterPathExpression) + { + this.AppendUpTo(filterPathExpression.Start + filterPathExpression.Length); + } + public void Visit(VariableExpression variableExpression) { this.AppendUpTo(variableExpression.Start + variableExpression.Length); @@ -493,6 +507,18 @@ public void Visit(CommandExpression commandExpression) this.currentCommand = previousCommand; } + public void Visit(FilterCallExpression filterCallExpression) + { + this.AppendUpTo(filterCallExpression.NameToken.Start); + this.result.Append(Theme.FormatCommand(filterCallExpression.NameToken.Value)); + this.currentPosition = filterCallExpression.NameToken.End; + + foreach (var argument in filterCallExpression.Arguments) + { + argument.Accept(this); + } + } + public void Visit(InterpolatedStringExpression interpolatedStringExpression) { foreach (var expr in interpolatedStringExpression.Expressions) diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Lsp/CosmosShellSemanticTokensHandler.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Lsp/CosmosShellSemanticTokensHandler.cs index a728c06..c7d10a3 100644 --- a/CosmosDBShell/Azure.Data.Cosmos.Shell.Lsp/CosmosShellSemanticTokensHandler.cs +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Lsp/CosmosShellSemanticTokensHandler.cs @@ -279,6 +279,13 @@ public void Visit(BinaryOperatorExpression binaryOperatorExpression) binaryOperatorExpression.Right.Accept(this); } + public void Visit(FilterPipeExpression filterPipeExpression) + { + filterPipeExpression.Left.Accept(this); + this.VisitToken(filterPipeExpression.PipeToken); + filterPipeExpression.Right.Accept(this); + } + public void Visit(UnaryOperatorExpression unaryOperatorExpression) { this.VisitToken(unaryOperatorExpression.OperatorToken); @@ -322,6 +329,20 @@ public void Visit(JsonArrayExpression jsonArrayExpression) this.addSpan(jsonArrayExpression.RBracketToken.Start, jsonArrayExpression.RBracketToken.Length, SemanticTokenType.Regexp); } + public void Visit(FilterPathExpression filterPathExpression) + { + this.addSpan(filterPathExpression.Start, filterPathExpression.Length, SemanticTokenType.Variable); + } + + public void Visit(FilterCallExpression filterCallExpression) + { + this.addSpan(filterCallExpression.NameToken.Start, filterCallExpression.NameToken.Length, SemanticTokenType.Function); + foreach (var argument in filterCallExpression.Arguments) + { + argument.Accept(this); + } + } + public void Visit(IfStatement ifStatement) { this.VisitKeyword(ifStatement.IfToken); diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/Expression.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/Expression.cs index 55160cc..d1a0d78 100644 --- a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/Expression.cs +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/Expression.cs @@ -17,7 +17,7 @@ internal abstract class Expression public static Expression Parse(Lexer lexer) { var parser = new ExpressionParser(lexer); - return parser.ParseExpression(); + return parser.ParseFilterExpression(); } /// diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterCallExpression.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterCallExpression.cs new file mode 100644 index 0000000..0fd4252 --- /dev/null +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterCallExpression.cs @@ -0,0 +1,207 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Azure.Data.Cosmos.Shell.Parser; + +using System.Linq; +using System.Text.Json; + +using Azure.Data.Cosmos.Shell.Core; + +internal class FilterCallExpression : Expression +{ + public FilterCallExpression(Token nameToken, IReadOnlyList arguments, int? length = null) + { + this.NameToken = nameToken ?? throw new ArgumentNullException(nameof(nameToken)); + this.Arguments = arguments ?? throw new ArgumentNullException(nameof(arguments)); + this.ExpressionLength = length ?? nameToken.Length; + } + + public Token NameToken { get; } + + public string Name => this.NameToken.Value; + + public IReadOnlyList Arguments { get; } + + public int ExpressionLength { get; } + + public override int Start => this.NameToken.Start; + + public override int Length => this.ExpressionLength; + + public override async Task EvaluateAsync(ShellInterpreter interpreter, CommandState currentState, CancellationToken cancellationToken) + { + var current = currentState.Result?.ConvertShellObject(DataType.Json); + if (current is not JsonElement currentJson) + { + throw new InvalidOperationException($"filter builtin '{this.Name}' requires a JSON pipeline value"); + } + + return this.Name switch + { + "length" => EvaluateLength(currentJson, this.Arguments.Count, this.Name), + "keys" => EvaluateKeys(currentJson, this.Arguments.Count, this.Name), + "type" => EvaluateType(currentJson, this.Arguments.Count, this.Name), + "contains" => await this.EvaluateContainsAsync(interpreter, currentState, currentJson, cancellationToken), + "map" => await this.EvaluateMapAsync(interpreter, currentJson, cancellationToken), + "select" => await this.EvaluateSelectAsync(interpreter, currentJson, cancellationToken), + "sort_by" => await this.EvaluateSortByAsync(interpreter, currentJson, cancellationToken), + _ => throw new InvalidOperationException($"Unsupported filter builtin '{this.Name}'"), + }; + } + + public override void Accept(IAstVisitor visitor) + { + visitor.Visit(this); + } + + private static ShellObject EvaluateLength(JsonElement currentJson, int argumentCount, string name) + { + RequireArgumentCount(argumentCount, name, 0); + return EvaluateLength(currentJson); + } + + private static ShellObject EvaluateKeys(JsonElement currentJson, int argumentCount, string name) + { + RequireArgumentCount(argumentCount, name, 0); + return EvaluateKeys(currentJson); + } + + private static ShellObject EvaluateType(JsonElement currentJson, int argumentCount, string name) + { + RequireArgumentCount(argumentCount, name, 0); + return EvaluateType(currentJson); + } + + private static ShellObject EvaluateLength(JsonElement currentJson) + { + return currentJson.ValueKind switch + { + JsonValueKind.Array => new ShellNumber(currentJson.GetArrayLength()), + JsonValueKind.Object => new ShellNumber(currentJson.EnumerateObject().Count()), + JsonValueKind.String => new ShellNumber((currentJson.GetString() ?? string.Empty).Length), + JsonValueKind.Null => new ShellNumber(0), + _ => throw new InvalidOperationException("length supports arrays, objects, strings, and null"), + }; + } + + private static ShellObject EvaluateKeys(JsonElement currentJson) + { + if (currentJson.ValueKind != JsonValueKind.Object) + { + throw new InvalidOperationException("keys requires an object input"); + } + + var keys = currentJson.EnumerateObject().Select(static p => p.Name).OrderBy(static p => p, StringComparer.Ordinal).ToArray(); + return new ShellJson(JsonSerializer.SerializeToElement(keys)); + } + + private static ShellObject EvaluateType(JsonElement currentJson) + { + var value = currentJson.ValueKind switch + { + JsonValueKind.Null => "null", + JsonValueKind.True or JsonValueKind.False => "boolean", + JsonValueKind.Number => "number", + JsonValueKind.String => "string", + JsonValueKind.Array => "array", + JsonValueKind.Object => "object", + _ => "null", + }; + + return new ShellText(value); + } + + private static void RequireArgumentCount(int actual, string name, int expected) + { + if (actual != expected) + { + throw new InvalidOperationException($"Builtin '{name}' expects {expected} argument(s)"); + } + } + + private async Task EvaluateContainsAsync(ShellInterpreter interpreter, CommandState currentState, JsonElement currentJson, CancellationToken cancellationToken) + { + this.RequireArgumentCount(1); + var target = await this.Arguments[0].EvaluateAsync(interpreter, currentState, cancellationToken); + return new ShellBool(FilterExpressionUtilities.Contains(currentJson, FilterExpressionUtilities.ToJsonElement(target))); + } + + private async Task EvaluateMapAsync(ShellInterpreter interpreter, JsonElement currentJson, CancellationToken cancellationToken) + { + this.RequireArgumentCount(1); + if (currentJson.ValueKind != JsonValueKind.Array) + { + throw new InvalidOperationException("map requires an array input"); + } + + var results = new List(); + foreach (var item in currentJson.EnumerateArray()) + { + var nestedState = new CommandState { Result = new ShellJson(item.Clone()) }; + var evaluated = await this.Arguments[0].EvaluateAsync(interpreter, nestedState, cancellationToken); + if (evaluated is ShellSequence sequence) + { + results.AddRange(sequence.Elements.Select(static e => e.Clone())); + } + else + { + results.Add(FilterExpressionUtilities.ToJsonElement(evaluated)); + } + } + + return new ShellJson(FilterExpressionUtilities.ToJsonArray(results)); + } + + private async Task EvaluateSelectAsync(ShellInterpreter interpreter, JsonElement currentJson, CancellationToken cancellationToken) + { + this.RequireArgumentCount(1); + if (currentJson.ValueKind != JsonValueKind.Array) + { + throw new InvalidOperationException("select requires an array input"); + } + + var results = new List(); + foreach (var item in currentJson.EnumerateArray()) + { + var nestedState = new CommandState { Result = new ShellJson(item.Clone()) }; + var evaluated = await this.Arguments[0].EvaluateAsync(interpreter, nestedState, cancellationToken); + if (evaluated is ShellBool shellBool && shellBool.Value) + { + results.Add(item.Clone()); + } + else if (evaluated is ShellJson json && json.Value.ValueKind == JsonValueKind.True) + { + results.Add(item.Clone()); + } + } + + return new ShellJson(FilterExpressionUtilities.ToJsonArray(results)); + } + + private async Task EvaluateSortByAsync(ShellInterpreter interpreter, JsonElement currentJson, CancellationToken cancellationToken) + { + this.RequireArgumentCount(1); + if (currentJson.ValueKind != JsonValueKind.Array) + { + throw new InvalidOperationException("sort_by requires an array input"); + } + + var items = new List<(JsonElement Item, JsonElement Key)>(); + foreach (var item in currentJson.EnumerateArray()) + { + var nestedState = new CommandState { Result = new ShellJson(item.Clone()) }; + var evaluated = await this.Arguments[0].EvaluateAsync(interpreter, nestedState, cancellationToken); + items.Add((item.Clone(), FilterExpressionUtilities.ToJsonElement(evaluated))); + } + + items.Sort(static (left, right) => FilterExpressionUtilities.Compare(left.Key, right.Key)); + return new ShellJson(FilterExpressionUtilities.ToJsonArray(items.Select(static item => item.Item))); + } + + private void RequireArgumentCount(int expected) + { + RequireArgumentCount(this.Arguments.Count, this.Name, expected); + } +} diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterExpressionUtilities.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterExpressionUtilities.cs new file mode 100644 index 0000000..54bea52 --- /dev/null +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterExpressionUtilities.cs @@ -0,0 +1,168 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Azure.Data.Cosmos.Shell.Parser; + +using System.Linq; +using System.Text.Json; + +internal static class FilterExpressionUtilities +{ + private static readonly JsonDocument NullJsonDocument = JsonDocument.Parse("null"); + + public static JsonElement NullElement() + { + return NullJsonDocument.RootElement; + } + + public static JsonElement ToJsonElement(ShellObject shellObject) + { + switch (shellObject) + { + case ShellJson shellJson: + return shellJson.Value.Clone(); + case ShellText shellText: + return JsonSerializer.SerializeToElement(shellText.Text); + case ShellNumber shellNumber: + return JsonSerializer.SerializeToElement(shellNumber.Value); + case ShellDecimal shellDecimal: + return JsonSerializer.SerializeToElement(shellDecimal.Value); + case ShellBool shellBool: + return JsonSerializer.SerializeToElement(shellBool.Value); + case ShellSequence shellSequence: + return ToJsonArray(shellSequence.Elements); + } + + var value = shellObject.ConvertShellObject(DataType.Json); + if (value is JsonElement jsonElement) + { + return jsonElement.Clone(); + } + + throw new InvalidOperationException($"Expected JSON value but got {shellObject.GetType().Name}"); + } + + public static JsonElement ToJsonArray(IEnumerable elements) + { + return JsonSerializer.SerializeToElement(elements.ToArray()); + } + + public static bool JsonEquals(JsonElement left, JsonElement right) + { + if (left.ValueKind != right.ValueKind) + { + return false; + } + + return left.ValueKind switch + { + JsonValueKind.Null or JsonValueKind.Undefined => true, + JsonValueKind.True => right.ValueKind == JsonValueKind.True, + JsonValueKind.False => right.ValueKind == JsonValueKind.False, + JsonValueKind.Number => left.GetDecimal() == right.GetDecimal(), + JsonValueKind.String => string.Equals(left.GetString(), right.GetString(), StringComparison.Ordinal), + JsonValueKind.Array => left.EnumerateArray().SequenceEqual(right.EnumerateArray(), JsonElementComparer.Instance), + JsonValueKind.Object => ObjectEquals(left, right), + _ => left.GetRawText() == right.GetRawText(), + }; + } + + public static bool Contains(JsonElement source, JsonElement target) + { + if (source.ValueKind == JsonValueKind.String && target.ValueKind == JsonValueKind.String) + { + return (source.GetString() ?? string.Empty).Contains(target.GetString() ?? string.Empty, StringComparison.Ordinal); + } + + if (source.ValueKind == JsonValueKind.Array) + { + return source.EnumerateArray().Any(item => JsonEquals(item, target)); + } + + if (source.ValueKind == JsonValueKind.Object && target.ValueKind == JsonValueKind.Object) + { + foreach (var property in target.EnumerateObject()) + { + if (!source.TryGetProperty(property.Name, out var sourceValue) || !Contains(sourceValue, property.Value)) + { + return false; + } + } + + return true; + } + + return JsonEquals(source, target); + } + + public static int Compare(JsonElement left, JsonElement right) + { + int leftRank = GetKindRank(left.ValueKind); + int rightRank = GetKindRank(right.ValueKind); + if (leftRank != rightRank) + { + return leftRank.CompareTo(rightRank); + } + + return left.ValueKind switch + { + JsonValueKind.Null or JsonValueKind.Undefined => 0, + JsonValueKind.False or JsonValueKind.True => left.GetBoolean().CompareTo(right.GetBoolean()), + JsonValueKind.Number => left.GetDecimal().CompareTo(right.GetDecimal()), + JsonValueKind.String => string.Compare(left.GetString(), right.GetString(), StringComparison.Ordinal), + _ => string.Compare(left.GetRawText(), right.GetRawText(), StringComparison.Ordinal), + }; + } + + private static bool ObjectEquals(JsonElement left, JsonElement right) + { + var leftProperties = left.EnumerateObject().OrderBy(static p => p.Name, StringComparer.Ordinal).ToArray(); + var rightProperties = right.EnumerateObject().OrderBy(static p => p.Name, StringComparer.Ordinal).ToArray(); + if (leftProperties.Length != rightProperties.Length) + { + return false; + } + + for (int i = 0; i < leftProperties.Length; i++) + { + if (!string.Equals(leftProperties[i].Name, rightProperties[i].Name, StringComparison.Ordinal) || + !JsonEquals(leftProperties[i].Value, rightProperties[i].Value)) + { + return false; + } + } + + return true; + } + + private static int GetKindRank(JsonValueKind valueKind) + { + return valueKind switch + { + JsonValueKind.Null or JsonValueKind.Undefined => 0, + JsonValueKind.False => 1, + JsonValueKind.True => 2, + JsonValueKind.Number => 3, + JsonValueKind.String => 4, + JsonValueKind.Array => 5, + JsonValueKind.Object => 6, + _ => 7, + }; + } + + private sealed class JsonElementComparer : IEqualityComparer + { + public static readonly JsonElementComparer Instance = new(); + + public bool Equals(JsonElement x, JsonElement y) + { + return JsonEquals(x, y); + } + + public int GetHashCode(JsonElement obj) + { + return obj.GetRawText().GetHashCode(StringComparison.Ordinal); + } + } +} \ No newline at end of file diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterIndexSegment.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterIndexSegment.cs new file mode 100644 index 0000000..3e5b6cc --- /dev/null +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterIndexSegment.cs @@ -0,0 +1,7 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Azure.Data.Cosmos.Shell.Parser; + +internal sealed record FilterIndexSegment(int Index, bool Optional) : FilterPathSegment(Optional); \ No newline at end of file diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterIterateSegment.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterIterateSegment.cs new file mode 100644 index 0000000..50cbcc5 --- /dev/null +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterIterateSegment.cs @@ -0,0 +1,7 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Azure.Data.Cosmos.Shell.Parser; + +internal sealed record FilterIterateSegment(bool Optional) : FilterPathSegment(Optional); \ No newline at end of file diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterPathExpression.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterPathExpression.cs new file mode 100644 index 0000000..c3db485 --- /dev/null +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterPathExpression.cs @@ -0,0 +1,143 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Azure.Data.Cosmos.Shell.Parser; + +using System.Text.Json; + +using Azure.Data.Cosmos.Shell.Core; + +internal class FilterPathExpression : Expression +{ + public FilterPathExpression(Token rootToken, IReadOnlyList segments, int? length = null) + { + this.RootToken = rootToken ?? throw new ArgumentNullException(nameof(rootToken)); + this.Segments = segments ?? throw new ArgumentNullException(nameof(segments)); + this.ExpressionLength = length ?? rootToken.Length; + } + + public Token RootToken { get; } + + public IReadOnlyList Segments { get; } + + public int ExpressionLength { get; } + + public override int Start => this.RootToken.Start; + + public override int Length => this.ExpressionLength; + + public static FilterPathExpression CreateShorthand(Token sourceToken, string propertyName) + { + return new FilterPathExpression(sourceToken, [new FilterPropertySegment(propertyName, false)], propertyName.Length); + } + + public override Task EvaluateAsync(ShellInterpreter interpreter, CommandState currentState, CancellationToken cancellationToken) + { + var evaluatedResult = currentState.Result?.ConvertShellObject(DataType.Json); + if (evaluatedResult is not JsonElement root) + { + throw new InvalidOperationException("filter path requires a JSON pipeline value"); + } + + var values = new List { root.Clone() }; + bool sequence = false; + + foreach (var segment in this.Segments) + { + var next = new List(); + + foreach (var value in values) + { + switch (segment) + { + case FilterPropertySegment propertySegment: + if (value.ValueKind == JsonValueKind.Object) + { + if (value.TryGetProperty(propertySegment.Name, out var propertyValue)) + { + next.Add(propertyValue.Clone()); + } + else + { + next.Add(FilterExpressionUtilities.NullElement()); + } + } + else if (propertySegment.Optional) + { + next.Add(FilterExpressionUtilities.NullElement()); + } + else + { + throw new InvalidOperationException($"Cannot read property '{propertySegment.Name}' from {value.ValueKind}"); + } + + break; + + case FilterIndexSegment indexSegment: + if (value.ValueKind == JsonValueKind.Array) + { + if (indexSegment.Index >= 0 && indexSegment.Index < value.GetArrayLength()) + { + next.Add(value[indexSegment.Index].Clone()); + } + else + { + next.Add(FilterExpressionUtilities.NullElement()); + } + } + else if (indexSegment.Optional) + { + next.Add(FilterExpressionUtilities.NullElement()); + } + else + { + throw new InvalidOperationException($"Cannot index {value.ValueKind} with [{indexSegment.Index}]"); + } + + break; + + case FilterIterateSegment iterateSegment: + if (value.ValueKind == JsonValueKind.Array) + { + foreach (var item in value.EnumerateArray()) + { + next.Add(item.Clone()); + } + + sequence = true; + } + else if (iterateSegment.Optional) + { + next.Add(FilterExpressionUtilities.NullElement()); + } + else + { + throw new InvalidOperationException($"Cannot iterate over {value.ValueKind}"); + } + + break; + } + } + + values = next; + } + + if (sequence) + { + return Task.FromResult(new ShellSequence(values)); + } + + if (values.Count == 0) + { + return Task.FromResult(new ShellJson(FilterExpressionUtilities.NullElement())); + } + + return Task.FromResult(new ShellJson(values[0])); + } + + public override void Accept(IAstVisitor visitor) + { + visitor.Visit(this); + } +} \ No newline at end of file diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterPathSegment.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterPathSegment.cs new file mode 100644 index 0000000..84854c1 --- /dev/null +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterPathSegment.cs @@ -0,0 +1,7 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Azure.Data.Cosmos.Shell.Parser; + +internal abstract record FilterPathSegment(bool Optional); \ No newline at end of file diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterPipeExpression.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterPipeExpression.cs new file mode 100644 index 0000000..96f8411 --- /dev/null +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterPipeExpression.cs @@ -0,0 +1,61 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Azure.Data.Cosmos.Shell.Parser; + +using System.Linq; +using Azure.Data.Cosmos.Shell.Core; + +internal class FilterPipeExpression : Expression +{ + public FilterPipeExpression(Expression left, Token pipeToken, Expression right) + { + this.Left = left ?? throw new ArgumentNullException(nameof(left)); + this.PipeToken = pipeToken ?? throw new ArgumentNullException(nameof(pipeToken)); + this.Right = right ?? throw new ArgumentNullException(nameof(right)); + } + + public Expression Left { get; } + + public Token PipeToken { get; } + + public Expression Right { get; } + + public override int Start => this.Left.Start; + + public override int Length => this.Right.End - this.Left.Start; + + public override async Task EvaluateAsync(ShellInterpreter interpreter, CommandState currentState, CancellationToken cancellationToken) + { + var leftResult = await this.Left.EvaluateAsync(interpreter, currentState, cancellationToken); + + if (leftResult is ShellSequence sequence) + { + var outputs = new List(); + foreach (var element in sequence.Elements) + { + var nestedState = new CommandState { Result = new ShellJson(element.Clone()) }; + var rightResult = await this.Right.EvaluateAsync(interpreter, nestedState, cancellationToken); + if (rightResult is ShellSequence nestedSequence) + { + outputs.AddRange(nestedSequence.Elements.Select(static e => e.Clone())); + } + else + { + outputs.Add(FilterExpressionUtilities.ToJsonElement(rightResult)); + } + } + + return new ShellSequence(outputs); + } + + var state = new CommandState { Result = leftResult }; + return await this.Right.EvaluateAsync(interpreter, state, cancellationToken); + } + + public override void Accept(IAstVisitor visitor) + { + visitor.Visit(this); + } +} \ No newline at end of file diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterPropertySegment.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterPropertySegment.cs new file mode 100644 index 0000000..35005c0 --- /dev/null +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Expression/FilterPropertySegment.cs @@ -0,0 +1,7 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Azure.Data.Cosmos.Shell.Parser; + +internal sealed record FilterPropertySegment(string Name, bool Optional) : FilterPathSegment(Optional); \ No newline at end of file diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ExpressionParser.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ExpressionParser.cs index 50a8d95..4ad2d0d 100644 --- a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ExpressionParser.cs +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ExpressionParser.cs @@ -14,6 +14,7 @@ internal class ExpressionParser private bool initialized = false; private Token? lastNonNullToken; private bool aborted = false; + private bool inFilterMode = false; public ExpressionParser(Lexer lexer) { @@ -175,6 +176,32 @@ public Expression ParseExpression() return this.ParseOr(); } + /// + /// Parses an expression that allows the jq-style filter pipe operator (|) at the + /// top level. This is used by the filter command's expression argument; the regular + /// entry point intentionally stops at the outer pipe so it + /// does not swallow shell-level | separators in assignments, conditions, etc. + /// + public Expression ParseFilterExpression() + { + this.Initialize(); + if (this.aborted) + { + return this.CreateAbortExpression(); + } + + var previous = this.inFilterMode; + this.inFilterMode = true; + try + { + return this.ParsePipeExpression(); + } + finally + { + this.inFilterMode = previous; + } + } + public Expression ParsePrimaryExpression() { this.Initialize(); @@ -186,6 +213,32 @@ public Expression ParsePrimaryExpression() return this.ParsePrimary(); } + private Expression ParsePipeExpression() + { + if (this.aborted) + { + return this.CreateAbortExpression(); + } + + var left = this.ParseOr(); + + while (!this.aborted && this.Check(TokenType.Pipe)) + { + var pipeToken = this.Current; + if (pipeToken == null) + { + this.AbortUnexpectedEnd(); + return this.CreateAbortExpression(); + } + + this.Advance(); + var right = this.ParseOr(); + left = new FilterPipeExpression(left, pipeToken, right); + } + + return left; + } + private bool Check(TokenType type) { this.Initialize(); @@ -500,7 +553,7 @@ private Expression ParsePrimary() } // Regular expression parsing - var expr = this.ParseOr(); + var expr = this.ParsePipeExpression(); var closeTokenExpr = this.Consume(TokenType.CloseParenthesis, MessageService.GetString("expression_error_expected_close_paren")); return new ParensExpression(openToken, expr, closeTokenExpr); } @@ -620,6 +673,16 @@ private Expression ParsePrimary() return new ConstantExpression(token, new ShellBool(false)); } + if (string.Equals(token.Value, "null", StringComparison.OrdinalIgnoreCase)) + { + return new ConstantExpression(token, new ShellJson(FilterExpressionUtilities.NullElement())); + } + + if (this.inFilterMode && token.Value.StartsWith(".", StringComparison.Ordinal)) + { + return this.ParseFilterPathExpression(token); + } + // Check for variables if (token.Value.StartsWith("$") && token.Value.Length > 1) { @@ -646,6 +709,16 @@ private Expression ParsePrimary() return new VariableExpression(token, varValue); } + if (this.inFilterMode && this.Check(TokenType.OpenParenthesis)) + { + return this.ParseFilterCallExpression(token); + } + + if (this.inFilterMode && this.IsFilterZeroArgBuiltin(token.Value)) + { + return new FilterCallExpression(token, []); + } + return new ConstantExpression(token, new ShellIdentifier(token.Value)); } @@ -923,6 +996,32 @@ void SkipTrivia() SkipTrivia(); + // Shorthand object property: {id, status} + if (!this.IsAtEnd && this.currentToken != null && + (this.currentToken.Type == TokenType.Comma || this.currentToken.Type == TokenType.CloseBrace) && + keyToken.Type == TokenType.Identifier) + { + properties[key] = FilterPathExpression.CreateShorthand(keyToken, keyToken.Value); + + if (this.currentToken.Type == TokenType.Comma) + { + this.Advance(); + SkipTrivia(); + if (!this.IsAtEnd && this.currentToken?.Type == TokenType.CloseBrace) + { + var shorthandTrailingBrace = this.currentToken!; + this.Advance(); + return new JsonExpression(lbrace, shorthandTrailingBrace, properties); + } + + continue; + } + + var shorthandCloseBrace = this.currentToken!; + this.Advance(); + return new JsonExpression(lbrace, shorthandCloseBrace, properties); + } + // Expect colon if (this.IsAtEnd || this.currentToken?.Type != TokenType.Colon) { @@ -950,7 +1049,7 @@ void SkipTrivia() } // Parse property value as a full expression - var value = this.ParseOr(); + var value = this.ParsePipeExpression(); // Add to properties if key valid if (key != null) @@ -1070,7 +1169,7 @@ private Expression ParseJsonArray() } // Parse next element as a full expression - var expr = this.ParseOr(); + var expr = this.ParsePipeExpression(); elements.Add(expr); this.SkipWhitespace(); @@ -1268,4 +1367,164 @@ private CommandShellWordParser CreateCommandShellWordParser() () => this.Advance(), () => this.ParsePrimary(), token => token.Type == TokenType.CloseParenthesis); + + private bool IsFilterZeroArgBuiltin(string value) + { + return string.Equals(value, "length", StringComparison.Ordinal) || + string.Equals(value, "keys", StringComparison.Ordinal) || + string.Equals(value, "type", StringComparison.Ordinal); + } + + private Expression ParseFilterCallExpression(Token nameToken) + { + var arguments = new List(); + this.Consume(TokenType.OpenParenthesis, MessageService.GetString("expression_error_expected_open_paren")); + + while (!this.IsAtEnd && !this.Check(TokenType.CloseParenthesis)) + { + arguments.Add(this.ParsePipeExpression()); + if (this.Check(TokenType.Comma)) + { + this.Advance(); + continue; + } + + break; + } + + var closeToken = this.Consume(TokenType.CloseParenthesis, MessageService.GetString("expression_error_expected_close_paren")); + return new FilterCallExpression(nameToken, arguments, (closeToken.Start + closeToken.Length) - nameToken.Start); + } + + private Expression ParseFilterPathExpression(Token firstToken) + { + var segments = new List(); + int end = firstToken.Start + firstToken.Length; + this.AddDotIdentifierSegments(firstToken, segments); + + while (!this.IsAtEnd) + { + if (this.Check(TokenType.OpenBracket)) + { + var openBracket = this.Current; + this.Advance(); + + if (this.Check(TokenType.String)) + { + var propertyToken = this.Current; + this.Advance(); + var propertyCloseBracket = this.Consume(TokenType.CloseBracket, MessageService.GetString("expression_error_expected_close_bracket")); + var questionToken = this.TryConsumeQuestion(); + end = propertyCloseBracket.Start + propertyCloseBracket.Length; + if (questionToken != null) + { + end = questionToken.Start + questionToken.Length; + } + + segments.Add(new FilterPropertySegment(propertyToken?.Value ?? string.Empty, questionToken != null)); + continue; + } + + if (this.Check(TokenType.CloseBracket)) + { + var closeBracket = this.Current; + this.Advance(); + var questionToken = this.TryConsumeQuestion(); + end = (closeBracket?.Start ?? openBracket?.Start ?? end) + (closeBracket?.Length ?? 1); + if (questionToken != null) + { + end = questionToken.Start + questionToken.Length; + } + + segments.Add(new FilterIterateSegment(questionToken != null)); + continue; + } + + var indexToken = this.Consume(TokenType.Number, MessageService.GetString("expression_error_expected_array_index")); + int index = int.TryParse(indexToken.Value, out var parsedIndex) ? parsedIndex : 0; + var indexedCloseBracket = this.Consume(TokenType.CloseBracket, MessageService.GetString("expression_error_expected_close_bracket")); + var indexQuestionToken = this.TryConsumeQuestion(); + end = indexedCloseBracket.Start + indexedCloseBracket.Length; + if (indexQuestionToken != null) + { + end = indexQuestionToken.Start + indexQuestionToken.Length; + } + + segments.Add(new FilterIndexSegment(index, indexQuestionToken != null)); + continue; + } + + if (this.Check(TokenType.String)) + { + var token = this.currentToken; + this.Advance(); + var questionToken = this.TryConsumeQuestion(); + end = token!.Start + token.Length; + if (questionToken != null) + { + end = questionToken.Start + questionToken.Length; + } + + segments.Add(new FilterPropertySegment(token.Value, questionToken != null)); + continue; + } + + if (this.Check(TokenType.Identifier) && this.currentToken != null && this.currentToken.Value.StartsWith(".", StringComparison.Ordinal)) + { + var token = this.currentToken; + this.Advance(); + var questionToken = this.AddDotIdentifierSegments(token, segments); + end = token.Start + token.Length; + if (questionToken != null) + { + end = questionToken.Start + questionToken.Length; + } + + continue; + } + + break; + } + + return new FilterPathExpression(firstToken, segments, end - firstToken.Start); + } + + private Token? AddDotIdentifierSegments(Token token, List segments) + { + var value = token.Value; + if (value == ".") + { + return null; + } + + var parts = value.Split('.', StringSplitOptions.RemoveEmptyEntries); + foreach (var part in parts) + { + segments.Add(new FilterPropertySegment(part, false)); + } + + if (parts.Length > 0) + { + var questionToken = this.TryConsumeQuestion(); + if (questionToken != null) + { + segments[^1] = ((FilterPropertySegment)segments[^1]) with { Optional = true }; + return questionToken; + } + } + + return null; + } + + private Token? TryConsumeQuestion() + { + if (this.Check(TokenType.Question)) + { + var token = this.Current; + this.Advance(); + return token; + } + + return null; + } } \ No newline at end of file diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/IAstVisitor.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/IAstVisitor.cs index 1884f22..a14a27a 100644 --- a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/IAstVisitor.cs +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/IAstVisitor.cs @@ -16,6 +16,8 @@ internal interface IAstVisitor void Visit(BinaryOperatorExpression binaryOperatorExpression); + void Visit(FilterPipeExpression filterPipeExpression); + void Visit(ParensExpression parensExpression); void Visit(JsonExpression jsonExpression); @@ -24,6 +26,10 @@ internal interface IAstVisitor void Visit(JSonPathExpression jSonPathExpression); + void Visit(FilterPathExpression filterPathExpression); + + void Visit(FilterCallExpression filterCallExpression); + void Visit(InterpolatedStringExpression interpolatedStringExpression); void Visit(VariableExpression variableExpression); @@ -90,6 +96,13 @@ public virtual void Visit(BinaryOperatorExpression binaryOperatorExpression) binaryOperatorExpression.Right.Accept(this); } + public virtual void Visit(FilterPipeExpression filterPipeExpression) + { + filterPipeExpression.Left.Accept(this); + this.VisitToken(filterPipeExpression.PipeToken); + filterPipeExpression.Right.Accept(this); + } + public virtual void Visit(ParensExpression parensExpression) { this.VisitToken(parensExpression.LParToken); @@ -106,6 +119,20 @@ public virtual void Visit(JSonPathExpression jSonPathExpression) this.VisitToken(jSonPathExpression.VariableToken); } + public virtual void Visit(FilterPathExpression filterPathExpression) + { + this.VisitToken(filterPathExpression.RootToken); + } + + public virtual void Visit(FilterCallExpression filterCallExpression) + { + this.VisitToken(filterCallExpression.NameToken); + foreach (var argument in filterCallExpression.Arguments) + { + argument.Accept(this); + } + } + public virtual void Visit(JsonArrayExpression jSonArrayExpression) { this.VisitToken(jSonArrayExpression.LBracketToken); diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Lexer.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Lexer.cs index c28b170..9758908 100644 --- a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Lexer.cs +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/Lexer.cs @@ -206,6 +206,11 @@ public enum TokenType /// Not, + /// + /// Optional access token ('?'). + /// + Question, + /// /// End of line token (newline characters). /// @@ -387,6 +392,10 @@ private static bool IsVariableIdentifierPart(char ch) this.Advance(); return new Token(TokenType.Not, "!", startPosition, 1); + case '?': + this.Advance(); + return new Token(TokenType.Question, "?", startPosition, 1); + case '\n': case '\r': var eolStartPos = this.position; diff --git a/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellSequence.cs b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellSequence.cs new file mode 100644 index 0000000..fff19bc --- /dev/null +++ b/CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellSequence.cs @@ -0,0 +1,30 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Azure.Data.Cosmos.Shell.Parser; + +using System.Linq; +using System.Text.Json; + +internal class ShellSequence : ShellObject +{ + public ShellSequence(IEnumerable elements) + : base(DataType.Json) + { + this.Elements = elements.Select(static e => e.Clone()).ToList(); + } + + public IReadOnlyList Elements { get; } + + public override object ConvertShellObject(DataType type) + { + var array = FilterExpressionUtilities.ToJsonArray(this.Elements); + return type switch + { + DataType.Json => array, + DataType.Text => array.GetRawText(), + _ => new ShellJson(array).ConvertShellObject(type), + }; + } +} \ No newline at end of file diff --git a/CosmosDBShell/lang/en.ftl b/CosmosDBShell/lang/en.ftl index 50c60b8..1729f09 100644 --- a/CosmosDBShell/lang/en.ftl +++ b/CosmosDBShell/lang/en.ftl @@ -270,6 +270,11 @@ command-results-limit_reached = command-jq-description = Commandline JSON processor command-jq-description-args = Arguments for the jq command +command-filter-description = Filter and transform piped JSON with the native filter expression language +command-filter-description-expression = Filter expression to evaluate against the piped JSON input +command-filter-error-no_expression = Filter expression is missing. +command-filter-error-no_input = The filter command requires piped JSON input. +command-filter-error-invalid_input = The filter command can only process JSON input. command-ftab-description = Render piped JSON as a table command-ftab-description-fields = Comma-separated field names to include in the table (Optional) command-ftab-description-take = Limit the number of rendered rows (Optional) @@ -495,7 +500,10 @@ json_error_unclosed_array_bracket = Unclosed array bracket. json_error_result_evaluation_null = Result evaluation returned null. expression_error_no_more_tokens = No more tokens +expression_error_expected_open_paren = Expected '(' expression_error_expected_close_paren = Expected ')' after expression +expression_error_expected_close_bracket = Expected ']' +expression_error_expected_array_index = Expected array index expression_error_invalid_number = Invalid number format: {$value} expression_error_unexpected_end = Unexpected end of expression expression_error_unexpected_token = Unexpected token: {$type} '{$value}' diff --git a/docs/commands.md b/docs/commands.md index 7e20874..9ed7fc4 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -432,6 +432,31 @@ Arguments: [args] Arguments to pass to jq (Optional) ``` +### filter + +Native JSON filter and transformation command. + +```text +Usage: filter expression + +Arguments: + expression Filter expression to evaluate against piped JSON input +``` + +Notes: + +- `filter` uses the built-in filter expression language documented in `docs/filter-v1-spec.md`. +- `filter` keeps structured JSON results in the shell pipeline. +- `jq` remains available as the external full-featured option when installed. + +Examples: + +```text +ls | filter '.items | length' +ls | filter '.items | map({"Volcano Name": .["Volcano Name"], Country})' +query "SELECT * FROM c" | filter '.items | select(.status == "active")' +``` + ### ftab JSON to table processor. diff --git a/docs/filter-v1-spec.md b/docs/filter-v1-spec.md new file mode 100644 index 0000000..5df0d1c --- /dev/null +++ b/docs/filter-v1-spec.md @@ -0,0 +1,447 @@ +# Filter Expression Language v1 + +This document defines the v1 grammar and semantics for the native `filter` command. + +`filter` is a built-in JSON transformation command for CosmosDBShell. It uses a jq-inspired expression language, but it is not a jq implementation and does not attempt full jq compatibility. + +## Goals + +- Support common JSON shaping and filtering workflows inside CosmosDBShell. +- Preserve structured pipeline results so `filter` can feed later commands. +- Keep the language small enough to implement and document clearly. + +## Non-Goals + +- Full jq compatibility. +- jq modules, variables, function definitions, or streaming mode. +- Regex support in v1. +- Update-assignment operators such as `|=`, `+=`, `del`, or `setpath`. +- Exact jq multi-result generator semantics. + +## Command Contract + +The command surface for v1 is: + +```text +filter +``` + +- `expression` is required. +- The expression is usually quoted at the shell level, for example: `filter '.items[0]'`. +- Input comes from the current pipeline value. +- Output is written back into the shell's structured command result. + +## Data Model + +The language operates on JSON values: + +- `null` +- booleans +- numbers +- strings +- arrays +- objects + +The current pipeline value is referred to as `.`. + +## Evaluation Model + +`filter` evaluates expressions eagerly. + +- Expressions consume one input JSON value. +- Expressions produce one JSON value. +- Operations that would naturally produce multiple values in jq are materialized as arrays in v1 when needed for shell-safe behavior. +- Intermediate results remain JSON values and can be passed to downstream commands. + +## Lexical Rules + +### Whitespace + +Whitespace may appear between tokens and is ignored unless it appears inside a string literal. + +### Identifiers + +Identifiers are used for object shorthand fields and builtin names. + +```text +identifier = letter , { letter | digit | '_' } +``` + +Examples: + +- `id` +- `items` +- `sort_by` + +### Literals + +Supported literals: + +- `null` +- `true` +- `false` +- integer and decimal numbers +- double-quoted strings with JSON-style escapes + +Examples: + +- `null` +- `true` +- `42` +- `3.14` +- `"active"` + +## Grammar + +The grammar below uses a compact EBNF-style notation. + +```text +filter-expression = expression ; + +expression = pipe-expression ; + +pipe-expression = comparison-expression , { '|' , comparison-expression } ; + +comparison-expression + = primary-expression , [ comparison-operator , primary-expression ] ; + +comparison-operator + = '==' | '!=' | '<' | '<=' | '>' | '>=' ; + +primary-expression = path-expression + | literal + | builtin-expression + | array-constructor + | object-constructor + | '(' , expression , ')' ; + +path-expression = '.' , { path-segment } ; + +path-segment = '.' , identifier , [ '?' ] + | '.' , string-literal , [ '?' ] + | '[' , string-literal , ']' , [ '?' ] + | '[' , integer-literal , ']' , [ '?' ] + | '[' , ']' , [ '?' ] ; + +builtin-expression = 'length' + | 'keys' + | 'type' + | 'contains' , '(' , expression , ')' + | 'map' , '(' , expression , ')' + | 'select' , '(' , expression , ')' + | 'sort_by' , '(' , expression , ')' ; + +array-constructor = '[' , [ expression , { ',' , expression } ] , ']' ; + +object-constructor = '{' , [ object-field , { ',' , object-field } ] , '}' ; + +object-field = identifier + | identifier , ':' , expression + | string-literal , ':' , expression ; +``` + +## Semantics + +### Identity + +`.` returns the current input unchanged. + +Examples: + +- `.` +- `. | type` + +### Property Access + +`.name` reads the property `name` from an object. + +- If the input is an object and the property exists, the property value is returned. +- If the property does not exist, v1 returns `null`. +- If the input is not an object, evaluation fails unless optional access is used. + +Examples: + +- `.id` +- `.items` +- `.metadata.status` +- `.["Volcano Name"]` +- `."Volcano Name"` + +### Optional Property Access + +`.name?` behaves like `.name`, but it suppresses type/access errors. + +- If the input is not an object, the result is `null`. +- If the property does not exist, the result is `null`. + +### Array Index Access + +`.[n]` reads the array element at zero-based index `n`. + +- If the input is an array and the index exists, the element is returned. +- If the index is out of range, the result is `null`. +- If the input is not an array, evaluation fails unless optional access is used. + +Examples: + +- `.[0]` +- `.items[0]` + +### Optional Array Index Access + +`.[n]?` suppresses type/access errors and returns `null` when the access cannot be satisfied. + +### Array Iteration + +`.[]` iterates the values of an array. + +In v1, iteration is materialized for shell-safe semantics: + +- if the input is an array, the result is an array containing the iterated values +- if the input is an object, `.[]` is not supported in v1 unless object iteration is explicitly added later +- if the input is not an array, evaluation fails unless optional iteration is used + +Examples: + +- `.items[]` +- `[.items[] | .id]` + +### Optional Iteration + +`.[]?` returns `null` when the input is not an array. + +### Pipe + +`a | b` evaluates `a` first, then evaluates `b` against the result of `a`. + +Examples: + +- `.items | length` +- `.items | map(.id)` +- `.items[0] | {id, status}` + +### Comparison + +Comparisons produce booleans. + +Supported operators: + +- `==` +- `!=` +- `<` +- `<=` +- `>` +- `>=` + +Examples: + +- `.status == "active"` +- `.count > 10` + +### Array Construction + +`[expr1, expr2, ...]` evaluates each expression against the current input and constructs a JSON array from the results. + +Examples: + +- `[.id, .status]` +- `[.items[0], .items[1]]` + +### Object Construction + +`{...}` constructs a JSON object. + +Supported forms: + +- shorthand field capture: `{id, status}` +- explicit mapping: `{id: .id, state: .status}` +- string keys: `{"item-id": .id}` + +For shorthand fields, `{id}` is equivalent to `{id: .id}`. + +## Builtins + +### `length` + +Returns the length of the input value. + +- array: number of elements +- object: number of properties +- string: number of characters +- null: `0` +- number and boolean: runtime error in v1 + +Examples: + +- `.items | length` +- `.name | length` + +### `keys` + +Returns an array of object property names. + +- input must be an object +- result ordering should be deterministic + +Example: + +- `.item | keys` + +### `type` + +Returns one of the strings: + +- `"null"` +- `"boolean"` +- `"number"` +- `"string"` +- `"array"` +- `"object"` + +Example: + +- `.payload | type` + +### `contains(expr)` + +Evaluates `expr` against the current input and returns whether the input contains the resulting value. + +v1 behavior: + +- string contains string substring +- array contains element by JSON equality +- object contains object subset by matching keys and values +- other types use JSON equality + +Examples: + +- `.tags | contains("prod")` +- `.item | contains({status: "active"})` + +### `map(expr)` + +Applies `expr` to each element of the input array and returns an array of transformed values. + +- input must be an array +- each element becomes the current input while evaluating `expr` + +Examples: + +- `.items | map(.id)` +- `.items | map({id, status})` + +### `select(expr)` + +Filters an input array by applying `expr` to each element and keeping elements where the result is `true`. + +- input must be an array +- `expr` is evaluated per element +- only boolean `true` keeps an element in v1 + +Examples: + +- `.items | select(.status == "active")` +- `.items | select(.count > 10)` + +### `sort_by(expr)` + +Sorts an input array using the value produced by `expr` for each element. + +- input must be an array +- keys must be mutually comparable +- stable sorting is preferred + +Examples: + +- `.items | sort_by(.id)` +- `.items | sort_by(.timestamp)` + +## Type Rules + +- Property access requires an object unless optional access is used. +- Index access and array builtins require an array unless optional access is used. +- `map`, `select`, and `sort_by` require arrays. +- `keys` requires an object. +- `length` supports arrays, objects, strings, and null. +- Comparisons require values that the implementation can compare deterministically. + +## Error Behavior + +v1 should distinguish these error classes: + +### Parse Errors + +Invalid syntax in the expression. + +Examples: + +- `.items[` +- `{id: }` + +### Unsupported Feature Errors + +Syntax that looks jq-like but is outside the v1 contract. + +Examples: + +- `.items[] | .id, .status` +- `reduce .items[] as $x (...)` +- `.name |= "x"` +- `test("abc")` + +The diagnostic should say the construct is not supported by `filter` v1 and should suggest using `jq` when full jq behavior is required. + +### Runtime Type Errors + +The expression is syntactically valid but is applied to the wrong input shape. + +Examples: + +- `.id` applied to an array +- `map(.id)` applied to an object + +## Supported Examples + +```text +. +.items[0] +.items[0]?.id +.items | length +.items | map(.id) +.items | select(.status == "active") +.items | sort_by(.id) +.items | map({id, status}) +{id, status} +[.id, .status] +``` + +## Unsupported Examples + +```text +.[] | .id, .status +reduce .items[] as $item (0; . + $item) +.items |= map(.id) +def pickId: .id; pickId +test("^abc") +inputs +``` + +## Implementation Notes + +The intended implementation model for v1 is: + +- parse into a small AST +- evaluate against `JsonElement` +- materialize iteration results as arrays where needed +- store the final JSON value back into `CommandState.Result` + +This language should be implemented as a native shell feature, not as a compatibility layer over the external `jq` executable. + +## Open Questions For Later Versions + +- Should object iteration with `.[]` be supported? +- Should `select(expr)` accept jq-like truthiness or only strict `true`? +- Should negative array indices be supported? +- Should array slicing be added? +- Should string helper functions be added before regex support? +- Should multi-result semantics be expanded beyond array materialization? diff --git a/docs/navigation.md b/docs/navigation.md index 59c4976..87e72cc 100644 --- a/docs/navigation.md +++ b/docs/navigation.md @@ -177,6 +177,7 @@ These commands accept and process piped JSON: | `echo` | Outputs piped value or extracts path | | `cd` | Can use path to select target | | `delete` | Deletes item specified by piped JSON | +| `filter` | Filters/transforms piped JSON with the native filter language | | `jq` | Filters/transforms piped JSON | | `ftab` | Formats piped JSON as table |