diff --git a/.github/workflows/parsers3.yml b/.github/workflows/parsers3.yml new file mode 100644 index 00000000..71f21b9d --- /dev/null +++ b/.github/workflows/parsers3.yml @@ -0,0 +1,72 @@ +name: Parsers Compatibility + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +env: + # Arrow -> TimeZones >=1.6 -> InlineStrings requires Parsers 2. The pinned + # Parsers 1/3 lanes therefore resolve TimeZones 1.5.9, whose build step is + # incompatible with multiple Julia thread pools (JuliaTime/TimeZones.jl#429). + JULIA_NUM_THREADS: '1' + +jobs: + parsers-1: + name: Julia 1.9 - Parsers 1.1.2 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: julia-actions/setup-julia@v3 + with: + version: '1.9' + - uses: julia-actions/cache@v3 + - name: Test with Parsers 1 + shell: julia --project=. --color=yes {0} + run: | + import Pkg + Pkg.add(Pkg.PackageSpec(name = "Parsers", version = v"1.1.2")) + Pkg.test(; coverage = true) + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v7 + with: + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true + + parsers-3: + name: Julia ${{ matrix.version }} - Parsers 3 + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + version: + - '1.10' + - '1' + steps: + - uses: actions/checkout@v7 + - uses: actions/checkout@v7 + with: + repository: JuliaData/Parsers.jl + ref: 83c7142fb714cb87261ef38eec7ab103444eb30d + path: .ci/Parsers-v3 + - uses: julia-actions/setup-julia@v3 + with: + version: ${{ matrix.version }} + - uses: julia-actions/cache@v3 + - name: Test with Parsers 3 PR head + env: + JSON_TEST_PARSERS_PATH: ${{ github.workspace }}/.ci/Parsers-v3 + shell: julia --project=. --color=yes {0} + run: | + import Pkg + Pkg.develop(Pkg.PackageSpec(path = ENV["JSON_TEST_PARSERS_PATH"])) + Pkg.test(; coverage = true) + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v7 + with: + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true diff --git a/Project.toml b/Project.toml index de0e3b26..04298ba2 100644 --- a/Project.toml +++ b/Project.toml @@ -20,7 +20,7 @@ JSONArrowExt = ["ArrowTypes"] [compat] Arrow = "2.8.0" ArrowTypes = "2.2" -Parsers = "1, 2" +Parsers = "1, 2, 3" PrecompileTools = "1" StructUtils = "2.8.4" julia = "1.9" @@ -29,8 +29,9 @@ julia = "1.9" Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45" ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Arrow", "Pkg", "Tar", "Test"] +test = ["Arrow", "Pkg", "Random", "Tar", "Test"] diff --git a/README.md b/README.md index e8ef0593..1fe19425 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ JSON.json("test.json", j) # Download json data and parse into a DataFrame using HTTP, JSON, Tables, DataFrames resp = HTTP.get("https://raw.githubusercontent.com/altair-viz/vega_datasets/master/vega_datasets/_data/wheat.json") -# null=missing will read json `null` as Julia `missing; `allownan=true` parses all numbers as Float64 +# null=missing reads JSON `null` as Julia `missing`; allownan=true also accepts NaN and infinity df = DataFrame(Tables.dictrowtable(JSON.parse(resp.body; null=missing, allownan=true))) ``` diff --git a/docs/src/migrate.md b/docs/src/migrate.md index 21e005ca..e650df8a 100644 --- a/docs/src/migrate.md +++ b/docs/src/migrate.md @@ -4,6 +4,40 @@ This guide provides an overview of how to migrate your code from either the pre- --- +## Number parsing with Parsers 3 + +JSON.jl accepts Parsers 1, 2, and 3. Parsers 3 requires Julia 1.10 or later; +Julia 1.9 continues to use the compatibility path. This dependency update does +not change standards-compliant JSON syntax or the four adaptive untyped result +types. + +There are six observable corrections: + +- `typemin(Int64)` now materializes as `Int64` instead of `BigInt`. +- `allownan=true` only enables configured and native special-value spellings. + It no longer forces finite integers through `Float64`. +- A leading `+` on a finite number is rejected. This removes a nonstandard + extension; `+Inf` remains available behind `allownan=true`. +- Negative zero keeps its sign, including extreme exponent forms such as + `-0e291` that lost the sign on the Parsers 2 compatibility path. +- Long finite decimals now use correct Base-compatible rounding. This also + avoids rare Parsers 1/2 failures and wrong results on valid JSON numbers. +- Supported built-in typed numeric targets use the original token bytes with + Parsers 3. This avoids intermediate `Float64` rounding for `Float32` and + `BigFloat`. Custom `StructUtils` styles and field lifts keep the prior + adaptive conversion path. + +On the Parsers 1 and 2 compatibility paths, JSON gives validated finite float +and overflowed integer spans to Base. A byte-vector input therefore needs a +temporary `String` for these legacy dependency versions. Parsers 3 converts the +original string or byte span directly and does not have this compatibility cost. + +Code that depended on `allownan=true` to coerce every finite number to +`Float64` should request a typed target instead, for example +`JSON.parse(source, Vector{Float64}; allownan=true)`. + +--- + ## Migration guide from pre-1.0 -> 1.0 ### Writing JSON diff --git a/docs/src/reading.md b/docs/src/reading.md index 25a10ba2..4078c114 100644 --- a/docs/src/reading.md +++ b/docs/src/reading.md @@ -25,10 +25,10 @@ The core JSON parsing machinery is hence built around having an `AbstractVector{ Each entrypoint function first calls [`JSON.lazy`](@ref), which will consume the JSON input until the type of the next JSON value can be identified (`{` for objects, `[` for arrays, `"` for strings, `t` for true, `f` for false, `n` for null, and `-` or a digit for numbers). [`JSON.lazy`](@ref) returns a [`JSON.LazyValue`](@ref), which wraps the JSON input buffer (`AbstractVector{UInt8}` or `AbstractString`), and marks the byte position the value starts at, the type of the value, and any keyword arguments that were provided that may affect parsing. Currently supported parsing-specific keyword arguments to [`JSON.lazy`](@ref) (and thus all other entrypoint functions) include: - - `allownan::Bool = false`: whether "special" float values shoudl be allowed while parsing (`NaN`, `Inf`, `-Inf`); these values are specifically _not allowed_ in the JSON spec, but many JSON libraries allow reading/writing + - `allownan::Bool = false`: whether "special" float values should be allowed while parsing (`NaN`, `Inf`, `-Inf`); these values are specifically _not allowed_ in the JSON spec, but many JSON libraries allow reading/writing. Finite numbers keep their normal adaptive type selection. - `ninf::String = "-Infinity"`: the string that will be used to parse `-Inf` if `allownan=true` - `inf::String = "Infinity"`: the string that will be used to parse `Inf` if `allownan=true` - - `nan::String = "NaN"`: the string that will be sued to parse `NaN` if `allownan=true` + - `nan::String = "NaN"`: the string that will be used to parse `NaN` if `allownan=true` - `jsonlines::Bool = false`: whether the JSON input should be treated as an implicit array, with newlines separating individual JSON elements with no leading `'['` or trailing `']'` characters. Common in logging or streaming workflows. Defaults to `true` when used with [`JSON.parsefile`](@ref) and the filename extension is `.jsonl` or `ndjson`. Note this ensures that parsing will _always_ return an array at the root-level. - Materialization-specific keyword arguments (i.e. they affect materialization, but not parsing) - `dicttype = JSON.Object{String, Any}`: type to parse JSON objects as by default (recursively) @@ -135,6 +135,12 @@ Under the hood, this `getindex` call is really calling `JSON.parse(lazyvalue)`. | `null` | `nothing` | | `true/false` | `Bool` | +Finite integers use `Int64` when possible, including `typemin(Int64)`, and +promote to `BigInt` outside that range. Finite decimal or exponent forms use +`Float64` when representable and promote overflowed values to `BigFloat`. +Setting `allownan=true` only adds special-value spellings; it does not force +finite values through `Float64`. + Mostly vanilla, but what is `JSON.Object`? It is a custom `AbstractDict` using an internal linked-list implementation that preserves insertion order, behaves as a drop-in replacement for `Dict`, and allows memory and performance benefits vs. `Dict` for small # of entries. It also supports natural JSON-object-like syntax for accessing or setting values, like `x.g.h.i` and `x.c = false`. @@ -255,6 +261,13 @@ date = JSON.parse("\"2023-05-08\"", Date) # Date("2023-05-08") ``` +With Parsers 3 on Julia 1.10 or later, supported built-in numeric targets are +converted from the original JSON token bytes. In particular, `Float32` and +`BigFloat` do not pass through an intermediate `Float64`. Numeric fields with +a custom `StructUtils` style or field tag keep the adaptive value-and-lift +path, so existing custom conversions continue to receive `Int64`, `BigInt`, +`Float64`, or `BigFloat` values. + ### Type conversions and handling nulls JSON.jl provides smart handling for Union types, especially for dealing with potentially null values: diff --git a/src/lazy.jl b/src/lazy.jl index e1226994..743af1ae 100644 --- a/src/lazy.jl +++ b/src/lazy.jl @@ -54,10 +54,10 @@ In this example, we only parsed as much of the `very_large_json_object` as was r Then we fully materialized `y` into `z`, which is now a normal Julia object. We can now mutate or access values in `z`. Currently supported keyword arguments include: - - `allownan::Bool = false`: whether "special" float values shoudl be allowed while parsing (`NaN`, `Inf`, `-Inf`); these values are specifically _not allowed_ in the JSON spec, but many JSON libraries allow reading/writing + - `allownan::Bool = false`: whether "special" float values should be allowed while parsing (`NaN`, `Inf`, `-Inf`); these values are specifically _not allowed_ in the JSON spec, but many JSON libraries allow reading/writing. Finite numbers keep their normal `Int64`/`BigInt`/`Float64`/`BigFloat` classification. - `ninf::String = "-Infinity"`: the string that will be used to parse `-Inf` if `allownan=true` - `inf::String = "Infinity"`: the string that will be used to parse `Inf` if `allownan=true` - - `nan::String = "NaN"`: the string that will be sued to parse `NaN` if `allownan=true` + - `nan::String = "NaN"`: the string that will be used to parse `NaN` if `allownan=true` - `jsonlines::Bool = false`: whether the JSON input should be treated as an implicit array, with newlines separating individual JSON elements with no leading `'['` or trailing `']'` characters. Common in logging or streaming workflows. Defaults to `true` when used with `JSON.parsefile` and the filename extension is `.jsonl` or `ndjson`. Note this ensures that parsing will _always_ return an array at the root-level. - `duplicate_keys::Symbol = :overwrite`: how repeated object keys are handled. `:overwrite` preserves the default last-value-wins behavior. `:error` throws [`JSON.DuplicateKeyError`](@ref). - `isroot::Bool = true`: whether this is the root LazyValue encompassing the entire json buffer. If `false` parses only the first JSON value and ignores trailing characters. @@ -242,7 +242,9 @@ function _lazy(buf, pos::Int, len, b, opts, isroot=false) getbyte(buf, pos + 3) == UInt8('s') && getbyte(buf, pos + 4) == UInt8('e') return LazyValue(buf, pos, JSONTypes.FALSE, opts, isroot) - elseif b == UInt8('-') || (UInt8('0') <= b <= UInt8('9')) || (opts.allownan && (b == UInt8('+') || firstbyteeq(opts.nan, b) || firstbyteeq(opts.ninf, b) || firstbyteeq(opts.inf, b))) + elseif b == UInt8('-') || (UInt8('0') <= b <= UInt8('9')) || + (opts.allownan && (b in (UInt8('+'), UInt8('I'), UInt8('i'), UInt8('N'), UInt8('n')) || + firstbyteeq(opts.nan, b) || firstbyteeq(opts.ninf, b) || firstbyteeq(opts.inf, b))) return LazyValue(buf, pos, JSONTypes.NUMBER, opts, isroot) else error = InvalidJSON @@ -526,34 +528,31 @@ function parsestring(x::LazyValue) invalid(error, buf, pos, "string") end -# core JSON number parsing function -# we rely on functionality in Parsers to help infer what kind -# of number we're parsing; valid return types include: -# Int64, BigInt, Float64 or BigFloat -const INT64_OVERFLOW_VAL = div(typemax(Int64), 10) -const INT64_OVERFLOW_DIGIT = typemax(Int64) % 10 +# Core JSON number parsing. JSON owns token classification so Parsers only +# converts spans that already satisfy the JSON number grammar. This keeps JSON +# syntax independent of Parsers' broader Base-compatible grammar. +const _PARSERS_V3 = isdefined(Parsers, :parsenext) + +@inline _numberbytes(buf::AbstractString) = codeunits(buf) +@inline _numberbytes(buf) = buf +@inline _numberspan(buf::Union{String,SubString{String}}, first::Int, last::Int) = + SubString(buf, first, last) +@inline function _numberspan(buf::AbstractString, first::Int, last::Int) + str = String(buf) + return SubString(str, first, last) +end +@inline _numberspan(buf, first::Int, last::Int) = String(view(buf, first:last)) -macro check_special(special, value) - esc(quote - pos = startpos - b = getbyte(buf, pos) - bytes = codeunits($special) - i = 1 - while b == @inbounds(bytes[i]) - pos += 1 - i += 1 - i > length(bytes) && break - if pos > len - error = UnexpectedEOF - @goto invalid - end - b = getbyte(buf, pos) - i += 1 - end - if i > length(bytes) - return NumberResult($value), pos - end - end) +@noinline function _legacyfloatfallback(buf, first::Int, last::Int) + source = _numberspan(buf, first, last) + value = Base.tryparse(Float64, source) + if value !== nothing && isfinite(value) + return NumberResult(value) + end + big = Base.tryparse(BigFloat, source) + big === nothing && return nothing + value = Float64(big) + return isfinite(value) ? NumberResult(value) : NumberResult(big) end const INT = 0x00 @@ -579,108 +578,189 @@ isfloat(x::NumberResult) = x.tag == FLOAT isbigint(x::NumberResult) = x.tag == BIGINT isbigfloat(x::NumberResult) = x.tag == BIGFLOAT -@inline function parsenumber(x::LazyValue) - buf = getbuf(x) - pos::Int = getpos(x) - len = getlength(buf) - opts = getopts(x) - b = getbyte(buf, pos) - startpos = pos - isneg = isfloat = overflow = false - if !opts.allownan - val = Int64(0) - isneg = b == UInt8('-') - if isneg || b == UInt8('+') # spec doesn't allow leading +, but we do +@inline function _specialend(buf, pos::Int, len::Int, special::String) + bytes = codeunits(special) + n = length(bytes) + (n == 0 || pos + n - 1 > len) && return 0 + @inbounds for i in 1:n + getbyte(buf, pos + i - 1) == bytes[i] || return 0 + end + return pos + n +end + +@inline function _parselegacyspecial(buf, startpos::Int, len::Int) + pos = startpos + @inbounds if getbyte(buf, pos) == UInt8('-') || getbyte(buf, pos) == UInt8('+') + pos += 1 + pos > len && return nothing + end + wordstart = pos + @inbounds while pos <= len + b = getbyte(buf, pos) | UInt8(0x20) + UInt8('a') <= b <= UInt8('z') || break + pos += 1 + end + pos == wordstart && return nothing + value = Base.tryparse(Float64, _numberspan(buf, startpos, pos - 1)) + (value !== nothing && !isfinite(value)) || return nothing + return NumberResult(value), pos +end + +# Return the first byte after the token and whether the token is floating-point. +# A return position equal to `startpos` means invalid JSON number syntax. +@inline function _numbertoken(buf, startpos::Int, len::Int) + pos = startpos + @inbounds if getbyte(buf, pos) == UInt8('-') + pos += 1 + pos > len && return startpos, false + end + + @inbounds b = getbyte(buf, pos) + if b == UInt8('0') + pos += 1 + if pos <= len + @inbounds b = getbyte(buf, pos) + UInt8('0') <= b <= UInt8('9') && return startpos, false + end + elseif UInt8('1') <= b <= UInt8('9') + pos += 1 + @inbounds while pos <= len && UInt8('0') <= getbyte(buf, pos) <= UInt8('9') + pos += 1 + end + else + return startpos, false + end + + isfloat = false + if pos <= len && @inbounds(getbyte(buf, pos) == UInt8('.')) + isfloat = true + pos += 1 + (pos > len || @inbounds(!(UInt8('0') <= getbyte(buf, pos) <= UInt8('9')))) && + return startpos, false + pos += 1 + @inbounds while pos <= len && UInt8('0') <= getbyte(buf, pos) <= UInt8('9') pos += 1 - if pos > len - error = UnexpectedEOF - @goto invalid - end - b = getbyte(buf, pos) end - # Parse integer part, check for leading zeros (invalid JSON) - if b == UInt8('0') + end + + if pos <= len + @inbounds b = getbyte(buf, pos) + if b == UInt8('e') || b == UInt8('E') + isfloat = true pos += 1 if pos <= len - b = getbyte(buf, pos) - if UInt8('0') <= b <= UInt8('9') - error = InvalidNumber - @goto invalid - end + @inbounds b = getbyte(buf, pos) + (b == UInt8('+') || b == UInt8('-')) && (pos += 1) end - elseif UInt8('1') <= b <= UInt8('9') - while UInt8('0') <= b <= UInt8('9') - digit = Int64(b - UInt8('0')) - if val > INT64_OVERFLOW_VAL || (val == INT64_OVERFLOW_VAL && digit > INT64_OVERFLOW_DIGIT) - overflow = true - break - end - val = Int64(10) * val + digit + (pos > len || @inbounds(!(UInt8('0') <= getbyte(buf, pos) <= UInt8('9')))) && + return startpos, false + pos += 1 + @inbounds while pos <= len && UInt8('0') <= getbyte(buf, pos) <= UInt8('9') pos += 1 - pos > len && break - b = getbyte(buf, pos) end - if overflow - bval = BigInt(val) - while UInt8('0') <= b <= UInt8('9') - digit = BigInt(b - UInt8('0')) - bval = BigInt(10) * bval + digit - pos += 1 - pos > len && break - b = getbyte(buf, pos) - end + end + end + return pos, isfloat +end + +@static if isdefined(Parsers, :parsenext) + @inline function _parsefinitenumber(buf, startpos::Int, nextpos::Int, isfloat::Bool) + bytes = _numberbytes(buf) + last = nextpos - 1 + if isfloat + value, code = Parsers.parsefloat(Float64, bytes, startpos, last) + if code == Parsers.RC_OK || code == Parsers.RC_UNDERFLOW + return NumberResult(value) + elseif code == Parsers.RC_OVERFLOW + return NumberResult(Parsers.parse(BigFloat, bytes, startpos, last)) end else - error = InvalidNumber - @goto invalid - end - # Check for decimal or exponent - if b == UInt8('.') || b == UInt8('e') || b == UInt8('E') - isfloat = true - # in strict JSON spec, we need at least one digit after the decimal - if b == UInt8('.') - pos += 1 - if pos > len - error = UnexpectedEOF - @goto invalid - end - b = getbyte(buf, pos) - if !(UInt8('0') <= b <= UInt8('9')) - error = InvalidNumber - @goto invalid - end + value, code = Parsers.parseint(Int64, bytes, startpos, last) + code == Parsers.RC_OK && return NumberResult(value) + if code == Parsers.RC_OVERFLOW + value, code = Parsers.parsebigint(bytes, startpos, last) + code == Parsers.RC_OK && return NumberResult(value) end end + return nothing end - if isfloat || opts.allownan - if opts.allownan - # check for NaN, Inf, -Inf - @check_special(opts.nan, NaN) - @check_special(opts.inf, Inf) - @check_special(opts.ninf, -Inf) - end - res = Parsers.xparse2(Float64, buf, startpos, len) - if !opts.allownan && Parsers.specialvalue(res.code) - # if we overflowed, then let's try BigFloat - bres = Parsers.xparse2(BigFloat, buf, startpos, len) - if !Parsers.invalid(bres.code) - return NumberResult(bres.val), startpos + Int(bres.tlen) - end - end - if Parsers.invalid(res.code) - error = InvalidNumber - @goto invalid + + @inline function _parsenativespecial(buf, startpos::Int, len::Int) + bytes = _numberbytes(buf) + value, nextpos, code = Parsers.parsenext(Float64, bytes, startpos, len) + code == Parsers.RC_OK && !isfinite(value) || return nothing + return NumberResult(value), nextpos + end +elseif isdefined(Parsers, :xparse2) + @inline function _parsefinitenumber(buf, startpos::Int, nextpos::Int, isfloat::Bool) + last = nextpos - 1 + isfloat && return _legacyfloatfallback(buf, startpos, last) + + bytes = _numberbytes(buf) + span = nextpos - startpos + res = Parsers.xparse2(Int64, bytes, startpos, last) + if !Parsers.invalid(res.code) && Int(res.tlen) == span + return NumberResult(res.val) end - return NumberResult(res.val), Int(startpos + res.tlen) - else - if overflow - return NumberResult(isneg ? -bval : bval), pos - else - return NumberResult(isneg ? -val : val), pos + value = Base.tryparse(BigInt, _numberspan(buf, startpos, last)) + value === nothing && return nothing + value == typemin(Int64) && return NumberResult(typemin(Int64)) + return NumberResult(value) + end + + @inline _parsenativespecial(buf, startpos::Int, len::Int) = + _parselegacyspecial(buf, startpos, len) +else + # Parsers 1 does not provide xparse2. Its integer xparse kernel is safe for + # exact spans, but its float kernel can abort on valid extreme decimals. + # Keep the legacy adapter narrow: JSON validates syntax, xparse converts + # Int64 values, and Base converts wider integers and finite float spans. + const _PARSERS1_OPTIONS = Parsers.Options() + + @inline function _parsefinitenumber(buf, startpos::Int, nextpos::Int, isfloat::Bool) + last = nextpos - 1 + isfloat && return _legacyfloatfallback(buf, startpos, last) + + bytes = _numberbytes(buf) + span = nextpos - startpos + value, code, vpos, vlen, tlen = + Parsers.xparse(Int64, bytes, startpos, last, _PARSERS1_OPTIONS) + if Parsers.ok(code) && vpos == startpos && vlen == span && tlen == span + return NumberResult(value) end + value = Base.tryparse(BigInt, _numberspan(buf, startpos, last)) + value === nothing && return nothing + value == typemin(Int64) && return NumberResult(typemin(Int64)) + return NumberResult(value) end -@label invalid + @inline _parsenativespecial(buf, startpos::Int, len::Int) = + _parselegacyspecial(buf, startpos, len) +end + +@inline function parsenumber(x::LazyValue) + buf = getbuf(x) + startpos::Int = getpos(x) + len = getlength(buf) + opts = getopts(x) + + if opts.allownan + pos = _specialend(buf, startpos, len, opts.nan) + pos != 0 && return NumberResult(NaN), pos + pos = _specialend(buf, startpos, len, opts.inf) + pos != 0 && return NumberResult(Inf), pos + pos = _specialend(buf, startpos, len, opts.ninf) + pos != 0 && return NumberResult(-Inf), pos + end + + nextpos, isfloat = _numbertoken(buf, startpos, len) + if nextpos != startpos + result = _parsefinitenumber(buf, startpos, nextpos, isfloat) + result === nothing || return result, nextpos + elseif opts.allownan + result = _parsenativespecial(buf, startpos, len) + result === nothing || return result + end invalid(InvalidNumber, buf, startpos, "number") end diff --git a/src/parse.jl b/src/parse.jl index 36dba098..4e79da79 100644 --- a/src/parse.jl +++ b/src/parse.jl @@ -451,6 +451,89 @@ function customlazylift(style::JSONReadStyle, ::Type{T}, x::LazyValues, tags) wh return nothing end +struct DirectNumberResult{T} + value::T + pos::Int +end + +@inline _directnumber(::Type, ::LazyValue) = nothing + +@inline function _hasconfiguredspecial(x::LazyValue) + opts = getopts(x) + opts.allownan || return false + buf = getbuf(x) + pos = getpos(x) + len = getlength(buf) + return _specialend(buf, pos, len, opts.nan) != 0 || + _specialend(buf, pos, len, opts.inf) != 0 || + _specialend(buf, pos, len, opts.ninf) != 0 +end + +@static if isdefined(Parsers, :parsenext) + const _JSONFixedInteger = Union{ + Int8, Int16, Int32, Int64, Int128, + UInt8, UInt16, UInt32, UInt64, UInt128, + } + const _JSONFixedFloat = Union{Float32, Float64} + + @inline function _directnumber(::Type{T}, x::LazyValue) where {T<:_JSONFixedInteger} + buf = getbuf(x) + startpos = getpos(x) + nextpos, isfloat = _numbertoken(buf, startpos, getlength(buf)) + (nextpos == startpos || isfloat) && return nothing + bytes = _numberbytes(buf) + value, code = Parsers.parseint(T, bytes, startpos, nextpos - 1) + code == Parsers.RC_OK || return nothing + return DirectNumberResult(value, nextpos) + end + + @inline function _directnumber(::Type{BigInt}, x::LazyValue) + buf = getbuf(x) + startpos = getpos(x) + nextpos, isfloat = _numbertoken(buf, startpos, getlength(buf)) + (nextpos == startpos || isfloat) && return nothing + bytes = _numberbytes(buf) + value, code = Parsers.parsebigint(bytes, startpos, nextpos - 1) + code == Parsers.RC_OK || return nothing + return DirectNumberResult(value, nextpos) + end + + @inline function _directnumber(::Type{T}, x::LazyValue) where {T<:_JSONFixedFloat} + buf = getbuf(x) + startpos = getpos(x) + nextpos, _ = _numbertoken(buf, startpos, getlength(buf)) + nextpos == startpos && return nothing + bytes = _numberbytes(buf) + value, code = Parsers.parsefloat(T, bytes, startpos, nextpos - 1) + (code == Parsers.RC_OK || code == Parsers.RC_OVERFLOW || + code == Parsers.RC_UNDERFLOW) || return nothing + return DirectNumberResult(value, nextpos) + end + + @inline function _directnumber(::Type{Float16}, x::LazyValue) + buf = getbuf(x) + startpos = getpos(x) + nextpos, _ = _numbertoken(buf, startpos, getlength(buf)) + nextpos == startpos && return nothing + bytes = _numberbytes(buf) + value, parsedpos, code = Parsers.parsenext(Float16, bytes, startpos, nextpos - 1) + (parsedpos == nextpos && (code == Parsers.RC_OK || code == Parsers.RC_OVERFLOW || + code == Parsers.RC_UNDERFLOW)) || return nothing + return DirectNumberResult(value, nextpos) + end + + @inline function _directnumber(::Type{BigFloat}, x::LazyValue) + buf = getbuf(x) + startpos = getpos(x) + nextpos, _ = _numbertoken(buf, startpos, getlength(buf)) + nextpos == startpos && return nothing + bytes = _numberbytes(buf) + value = Parsers.tryparse(BigFloat, bytes, startpos, nextpos - 1) + value === nothing && return nothing + return DirectNumberResult(value, nextpos) + end +end + function StructUtils.lift(style::JSONReadStyle, ::Type{T}, x::LazyValues) where {T<:AbstractArray{E,0}} where {E} m = T(undef) m[1], pos = StructUtils.lift(style, E, x) @@ -471,6 +554,15 @@ function StructUtils.lift(style::JSONReadStyle, ::Type{T}, x::LazyValues, tags=( end return str, pos elseif type == JSONTypes.NUMBER + # Parsers 3 can convert supported built-in numeric targets directly + # from the original bytes. Keep custom style and tagged lift calls on + # the adaptive path below, where they continue to receive JSON's + # default Int64/BigInt/Float64/BigFloat value. + if _PARSERS_V3 && style.style isa StructUtils.DefaultStyle && isempty(tags) && + !_hasconfiguredspecial(x) + direct = _directnumber(T, x) + direct === nothing || return direct.value, direct.pos + end num, pos = parsenumber(x) if isint(num) T === Int64 && return num.int, pos diff --git a/test/parse.jl b/test/parse.jl index bbea56ba..d495eaf9 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -1,10 +1,19 @@ -using JSON, StructUtils, UUIDs, Dates, Test +using JSON, StructUtils, UUIDs, Dates, Random, Test struct CustomJSONStyle <: JSON.JSONStyle end struct RefValueStyle <: JSON.JSONStyle end struct DateStringStyle <: JSON.JSONStyle end struct DateObjectStyle <: JSON.JSONStyle end struct DateMaterializedObjectStyle <: JSON.JSONStyle end +struct NumericLiftStyle <: JSON.JSONStyle end + +struct SpecialFloatField + value::Float32 +end + +struct SpecialIntField + value::Int +end struct A a::Int @@ -231,6 +240,10 @@ end any::Any &(choosetype=x -> x.type[] == "int" ? @NamedTuple{type::String, value::Int} : x.type[] == "float" ? @NamedTuple{type::String, value::Float64} : @NamedTuple{type::String, value::String},) end +@tags struct TaggedNumeric + value::Float32 &(json=(lift=x -> Float32(x) + 2.0f0, route=:adaptive)) +end + # https://github.com/JuliaIO/JSON.jl/issues/453 - custom JSONStyle dictlike dispatch @kwdef struct DictlikeViaCustomStyle vals::Dict{String,Int} = Dict{String,Int}() @@ -243,6 +256,8 @@ StructUtils.dictlike(::CustomJSONStyle, ::Type{DictlikeViaCustomStyle}) = true StructUtils.structlike(::RefValueStyle, ::Type{Base.RefValue{Int}}) = false StructUtils.lower(::RefValueStyle, x::Base.RefValue{Int}) = x[] StructUtils.lift(::RefValueStyle, ::Type{Base.RefValue{Int}}, x::Integer) = Ref{Int}(x), nothing +StructUtils.lift(::NumericLiftStyle, ::Type{Float32}, x::Float64, tags) = + (Float32(x) + 1.0f0, nothing) JSON.lower(::DateStringStyle, d::Date) = string(d) JSON.lift(::DateStringStyle, ::Type{Date}, x::String) = Date(x) @@ -404,9 +419,21 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ @test_throws ArgumentError JSON.parse("nula") @test_throws ArgumentError JSON.parse("nul") @test_throws ArgumentError JSON.parse("trub") - # allownan for parsing normally invalid json values + # allownan changes special-value acceptance, not finite-number promotion @test JSON.parse("NaN"; allownan=true) === NaN - @test JSON.parse("Inf"; inf="Inf", allownan=true) === Inf + @test JSON.parse("Inf"; allownan=true) === Inf + @test JSON.parse("-Inf"; allownan=true) === -Inf + @test JSON.parse("Infinity"; allownan=true) === Inf + @test JSON.parse("-Infinity"; allownan=true) === -Inf + @test isequal(JSON.parse("[Inf,NaN,-Infinity]"; allownan=true), [Inf, NaN, -Inf]) + @test JSON.parse("custom-inf"; inf="custom-inf", allownan=true) === Inf + @test JSON.parse("Inf"; inf="custom-inf", allownan=true) === Inf + @test_throws ArgumentError JSON.parse("+1"; allownan=true) + @test JSON.parse(string(typemax(Int64)); allownan=true) === typemax(Int64) + @test JSON.parse("9007199254740993"; allownan=true) === Int64(9007199254740993) + @test JSON.parse(string(typemax(Int64)), Int64; allownan=true) === typemax(Int64) + @test JSON.parse(string(typemax(Int128)), Int128; allownan=true) === typemax(Int128) + @test JSON.parse(string(typemax(UInt64)), UInt64; allownan=true) === typemax(UInt64) # jsonlines support @test JSON.parse("1"; jsonlines=true) == [1] @test JSON.parse("1 \t"; jsonlines=true) == [1] @@ -493,6 +520,10 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ @test JSON.parse("9223372036854775805") === 9223372036854775805 @test JSON.parse("9223372036854775806") === 9223372036854775806 @test JSON.parse("9223372036854775807") === 9223372036854775807 + @test JSON.parse("-9223372036854775808") === typemin(Int64) + @test JSON.parse("[7,-9223372036854775808]")[2] === typemin(Int64) + x = JSON.parse("-9223372036854775809") + @test x isa BigInt && x == -9223372036854775809 # promote to BigInt x = JSON.parse("9223372036854775808") # only == here because BigInt don't compare w/ === @@ -501,8 +532,79 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ @test x isa BigInt && x == 170141183460469231731687303715884105727 x = JSON.parse("170141183460469231731687303715884105728") @test x isa BigInt && x == 170141183460469231731687303715884105728 + # Every returned BigInt owns its storage, including the Parsers 1/2 fallback. + wide = "1234567890123456789012345678901234567890" + x = JSON.parse(wide) + y = JSON.parse(wide) + @test x !== y + Base.GMP.MPZ.add_ui!(x, 1) + @test y == Base.parse(BigInt, wide) + @test JSON.parse(wide) == y # BigFloat @test JSON.parse("1.7976931348623157e310") == big"1.7976931348623157e310" + @test JSON.parse("1e310") isa BigFloat + + # Parsers 1/2 can fail or misparse valid long decimals. Their + # compatibility path converts every validated finite float span with Base. + @test JSON.parse("295574326048237151328925.8099133506971425945276929554326e-440") === 0.0 + @test isequal( + JSON.parse("-645846793726181672171.9101155724413627413656362746354480124e-379"), + -0.0, + ) + bounds_source = "-773185451005006305224330936226383685.195e3" + @test JSON.parse(bounds_source) === Base.parse(Float64, bounds_source) + for source in ( + "-75738806850214820018096823497.7e229", + "0.72741733550162454424961322208253163690E+61", + "0.0725793004898340574524597074187135e294", + "-0.02180916574168124355037332825156547e110", + "0.73032752417478587296191914738492447036E+169", + "0.95599741872715652854E+201", + "-0.056906042106688574231E+248", + ) + expected = Base.parse(Float64, source) + @test JSON.parse(source) === expected + @test JSON.parse(Vector{UInt8}(codeunits(source))) === expected + end + @test isequal(JSON.parse("-0.0e100"), -0.0) + + # A fixed-seed differential gate covers wide integers and the complete + # JSON float grammar against a high-precision rounding oracle. + rng = MersenneTwister(0x8259_2026) + for _ in 1:20_000 + ndigits = rand(rng, 1:90) + digits = string( + rand(rng, '1':'9'), + String(rand(rng, '0':'9', ndigits - 1)), + ) + source = (rand(rng, Bool) ? "-" : "") * digits + @test JSON.parse(source) == Base.parse(BigInt, source) + end + setprecision(BigFloat, 512) do + for _ in 1:20_000 + intpart = rand(rng, Bool) ? "0" : string( + rand(rng, '1':'9'), + String(rand(rng, '0':'9', rand(rng, 0:35))), + ) + frac = "." * String(rand(rng, '0':'9', rand(rng, 1:40))) + expo = rand(rng, Bool) ? "" : string( + rand(rng, Bool) ? 'e' : 'E', + rand(rng, ("", "+", "-")), + rand(rng, 0:500), + ) + source = (rand(rng, Bool) ? "-" : "") * intpart * frac * expo + exact = Base.parse(BigFloat, source) + rounded = Float64(exact) + got = JSON.parse(source) + if isinf(rounded) + @test got isa BigFloat + @test got == exact + else + @test got isa Float64 + @test isequal(got, rounded) + end + end + end # zeros @test JSON.parse("0") === Int64(0) @@ -518,12 +620,14 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ @test JSON.parse("-0.00e-01234567890123456789") == big"0.0" @test JSON.parse("0e291") === 0.0 @test JSON.parse("0e292") === 0.0 - @test JSON.parse("0e347") == big"0.0" - @test JSON.parse("0e348") == big"0.0" - @test JSON.parse("-0e291") === 0.0 - @test JSON.parse("-0e292") === 0.0 - @test JSON.parse("-0e347") == big"0.0" - @test JSON.parse("-0e348") == big"0.0" + @test JSON.parse("0e347") === 0.0 + @test JSON.parse("0e348") === 0.0 + for source in ("-0e291", "-0e292") + @test JSON.parse(source) === -0.0 + @test JSON.parse(Vector{UInt8}(codeunits(source))) === -0.0 + end + @test JSON.parse("-0e347") === -0.0 + @test JSON.parse("-0e348") === -0.0 @test JSON.parse("2e-324") === 0.0 # extremes @test JSON.parse("1e310") == big"1e310" @@ -555,6 +659,59 @@ JSON.lift(::DateMaterializedObjectStyle, ::Type{Date}, x::JSON.Object) = Date(x[ @test_throws ArgumentError JSON.parse("+1") end @testset "JSON.parse with types" begin + @test JSON.parse("1.0", Int) == 1 + @test_throws InexactError JSON.parse("1.5", Int) + @test JSON.parse(string(typemax(Int128)), Int128) == typemax(Int128) + @test JSON.parse(Vector{UInt8}(codeunits(string(typemax(Int128)))), Int128) == typemax(Int128) + + for (T, overflow, underflow) in ( + (Float16, "1e10", "-1e-20"), + (Float32, "1e50", "-1e-100"), + (Float64, "1e400", "-1e-400"), + ) + over = JSON.parse(overflow, T) + under = JSON.parse(underflow, T) + @test over === T(Inf) + @test iszero(under) && signbit(under) + end + @test JSON.parse("1.5", Float16) === Float16(1.5) + + for T in (Float16, Float32, Float64, BigFloat) + @test isinf(JSON.parse("123", T; inf="123", allownan=true)) + @test JSON.parse("456", T; ninf="456", allownan=true) == T(-Inf) + @test isnan(JSON.parse("789", T; nan="789", allownan=true)) + @test JSON.parse("1x", T; inf="1x", allownan=true) == T(Inf) + end + @test_throws InexactError JSON.parse("123", Int; inf="123", allownan=true) + @test JSON.parse( + "{\"value\":123}", SpecialFloatField; inf="123", allownan=true + ).value === Inf32 + @test_throws InexactError JSON.parse( + "{\"value\":123}", SpecialIntField; inf="123", allownan=true + ) + + @test JSON.parse("0.5", Float32; style=NumericLiftStyle()) === 1.5f0 + @test JSON.parse("{\"value\":0.5}", TaggedNumeric) == TaggedNumeric(2.5f0) + + if JSON._PARSERS_V3 + f32source = "1.000000059604644830901776231257827021181583404541015625" + expected32 = Base.parse(Float32, f32source) + @test expected32 != Float32(Base.parse(Float64, f32source)) + @test JSON.parse(f32source, Float32) === expected32 + @test JSON.parse(Vector{UInt8}(codeunits(f32source)), Float32) === expected32 + + f16source = "1.0004882812500000000000000000000000000000000000000000001" + @test JSON.parse(f16source, Float16) === Base.parse(Float16, f16source) + + setprecision(BigFloat, 256) do + source = "0.1000000000000000000000000000000000000000000000000000001" + expected = Base.parse(BigFloat, source) + @test expected != BigFloat(Base.parse(Float64, source)) + @test JSON.parse(source, BigFloat) == expected + @test JSON.parse(Vector{UInt8}(codeunits(source)), BigFloat) == expected + end + end + obj = JSON.parse("""{ "a": 1,"b": 2,"c": 3,"d": 4}""", A) @test obj == A(1, 2, 3, 4) @test JSON.parse("""{ "a": 1,"b": 2,"c": 3,"d": 4, "e": 5}""", A) == A(1, 2, 3, 4) diff --git a/test/trim_compile_tests.jl b/test/trim_compile_tests.jl index 8caddc0c..20360c25 100644 --- a/test/trim_compile_tests.jl +++ b/test/trim_compile_tests.jl @@ -18,6 +18,8 @@ function _prepare_trim_project(project_path::String, trim_project::String)::Noth try Pkg.activate(trim_project) Pkg.develop(Pkg.PackageSpec(path = project_path)) + parsers_path = get(ENV, "JSON_TEST_PARSERS_PATH", "") + isempty(parsers_path) || Pkg.develop(Pkg.PackageSpec(path = parsers_path)) Pkg.instantiate() finally if original_project !== nothing