From 0ac6e7810dd528a49e19e9718d6c8bcc882a9cf2 Mon Sep 17 00:00:00 2001 From: Josh <59419126+this-josh@users.noreply.github.com> Date: Tue, 22 Mar 2022 14:47:01 +0000 Subject: [PATCH 1/9] Enable quantile to determine the quantile over an entire matrix --- src/Statistics.jl | 1 + test/runtests.jl | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Statistics.jl b/src/Statistics.jl index 33fc737e..fc285377 100644 --- a/src/Statistics.jl +++ b/src/Statistics.jl @@ -958,6 +958,7 @@ function quantile!(v::AbstractVector, p::Union{AbstractArray, Tuple{Vararg{Real} end return map(x->_quantile(v, x, alpha=alpha, beta=beta), p) end +quantile!(arr::AbstractArray, p::Union{AbstractArray,Tuple{Vararg{Real}}}; sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = quantile!(arr[:], p, sorted=sorted, alpha=alpha, beta=alpha) quantile!(v::AbstractVector, p::Real; sorted::Bool=false, alpha::Real=1., beta::Real=alpha) = _quantile(_quantilesort!(v, sorted, p, p), p, alpha=alpha, beta=beta) diff --git a/test/runtests.jl b/test/runtests.jl index 783f6302..161dcba5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -612,6 +612,7 @@ end @test quantile(Any[1, 2, 3], Float16(0.5)) isa Float16 @test quantile(Any[1, Float16(2), 3], Float16(0.5)) isa Float16 @test quantile(Any[1, big(2), 3], Float16(0.5)) isa BigFloat + @test quantile(reshape(collect(1:100), (10, 10)), [0.00, 0.25, 0.50, 0.75, 1.00]) == [1.0, 25.75, 50.5, 75.25, 100.0] # Need a large vector to actually check consequences of partial sorting x = rand(50) From d6a32e1ffa086651735b1e96aeeaf7f7f8674a48 Mon Sep 17 00:00:00 2001 From: Josh <59419126+this-josh@users.noreply.github.com> Date: Tue, 22 Mar 2022 14:48:39 +0000 Subject: [PATCH 2/9] broke method to be over two lines --- src/Statistics.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Statistics.jl b/src/Statistics.jl index fc285377..76997faf 100644 --- a/src/Statistics.jl +++ b/src/Statistics.jl @@ -958,7 +958,8 @@ function quantile!(v::AbstractVector, p::Union{AbstractArray, Tuple{Vararg{Real} end return map(x->_quantile(v, x, alpha=alpha, beta=beta), p) end -quantile!(arr::AbstractArray, p::Union{AbstractArray,Tuple{Vararg{Real}}}; sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = quantile!(arr[:], p, sorted=sorted, alpha=alpha, beta=alpha) +quantile!(arr::AbstractArray, p::Union{AbstractArray,Tuple{Vararg{Real}}}; sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = + quantile!(arr[:], p, sorted=sorted, alpha=alpha, beta=alpha) quantile!(v::AbstractVector, p::Real; sorted::Bool=false, alpha::Real=1., beta::Real=alpha) = _quantile(_quantilesort!(v, sorted, p, p), p, alpha=alpha, beta=beta) From 69988834b8fec5cebacadc2bace7ba0aed862d87 Mon Sep 17 00:00:00 2001 From: this-josh <59419126+this-josh@users.noreply.github.com> Date: Wed, 30 Mar 2022 13:58:10 +0100 Subject: [PATCH 3/9] Change to ensure quantile is determined in place Co-authored-by: Milan Bouchet-Valat --- src/Statistics.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Statistics.jl b/src/Statistics.jl index 76997faf..2a00d8a5 100644 --- a/src/Statistics.jl +++ b/src/Statistics.jl @@ -958,8 +958,9 @@ function quantile!(v::AbstractVector, p::Union{AbstractArray, Tuple{Vararg{Real} end return map(x->_quantile(v, x, alpha=alpha, beta=beta), p) end -quantile!(arr::AbstractArray, p::Union{AbstractArray,Tuple{Vararg{Real}}}; sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = - quantile!(arr[:], p, sorted=sorted, alpha=alpha, beta=alpha) +quantile!(arr::AbstractArray, p::Union{AbstractArray,Tuple{Vararg{Real}}}; + sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = + quantile!(vec(arr), p, sorted=sorted, alpha=alpha, beta=alpha) quantile!(v::AbstractVector, p::Real; sorted::Bool=false, alpha::Real=1., beta::Real=alpha) = _quantile(_quantilesort!(v, sorted, p, p), p, alpha=alpha, beta=beta) From 9924bc75844444883ae5517a9964950ab2d1922e Mon Sep 17 00:00:00 2001 From: Josh Kirk <59419126+this-josh@users.noreply.github.com> Date: Wed, 30 Mar 2022 14:35:13 +0100 Subject: [PATCH 4/9] Change from quantile! to quantile Not sure what a meaningful way to work out the quantiles of a matrix inplace would look like, so remove the ! to make it clear it isn't inplace --- src/Statistics.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Statistics.jl b/src/Statistics.jl index 2a00d8a5..532bc6ff 100644 --- a/src/Statistics.jl +++ b/src/Statistics.jl @@ -958,9 +958,6 @@ function quantile!(v::AbstractVector, p::Union{AbstractArray, Tuple{Vararg{Real} end return map(x->_quantile(v, x, alpha=alpha, beta=beta), p) end -quantile!(arr::AbstractArray, p::Union{AbstractArray,Tuple{Vararg{Real}}}; - sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = - quantile!(vec(arr), p, sorted=sorted, alpha=alpha, beta=alpha) quantile!(v::AbstractVector, p::Real; sorted::Bool=false, alpha::Real=1., beta::Real=alpha) = _quantile(_quantilesort!(v, sorted, p, p), p, alpha=alpha, beta=beta) @@ -1076,6 +1073,10 @@ quantile(itr, p; sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = quantile(v::AbstractVector, p; sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = quantile!(sorted ? v : Base.copymutable(v), p; sorted=sorted, alpha=alpha, beta=beta) +quantile(arr::AbstractArray, p::Union{AbstractArray,Tuple{Vararg{Real}}}; + sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = + quantile!(vec(arr), p, sorted=sorted, alpha=alpha, beta=alpha) + ##### SparseArrays optimizations ##### From 9d0ba4c80b35aba9a32f71bfe641372ac1b3c179 Mon Sep 17 00:00:00 2001 From: Josh Kirk <59419126+this-josh@users.noreply.github.com> Date: Wed, 30 Mar 2022 14:52:43 +0100 Subject: [PATCH 5/9] made the array method nearly identical to the vector method Previous method caused type ambiguity, this is okay. --- src/Statistics.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Statistics.jl b/src/Statistics.jl index 532bc6ff..5c29ec11 100644 --- a/src/Statistics.jl +++ b/src/Statistics.jl @@ -1073,9 +1073,9 @@ quantile(itr, p; sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = quantile(v::AbstractVector, p; sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = quantile!(sorted ? v : Base.copymutable(v), p; sorted=sorted, alpha=alpha, beta=beta) -quantile(arr::AbstractArray, p::Union{AbstractArray,Tuple{Vararg{Real}}}; - sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = - quantile!(vec(arr), p, sorted=sorted, alpha=alpha, beta=alpha) +quantile(arr::AbstractArray, p; sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = + quantile!(sorted ? vec(arr) : Base.copymutable(vec(arr)), p; sorted=sorted, alpha=alpha, beta=beta) + ##### SparseArrays optimizations ##### From 819e65369e7fff911d913ea19b24e9d50933b031 Mon Sep 17 00:00:00 2001 From: Josh Kirk <59419126+this-josh@users.noreply.github.com> Date: Thu, 31 Mar 2022 09:20:37 +0100 Subject: [PATCH 6/9] Revert 9924bc7 Change from quantile to quantile! --- src/Statistics.jl | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Statistics.jl b/src/Statistics.jl index 5c29ec11..0686a6fe 100644 --- a/src/Statistics.jl +++ b/src/Statistics.jl @@ -958,6 +958,9 @@ function quantile!(v::AbstractVector, p::Union{AbstractArray, Tuple{Vararg{Real} end return map(x->_quantile(v, x, alpha=alpha, beta=beta), p) end +quantile!(arr::AbstractArray, p::Union{AbstractArray,Tuple{Vararg{Real}}}; + sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = + quantile!(vec(arr), p, sorted=sorted, alpha=alpha, beta=alpha) quantile!(v::AbstractVector, p::Real; sorted::Bool=false, alpha::Real=1., beta::Real=alpha) = _quantile(_quantilesort!(v, sorted, p, p), p, alpha=alpha, beta=beta) @@ -1073,11 +1076,6 @@ quantile(itr, p; sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = quantile(v::AbstractVector, p; sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = quantile!(sorted ? v : Base.copymutable(v), p; sorted=sorted, alpha=alpha, beta=beta) -quantile(arr::AbstractArray, p; sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = - quantile!(sorted ? vec(arr) : Base.copymutable(vec(arr)), p; sorted=sorted, alpha=alpha, beta=beta) - - - ##### SparseArrays optimizations ##### function cov(X::SparseMatrixCSC; dims::Int=1, corrected::Bool=true) From 4c8d6f1eaa4910ead14a65aa79a05c056918b813 Mon Sep 17 00:00:00 2001 From: Josh Kirk <59419126+this-josh@users.noreply.github.com> Date: Thu, 31 Mar 2022 09:30:50 +0100 Subject: [PATCH 7/9] Quantile!(matrix) Added a method which accepts q when determining quantiles over a matrix. This q is modified in place. Added a test to confirm this behaviour --- src/Statistics.jl | 8 ++++++-- test/runtests.jl | 5 +++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Statistics.jl b/src/Statistics.jl index 0686a6fe..bda1724a 100644 --- a/src/Statistics.jl +++ b/src/Statistics.jl @@ -959,10 +959,14 @@ function quantile!(v::AbstractVector, p::Union{AbstractArray, Tuple{Vararg{Real} return map(x->_quantile(v, x, alpha=alpha, beta=beta), p) end quantile!(arr::AbstractArray, p::Union{AbstractArray,Tuple{Vararg{Real}}}; - sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = + sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = quantile!(vec(arr), p, sorted=sorted, alpha=alpha, beta=alpha) -quantile!(v::AbstractVector, p::Real; sorted::Bool=false, alpha::Real=1., beta::Real=alpha) = +quantile!(q::AbstractArray, arr::AbstractArray, p::Union{AbstractArray,Tuple{Vararg{Real}}}; + sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = + quantile!(q, vec(arr), p, sorted=sorted, alpha=alpha, beta=alpha) + +quantile!(v::AbstractVector, p::Real; sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = _quantile(_quantilesort!(v, sorted, p, p), p, alpha=alpha, beta=beta) # Function to perform partial sort of v for quantiles in given range diff --git a/test/runtests.jl b/test/runtests.jl index 161dcba5..0768aeb5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -640,6 +640,11 @@ end @test quantile!(y, x, [0.1, 0.5, 0.9]) === y @test y ≈ [1.2, 2.0, 2.8] + x = reshape(collect(1:100), (10, 10)) + y = zeros(5) + @test quantile!(y, x, [0.00, 0.25, 0.50, 0.75, 1.00]) === y + @test y ≈ [1.0, 25.75, 50.5, 75.25, 100.0] + #tests for quantile calculation with configurable alpha and beta parameters v = [2, 3, 4, 6, 9, 2, 6, 2, 21, 17] From 36640881358fdd0e009130678629553211fe5feb Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Thu, 9 Jun 2022 09:41:49 +0200 Subject: [PATCH 8/9] Stylistic changes --- src/Statistics.jl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Statistics.jl b/src/Statistics.jl index bda1724a..30da0adc 100644 --- a/src/Statistics.jl +++ b/src/Statistics.jl @@ -958,19 +958,19 @@ function quantile!(v::AbstractVector, p::Union{AbstractArray, Tuple{Vararg{Real} end return map(x->_quantile(v, x, alpha=alpha, beta=beta), p) end -quantile!(arr::AbstractArray, p::Union{AbstractArray,Tuple{Vararg{Real}}}; - sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = - quantile!(vec(arr), p, sorted=sorted, alpha=alpha, beta=alpha) +quantile!(a::AbstractArray, p::Union{AbstractArray,Tuple{Vararg{Real}}}; + sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = + quantile!(vec(a), p, sorted=sorted, alpha=alpha, beta=alpha) -quantile!(q::AbstractArray, arr::AbstractArray, p::Union{AbstractArray,Tuple{Vararg{Real}}}; - sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = - quantile!(q, vec(arr), p, sorted=sorted, alpha=alpha, beta=alpha) +quantile!(q::AbstractArray, a::AbstractArray, p::Union{AbstractArray,Tuple{Vararg{Real}}}; + sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = + quantile!(q, vec(a), p, sorted=sorted, alpha=alpha, beta=alpha) quantile!(v::AbstractVector, p::Real; sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = _quantile(_quantilesort!(v, sorted, p, p), p, alpha=alpha, beta=beta) # Function to perform partial sort of v for quantiles in given range -function _quantilesort!(v::AbstractArray, sorted::Bool, minp::Real, maxp::Real) +function _quantilesort!(v::AbstractVector, sorted::Bool, minp::Real, maxp::Real) isempty(v) && throw(ArgumentError("empty data vector")) require_one_based_indexing(v) @@ -1080,6 +1080,7 @@ quantile(itr, p; sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = quantile(v::AbstractVector, p; sorted::Bool=false, alpha::Real=1.0, beta::Real=alpha) = quantile!(sorted ? v : Base.copymutable(v), p; sorted=sorted, alpha=alpha, beta=beta) + ##### SparseArrays optimizations ##### function cov(X::SparseMatrixCSC; dims::Int=1, corrected::Bool=true) From 847dff48452805381b0fe2cdd98c534656060a3f Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Thu, 9 Jun 2022 09:42:55 +0200 Subject: [PATCH 9/9] Apply suggestions from code review --- test/runtests.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 0768aeb5..e4d4ba81 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -612,7 +612,9 @@ end @test quantile(Any[1, 2, 3], Float16(0.5)) isa Float16 @test quantile(Any[1, Float16(2), 3], Float16(0.5)) isa Float16 @test quantile(Any[1, big(2), 3], Float16(0.5)) isa BigFloat - @test quantile(reshape(collect(1:100), (10, 10)), [0.00, 0.25, 0.50, 0.75, 1.00]) == [1.0, 25.75, 50.5, 75.25, 100.0] + + @test quantile(reshape(1:100, (10, 10)), [0.00, 0.25, 0.50, 0.75, 1.00]) == + [1.0, 25.75, 50.5, 75.25, 100.0] # Need a large vector to actually check consequences of partial sorting x = rand(50) @@ -643,7 +645,7 @@ end x = reshape(collect(1:100), (10, 10)) y = zeros(5) @test quantile!(y, x, [0.00, 0.25, 0.50, 0.75, 1.00]) === y - @test y ≈ [1.0, 25.75, 50.5, 75.25, 100.0] + @test y ≈ [1.0, 25.75, 50.5, 75.25, 100.0] #tests for quantile calculation with configurable alpha and beta parameters v = [2, 3, 4, 6, 9, 2, 6, 2, 21, 17]