diff --git a/src/FactoredMatrices.jl b/src/FactoredMatrices.jl index cbe2285..0fb1a6f 100644 --- a/src/FactoredMatrices.jl +++ b/src/FactoredMatrices.jl @@ -16,18 +16,19 @@ Multiplications exploit the factored structure without forming the full dense ma For allocation-free repeated multiplications, create a [`Workspace`](@ref) and pass it as `mul!(C, A, B; cache=ws)`. """ -struct FactoredMatrix{T} <: Factorization{T} - U::Matrix{T} # m × k - V::Matrix{T} # k × n +struct FactoredMatrix{T, UT<:AbstractMatrix{T}, VT<:AbstractMatrix{T}} <: Factorization{T} + U::UT # m × k + V::VT # k × n - function FactoredMatrix{T}(U::Matrix{T}, V::Matrix{T}) where T - # This deliberately requires `Matrix{T}` to ensure these precise buffers will be used + function FactoredMatrix{T, UT, VT}(U::UT, V::VT) where {T, UT, VT} + # This deliberately requires an exact match to ensure these buffers will be used size(U, 2) == size(V, 1) || throw(DimensionMismatch("U is $(size(U,1))×$(size(U,2)) but V is $(size(V,1))×$(size(V,2))")) - new{T}(U, V) + new{T, UT, VT}(U, V) end end -FactoredMatrix(U::Matrix{T}, V::Matrix{T}) where T = FactoredMatrix{T}(U, V) +FactoredMatrix{T}(U::AbstractMatrix{T}, V::AbstractMatrix{T}) where T = FactoredMatrix{T, typeof(U), typeof(V)}(U, V) +FactoredMatrix(U::AbstractMatrix{T}, V::AbstractMatrix{T}) where T = FactoredMatrix{T}(U, V) function FactoredMatrix(U::Matrix{T}, V::Matrix{T}, ::Integer) where T Base.depwarn( @@ -38,34 +39,42 @@ function FactoredMatrix(U::Matrix{T}, V::Matrix{T}, ::Integer) where T end """ - Workspace(A::FactoredMatrix, j::Integer) + Workspace{T}(k::Int, p::Int) + Workspace(A::FactoredMatrix, p::Int) -Pre-allocated scratch buffers for allocation-free `mul!` with a [`FactoredMatrix`](@ref). -`j` is the number of columns (or rows) in the non-`FactoredMatrix` operand. +Pre-allocated scratch buffers for allocation-free `mul!` with a +[`FactoredMatrix`](@ref). `k` is the rank of the factorization (the "inner" size +of `U` and `V`), and `p` is the "outer" size (the one that can't be guessed from +`A`) in the non-`FactoredMatrix` operand. For example, to use `A` for +matrix-vector products, set `p = 1`. -Create a separate `Workspace` per thread when multiplying concurrently against the same `A`. +When multiplying two `FactoredMatrix`es, set `k` to the rank of the first and +`p` to the rank of the second. This will eliminate all allocations if the output +`C` is a `FactoredMatrix`, and will eliminate one of two allocations otherwise. + +Create a separate `Workspace` per thread when multiplying concurrently against +the same `A`. # Example ```julia ws = Workspace(A, size(B, 2)) -mul!(C, A, B; cache=ws) # writes C = A*B with no allocation +mul!(C, A, B; cache=ws) # writes C = A*B with no allocation +mul!(D, B', A'; cache=ws) # same `ws` writes D = B'*A' with no allocation ``` """ struct Workspace{T} - temp::Matrix{T} # k × j: intermediate for (left FM or its adjoint/transpose) × B - tempT::Matrix{T} # j × k: intermediate for A × (right FM or its adjoint/transpose) + templeft::Matrix{T} # k × p: intermediate for (left FM or its adjoint/transpose) × B + tempright::Matrix{T} # p × k: intermediate for A × (right FM or its adjoint/transpose) end -function Workspace(A::FactoredMatrix{T}, j::Integer) where T - k = size(A.U, 2) - j = Int(j) - Workspace{T}(Matrix{T}(undef, k, j), Matrix{T}(undef, j, k)) -end +Workspace{T}(k::Int, p::Int) where T = Workspace{T}(Matrix{T}(undef, k, p), Matrix{T}(undef, p, k)) + +Workspace(A::FactoredMatrix{T}, p::Int) where T = Workspace{T}(size(A.U, 2), p) size(A::FactoredMatrix) = size(A.U, 1), size(A.V, 2) size(A::FactoredMatrix, d::Integer) = d == 1 ? size(A.U, 1) : (d == 2 ? size(A.V, 2) : 1) similar(A::FactoredMatrix, dims) = Array{eltype(A.U)}(undef, dims) -similar(A::FactoredMatrix, T, dims) = Array{T}(undef, dims) +similar(::FactoredMatrix, T, dims) = Array{T}(undef, dims) length(A::FactoredMatrix) = size(A.U, 1) * size(A.V, 2) Array(A::FactoredMatrix) = A.U * A.V @@ -79,116 +88,131 @@ getindex(A::Transpose{T,FactoredMatrix{T}}, i, j) where T = # --- mul! --- -mul!(C, A::FactoredMatrix, B::FactoredMatrix; cache::Union{Nothing,Workspace}=nothing) = - error("not defined") +# Because `Adjoint` and `Transpose` are subtypes of `AbstractMatrix`, we have a +# royal mess of ambiguities that need handling. Signatures have to be specific +# enough to intercept dispatch. Use `rewrap` to limit the number of actual +# implementations needed. -function mul!(C, A::FactoredMatrix, B; cache::Union{Nothing,Workspace}=nothing) - if cache === nothing - mul!(C, A.U, A.V * B) - else - mul!(C, A.U, mul!(cache.temp, A.V, B)) +rewrap(A::FactoredMatrix) = A +rewrap(A::Transpose{T, <:FactoredMatrix{T}}) where {T} = FactoredMatrix{T}(transpose(parent(A).V), transpose(parent(A).U)) +rewrap(A::Adjoint{T, <:FactoredMatrix{T}}) where {T} = FactoredMatrix{T}(adjoint(parent(A).V), adjoint(parent(A).U)) + +const _multypes = (FactoredMatrix, Adjoint{T, <:FactoredMatrix{T}} where T, Transpose{T, <:FactoredMatrix{T}} where T) + +for AT in _multypes, BT in _multypes + @eval begin + mul!(C, A::$AT, B::$BT; cache::Union{Nothing,Workspace}=nothing) = _mul!(C, rewrap(A), rewrap(B), cache) end end - -function mul!(C, A::FactoredMatrix, b::AbstractVector; cache::Union{Nothing,Workspace}=nothing) - if cache === nothing - mul!(C, A.U, A.V * b) - else - mul!(C, A.U, mul!(view(cache.temp, :, 1), A.V, b)) +for T in _multypes + @eval begin + mul!(C, A::$T, B::AbstractMatrix; cache::Union{Nothing,Workspace}=nothing) = _mul!(C, rewrap(A), B, cache) + mul!(C, A::$T, b::AbstractVector; cache::Union{Nothing,Workspace}=nothing) = _mul!(C, rewrap(A), b, cache) + mul!(C, A::AbstractMatrix, B::$T; cache::Union{Nothing,Workspace}=nothing) = _mul!(C, A, rewrap(B), cache) end end -function mul!(C, A, B::FactoredMatrix; cache::Union{Nothing,Workspace}=nothing) - if cache === nothing - mul!(C, A * B.U, B.V) - else - mul!(C, mul!(cache.tempT, A, B.U), B.V) +# Now with a FactoredMatrix output +for CT in _multypes, AT in _multypes, BT in _multypes + @eval begin + mul!(C::$CT, A::$AT, B::$BT; cache::Union{Nothing,Workspace}=nothing) = _mul!(C, rewrap(A), rewrap(B), cache) + end +end +for CT in _multypes, T in _multypes + @eval begin + mul!(C::$CT, A::$T, B::AbstractMatrix; cache::Union{Nothing,Workspace}=nothing) = _mul!(C, rewrap(A), B, nothing) + mul!(C::$CT, A::AbstractMatrix, B::$T; cache::Union{Nothing,Workspace}=nothing) = _mul!(C, A, rewrap(B), nothing) + end +end +for CT in _multypes + @eval begin + mul!(C::$CT, A::AbstractMatrix, B::AbstractMatrix; cache::Union{Nothing,Workspace}=nothing) = _mul!(C, A, B, nothing) end end -mul!(C, A::Transpose{T,FactoredMatrix{T}}, B::FactoredMatrix; cache::Union{Nothing,Workspace}=nothing) where T = - error("not defined") +# Internals -function mul!(C::Array{T,N}, A::Transpose{T,FactoredMatrix{T}}, B::Array{T,N}; - cache::Union{Nothing,Workspace}=nothing) where {T,N} - if cache === nothing - mul!(C, transpose(A.parent.V), transpose(A.parent.U) * B) +# When the output is an AbstractMatrix: +function _mul!(C, A::FactoredMatrix, B::FactoredMatrix, cache::Union{Nothing,Workspace}) + tmp = if cache === nothing + A.V * B.U else - mul!(C, transpose(A.parent.V), mul!(cache.temp, transpose(A.parent.U), B)) + mul!(cache.templeft, A.V, B.U) end + k, p = size(tmp) + return k <= p ? mul!(C, A.U * tmp, B.V) : mul!(C, A.U, tmp * B.V) end -function mul!(C, A::Transpose, B::FactoredMatrix; cache::Union{Nothing,Workspace}=nothing) +function _mul!(C, A::FactoredMatrix, B::AbstractMatrix, cache::Union{Nothing,Workspace}) if cache === nothing - mul!(C, A * B.U, B.V) + mul!(C, A.U, A.V * B) else - mul!(C, mul!(cache.tempT, A, B.U), B.V) + mul!(C, A.U, mul!(cache.templeft, A.V, B)) end + return C end -mul!(C, A::Adjoint{T,FactoredMatrix{T}}, B::FactoredMatrix; cache::Union{Nothing,Workspace}=nothing) where T = - error("not defined") - -function mul!(C::Array{T,N}, A::Adjoint{T,FactoredMatrix{T}}, B::Array{T,N}; - cache::Union{Nothing,Workspace}=nothing) where {T,N} +function _mul!(C, A::FactoredMatrix, b::AbstractVector, cache::Union{Nothing,Workspace}) if cache === nothing - mul!(C, A.parent.V', A.parent.U' * B) + mul!(C, A.U, A.V * b) else - mul!(C, A.parent.V', mul!(cache.temp, A.parent.U', B)) + mul!(C, A.U, mul!(view(cache.templeft, :, 1), A.V, b)) end + return C end -function mul!(C, A::Adjoint, B::FactoredMatrix; cache::Union{Nothing,Workspace}=nothing) +function _mul!(C, A::AbstractMatrix, B::FactoredMatrix, cache::Union{Nothing,Workspace}) if cache === nothing mul!(C, A * B.U, B.V) else - mul!(C, mul!(cache.tempT, A, B.U), B.V) + mul!(C, mul!(cache.tempright, A, B.U), B.V) end + return C end -mul!(C, A::FactoredMatrix, B::Transpose{T,FactoredMatrix{T}}; cache::Union{Nothing,Workspace}=nothing) where T = - error("not defined") - -function mul!(C, A::FactoredMatrix, B::Transpose; cache::Union{Nothing,Workspace}=nothing) - if cache === nothing - mul!(C, A.U, A.V * B) +# When the output is a FactoredMatrix: +function _mul!(C::FactoredMatrix, A::FactoredMatrix, B::FactoredMatrix, cache::Union{Nothing,Workspace}) + tmp = if cache === nothing + A.V * B.U else - mul!(C, A.U, mul!(cache.temp, A.V, B)) + mul!(cache.templeft, A.V, B.U) end -end - -function mul!(C::Array{T,N}, A::Array{T,N}, B::Transpose{T,FactoredMatrix{T}}; - cache::Union{Nothing,Workspace}=nothing) where {T,N} - if cache === nothing - mul!(C, A * transpose(B.parent.V), transpose(B.parent.U)) + k, p = size(tmp) + if size(C.U, 2) == size(C.V, 1) == p + mul!(C.U, A.U, tmp) + copyto!(C.V, B.V) + elseif size(C.U, 2) == size(C.V, 1) == k + copyto!(C.U, A.U) + mul!(C.V, tmp, B.V) else - mul!(C, mul!(cache.tempT, A, transpose(B.parent.V)), transpose(B.parent.U)) + throw(DimensionMismatch("C's factorization size doesn't match the result of A*B")) end + return C end -mul!(C, A::FactoredMatrix, B::Adjoint{T,FactoredMatrix{T}}; cache::Union{Nothing,Workspace}=nothing) where T = - error("not defined") +function _mul!(C::FactoredMatrix, A::FactoredMatrix, B::AbstractMatrix, ::Union{Nothing,Workspace}) + # cache is unused + copyto!(C.U, A.U) + mul!(C.V, A.V, B) + return C +end -function mul!(C, A::FactoredMatrix, B::Adjoint; cache::Union{Nothing,Workspace}=nothing) - if cache === nothing - mul!(C, A.U, A.V * B) - else - mul!(C, A.U, mul!(cache.temp, A.V, B)) - end +function _mul!(C::FactoredMatrix, A::AbstractMatrix, B::FactoredMatrix, ::Union{Nothing,Workspace}) + # cache is unused + mul!(C.U, A, B.U) + copyto!(C.V, B.V) + return C end -function mul!(C::Array{T,N}, A::Array{T,N}, B::Adjoint{T,FactoredMatrix{T}}; - cache::Union{Nothing,Workspace}=nothing) where {T,N} - if cache === nothing - mul!(C, A * B.parent.V', B.parent.U') - else - mul!(C, mul!(cache.tempT, A, B.parent.V'), B.parent.U') - end +function _mul!(C::FactoredMatrix, A::AbstractMatrix, B::AbstractMatrix, ::Union{Nothing,Workspace}) + copyto!(C.U, A) + copyto!(C.V, B) + return C end # --- * (allocating) --- -*(A::FactoredMatrix, B::FactoredMatrix) = error("not defined") +*(A::FactoredMatrix, B::FactoredMatrix) = A.U * ((A.V * B.U) * B.V) *(A::FactoredMatrix, B) = A.U * (A.V * B) *(A, B::FactoredMatrix) = (A * B.U) * B.V diff --git a/test/runtests.jl b/test/runtests.jl index baa796b..aa38414 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,6 +6,14 @@ myrand(::Type{Complex{T}}, m, n) where {T<:Real} = rand(T, m, n) + im*rand(T, m, # Wrapper so the compiler sees ws as a concrete positional argument and can elide # the keyword NamedTuple that would otherwise allocate in Julia 1.10's kwarg machinery. +function check_fmfm_fm_alloc(Mf, Mf_A, ws, Cf15x8) + @test @allocated(mul!(Cf15x8, Mf, Mf_A; cache=ws)) == 0 +end + +function check_vec_alloc(Mf, b, ws, c) + @test @allocated(mul!(c, Mf, b; cache=ws)) == 0 +end + function check_allocs(Mf, ws, C, D, E, r15x5, r10x5, r5x10, r5x15) @test @allocated(mul!(r15x5, Mf, C; cache=ws)) == 0 @test @allocated(mul!(r10x5, Mf', D; cache=ws)) == 0 @@ -52,17 +60,6 @@ end r5x10 = Matrix{T}(undef, 5, 10) # D'*Mf, transpose(D)*Mf r5x15 = Matrix{T}(undef, 5, 15) # E*Mf', E*transpose(Mf) - # warm up to avoid counting compilation - mul!(r15x5, Mf, C; cache=ws) - mul!(r10x5, Mf', D; cache=ws) - mul!(r10x5, transpose(Mf), D; cache=ws) - mul!(r5x10, D', Mf; cache=ws) - mul!(r5x10, transpose(D), Mf; cache=ws) - mul!(r15x5, Mf, E'; cache=ws) - mul!(r15x5, Mf, transpose(E); cache=ws) - mul!(r5x15, E, Mf'; cache=ws) - mul!(r5x15, E, transpose(Mf); cache=ws) - mul!(r15x5, Mf, C; cache=ws); @test r15x5 ≈ M*C mul!(r10x5, Mf', D; cache=ws); @test r10x5 ≈ M'*D mul!(r10x5, transpose(Mf), D; cache=ws); @test r10x5 ≈ transpose(M)*D @@ -80,5 +77,111 @@ end M1 = U1*V1 M1f = FactoredMatrix(U1, V1) @test sum(abs2, M-M1) ≈ FactoredMatrices.ssd(Mf, M1f) + + # FM × FM (all 9 op combinations); Mf is 15×10 + # Compatible right-hand FMs: + # Mf_A (10×8): right plain FM for Mf + # Mf_B (15×8): right plain FM for Mf' / transpose(Mf) + # Mf_C (8×10): right FM whose adj/transpose (10×8) is right-compatible with Mf + # Mf_D (8×15): right FM whose adj/transpose (15×8) is right-compatible with Mf'/transpose(Mf) + let + Mf_A = FactoredMatrix(myrand(T, 10, 3), myrand(T, 3, 8)); MA = Array(Mf_A) + Mf_B = FactoredMatrix(myrand(T, 15, 3), myrand(T, 3, 8)); MB = Array(Mf_B) + Mf_C = FactoredMatrix(myrand(T, 8, 3), myrand(T, 3, 10)); MC = Array(Mf_C) + Mf_D = FactoredMatrix(myrand(T, 8, 3), myrand(T, 3, 15)); MD = Array(Mf_D) + r15x8 = Matrix{T}(undef, 15, 8) + r10x8 = Matrix{T}(undef, 10, 8) + @test Mf * Mf_A ≈ M * MA + @test Mf' * Mf_B ≈ M' * MB + @test transpose(Mf) * Mf_B ≈ transpose(M) * MB + @test Mf * Mf_C' ≈ M * MC' + @test Mf * transpose(Mf_C) ≈ M * transpose(MC) + @test Mf' * Mf_D' ≈ M' * MD' + @test Mf' * transpose(Mf_D) ≈ M' * transpose(MD) + @test transpose(Mf) * Mf_D' ≈ transpose(M) * MD' + @test transpose(Mf) * transpose(Mf_D) ≈ transpose(M) * transpose(MD) + mul!(r15x8, Mf, Mf_A); @test r15x8 ≈ M * MA + mul!(r10x8, Mf', Mf_B); @test r10x8 ≈ M' * MB + mul!(r10x8, transpose(Mf), Mf_B); @test r10x8 ≈ transpose(M) * MB + mul!(r15x8, Mf, Mf_C'); @test r15x8 ≈ M * MC' + mul!(r15x8, Mf, transpose(Mf_C)); @test r15x8 ≈ M * transpose(MC) + mul!(r10x8, Mf', Mf_D'); @test r10x8 ≈ M' * MD' + mul!(r10x8, Mf', transpose(Mf_D)); @test r10x8 ≈ M' * transpose(MD) + mul!(r10x8, transpose(Mf), Mf_D'); @test r10x8 ≈ transpose(M) * MD' + mul!(r10x8, transpose(Mf), transpose(Mf_D)); @test r10x8 ≈ transpose(M) * transpose(MD) + end + + # mul! FM×FM→AM with cache + let + Mf_A = FactoredMatrix(myrand(T, 10, 3), myrand(T, 3, 8)); MA = Array(Mf_A) + Mf_B = FactoredMatrix(myrand(T, 15, 3), myrand(T, 3, 8)); MB = Array(Mf_B) + r15x8 = Matrix{T}(undef, 15, 8) + r10x8 = Matrix{T}(undef, 10, 8) + ws3x3 = FactoredMatrices.Workspace{T}(3, 3) + mul!(r15x8, Mf, Mf_A; cache=ws3x3); @test r15x8 ≈ M * MA + mul!(r10x8, Mf', Mf_B; cache=ws3x3); @test r10x8 ≈ M' * MB + mul!(r10x8, transpose(Mf), Mf_B; cache=ws3x3); @test r10x8 ≈ transpose(M) * MB + end + + # mul! with FactoredMatrix output: FM×AM, AM×FM, AM×AM, adj/transpose variants + let + C_mat = myrand(T, 10, 5) + D_mat = myrand(T, 15, 5) + E_mat = myrand(T, 5, 10) + + # FM × AM → FM + Cf = FactoredMatrix(Matrix{T}(undef, 15, 3), Matrix{T}(undef, 3, 5)) + mul!(Cf, Mf, C_mat); @test Array(Cf) ≈ M * C_mat + # adj(FM) × AM → FM + Cf2 = FactoredMatrix(Matrix{T}(undef, 10, 3), Matrix{T}(undef, 3, 5)) + mul!(Cf2, Mf', D_mat); @test Array(Cf2) ≈ M' * D_mat + mul!(Cf2, transpose(Mf), D_mat); @test Array(Cf2) ≈ transpose(M) * D_mat + + # AM × FM → FM + A_mat = myrand(T, 5, 15) + Cf3 = FactoredMatrix(Matrix{T}(undef, 5, 3), Matrix{T}(undef, 3, 10)) + mul!(Cf3, A_mat, Mf); @test Array(Cf3) ≈ A_mat * M + # AM × adj(FM) → FM + Cf4 = FactoredMatrix(Matrix{T}(undef, 5, 3), Matrix{T}(undef, 3, 15)) + mul!(Cf4, E_mat, Mf'); @test Array(Cf4) ≈ E_mat * M' + mul!(Cf4, E_mat, transpose(Mf)); @test Array(Cf4) ≈ E_mat * transpose(M) + + # AM × AM → FM + A2 = myrand(T, 5, 3); B2 = myrand(T, 3, 7) + Cf5 = FactoredMatrix(Matrix{T}(undef, 5, 3), Matrix{T}(undef, 3, 7)) + mul!(Cf5, A2, B2); @test Array(Cf5) ≈ A2 * B2 + end + + # mul! FM×FM→FM (without and with cache, plain and adj/transpose) + let + Mf_A = FactoredMatrix(myrand(T, 10, 3), myrand(T, 3, 8)); MA = Array(Mf_A) + Mf_B = FactoredMatrix(myrand(T, 15, 3), myrand(T, 3, 8)); MB = Array(Mf_B) + Mf_C = FactoredMatrix(myrand(T, 8, 3), myrand(T, 3, 10)); MC = Array(Mf_C) + ws3x3 = FactoredMatrices.Workspace{T}(3, 3) + Cf15x8 = FactoredMatrix(Matrix{T}(undef, 15, 3), Matrix{T}(undef, 3, 8)) + Cf10x8 = FactoredMatrix(Matrix{T}(undef, 10, 3), Matrix{T}(undef, 3, 8)) + + mul!(Cf15x8, Mf, Mf_A); @test Array(Cf15x8) ≈ M * MA + mul!(Cf10x8, Mf', Mf_B); @test Array(Cf10x8) ≈ M' * MB + mul!(Cf10x8, transpose(Mf), Mf_B); @test Array(Cf10x8) ≈ transpose(M) * MB + mul!(Cf15x8, Mf, Mf_C'); @test Array(Cf15x8) ≈ M * MC' + mul!(Cf15x8, Mf, transpose(Mf_C)); @test Array(Cf15x8) ≈ M * transpose(MC) + + mul!(Cf15x8, Mf, Mf_A; cache=ws3x3); @test Array(Cf15x8) ≈ M * MA + mul!(Cf10x8, Mf', Mf_B; cache=ws3x3); @test Array(Cf10x8) ≈ M' * MB + mul!(Cf10x8, transpose(Mf), Mf_B; cache=ws3x3); @test Array(Cf10x8) ≈ transpose(M) * MB + mul!(Cf15x8, Mf, Mf_C'; cache=ws3x3); @test Array(Cf15x8) ≈ M * MC' + check_fmfm_fm_alloc(Mf, Mf_A, ws3x3, Cf15x8) + end + + # mul! vector (with and without cache) + let + b = vec(myrand(T, 10, 1)) + c = Vector{T}(undef, 15) + mul!(c, Mf, b); @test c ≈ M * b + ws_vec = FactoredMatrices.Workspace(Mf, 1) + mul!(c, Mf, b; cache=ws_vec); @test c ≈ M * b + check_vec_alloc(Mf, b, ws_vec, c) + end end end