diff --git a/docs/src/man/julia_differences.md b/docs/src/man/julia_differences.md index 6affaa32..91b8af3f 100644 --- a/docs/src/man/julia_differences.md +++ b/docs/src/man/julia_differences.md @@ -12,6 +12,17 @@ one becomes a compile-time diagnostic. `try`/`catch` is not supported. Use `ct.@assert` for explicit runtime checks. +## Per-element branches run both arms + +A scalar function broadcast or mapped over a tile (`f.(tile)`) runs once for +the whole tile, so a branch on an element's value (`x < 0 ? a : b`, or the +special cases in Base's `asin` and `hypot`) cannot jump. Instead, both arms are +computed for every element and each element selects its own result. This +requires both arms to be free of side effects such as memory accesses or +printing. A `throw` in an arm becomes an assertion that no element takes it. +Loops whose trip count depends on the element's value are not supported. + + ## Some operations are non-throwing Float-to-integer conversions such as `Int32(x)`, `trunc(Int32, x)`, and diff --git a/src/compiler/codegen/control_flow.jl b/src/compiler/codegen/control_flow.jl index 9dc77896..77143b55 100644 --- a/src/compiler/codegen/control_flow.jl +++ b/src/compiler/codegen/control_flow.jl @@ -94,6 +94,10 @@ function emit_if_op!(ctx::CGCtx, op::IfOp, @nospecialize(parent_result_type), ss # Get condition value cond_tv = emit_value!(ctx, op.condition) cond_tv === nothing && throw(IRError("Cannot resolve condition for IfOp")) + isempty(cond_tv.shape) || throw(IRError( + "Cannot branch on a per-element condition (a Bool tile of size " * + "$(Tuple(ColMajorShape(cond_tv.shape).dims))); a branch in a function lifted over " * + "a tile must be free of loops and side effects in both arms, or throw in one")) # Determine result types from parent_result_type (token_order_pass! already # updated the type to include any token carries via inst[:type] = …). @@ -295,6 +299,10 @@ function emit_while_op!(ctx::CGCtx, op::WhileOp, @nospecialize(parent_result_typ cond_tv = emit_value!(ctx, cond_op.condition) (cond_tv === nothing || cond_tv.v === nothing) && throw(IRError("Cannot resolve WhileOp condition")) + isempty(cond_tv.shape) || throw(IRError( + "Cannot loop on a per-element condition (a Bool tile of size " * + "$(Tuple(ColMajorShape(cond_tv.shape).dims))); a loop in a function lifted over " * + "a tile cannot depend on the element's value")) # Emit conditional break: if (cond) { yield } else { break } # This keeps nested loops in "after" at LoopOp body level diff --git a/src/compiler/interpreter.jl b/src/compiler/interpreter.jl index c1d7fae8..0d23a161 100644 --- a/src/compiler/interpreter.jl +++ b/src/compiler/interpreter.jl @@ -26,7 +26,14 @@ function cuTileInterpreter(cache::CacheView; always_inline::Bool=true) else inf_cache = Vector{CC.InferenceResult}() end - inf_params = CC.InferenceParams() + # Before 1.12, calls in a block that ends in `throw` are left uninferred and + # uninlined by default. Kernels need them inferred: an exception + # constructor typed `Any` cannot be removed, and its message is lost. + inf_params = @static if VERSION >= v"1.12.0-DEV.1017" + CC.InferenceParams() + else + CC.InferenceParams(; unoptimize_throw_blocks=false) + end opt_params = if always_inline CC.OptimizationParams(; inline_cost_threshold=typemax(Int)) else diff --git a/src/compiler/transform/canonicalize.jl b/src/compiler/transform/canonicalize.jl index 8f20b7cc..869b39dd 100644 --- a/src/compiler/transform/canonicalize.jl +++ b/src/compiler/transform/canonicalize.jl @@ -28,6 +28,8 @@ const INTRINSIC_RULES = RewriteRule[ Intrinsics.cmpi(~x, ~y, $(ComparisonPredicate.LessThanOrEqual), $(Signedness.Signed)) @rewrite Core.Intrinsics.ult_int(~x, ~y) => Intrinsics.cmpi(~x, ~y, $(ComparisonPredicate.LessThan), $(Signedness.Unsigned)) + @rewrite Core.Intrinsics.ule_int(~x, ~y) => + Intrinsics.cmpi(~x, ~y, $(ComparisonPredicate.LessThanOrEqual), $(Signedness.Unsigned)) # Bitwise @rewrite Core.Intrinsics.and_int(~x, ~y) => Intrinsics.andi(~x, ~y) @@ -52,6 +54,8 @@ const INTRINSIC_RULES = RewriteRule[ # `muladd` may be computed fused (rounded once) or as a multiply and an add; # Tile IR has a native `fma`, so always fuse. @rewrite Core.Intrinsics.muladd_float(~x, ~y, ~z) => Intrinsics.fma(~x, ~y, ~z) + # Base's `asin` and `acos` call `sqrt_llvm` directly. + @rewrite Core.Intrinsics.sqrt_llvm(~x) => Intrinsics.sqrt(~x) # Float comparison @rewrite Core.Intrinsics.lt_float(~x, ~y) => @@ -87,44 +91,78 @@ lower_intr_pass!(sci::StructuredIRCode) = rewrite_patterns!(sci, INTRINSIC_RULES # # This eliminates the scalar/tile duality that complicates codegen and pattern # matching. Scalars now only can exist as literal values embedded in the IR. +# +# A scalar function lifted over a tile (`f.(tile)`, `map(f, tile)`) becomes +# scalar code operating on `to_scalar` results. Its control flow ends up in +# nested regions, so shapes are propagated over the whole tree in program +# order. A branch on a shaped condition is later flattened by `if_convert!`. function scalar_elim_pass!(sci::StructuredIRCode) - scalar_elim_block!(sci.entry) + # Phase 1: Eliminate to_scalar (forward tile-typed operand). Uses may be + # in nested regions, so this must happen before any shape propagation. + eliminate_calls!(sci, Intrinsics.to_scalar) + + # Phase 2: Propagate tile types from operands to results. + propagate_tile_types!(sci.entry) + + # Phase 3: Eliminate from_scalar (operand now has correct tile type) + eliminate_calls!(sci, Intrinsics.from_scalar) + + for block in eachblock(sci) + promote_scalar_types!(block) + end end -function scalar_elim_block!(block::Block) - # Recurse into nested control flow first - for inst in instructions(block) - s = inst[:stmt] - if s isa ForOp - scalar_elim_block!(s.body) - elseif s isa IfOp - scalar_elim_block!(s.then_region) - scalar_elim_block!(s.else_region) - elseif s isa WhileOp - scalar_elim_block!(s.before) - scalar_elim_block!(s.after) - elseif s isa LoopOp - scalar_elim_block!(s.body) +# Replace every call to `func` with its first operand. +function eliminate_calls!(sci::StructuredIRCode, func) + for block in eachblock(sci) + to_delete = Instruction[] + for inst in instructions(block) + call = resolve_call(block, inst[:stmt]) + call === nothing && continue + call[1] === func || continue + replace_uses!(block, ssavalue(inst), call[2][1]) + push!(to_delete, inst) + end + for inst in to_delete + delete!(block, inst) end end +end - # Phase 1: Eliminate to_scalar (forward tile-typed operand) - to_delete = Instruction[] - for inst in instructions(block) - call = resolve_call(block, inst[:stmt]) - call === nothing && continue - func, ops = call - func === Intrinsics.to_scalar || continue - replace_uses!(block, ssavalue(inst), ops[1]) - push!(to_delete, inst) +# Broadcast shape of the shaped tiles among `types`, or `nothing` when no type +# is a tile. Incompatible shapes keep the first one; codegen reports those. +function broadcast_tile_shape(types) + shape = nothing + for T in types + T isa DataType && T <: Tile || continue + s = size(T) + shape = shape === nothing ? s : + try Base.Broadcast.broadcast_shape(shape, s) catch; shape end end + return shape +end - # Phase 2: Propagate tile types from operands to results. - # Instructions that consumed to_scalar results still have scalar type - # annotations but now receive tile-typed operands — inherit their shape. +# Type of an operand, or `nothing` for tokens and unknown types. +function tile_operand_type(block::Block, @nospecialize(op)) + T = value_type(block, op) + (T === nothing || is_token_type(T)) ? nothing : CC.widenconst(T) +end + +# Instructions that consumed to_scalar results still have scalar type +# annotations but now receive tile-typed operands; give them the broadcast +# shape of those operands. Regions are visited in program order, so values +# defined outside a region are typed before the region uses them. +function propagate_tile_types!(block::Block) for inst in instructions(block) - call = resolve_call(block, inst[:stmt]) + stmt = inst[:stmt] + if stmt isa ControlFlowOp + foreach(propagate_tile_types!, IRStructurizer.blocks(stmt)) + stmt isa IfOp && propagate_if_type!(block, inst, stmt) + continue + end + + call = resolve_call(block, stmt) call === nothing && continue func, ops = call @@ -140,32 +178,45 @@ function scalar_elim_block!(block::Block) T <: Tile && continue # already tile-typed T <: Number || continue # only promote scalar number types - for op in ops - op_type = value_type(block, op) - op_type === nothing && continue - is_token_type(op_type) && continue - OT = CC.widenconst(op_type) - OT <: Tile || continue - S = OT.parameters[2] - inst[:type] = Tile{T, S} - break + # A field of a control-flow result, which carries its own shape. + if func === getfield && length(ops) >= 2 && ops[2] isa Int + PT = tile_operand_type(block, ops[1]) + if PT isa DataType && PT <: Tuple && 1 <= ops[2] <= length(PT.parameters) + FT = CC.widenconst(PT.parameters[ops[2]]) + FT <: Tile{T} && (inst[:type] = FT) + continue + end end - end - # Phase 3: Eliminate from_scalar (operand now has correct tile type) - for inst in instructions(block) - call = resolve_call(block, inst[:stmt]) - call === nothing && continue - func, ops = call - func === Intrinsics.from_scalar || continue - replace_uses!(block, ssavalue(inst), ops[1]) - push!(to_delete, inst) + shape = broadcast_tile_shape(tile_operand_type(block, op) for op in ops) + shape === nothing && continue + inst[:type] = Tile{T, Tuple{shape...}} end +end - for inst in to_delete - delete!(block, inst) +# Give each scalar result of an `IfOp` the broadcast shape of the values its +# arms yield. A shaped condition will be flattened into selects, whose result +# also has the condition's shape. +function propagate_if_type!(block::Block, inst::Instruction, op::IfOp) + RT = CC.widenconst(inst[:type]) + RT <: Tuple || return + cond_type = tile_operand_type(block, op.condition) + params = map(enumerate(RT.parameters)) do (i, P) + P = CC.widenconst(P) + P <: Number && !(P <: Tile) || return P + types = Any[cond_type] + for region in (op.then_region, op.else_region) + term = terminator(region) + term isa YieldOp && i <= length(term.values) && + push!(types, tile_operand_type(region, term.values[i])) + end + shape = broadcast_tile_shape(types) + shape === nothing ? P : Tile{P, Tuple{shape...}} end + inst[:type] = Tuple{params...} +end +function promote_scalar_types!(block::Block) # Phase 4: Promote any remaining scalar Number types to 0D tiles. # This catches intrinsics that natively return scalars (get_tile_block_id, # item, etc.) and any scalar arithmetic on them (addi of two block IDs). diff --git a/src/compiler/transform/if_conversion.jl b/src/compiler/transform/if_conversion.jl new file mode 100644 index 00000000..ed5526c2 --- /dev/null +++ b/src/compiler/transform/if_conversion.jl @@ -0,0 +1,216 @@ +# If-conversion of branches on shaped conditions +# +# A scalar function lifted over a tile (`f.(tile)`, `map(f, tile)`) may branch +# on a value derived from its arguments: `x > 0 ? a : b`, or the special cases +# in Base's `asin` and `hypot`. After scalar elimination such a branch has a +# shaped `Tile{Bool}` condition, which Tile IR's `if` cannot take. Both arms +# are executed instead and their results combined lane-wise with `select`. +# Arms executed this way may only contain operations without side effects, and +# nested branches must have been flattened already. Integer divisions are +# made safe for the inputs they are undefined for, which lanes that do not +# take the arm may well have. +# +# A throw becomes an assertion that no lane takes its path. Julia ends a +# throwing arm in `unreachable`, so the arm yields nothing. Once the assertion +# holds, every lane takes the other arm, which therefore runs unchanged even if +# it has side effects. When nothing follows the branch, Julia places the rest +# of the enclosing block in that arm. + +function is_shaped_condition(block::Block, @nospecialize(cond)) + T = tile_operand_type(block, cond) + T isa DataType && T <: Tile && !isempty(size(T)) +end + +""" + if_convert!(sci) + +Flatten every `IfOp` whose condition is a shaped tile into its parent block, +innermost first. Branches that cannot be flattened are left in place. +""" +function if_convert!(sci::StructuredIRCode) + if_convert_block!(Rewriter(sci), sci.entry) + return sci +end + +function if_convert_block!(r::Rewriter, block::Block) + for inst in collect(instructions(block)) + # Flattening a branch deletes the extractions of its results. + haskey(block, inst.ssa_idx) || continue + stmt = inst[:stmt] + stmt isa ControlFlowOp || continue + for region in IRStructurizer.blocks(stmt) + if_convert_block!(r, region) + end + stmt isa IfOp && is_shaped_condition(block, stmt.condition) || continue + convert_if!(r, block, inst, stmt) + end +end + +# Whether an instruction may also run for lanes that do not take its arm. +# Assertions are weakened for those lanes, and `@fpmode` markers only scope +# codegen. Loads are removable but not speculatable: the lanes that do not take +# the arm may not be allowed to access that memory. +function speculatable(block::Block, inst::Instruction) + call = resolve_call(block, inst[:stmt]) + if call !== nothing + func = call[1] + func in (Intrinsics.assert, Intrinsics.fpmode_begin, Intrinsics.fpmode_end) && + return true + classify_memory_op(func) === MEM_NONE || return false + end + return !must_keep(block, inst) +end + +# `:pure` for an arm without side effects that yields, `:throw` for one that +# ends in a throw, `:other` for any other arm. +function arm_kind(region::Block) + throws = false + for inst in instructions(region) + stmt = inst[:stmt] + if stmt isa ThrowNode + throws = true + elseif stmt isa ControlFlowOp || !speculatable(region, inst) + return :other + end + end + # Nothing after a throw executes, whatever the terminator (`unreachable`, or + # a yield once `hoist_returns!` has rewritten it). + throws && return :throw + return plain_yield(region) ? :pure : :other +end + +# Integer division is undefined for a zero divisor, and signed division also +# for MIN / -1, even if the quotient is then discarded: tileiras assumes the +# divisor is valid and drops the select that guarded it. Divide by 1 instead +# for those inputs, which only changes results that were undefined anyway. +const INTEGER_DIVISIONS = (Intrinsics.divi, Intrinsics.fldi, Intrinsics.cldi, Intrinsics.remi) + +function make_division_safe!(r::Rewriter, block::Block, inst::Instruction, func, ops) + x, y, signedness = ops + y isa Integer && !iszero(y) && y != -1 && return + T = eltype(something(tile_operand_type(block, y), typeof(y))) + shaped(S, vals...) = Tile{S, Tuple{something( + broadcast_tile_shape(Any[tile_operand_type(block, v) for v in vals]), ())...}} + insert(stmt, type, f) = ssavalue(insert_before!(r, block, inst, stmt, type; + flag=inferred_flags(f))) + cmp(a, b) = insert(Expr(:call, Intrinsics.cmpi, a, b, ComparisonPredicate.Equal, + signedness), shaped(Bool, a, b), Intrinsics.cmpi) + invalid = cmp(y, zero(T)) + if signedness === Signedness.Signed && func !== Intrinsics.remi + overflow = cmp(x, typemin(T)) + minus_one = cmp(y, -one(T)) + overflow = insert(Expr(:call, Intrinsics.andi, overflow, minus_one), + shaped(Bool, x, y), Intrinsics.andi) + invalid = insert(Expr(:call, Intrinsics.ori, invalid, overflow), + shaped(Bool, x, y), Intrinsics.ori) + end + divisor = insert(Expr(:call, Intrinsics.select, invalid, one(T), y), + shaped(T, invalid, y), Intrinsics.select) + replace_stmt!(r, block, ssavalue(inst), Expr(:call, func, x, divisor, signedness)) +end + +function convert_if!(r::Rewriter, parent::Block, inst::Instruction, op::IfOp) + sci = r.sci + kinds = (arm_kind(op.then_region), arm_kind(op.else_region)) + # Either both arms are selected from, or one throws and the other continues. + kinds === (:pure, :pure) || count(==(:throw), kinds) == 1 || return false + selected = kinds === (:pure, :pure) + + if_ssa = ssavalue(inst) + RT = CC.widenconst(inst[:type]) + RT === Nothing || RT <: Tuple || return false + result_types = RT === Nothing ? Any[] : collect(Any, RT.parameters) + extracts = Tuple{Block, SSAValue, Int}[] + if !isempty(result_types) + extracts = tuple_result_extracts(sci, r, if_ssa, length(result_types)) + extracts === nothing && return false + end + + # An arm that continues past the branch replaces the parent's terminator. + continuing = selected ? nothing : + kinds[1] === :throw ? op.else_region : op.then_region + new_terminator = nothing + if continuing !== nothing && !plain_yield(continuing) + isempty(result_types) || return false + last(collect(instructions(parent))).ssa_idx == inst.ssa_idx || return false + new_terminator = terminator(continuing) + end + + then_vals = kinds[1] === :throw ? nothing : yield_values(op.then_region) + else_vals = kinds[2] === :throw ? nothing : yield_values(op.else_region) + if new_terminator === nothing + for vals in (then_vals, else_vals) + vals === nothing || length(vals) == length(result_types) || return false + end + end + # Only tiles can be selected; any other result must be the same in both arms. + if selected + for (i, T) in enumerate(result_types) + CC.widenconst(T) <: Tile || then_vals[i] === else_vals[i] || return false + end + end + + cond = op.condition + cond_type = tile_operand_type(parent, cond) + not_cond = ssavalue(insert_before!(r, parent, inst, Expr(:call, Intrinsics.xori, cond, true), + cond_type; flag=inferred_flags(Intrinsics.xori))) + + # Move both arms into the parent. An assertion inside an arm (from a throw, + # possibly in an already flattened nested branch) must only fail for the + # lanes that take the arm. + for (region, off_arm) in ((op.then_region, not_cond), (op.else_region, cond)) + for rinst in collect(instructions(region)) + stmt = rinst[:stmt] + if stmt isa ThrowNode + replace_stmt!(r, region, ssavalue(rinst), + Expr(:call, Intrinsics.assert, off_arm, stmt.message); + flag=inferred_flags(Intrinsics.assert)) + rinst[:type] = Nothing + continue + end + call = resolve_call(region, stmt) + call === nothing && continue + func, ops = call + if func === Intrinsics.assert + ok = ops[1] + ok_type = Tile{Bool, Tuple{broadcast_tile_shape( + (tile_operand_type(region, ok), cond_type))...}} + weakened = insert_before!(r, region, rinst, + Expr(:call, Intrinsics.ori, ok, off_arm), ok_type; + flag=inferred_flags(Intrinsics.ori)) + replace_stmt!(r, region, ssavalue(rinst), + Expr(:call, Intrinsics.assert, ssavalue(weakened), ops[2])) + elseif selected && func in INTEGER_DIVISIONS + make_division_safe!(r, region, rinst, func, ops) + end + end + for rinst in collect(instructions(region)) + move_before!(rinst, inst) + end + end + + results = map(enumerate(result_types)) do (i, T) + T = CC.widenconst(T) + if selected + then_vals[i] === else_vals[i] && return then_vals[i] + return ssavalue(insert_before!(r, parent, inst, + Expr(:call, Intrinsics.select, cond, then_vals[i], else_vals[i]), T; + flag=inferred_flags(Intrinsics.select))) + end + val = something(then_vals, else_vals)[i] + # The surviving value may be narrower than the branch result, which + # has the condition's shape. + T <: Tile && tile_operand_type(parent, val) !== T || return val + ssavalue(insert_before!(r, parent, inst, + Expr(:call, Intrinsics.broadcast, val, size(T)), T; + flag=inferred_flags(Intrinsics.broadcast))) + end + + for (block, extract_ssa, field) in extracts + replace_uses!(r, extract_ssa, results[field]) + erase!(r, block, extract_ssa) + end + erase!(r, parent, if_ssa) + new_terminator === nothing || terminator!(parent, new_terminator) + return true +end diff --git a/src/compiler/transform/pipeline.jl b/src/compiler/transform/pipeline.jl index d44fa542..2bcdb635 100644 --- a/src/compiler/transform/pipeline.jl +++ b/src/compiler/transform/pipeline.jl @@ -276,6 +276,9 @@ function run_passes!(sci::StructuredIRCode; alias_groups::AliasGroups=()) # No current rewrite rule makes a branch condition newly constant, so the # fold only needs to run here, before normalization and token threading. fold_constant_branches!(sci) + # After the fold, so that arms holding a constant branch (`@boundscheck`) + # can be flattened. + if_convert!(sci) rewrite_patterns!(sci, PRINT_FUSION_RULES) diff --git a/src/cuTile.jl b/src/cuTile.jl index d07aa72f..bca1d39d 100644 --- a/src/cuTile.jl +++ b/src/cuTile.jl @@ -62,6 +62,7 @@ include("compiler/transform/boundscheck.jl") include("compiler/transform/throws.jl") include("compiler/transform/canonicalize.jl") include("compiler/transform/constant_branches.jl") +include("compiler/transform/if_conversion.jl") include("compiler/transform/control_flow.jl") include("compiler/transform/token_keys.jl") include("compiler/transform/token_order.jl") diff --git a/src/language/math.jl b/src/language/math.jl index 6782d04b..e8a36073 100644 --- a/src/language/math.jl +++ b/src/language/math.jl @@ -42,6 +42,117 @@ end reinterpret(T, (reinterpret(U, x) & ~mask) | (reinterpret(U, y) & mask)) end +# A quotient as `a * (1 / b)` (`div.approx.f32`, within 2 ulp), without the +# range check and slow-path call IEEE division makes for every element. For a +# divisor in a narrow range whose quotient only corrects a larger term. +function approx_div(a::Float32, b::Float32) + Intrinsics.fpmode_begin(Rounding.Approx, nothing) + q = a / b + Intrinsics.fpmode_end() + q +end + +# Base's `log1p` and `expm1` look up tables with a per-element index. Float32 +# uses the table-free algorithms of fdlibm (as in musl's `log1pf` and `expm1f`, +# within 0.86 ulp), written without early returns so that every element +# computes each path and selects its own. Float16 and BFloat16 round the +# Float32 result. Other types use Kahan's formulations on top of `log` and +# `exp`: the rounding error of `u = 1 + x` (or `u = exp(x)`) cancels in the +# quotient, leaving a few ulp (2.4 in Float32). Like the other overlays, +# out-of-domain arguments give NaN rather than throwing. +@overlay function Base.log1p(x::Float32) + ix = reinterpret(UInt32, x) + # For 1 + x in [sqrt(2)/2, sqrt(2)), use x directly. Otherwise split + # 1 + x = 2^k * (1 + f) and carry the rounding error of 1 + x in c. + direct = (ix < 0x3ed413d0) | ((ix >= 0x80000000) & (ix <= 0xbe95f619)) + u = 1f0 + x + iu = reinterpret(UInt32, u) + (0x3f800000 - 0x3f3504f3) + k = reinterpret(Int32, iu >> UInt32(23)) - Int32(0x7f) + c = ifelse(k < Int32(25), approx_div(ifelse(k >= Int32(2), 1f0 - (u - x), x - (u - 1f0)), u), 0f0) + f = reinterpret(Float32, (iu & 0x007fffff) + 0x3f3504f3) - 1f0 + k = ifelse(direct, Int32(0), k) + c = ifelse(direct, 0f0, c) + f = ifelse(direct, x, f) + + # log(1 + f) = f - f^2/2 + s*(f^2/2 + R(s^2)), with s = f / (2 + f) + s = approx_div(f, 2f0 + f) + z = s * s + w = z * z + t1 = w * (reinterpret(Float32, 0x3eccce13) + w * reinterpret(Float32, 0x3e789e26)) + t2 = z * (reinterpret(Float32, 0x3f2aaaaa) + w * reinterpret(Float32, 0x3e91e9ee)) + hfsq = 0.5f0 * f * f + dk = Float32(k) + ln2_hi = reinterpret(Float32, 0x3f317180) + ln2_lo = reinterpret(Float32, 0x3717f7d1) + r = s * (hfsq + (t2 + t1)) + (dk * ln2_lo + c) - hfsq + f + dk * ln2_hi + + # |x| < 2^-24, +Inf and NaN give x; -1 gives -Inf; x < -1 gives NaN. + r = ifelse((ix << UInt32(1)) < (0x33800000 << UInt32(1)), x, r) + r = ifelse((ix >= 0x7f800000) & (ix < 0x80000000), x, r) + ifelse(ix >= 0xbf800000, ifelse(x == -1f0, -Inf32, NaN32), r) +end +@overlay function Base.expm1(x::Float32) + ix = reinterpret(UInt32, x) + hx = ix & 0x7fffffff + neg = ix >= 0x80000000 + + # For |x| > ln2/2, reduce to hi - lo = x - k*ln2, with c the rounding error. + ln2_hi = reinterpret(Float32, 0x3f317180) + ln2_lo = reinterpret(Float32, 0x3717f7d1) + kf = reinterpret(Float32, 0x3fb8aa3b) * x + ifelse(neg, -0.5f0, 0.5f0) + k = ifelse(hx < 0x3f851592, ifelse(neg, Int32(-1), Int32(1)), unsafe_trunc(Int32, kf)) + t = Float32(k) + hi = x - t * ln2_hi # exact for these k + lo = t * ln2_lo + reduced = hx > 0x3eb17218 + xr = ifelse(reduced, hi - lo, x) + c = ifelse(reduced, (hi - xr) - lo, 0f0) + k = ifelse(reduced, k, Int32(0)) + + # expm1(xr) from a rational approximation, then scaled by 2^k. + hfx = 0.5f0 * xr + hxs = xr * hfx + r1 = 1f0 + hxs * (reinterpret(Float32, 0xbd088868) + hxs * reinterpret(Float32, 0x3acf3010)) + t3 = 3f0 - r1 * hfx + e = hxs * approx_div(r1 - t3, 6f0 - xr * t3) + r0 = xr - (xr * e - hxs) + e = xr * (e - c) - c - hxs + rm1 = 0.5f0 * (xr - e) - 0.5f0 + rp1 = ifelse(xr < -0.25f0, -2f0 * (e - (xr + 0.5f0)), 1f0 + 2f0 * (xr - e)) + twopk = reinterpret(Float32, reinterpret(UInt32, Int32(0x7f) + k) << UInt32(23)) + twomk = reinterpret(Float32, reinterpret(UInt32, Int32(0x7f) - k) << UInt32(23)) + y = xr - e + 1f0 + # 2^128 is not a Float32, so scale by 2 * 2^127 for k = 128. + rfar = ifelse(k == Int32(128), y * 2f0 * reinterpret(Float32, 0x7f000000), y * twopk) - 1f0 + rmid = ifelse(k < Int32(23), (xr - e + (1f0 - twomk)) * twopk, + (xr - (e + twomk) + 1f0) * twopk) + r = ifelse(k == Int32(0), r0, + ifelse(k == Int32(-1), rm1, + ifelse(k == Int32(1), rp1, + ifelse((k < Int32(0)) | (k > Int32(56)), rfar, rmid)))) + + # |x| < 2^-25 and NaN give x; below -27 ln2, -1; above the overflow + # threshold (higher than fdlibm's 0x42b17180, which overflows early), Inf. + r = ifelse(hx < 0x33000000, x, r) + far = hx >= 0x4195b844 + r = ifelse(far & neg, -1f0, r) + r = ifelse(far & !neg & (x > reinterpret(Float32, 0x42b17217)), Inf32, r) + ifelse(hx > 0x7f800000, x, r) +end +@overlay Base.log1p(x::T) where {T <: Union{Float16, BFloat16}} = T(Base.log1p(Float32(x))) +@overlay Base.expm1(x::T) where {T <: Union{Float16, BFloat16}} = T(Base.expm1(Float32(x))) +@overlay function Base.log1p(x::T) where {T <: ScalarFloat} + u = one(T) + x + r = Intrinsics.log(u) * (x / (u - one(T))) + ifelse(u == one(T), x, ifelse(u == T(Inf), u, r)) +end +@overlay function Base.expm1(x::T) where {T <: ScalarFloat} + u = Intrinsics.exp(x) + um1 = u - one(T) + r = um1 * (x / Intrinsics.log(u)) + ifelse(u == one(T), x, ifelse((um1 == -one(T)) | (u == T(Inf)), um1, r)) +end + # max/min @overlay Base.max(x::T, y::T) where {T <: Signed} = Intrinsics.maxi(x, y, Signedness.Signed) @overlay Base.max(x::T, y::T) where {T <: Unsigned} = Intrinsics.maxi(x, y, Signedness.Unsigned) diff --git a/test/codegen/integration.jl b/test/codegen/integration.jl index 1cb9c18f..378f6bc2 100644 --- a/test/codegen/integration.jl +++ b/test/codegen/integration.jl @@ -794,6 +794,20 @@ end end end + @testset "throw guarded by a bounds check" begin + @test @filecheck begin + @check "assert {{.*}}ArgumentError: out of range" + code_tiled(Tuple{Int32}) do x + @inline function checked(y) + @boundscheck y > Int32(100) && throw(ArgumentError("out of range")) + return + end + checked(x) + return + end + end + end + @testset "runtime-dependent message falls back to the exception type" begin @test @filecheck begin @check "assert {{.*}}ArgumentError was thrown" diff --git a/test/codegen/operations.jl b/test/codegen/operations.jl index 15838615..a901cca0 100644 --- a/test/codegen/operations.jl +++ b/test/codegen/operations.jl @@ -2133,6 +2133,110 @@ end end end + @testset "branches in broadcast functions" begin + # A branch on a per-element condition executes both arms and selects. + @test @filecheck begin + @check_label "entry" + @check_not "if" + @check "sqrt" + @check "select" + code_tiled(Tuple{ct.TileArray{Float32,1,Int32,spec1d}}) do a + tile = ct.load(a, 1, (16,)) + f(x) = isinf(x) ? Inf32 : sqrt(x * x + 1f0) + ct.store(a, 1, f.(tile)) + return + end + end + + # A throw in such a branch asserts that no element takes it. + @test @filecheck begin + @check_label "entry" + @check_not "if" + @check "assert {{.*}}, \"DomainError" + code_tiled(Tuple{ct.TileArray{Float32,1,Int32,spec1d}}) do a + tile = ct.load(a, 1, (16,)) + g(x) = x < 0f0 ? throw(DomainError(x)) : sqrt(x) + ct.store(a, 1, g.(tile)) + return + end + end + + # A division in an arm also runs for lanes it is undefined for, so it + # divides by 1 for those. + @test @filecheck begin + @check_label "entry" + @check_not "if" + @check "[[DIVISOR:%.+]] = select" + @check "divi {{.*}}, [[DIVISOR]] signed" + code_tiled(Tuple{ct.TileArray{Int32,1,Int32,spec1d}}) do a + tile = ct.load(a, 1, (16,)) + d(x) = x != Int32(0) ? Int32(100) ÷ x : Int32(0) + ct.store(a, 1, d.(tile)) + return + end + end + + # A `@boundscheck` in an arm is folded before flattening. + @test @filecheck begin + @check_label "entry" + @check_not "if" + @check "select" + code_tiled(Tuple{ct.TileArray{Float32,1,Int32,spec1d}}) do a + tile = ct.load(a, 1, (16,)) + @inline function checked(x) + @boundscheck x > 100f0 && throw(ArgumentError("out of range")) + 2f0 * x + end + f(x) = x > 0f0 ? checked(x) : 0f0 + ct.store(a, 1, f.(tile)) + return + end + end + + # A branch that cannot be flattened is reported, not passed to tileiras. + @test_throws "Cannot branch on a per-element condition" code_tiled( + Tuple{ct.TileArray{Float32,1,Int32,spec1d}}) do a + tile = ct.load(a, 1, (16,)) + h(x) = (for i in 1:4; x > Float32(i) || return Float32(i); end; 0f0) + ct.store(a, 1, h.(tile)) + return + end + # Neither is a load, which lanes not taking the arm may not be allowed + # to perform. + @test_throws "Cannot branch on a per-element condition" code_tiled( + Tuple{ct.TileArray{Int32,1,Int32,spec1d}, + ct.TileArray{Int32,1,Int32,spec1d}}) do a, b + tile = ct.load(a, 1, (16,)) + l(x) = x > Int32(0) ? sum(ct.load(b, 1, (16,); check_bounds=false)) : Int32(0) + ct.store(a, 1, l.(tile)) + return + end + end + + @testset "log1p and expm1 divide approximately" begin + # Their quotients only correct larger terms; IEEE division would check + # its range and call a slow path for every element. + @test @filecheck begin + @check_label "entry" + @check "divf {{.*}} rounding" + @check "divf {{.*}} rounding" + code_tiled(Tuple{ct.TileArray{Float32,1,Int32,spec1d}}) do a + tile = ct.load(a, 1, (16,)) + ct.store(a, 1, log1p.(tile)) + return + end + end + @test @filecheck begin + @check_label "entry" + @check "divf {{.*}} rounding" + code_tiled(Tuple{ct.TileArray{Float32,1,Int32,spec1d}}) do a + tile = ct.load(a, 1, (16,)) + ct.store(a, 1, expm1.(tile)) + return + end + end + end + @testset "scalar literal promotion" begin @test @filecheck begin @check_label "entry" diff --git a/test/device/broadcast.jl b/test/device/broadcast.jl index 633e6df2..4beedde4 100644 --- a/test/device/broadcast.jl +++ b/test/device/broadcast.jl @@ -955,3 +955,77 @@ end @test O[:, 5:8] == 1f0 .- X @test O[:, 9:12] == Float32.(X .> 0f0) end + +@testset "scalar functions with branches" begin + # A branch on a per-element condition is flattened: both arms run and the + # results are selected per element. + hyp(x, y) = isinf(x) | isinf(y) ? Inf32 : sqrt(muladd(x, x, y * y)) + piecewise(x) = x < -1f0 ? -1f0 : x > 1f0 ? x * x : x + + function branch_kernel(out::ct.TileArray{Float32,2}, x::ct.TileArray{Float32,2}) + t = ct.load(x, (1, 1), (64, 4)) + ct.store(out, (1, 1), hyp.(t, t)) + ct.store(out, (1, 2), piecewise.(t)) + return + end + + x = CUDA.randn(Float32, 64, 4) + CUDA.@allowscalar x[1, 1] = Inf32 + out = CUDA.zeros(Float32, 64, 8) + @cuda backend=cuTile branch_kernel(out, x) + + X = Array(x) + O = Array(out) + @test O[:, 1:4] ≈ hyp.(X, X) + @test O[:, 5:8] == piecewise.(X) +end + +@testset "guarded integer division" begin + # Both arms run, so the divisions must not see the zero divisors (or + # typemin / -1) their branches exclude. + fs = (x -> x != Int32(0) ? Int32(100) ÷ x : Int32(0), + x -> x == Int32(0) ? Int32(-1) : rem(Int32(7), x), + x -> x != Int32(0) ? fld(Int32(100), x) : Int32(0), + x -> x != Int32(0) ? cld(Int32(100), x) : Int32(0), + x -> x != Int32(0) && x != Int32(-1) ? typemin(Int32) ÷ x : Int32(0)) + function div_kernel(out::ct.TileArray{Int32,2}, x::ct.TileArray{Int32,2}, y::Int32) + t = ct.load(x, (1, 1), (64, 1)) + ct.store(out, (1, 1), fs[1].(t)) + ct.store(out, (1, 2), fs[2].(t)) + ct.store(out, (1, 3), fs[3].(t)) + ct.store(out, (1, 4), fs[4].(t)) + ct.store(out, (1, 5), fs[5].(t)) + # A divisor that is the same for all elements. + ct.store(out, (1, 6), (x -> x > Int32(0) ? Int32(100) ÷ y : Int32(0)).(t)) + return + end + + x = Int32.(rand(-3:3, 64, 1)) + x[1:2] .= (0, -1) + out = CUDA.zeros(Int32, 64, 6) + @cuda backend=cuTile div_kernel(out, CuArray(x), Int32(0)) + + O = Array(out) + for (i, f) in enumerate(fs) + @test O[:, i] == f.(vec(x)) + end + @test all(iszero, O[vec(x) .<= 0, 6]) + + @cuda backend=cuTile div_kernel(out, CuArray(x), Int32(7)) + @test Array(out)[:, 6] == [x > 0 ? 100 ÷ Int32(7) : Int32(0) for x in vec(x)] +end + +@testset "assertions in flattened branches" begin + # The domain check of `asin` is only asserted for elements that call it. + clamped_asin(x) = abs(x) > 1f0 ? 0f0 : asin(x) + + function asin_kernel(out::ct.TileArray{Float32,1}, x::ct.TileArray{Float32,1}) + ct.store(out, 1, clamped_asin.(ct.load(x, 1, (64,)))) + return + end + + x = 4 .* rand(Float32, 64) .- 2 + out = CUDA.zeros(Float32, 64) + @cuda backend=cuTile asin_kernel(out, CuArray(x)) + @test Array(out) ≈ clamped_asin.(x) +end diff --git a/test/device/math.jl b/test/device/math.jl index dc6aa0c5..e4229d68 100644 --- a/test/device/math.jl +++ b/test/device/math.jl @@ -252,6 +252,13 @@ end @testset "Base math functions over tiles" begin for (name, f, gen) in [ + # These branch per element. Inputs stay within each domain, outside of + # which Base throws. + ("log1p", log1p, () -> 4 .* rand(Float32, 256) .- 0.999f0), + ("expm1", expm1, () -> 4 .* randn(Float32, 256)), + ("asin", asin, () -> 2 .* rand(Float32, 256) .- 1), + ("acos", acos, () -> 2 .* rand(Float32, 256) .- 1), + ("hypot", x -> hypot(x, 2x), () -> randn(Float32, 256)), ("atan", atan, () -> 4 .* randn(Float32, 256)), ("copysign", x -> copysign(2f0, x), () -> randn(Float32, 256)), ("muladd", x -> muladd(x, x, 1f0), () -> randn(Float32, 256)), @@ -270,4 +277,35 @@ end @test Array(out) ≈ f.(x) rtol=1f-5 end end + + # Edge cases: tiny arguments, the boundaries of the argument reduction, -1, + # overflow, and special values. Results are within 1 ulp of the correctly + # rounded value. + function edge_kernel(out::ct.TileArray{Float32,2}, x::ct.TileArray{Float32,2}) + ct.store(out, (1, 1), log1p.(ct.load(x, (1, 1), (32, 1)))) + ct.store(out, (1, 2), expm1.(ct.load(x, (1, 2), (32, 1)))) + return + end + small = Float32[0, -0f0, 1f-30, -1f-30, 1f-10, -1f-10, 1f-7, -1f-7, 3f-8, -3f-8, 1f-4, -1f-4] + log1p_x = vcat(small, Float32[0.4142, 0.4143, -0.2929, -0.293, -0.5, -0.9999, -1, 0.5, + 1, 10, 1f6, 1f30, floatmax(Float32), Inf, -2, NaN, 0.3, 3, 2, 0.7]) + expm1_x = vcat(small, Float32[0.34, 0.35, -0.35, 1.03, 1.04, -1.04, 18, 19, -18, -19, + 88.72, 88.7228, 88.723, 100, -100, Inf, -Inf, NaN, 5, -5]) + out = CUDA.zeros(Float32, 32, 2) + @cuda backend=cuTile edge_kernel(out, CuArray([log1p_x expm1_x])) + O = Array(out) + rounded(f, x) = f === log1p && x < -1 ? NaN32 : Float32(f(Float64(x))) + @test all(isapprox.(O[:, 1], rounded.(log1p, log1p_x); rtol=eps(Float32), nans=true)) + @test all(isapprox.(O[:, 2], rounded.(expm1, expm1_x); rtol=eps(Float32), nans=true)) + + # Float16 rounds the Float32 result. + function half_kernel(out::ct.TileArray{Float16,2}, x::ct.TileArray{Float16,2}) + ct.store(out, (1, 1), log1p.(ct.load(x, (1, 1), (256, 1)))) + ct.store(out, (1, 2), expm1.(ct.load(x, (1, 2), (256, 1)))) + return + end + xh = Float16.([rand(Float32, 256) .* 4 .- 0.99f0 randn(Float32, 256) .* 4]) + outh = CUDA.zeros(Float16, 256, 2) + @cuda backend=cuTile half_kernel(outh, CuArray(xh)) + @test Array(outh) ≈ [log1p.(xh[:, 1]) expm1.(xh[:, 2])] rtol=eps(Float16) end