From 99fce74798c1bb2c0c9b723e1e86d30ea534e410 Mon Sep 17 00:00:00 2001 From: Nery Campusano Date: Thu, 8 Oct 2026 14:50:34 -0400 Subject: [PATCH] add array element stride for frozen heap arrays --- array.c | 466 +++++++++++++- compile.c | 1 + gc.c | 8 +- hash.c | 15 + include/ruby/internal/core/rarray.h | 85 ++- internal/array.h | 101 ++- internal/hash.h | 1 + ractor.c | 5 + test/ruby/test_array_stride.rb | 941 ++++++++++++++++++++++++++++ vm_args.c | 14 + 10 files changed, 1595 insertions(+), 42 deletions(-) create mode 100644 test/ruby/test_array_stride.rb diff --git a/array.c b/array.c index 6add3d7fe4d125..c09cecf56991e9 100644 --- a/array.c +++ b/array.c @@ -34,6 +34,7 @@ #include "ruby/ractor.h" #include "shape.h" #include "vm_core.h" +#include "vm_sync.h" #include "builtin.h" #include "zjit.h" @@ -98,7 +99,17 @@ should_be_T_ARRAY(VALUE ary) (RUBY_ASSERT(ARY_EMBED_P(a)), \ (long)((RBASIC(a)->flags >> RARRAY_EMBED_LEN_SHIFT) & \ (RARRAY_EMBED_LEN_MASK >> RARRAY_EMBED_LEN_SHIFT))) -#define ARY_HEAP_SIZE(a) (RUBY_ASSERT(!ARY_EMBED_P(a)), RUBY_ASSERT(ARY_OWNS_HEAP_P(a)), ARY_CAPA(a) * sizeof(VALUE)) +static inline size_t +ary_stride_bytes(VALUE ary) +{ + switch (rb_array_stride(ary)) { + case RARRAY_STRIDE_W8: return 1; + case RARRAY_STRIDE_W16: return 2; + case RARRAY_STRIDE_W32: return 4; + default: return sizeof(VALUE); + } +} +#define ARY_HEAP_SIZE(a) (RUBY_ASSERT(!ARY_EMBED_P(a)), RUBY_ASSERT(ARY_OWNS_HEAP_P(a)), ARY_CAPA(a) * ary_stride_bytes(a)) #define ARY_OWNS_HEAP_P(a) (RUBY_ASSERT(should_be_T_ARRAY((VALUE)(a))), \ !FL_TEST_RAW((a), RARRAY_SHARED_FLAG|RARRAY_EMBED_FLAG)) @@ -225,8 +236,12 @@ rb_ary_embeddable_p(VALUE ary) * flag. * - Shared: we don't want to re-embed an array that points to a shared * root (to save memory). + * - Narrowed: the buffer is not VALUEs, so it cannot be copied into the + * slot as-is, and GC compaction (the caller) cannot widen because it + * must not allocate. */ - return !(ARY_SHARED_ROOT_P(ary) || OBJ_FROZEN(ary) || ARY_SHARED_P(ary)); + return !(ARY_SHARED_ROOT_P(ary) || OBJ_FROZEN(ary) || ARY_SHARED_P(ary) || + rb_array_stride(ary) != RARRAY_STRIDE_VALUE); } /* True when other arrays may read this array's elements out of its own slot, so the @@ -278,14 +293,16 @@ ary_verify_(VALUE ary, const char *file, int line) RUBY_ASSERT(RARRAY_LEN(ary) <= ary_embed_capa(ary)); } else { - const VALUE *ptr = RARRAY_CONST_PTR(ary); - long i, len = RARRAY_LEN(ary); - volatile VALUE v; - if (len > 1) len = 1; /* check only HEAD */ - for (i=0; i= RARRAY_LEN(ary)); + } + if (RARRAY_LEN(ary) > 0) { + /* RARRAY_AREF is narrow-aware and does not widen, so an assertion + * cannot change the array's representation. */ + volatile VALUE v = RARRAY_AREF(ary, 0); /* access check: HEAD only */ + v = v; } - v = v; } return ary; @@ -294,6 +311,225 @@ ary_verify_(VALUE ary, const char *file, int line) #define ary_verify(ary) ((void)0) #endif +/* ---- element stride ---------------------------------------------------- */ + +static inline void +ary_set_stride(VALUE ary, int stride) +{ + RBASIC(ary)->flags = (RBASIC(ary)->flags & ~(VALUE)RARRAY_STRIDE_MASK) | + ((VALUE)stride << RARRAY_STRIDE_SHIFT); +} + +static void +ary_widen_locked(VALUE ary) +{ + const long len = RARRAY_LEN(ary); + const size_t old_bytes = (size_t)ARY_CAPA(ary) * ary_stride_bytes(ary); + VALUE *wide = ALLOC_N(VALUE, len == 0 ? 1 : len); + for (long i = 0; i < len; i++) { + wide[i] = rarray_narrow_aref(ary, i); + } + + void *old = (void *)RARRAY(ary)->as.heap.ptr; + ary_set_stride(ary, RARRAY_STRIDE_VALUE); + RARRAY(ary)->as.heap.ptr = wide; + RARRAY(ary)->as.heap.aux.capa = len == 0 ? 1 : len; + RARRAY(ary)->as.heap.len = len; + ruby_xfree_sized(old, old_bytes); +} + +void +rb_ary_widen(VALUE ary) +{ + if (rb_array_stride(ary) == RARRAY_STRIDE_VALUE) return; + + /* A frozen narrow array may be Ractor-shareable, so two Ractors can reach + * this point in parallel through RARRAY_CONST_PTR. Widening swaps and + * frees the buffer, which must happen exactly once. Take the VM lock and + * re-check: whoever loses the race finds the array already widened. */ + RB_VM_LOCKING() { + if (rb_array_stride(ary) != RARRAY_STRIDE_VALUE) { + ary_widen_locked(ary); + } + } +} + +static int +ary_try_narrow(VALUE ary) +{ + if (rb_array_stride(ary) != RARRAY_STRIDE_VALUE) return rb_array_stride(ary); + if (ARY_SHARED_P(ary)) rb_ary_cancel_sharing(ary); + if (ARY_EMBED_P(ary) || ARY_SHARED_P(ary) || ARY_SHARED_ROOT_P(ary)) { + return RARRAY_STRIDE_VALUE; + } + + const long len = RARRAY_LEN(ary); + if (len == 0) return RARRAY_STRIDE_VALUE; + + /* one pass for the range, then pick the narrowest width that holds it */ + const VALUE *src = RARRAY_CONST_PTR(ary); + long lo = LONG_MAX, hi = LONG_MIN; + for (long i = 0; i < len; i++) { + if (!FIXNUM_P(src[i])) return RARRAY_STRIDE_VALUE; + long v = FIX2LONG(src[i]); + if (v < lo) lo = v; + if (v > hi) hi = v; + } + + int want, uns; + if (lo >= 0) { + uns = 1; + if (hi <= UINT8_MAX) want = RARRAY_STRIDE_W8; + else if (hi <= UINT16_MAX) want = RARRAY_STRIDE_W16; + else if (hi <= UINT32_MAX) want = RARRAY_STRIDE_W32; + else return RARRAY_STRIDE_VALUE; + } + else { + uns = 0; + if (lo >= INT8_MIN && hi <= INT8_MAX) want = RARRAY_STRIDE_W8; + else if (lo >= INT16_MIN && hi <= INT16_MAX) want = RARRAY_STRIDE_W16; + else if (lo >= INT32_MIN && hi <= INT32_MAX) want = RARRAY_STRIDE_W32; + else return RARRAY_STRIDE_VALUE; + } + + const size_t width = want == RARRAY_STRIDE_W8 ? 1 : want == RARRAY_STRIDE_W16 ? 2 : 4; + const size_t old_bytes = (size_t)ARY_CAPA(ary) * sizeof(VALUE); + void *buf = ruby_xmalloc((size_t)len * width); + for (long i = 0; i < len; i++) { + long v = FIX2LONG(src[i]); + switch (want) { + case RARRAY_STRIDE_W8: ((uint8_t *)buf)[i] = (uint8_t)v; break; + case RARRAY_STRIDE_W16: ((uint16_t *)buf)[i] = (uint16_t)v; break; + default: ((uint32_t *)buf)[i] = (uint32_t)v; break; + } + } + + void *old = (void *)RARRAY(ary)->as.heap.ptr; + ary_set_stride(ary, want); + if (uns) FL_SET_RAW(ary, RARRAY_STRIDE_UNSIGNED); + else FL_UNSET_RAW(ary, RARRAY_STRIDE_UNSIGNED); + RARRAY(ary)->as.heap.ptr = (const VALUE *)buf; + RARRAY(ary)->as.heap.aux.capa = len; + RARRAY(ary)->as.heap.len = len; + ruby_xfree_sized(old, old_bytes); + return want; +} + +int +rb_ary_narrow(VALUE ary) +{ + return ary_try_narrow(ary); +} + +/* ---- narrow fast paths ------------------------------------------------- */ + +/* Run BODY once with `const T *p` bound to a narrowed array's packed buffer, + * where T is the element's C type. This hoists the width/sign dispatch out + * of the caller's loop: inside BODY, p[i] is a plain typed load the compiler + * can vectorize, instead of a per-element RARRAY_AREF decode. */ +#define ARY_NARROW_DISPATCH(ary, T, p, BODY) do { \ + const void *narrow_raw_ = rarray_raw_ptr(ary); \ + if (rb_array_stride_unsigned_p(ary)) { \ + switch (rb_array_stride(ary)) { \ + case RARRAY_STRIDE_W8: { typedef uint8_t T; const T *p = narrow_raw_; BODY; } break; \ + case RARRAY_STRIDE_W16: { typedef uint16_t T; const T *p = narrow_raw_; BODY; } break; \ + default: { typedef uint32_t T; const T *p = narrow_raw_; BODY; } break; \ + } \ + } \ + else { \ + switch (rb_array_stride(ary)) { \ + case RARRAY_STRIDE_W8: { typedef int8_t T; const T *p = narrow_raw_; BODY; } break; \ + case RARRAY_STRIDE_W16: { typedef int16_t T; const T *p = narrow_raw_; BODY; } break; \ + default: { typedef int32_t T; const T *p = narrow_raw_; BODY; } break; \ + } \ + } \ +} while (0) + +/* Index of the first (or last) element equal to needle, or -1. Equality on + * unboxed integers is a plain compare, so this is only valid when Integer#== + * is not redefined; callers check. A needle the element type cannot + * represent cannot be present, which the round trip through T detects. */ +static long +ary_narrow_find(VALUE ary, long needle, bool from_end) +{ + const long len = RARRAY_LEN(ary); + long found = -1; + ARY_NARROW_DISPATCH(ary, T, p, { + if ((long)(T)needle == needle) { + const T n = (T)needle; + if (from_end) { + for (long i = len - 1; i >= 0; i--) if (p[i] == n) { found = i; break; } + } + else { + for (long i = 0; i < len; i++) if (p[i] == n) { found = i; break; } + } + } + }); + return found; +} + +static long +ary_narrow_count(VALUE ary, long needle) +{ + const long len = RARRAY_LEN(ary); + long count = 0; + ARY_NARROW_DISPATCH(ary, T, p, { + if ((long)(T)needle == needle) { + const T n = (T)needle; + for (long i = 0; i < len; i++) count += (p[i] == n); + } + }); + return count; +} + +/* Sum of a narrowed array as a long. Elements are at most 32 bits, so the + * accumulator cannot overflow for fewer than 2**31 elements; callers check. */ +static long +ary_narrow_sum(VALUE ary) +{ + const long len = RARRAY_LEN(ary); + long acc = 0; + ARY_NARROW_DISPATCH(ary, T, p, { + for (long i = 0; i < len; i++) acc += p[i]; + }); + return acc; +} + +/* Minimum (or maximum) of a non-empty narrowed array. Only valid when + * Integer#<=> is not redefined; callers check. */ +static long +ary_narrow_minmax(VALUE ary, bool want_max) +{ + const long len = RARRAY_LEN(ary); + RUBY_ASSERT(len > 0); + long best = 0; + ARY_NARROW_DISPATCH(ary, T, p, { + T b = p[0]; + if (want_max) { for (long i = 1; i < len; i++) if (p[i] > b) b = p[i]; } + else { for (long i = 1; i < len; i++) if (p[i] < b) b = p[i]; } + best = (long)b; + }); + return best; +} + +/* Decode a narrowed array's elements into dst[0..len) as VALUEs, in order or + * reversed. One dispatch, then a tight typed loop: ~8x faster than + * RARRAY_AREF per element. Elements are immediates, so dst needs no write + * barrier. */ +static void +ary_narrow_decode(VALUE src, VALUE *dst, long len, bool reversed) +{ + ARY_NARROW_DISPATCH(src, T, p, { + if (reversed) { for (long i = 0; i < len; i++) dst[len - 1 - i] = LONG2FIX(p[i]); } + else { for (long i = 0; i < len; i++) dst[i] = LONG2FIX(p[i]); } + }); +} + +/* True when the narrow fast paths may stand in for Integer#== / #<=>. */ +#define ARY_NARROW_EQ_OK(ary, item) \ + (RB_UNLIKELY(rb_array_stride(ary) != RARRAY_STRIDE_VALUE) && FIXNUM_P(item) && \ + BASIC_OP_UNREDEFINED_P(BOP_EQ, INTEGER_REDEFINED_OP_FLAG)) + VALUE * rb_ary_ptr_use_start(VALUE ary) { @@ -393,6 +629,8 @@ static size_t ary_heap_realloc(VALUE ary, size_t new_capa) { RUBY_ASSERT(!OBJ_FROZEN(ary)); + /* capacity is counted in VALUEs; a narrow buffer must be widened first */ + RUBY_ASSERT(rb_array_stride(ary) == RARRAY_STRIDE_VALUE); SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, new_capa, ARY_HEAP_CAPA(ary)); ary_verify(ary); @@ -552,6 +790,9 @@ rb_ary_modify_check(VALUE ary) RUBY_ASSERT(ruby_thread_has_gvl_p()); rb_check_frozen(ary); + if (RB_UNLIKELY(rb_array_stride(ary) != RARRAY_STRIDE_VALUE)) { + rb_ary_widen(ary); + } ary_verify(ary); } @@ -667,7 +908,9 @@ rb_ary_freeze(VALUE ary) if (OBJ_FROZEN(ary)) return ary; if (!ARY_EMBED_P(ary) && !ARY_SHARED_P(ary) && !ARY_SHARED_ROOT_P(ary)) { - ary_shrink_capa(ary); + if (ary_try_narrow(ary) == RARRAY_STRIDE_VALUE) { + ary_shrink_capa(ary); + } } return rb_obj_freeze(ary); @@ -941,7 +1184,7 @@ size_t rb_ary_memsize(VALUE ary) { if (ARY_OWNS_HEAP_P(ary)) { - return ARY_CAPA(ary) * sizeof(VALUE); + return ARY_CAPA(ary) * ary_stride_bytes(ary); } else { return 0; @@ -953,6 +1196,15 @@ ary_make_shared(VALUE ary) { ary_verify(ary); + /* A shared root's buffer is read as VALUEs by every array that borrows + * it, and borrowers copy the raw pointer. A narrow array must therefore + * never become a root; widen before handing the buffer out. This is the + * frozen case in practice: narrowing happens on freeze, and a frozen + * array is returned below as its own root. */ + if (RB_UNLIKELY(rb_array_stride(ary) != RARRAY_STRIDE_VALUE)) { + rb_ary_widen(ary); + } + if (ARY_SHARED_P(ary)) { return ARY_SHARED_ROOT(ary); } @@ -2235,6 +2487,10 @@ rb_ary_index(int argc, VALUE *argv, VALUE ary) val = argv[0]; if (rb_block_given_p()) rb_warn("given block not used"); + if (ARY_NARROW_EQ_OK(ary, val)) { + long found = ary_narrow_find(ary, FIX2LONG(val), false); + return found < 0 ? Qnil : LONG2NUM(found); + } for (i=0; i 0) { + if (len > 0 && RB_UNLIKELY(rb_array_stride(ary) != RARRAY_STRIDE_VALUE)) { + /* decode reversed, so the source stays narrow */ + RARRAY_PTR_USE(dup, p2, ary_narrow_decode(ary, p2, len, true)); + } + else if (len > 0) { const VALUE *p1 = RARRAY_CONST_PTR(ary); VALUE *p2 = (VALUE *)RARRAY_CONST_PTR(dup) + len - 1; do *p2-- = *p1++; while (--len > 0); @@ -5452,6 +5722,94 @@ rb_ary_rassoc(VALUE ary, VALUE value) return Qnil; } +/* Two wide heap arrays that share one buffer are trivially equal. A narrow + * array has no VALUE buffer to compare, and asking for one would widen it; + * narrow arrays never share a buffer, so the shortcut simply does not apply. */ +static inline bool +ary_same_buffer_p(VALUE ary1, VALUE ary2) +{ + return rb_array_stride(ary1) == RARRAY_STRIDE_VALUE && + rb_array_stride(ary2) == RARRAY_STRIDE_VALUE && + RARRAY_CONST_PTR(ary1) == RARRAY_CONST_PTR(ary2); +} + +/* Element-wise == from index i, re-reading both arrays every iteration. + * rb_equal() may run Ruby code that mutates, widens or shrinks either array, + * so nothing is cached across it. Lengths were equal on entry. */ +static VALUE +ary_equal_slow_from(VALUE ary1, VALUE ary2, long i, bool eql) +{ + for (; i < RARRAY_LEN(ary1); i++) { + VALUE e1 = RARRAY_AREF(ary1, i); + VALUE e2 = RARRAY_AREF(ary2, i); + if (e1 != e2) { + if (!(eql ? rb_eql(e1, e2) : rb_equal(e1, e2))) return Qfalse; + long len1 = RARRAY_LEN(ary1); + if (len1 != RARRAY_LEN(ary2)) return Qfalse; + if (len1 < i) return Qtrue; + } + } + return Qtrue; +} + +/* == where at least one side is narrow. Two equal-width narrow arrays are a + * memcmp. Otherwise the dispatch is hoisted onto the narrow side and the + * other side is read through RARRAY_AREF; a mismatch against a Fixnum is a + * definite inequality (Integer#== not redefined), anything else drops to the + * slow path, which is safe against rb_equal() mutating the arrays. */ +static VALUE +ary_narrow_equal(VALUE ary1, VALUE ary2, bool eql) +{ + const long len = RARRAY_LEN(ary1); + const bool n1 = rb_array_stride(ary1) != RARRAY_STRIDE_VALUE; + const bool n2 = rb_array_stride(ary2) != RARRAY_STRIDE_VALUE; + + if (n1 && n2 && + rb_array_stride(ary1) == rb_array_stride(ary2) && + rb_array_stride_unsigned_p(ary1) == rb_array_stride_unsigned_p(ary2)) { + return RBOOL(memcmp(rarray_raw_ptr(ary1), rarray_raw_ptr(ary2), (size_t)len * ary_stride_bytes(ary1)) == 0); + } + + /* Two Fixnums that are not identical are never == (unless Integer#== is + * redefined) and never eql?. Anything else goes to the slow path. */ + const bool fix_mismatch_decides = eql || BASIC_OP_UNREDEFINED_P(BOP_EQ, INTEGER_REDEFINED_OP_FLAG); + long i = 0; + if (n1 && n2) { + /* different widths: decode the second side per element */ + ARY_NARROW_DISPATCH(ary1, T, p, { + for (; i < len; i++) { + if (LONG2FIX(p[i]) == rarray_narrow_aref(ary2, i)) continue; + if (fix_mismatch_decides) return Qfalse; + break; + } + }); + } + else if (n1) { + const VALUE *p2 = RARRAY_CONST_PTR(ary2); /* ary2 is wide: no widen */ + ARY_NARROW_DISPATCH(ary1, T, p, { + for (; i < len; i++) { + VALUE e2 = p2[i]; + if (LONG2FIX(p[i]) == e2) continue; + if (FIXNUM_P(e2) && fix_mismatch_decides) return Qfalse; + break; /* non-Fixnum on the other side: slow path */ + } + }); + } + else { + const VALUE *p1 = RARRAY_CONST_PTR(ary1); /* ary1 is wide: no widen */ + ARY_NARROW_DISPATCH(ary2, T, q, { + for (; i < len; i++) { + VALUE e1 = p1[i]; + if (e1 == LONG2FIX(q[i])) continue; + if (FIXNUM_P(e1) && fix_mismatch_decides) return Qfalse; + break; + } + }); + } + if (i >= len) return Qtrue; + return ary_equal_slow_from(ary1, ary2, i, eql); +} + static VALUE recursive_equal(VALUE ary1, VALUE ary2, int recur) { @@ -5460,7 +5818,13 @@ recursive_equal(VALUE ary1, VALUE ary2, int recur) if (recur) return Qtrue; /* Subtle! */ - /* rb_equal() can evacuate ptrs */ + if (RB_UNLIKELY(rb_array_stride(ary1) != RARRAY_STRIDE_VALUE || + rb_array_stride(ary2) != RARRAY_STRIDE_VALUE)) { + return ary_narrow_equal(ary1, ary2, false); + } + + /* Both wide: the original cached-pointer loop. rb_equal() can evacuate + * ptrs, hence the re-fetch after each call that returns true. */ p1 = RARRAY_CONST_PTR(ary1); p2 = RARRAY_CONST_PTR(ary2); len1 = RARRAY_LEN(ary1); @@ -5521,7 +5885,7 @@ rb_ary_equal(VALUE ary1, VALUE ary2) return rb_equal(ary2, ary1); } if (RARRAY_LEN(ary1) != RARRAY_LEN(ary2)) return Qfalse; - if (RARRAY_CONST_PTR(ary1) == RARRAY_CONST_PTR(ary2)) return Qtrue; + if (ary_same_buffer_p(ary1, ary2)) return Qtrue; return rb_exec_recursive_paired(recursive_equal, ary1, ary2, ary2); } @@ -5531,6 +5895,10 @@ recursive_eql(VALUE ary1, VALUE ary2, int recur) long i; if (recur) return Qtrue; /* Subtle! */ + if (RB_UNLIKELY(rb_array_stride(ary1) != RARRAY_STRIDE_VALUE || + rb_array_stride(ary2) != RARRAY_STRIDE_VALUE)) { + return ary_narrow_equal(ary1, ary2, true); + } for (i=0; i= 0); + } + + /* RARRAY_AREF is narrow-aware inside core, so this loop reads a narrowed + * array in place without widening it. */ for (i=0; i 0) { + if (RB_UNLIKELY(rb_array_stride(ary) != RARRAY_STRIDE_VALUE) && CMP_OPTIMIZABLE(INTEGER)) { + return LONG2FIX(ary_narrow_minmax(ary, true)); + } result = RARRAY_AREF(ary, 0); if (n > 1) { if (FIXNUM_P(result) && CMP_OPTIMIZABLE(INTEGER)) { @@ -6412,6 +6827,9 @@ rb_ary_min(int argc, VALUE *argv, VALUE ary) } } else if (n > 0) { + if (RB_UNLIKELY(rb_array_stride(ary) != RARRAY_STRIDE_VALUE) && CMP_OPTIMIZABLE(INTEGER)) { + return LONG2FIX(ary_narrow_minmax(ary, false)); + } result = RARRAY_AREF(ary, 0); if (n > 1) { if (FIXNUM_P(result) && CMP_OPTIMIZABLE(INTEGER)) { @@ -6695,6 +7113,9 @@ rb_ary_count(int argc, VALUE *argv, VALUE ary) if (rb_block_given_p()) { rb_warn("given block not used"); } + if (ARY_NARROW_EQ_OK(ary, obj)) { + return LONG2NUM(ary_narrow_count(ary, FIX2LONG(obj))); + } for (i = 0; i < RARRAY_LEN(ary); i++) { if (rb_equal(RARRAY_AREF(ary, i), obj)) n++; } @@ -8363,6 +8784,15 @@ rb_ary_sum(int argc, VALUE *argv, VALUE ary) n = 0; r = Qundef; + /* Narrow fast path: unboxed elements are summed as plain longs. Mirrors + * the Fixnum loop below (which also adds natively); elements are <= 32 + * bits, so the accumulator is safe below 2**31 elements. */ + if (!block_given && FIXNUM_P(v) && + RB_UNLIKELY(rb_array_stride(ary) != RARRAY_STRIDE_VALUE) && + RARRAY_LEN(ary) < (1L << 30)) { + return rb_int_plus(LONG2NUM(ary_narrow_sum(ary)), v); + } + bool init_is_float = RB_FLOAT_TYPE_P(v); if (init_is_float) { v = LONG2FIX(0); @@ -9061,6 +9491,10 @@ Init_Array(void) rb_define_method(rb_cArray, "clear", rb_ary_clear, 0); rb_define_method(rb_cArray, "fill", rb_ary_fill, -1); rb_define_method(rb_cArray, "include?", rb_ary_includes, 1); + + /* Test hooks for element-stride narrowing. Not public API. */ + rb_define_method(rb_cArray, "__narrow!", rb_ary_narrow_bang, 0); + rb_define_method(rb_cArray, "__stride", rb_ary_stride_m, 0); rb_define_method(rb_cArray, "<=>", rb_ary_cmp, 1); rb_define_method(rb_cArray, "slice", rb_ary_aref, -1); diff --git a/compile.c b/compile.c index d719764b150b9d..c99d074d0b16ec 100644 --- a/compile.c +++ b/compile.c @@ -3676,6 +3676,7 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal if (vm_ci_simple(ci) && vm_ci_argc(ci) == 0 && blockiseq == NULL && vm_ci_mid(ci) == idFreeze) { VALUE ary = iobj->operands[0]; rb_obj_reveal(ary, rb_cArray); + rb_ary_narrow(ary); insn_replace_with_operands(iseq, iobj, BIN(opt_ary_freeze), 2, ary, (VALUE)ci); ELEM_REMOVE(next); diff --git a/gc.c b/gc.c index f618f4569c452f..d3f7aba3871283 100644 --- a/gc.c +++ b/gc.c @@ -3677,6 +3677,11 @@ rb_gc_mark_children(void *objspace, VALUE obj) gc_mark_and_pin_internal(root); } } + else if (rb_array_stride(obj) != RARRAY_STRIDE_VALUE) { + /* narrow storage holds unboxed values, never object references, so + * there is nothing to trace; widening here is impossible anyway + * because it allocates */ + } else { long len = RARRAY_LEN(obj); const VALUE *ptr = RARRAY_CONST_PTR(obj); @@ -4620,7 +4625,8 @@ gc_ref_update_array(void *objspace, VALUE v) else { long len = RARRAY_LEN(v); - if (len > 0) { + /* narrow storage holds no object references: nothing to re-point */ + if (len > 0 && rb_array_stride(v) == RARRAY_STRIDE_VALUE) { VALUE *ptr = (VALUE *)RARRAY_CONST_PTR(v); for (long i = 0; i < len; i++) { UPDATE_IF_MOVED(objspace, ptr[i]); diff --git a/hash.c b/hash.c index a980d05baa2fd0..469d8f4b6b5792 100644 --- a/hash.c +++ b/hash.c @@ -321,6 +321,21 @@ rb_objid_hash(st_index_t index) return (long)st_index_hash(index); } +/* The value NUM2LONG(rb_hash(fixnum)) would produce, without the type switch + * and the Fixnum round trip. For code that hashes many Fixnums in a row. */ +long +rb_fixnum_hash(VALUE fixnum) +{ + RUBY_ASSERT(FIXNUM_P(fixnum)); + /* same clamping as the tail of any_hash() */ + st_index_t hnum = rb_objid_hash((st_index_t)fixnum); + if ((SIGNED_VALUE)hnum > 0) + hnum &= FIXNUM_MAX; + else + hnum |= FIXNUM_MIN; + return (long)hnum; +} + static st_index_t objid_hash(VALUE obj) { diff --git a/include/ruby/internal/core/rarray.h b/include/ruby/internal/core/rarray.h index d70535e9d9ba41..90513ea6015add 100644 --- a/include/ruby/internal/core/rarray.h +++ b/include/ruby/internal/core/rarray.h @@ -111,7 +111,31 @@ enum ruby_rarray_flags { * store array elements. It was a bad idea to expose this to them. */ RARRAY_EMBED_LEN_MASK = RUBY_FL_USER9 | RUBY_FL_USER8 | RUBY_FL_USER7 | RUBY_FL_USER6 | - RUBY_FL_USER5 | RUBY_FL_USER4 | RUBY_FL_USER3 + RUBY_FL_USER5 | RUBY_FL_USER4 | RUBY_FL_USER3, + + /** + * Width of one element in the backend storage. Zero means the storage is + * an array of ::VALUE, which is the only representation 3rd parties can + * see. A non-zero value means the elements are stored unboxed, narrower + * than a ::VALUE, and the array must be widened before its buffer is + * handed out. See ::rb_ary_widen. + * + * @internal + * + * 3rd parties must not be aware that there even is more than one way to + * store array elements. It was a bad idea to expose this to them. + */ + RARRAY_STRIDE_MASK = RUBY_FL_USER11 | RUBY_FL_USER10, + + /** + * Set when a narrowed array's unboxed elements are to be read as unsigned. + * Doubles the non-negative range a given width can hold. + * + * @internal + * + * 3rd parties must not be aware of this. + */ + RARRAY_STRIDE_UNSIGNED = RUBY_FL_USER2 }; /** @@ -120,9 +144,43 @@ enum ruby_rarray_flags { */ enum ruby_rarray_consts { /** Where ::RARRAY_EMBED_LEN_MASK resides. */ - RARRAY_EMBED_LEN_SHIFT = RUBY_FL_USHIFT + 3 + RARRAY_EMBED_LEN_SHIFT = RUBY_FL_USHIFT + 3, + + /** Where ::RARRAY_STRIDE_MASK resides. */ + RARRAY_STRIDE_SHIFT = RUBY_FL_USHIFT + 10 +}; + +/** + * Element widths an array can use, in bits. ::RARRAY_STRIDE_VALUE is the + * default: elements are ::VALUE. + */ +enum ruby_rarray_stride { + RARRAY_STRIDE_VALUE = 0, + RARRAY_STRIDE_W8 = 1, + RARRAY_STRIDE_W16 = 2, + RARRAY_STRIDE_W32 = 3 }; +/** + * @private + * + * True when a narrowed array's elements are unsigned. + */ +static inline int +rb_array_stride_unsigned_p(VALUE a) +{ + return RB_FL_ANY_RAW(a, RARRAY_STRIDE_UNSIGNED) ? 1 : 0; +} + +/** + * Converts a narrowed array back to ::VALUE storage, in place. A no-op for an + * array that already uses ::RARRAY_STRIDE_VALUE. Allocates, so it must not be + * called while allocation is forbidden, for instance during GC marking. + * + * @param[out] ary An object of ::RArray. + */ +void rb_ary_widen(VALUE ary); + /** Ruby's array. */ struct RArray { @@ -282,13 +340,30 @@ RARRAY_LENINT(VALUE ary) return rb_long2int(RARRAY_LEN(ary)); } -RBIMPL_ATTR_PURE_UNLESS_DEBUG() +/** + * @private + * + * The element width currently used by the backend storage. + * + * @param[in] a An object of ::RArray. + * @return One of ::ruby_rarray_stride. + */ +static inline int +rb_array_stride(VALUE a) +{ + RBIMPL_ASSERT_TYPE(a, RUBY_T_ARRAY); + return (int)((RBASIC(a)->flags & RARRAY_STRIDE_MASK) >> RARRAY_STRIDE_SHIFT); +} + /** * @private * * This is an implementation detail of RARRAY_PTR(). People do not use it * directly. * + * This is deliberately not pure: a narrowed array is widened in place before + * its buffer is returned, because every caller expects ::VALUE elements. + * * @param[in] a An object of ::RArray. * @return Its backend storage. */ @@ -297,6 +372,10 @@ rb_array_const_ptr(VALUE a) { RBIMPL_ASSERT_TYPE(a, RUBY_T_ARRAY); + if (RB_UNLIKELY(rb_array_stride(a) != RARRAY_STRIDE_VALUE)) { + rb_ary_widen(a); + } + if (RB_FL_ANY_RAW(a, RARRAY_EMBED_FLAG)) { return FIX_CONST_VALUE_PTR(RARRAY(a)->as.ary); } diff --git a/internal/array.h b/internal/array.h index 9fffd9b6bc558a..b34de589cee212 100644 --- a/internal/array.h +++ b/internal/array.h @@ -23,6 +23,79 @@ #define RARRAY_PTR_IN_USE_FLAG FL_USER14 #define RARRAY_FAKEARY FL_USER19 +/* Address of a narrowed array's packed buffer. Only the narrow decode path + * may use this; everything else goes through RARRAY_CONST_PTR, which widens. + * Narrow arrays always own a heap buffer: embedded arrays are never narrowed. */ +static inline const void * +rarray_raw_ptr(VALUE ary) +{ + RUBY_ASSERT(rb_array_stride(ary) != RARRAY_STRIDE_VALUE); + RUBY_ASSERT(!FL_TEST_RAW(ary, RARRAY_EMBED_FLAG)); + return (const void *)RARRAY(ary)->as.heap.ptr; +} + +/* Load element i of a narrowed array as a C integer. This is the single + * place that knows the width/sign encoding. It serves any strategy whose + * unboxed payload is an integer; a strategy with a different payload type + * would add a sibling loader. */ +static inline long +rarray_narrow_load_int(VALUE ary, long i) +{ + const void *p = rarray_raw_ptr(ary); + if (rb_array_stride_unsigned_p(ary)) { + switch (rb_array_stride(ary)) { + case RARRAY_STRIDE_W8: return (long)((const uint8_t *)p)[i]; + case RARRAY_STRIDE_W16: return (long)((const uint16_t *)p)[i]; + default: return (long)((const uint32_t *)p)[i]; + } + } + switch (rb_array_stride(ary)) { + case RARRAY_STRIDE_W8: return ((const int8_t *)p)[i]; + case RARRAY_STRIDE_W16: return ((const int16_t *)p)[i]; + default: return ((const int32_t *)p)[i]; + } +} + +/* Decode element i of a narrowed array into a ::VALUE. This is where the + * per-strategy knowledge lives: how to rebuild a VALUE from the payload. */ +static inline VALUE +rarray_narrow_aref(VALUE ary, long i) +{ + return LONG2FIX(rarray_narrow_load_int(ary, i)); +} + +/* Core's RARRAY_AREF replaces the public macro (which must stay an lvalue- + * capable macro for extensions). It adds a bounds assertion, and it is + * narrow-aware: an element of a narrowed array is decoded in place rather + * than widening the whole array, so read-only iteration (each, map, sum, + * index, ==, ...) leaves a narrowed array narrow. Never allocates, so it is + * also safe where RARRAY_CONST_PTR is not (opt_aref, GC callbacks). + * + * Not PURE: it is still a pure function of (ary, i) in the narrow case, but + * the wide case reads through RARRAY_CONST_PTR, which is not pure either. */ +#undef RARRAY_AREF +RBIMPL_ATTR_ARTIFICIAL() +static inline VALUE +RARRAY_AREF(VALUE ary, long i) +{ + VALUE val; + RBIMPL_ASSERT_TYPE(ary, RUBY_T_ARRAY); + + RUBY_ASSERT(i < RARRAY_LEN(ary)); + + if (RB_UNLIKELY(rb_array_stride(ary) != RARRAY_STRIDE_VALUE)) { + return rarray_narrow_aref(ary, i); + } + + RBIMPL_WARNING_PUSH(); +#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ == 13 + RBIMPL_WARNING_IGNORED(-Warray-bounds); +#endif + val = RARRAY_CONST_PTR(ary)[i]; + RBIMPL_WARNING_POP(); + return val; +} + /* array.c */ VALUE rb_ary_hash_values(long len, const VALUE *elements); VALUE rb_ary_last(int, const VALUE *, VALUE); @@ -34,6 +107,11 @@ size_t rb_ary_memsize(VALUE); VALUE rb_to_array_type(VALUE obj); VALUE rb_to_array(VALUE obj); void rb_ary_cancel_sharing(VALUE ary); +/* Pack a heap array whose elements can all be stored unboxed to the narrowest + * element width that holds every value. Returns the resulting stride; + * RARRAY_STRIDE_VALUE means the array was left as-is. Used by the freeze + * trigger and by the compiler for frozen array literals. */ +int rb_ary_narrow(VALUE ary); size_t rb_ary_size_as_embedded(VALUE ary); void rb_ary_make_embedded(VALUE ary); bool rb_ary_embeddable_p(VALUE ary); @@ -59,7 +137,6 @@ static inline VALUE rb_ary_entry_internal(VALUE ary, long offset) { long len = RARRAY_LEN(ary); - const VALUE *ptr = RARRAY_CONST_PTR(ary); if (len == 0) return Qnil; if (offset < 0) { offset += len; @@ -68,7 +145,7 @@ rb_ary_entry_internal(VALUE ary, long offset) else if (len <= offset) { return Qnil; } - return ptr[offset]; + return RARRAY_AREF(ary, offset); /* narrow-aware; does not allocate */ } static inline bool @@ -135,24 +212,4 @@ ARY_SHARED_ROOT_REFCNT(VALUE ary) }) #endif -#undef RARRAY_AREF -RBIMPL_ATTR_PURE_UNLESS_DEBUG() -RBIMPL_ATTR_ARTIFICIAL() -static inline VALUE -RARRAY_AREF(VALUE ary, long i) -{ - VALUE val; - RBIMPL_ASSERT_TYPE(ary, RUBY_T_ARRAY); - - RUBY_ASSERT(i < RARRAY_LEN(ary)); - - RBIMPL_WARNING_PUSH(); -#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ == 13 - RBIMPL_WARNING_IGNORED(-Warray-bounds); -#endif - val = RARRAY_CONST_PTR(ary)[i]; - RBIMPL_WARNING_POP(); - return val; -} - #endif /* INTERNAL_ARRAY_H */ diff --git a/internal/hash.h b/internal/hash.h index 537913fd20ae93..c7996c63aaae35 100644 --- a/internal/hash.h +++ b/internal/hash.h @@ -75,6 +75,7 @@ VALUE rb_hash_default_value(VALUE hash, VALUE key); VALUE rb_hash_set_default(VALUE hash, VALUE ifnone); VALUE rb_hash_set_default_proc(VALUE hash, VALUE proc); long rb_dbl_long_hash(double d); +long rb_fixnum_hash(VALUE fixnum); st_table *rb_init_identtable(void); void rb_init_existing_identtable_with_size(st_table *tbl, st_index_t size); st_index_t rb_any_hash(VALUE a); diff --git a/ractor.c b/ractor.c index c68b5f9d0e9664..bd18f6472df2a0 100644 --- a/ractor.c +++ b/ractor.c @@ -1721,6 +1721,11 @@ obj_traverse_i(VALUE obj, struct obj_traverse_data *data) { rb_ary_cancel_sharing(obj); + /* Narrow storage holds unboxed values, never object references: + * every element would hit the RB_SPECIAL_CONST_P early return in + * obj_traverse_i, so there is nothing to visit. Skip the loop. */ + if (rb_array_stride(obj) != RARRAY_STRIDE_VALUE) break; + for (int i = 0; i < RARRAY_LENINT(obj); i++) { VALUE e = RARRAY_AREF(obj, i); if (obj_traverse_i(e, data)) return 1; diff --git a/test/ruby/test_array_stride.rb b/test/ruby/test_array_stride.rb new file mode 100644 index 00000000000000..03f0a78816a420 --- /dev/null +++ b/test/ruby/test_array_stride.rb @@ -0,0 +1,941 @@ +# frozen_string_literal: false +require 'test/unit' +require 'objspace' + +# Element-stride narrowing for arrays of Fixnums. +# +# The shapes here come from the after-boot heap census of Shopify Core and SFR: +# - mail gem Ragel transition tables, 229,766 / 52,489 / 34,684 / 22,106 elements +# - annex_29 Unicode word segmentation table, 13,773 elements +# - a 3,072-element JSON escape table, allocated 116 times +# - SPLIT_DELIVERY_INCOMPATIBLE_APP_IDS, 69,937 ids, some above 2**32 +# - Method#parameters pairs, [:req, :name], 53% of SFR's all-immediate arrays +# - 80% of all-immediate arrays are length 1 or 2 +# - of all-fixnum arrays: 78.5% hold only 0..255, 96.5% fit 0..65535, +# 99.8% fit signed 32-bit, none contain a bignum +class TestArrayStride < Test::Unit::TestCase + WIDE = 0 # elements are VALUE, 8 bytes + W8 = 1 + W16 = 2 + W32 = 3 + + # Arrays at or below this length are stored inside the object itself and own + # no buffer, so there is nothing to narrow. + EMBED_MAX = 126 + N = 200 # comfortably past EMBED_MAX + + def narrow(a) = a.__narrow! + def stride(a) = a.__stride + + # ---------------------------------------------------------------- widths + + def test_unsigned_byte_range + a = Array.new(N) { |i| i % 256 } + assert_equal W8, narrow(a) + assert_equal W8, stride(a) + N.times { |i| assert_equal i % 256, a[i] } + end + + def test_unsigned_byte_boundary + assert_equal W8, narrow(Array.new(N) { |i| i % 256 }) + assert_equal W16, narrow(Array.new(N) { |i| i == 0 ? 256 : i % 256 }) + end + + def test_signed_byte_boundary + assert_equal W8, narrow(Array.new(N) { |i| (i % 256) - 128 }) + assert_equal W16, narrow(Array.new(N) { |i| i == 0 ? -129 : 0 }) + end + + def test_unsigned_16_boundary + assert_equal W16, narrow(Array.new(N) { |i| i * 327 }) # max 65,073 + assert_equal W32, narrow(Array.new(N) { |i| i == 0 ? 65_536 : 0 }) + end + + def test_unsigned_32_boundary + assert_equal W32, narrow(Array.new(N) { |i| i == 0 ? 2**32 - 1 : 0 }) + assert_equal WIDE, narrow(Array.new(N) { |i| i == 0 ? 2**32 : 0 }) + end + + def test_signed_32_boundary + assert_equal W32, narrow(Array.new(N) { |i| i.zero? ? -(2**31) : 0 }) + assert_equal WIDE, narrow(Array.new(N) { |i| i.zero? ? -(2**31) - 1 : 0 }) + end + + def test_bignum_never_narrows + a = Array.new(N) { 2**64 } + assert_equal WIDE, narrow(a) + assert_equal WIDE, stride(a) + end + + def test_fixnum_above_32_bits_does_not_narrow + # SPLIT_DELIVERY_INCOMPATIBLE_APP_IDS holds 111_900_950_529 and friends + a = Array.new(1_000) { |i| i } + [111_900_950_529] + assert_equal WIDE, narrow(a) + end + + # ---------------------------------------------------------- non-candidates + + def test_symbols_do_not_narrow + assert_equal WIDE, narrow(Array.new(N) { |i| :"name_#{i % 7}" }) + end + + def test_method_parameters_pairs_do_not_narrow + pairs = Array.new(N) { [:req, :name] } + assert_equal WIDE, narrow(pairs) + pairs.each { |p| assert_equal WIDE, narrow(p) } + end + + def test_floats_do_not_narrow + assert_equal WIDE, narrow(Array.new(N) { |i| i * 1.5 }) + end + + def test_nil_does_not_narrow + a = Array.new(N) { |i| i % 100 } + a[5] = nil + assert_equal WIDE, narrow(a) + end + + def test_mixed_does_not_narrow + a = Array.new(N) { |i| i } + a << "not a number" + assert_equal WIDE, narrow(a) + end + + def test_embedded_arrays_are_left_alone + # 80% of the census population is length 1 or 2, embedded, nothing to save + [[], [1], [1, 2], [1, 2, 3], Array.new(EMBED_MAX) { 1 }].each do |a| + assert_equal WIDE, narrow(a), "length #{a.length} should not narrow" + assert_equal 0, ObjectSpace.memsize_of(a) - ObjectSpace.memsize_of(a) + end + end + + # ------------------------------------------------------- the census shapes + + def test_ragel_transition_table_shape + src = Array.new(229_766) { |i| (i * 7) % 300 } + a = src.dup + assert_equal W16, narrow(a) + assert_equal 229_766, a.length + assert_equal src[0], a[0] + assert_equal src[-1], a[-1] + assert_equal src[115_000], a[115_000] + assert_equal src.sum, a.sum + # 1.8 MB of VALUEs becomes ~460 KB of uint16_t + assert_operator ObjectSpace.memsize_of(a), :<=, 229_766 * 2 + 128 + end + + # The mail gem's Ragel tables are plain (unfrozen) literals assigned once: + # self._indicies = [ 0, 1, 1, 1, 2, ... ] + # The compiler turns a static literal into a frozen hidden template and + # `duparray` hands out a 40-byte borrower sharing its buffer. The 1.8 MB + # lives in the template, invisible to ObjectSpace, kept alive only by the + # borrower. Neither the freeze trigger nor the literal trigger reaches + # this shape (the borrower is unfrozen and shared; the template is a copy + # source). Narrowing it explicitly must see through the share, take + # ownership, and let the template die -- this is what a boot-time pass + # would do. (Census: Core's 153 all-immediate arrays >= 1024 are only + # 19.6% frozen -- these tables are why.) + def test_duparray_literal_table_narrows_and_frees_template + n = 20_000 + src = Array.new(n) { |i| (i * 7) % 300 } + mod = Module.new + mod.module_eval("class << self; attr_accessor :table; end\nself.table = [#{src.join(',')}]\n", "ragel_shape.rb", 1) + t = mod.table + GC.start + assert_equal n, t.length + refute_predicate t, :frozen? + assert_equal WIDE, stride(t) + assert_equal 40, ObjectSpace.memsize_of(t), "borrower owns no buffer" + template_alive = -> { + ObjectSpace.dump_all(output: :string).lines.any? { |l| + l.include?('"type":"ARRAY"') && l.include?(%("length":#{n})) && l.include?('"frozen":true') && !l.include?('"class"') + } + } + assert template_alive.call, "hidden template should exist before narrowing" + + assert_equal W16, narrow(t) + GC.start + assert_equal W16, stride(t) + assert_operator ObjectSpace.memsize_of(t), :<=, n * 2 + 128 + assert_equal src, t.map { |x| x } + refute template_alive.call, "template must be freed once the borrower owns its own buffer" + end + + # Census: homogeneous arrays >= 1024 elements are 93.6% (Core) / 99.3% + # (SFR) frozen. Those written as `CONST = [...].freeze` never reach + # Array#freeze; the compiler narrows the opt_ary_freeze operand instead. + def test_frozen_literal_table_is_narrow_and_owns_nothing_extra + n = 5_000 + mod = Module.new + mod.module_eval("TABLE = [#{Array.new(n) { |i| i % 256 }.join(',')}].freeze", "frozen_table.rb", 1) + t = mod::TABLE + assert_predicate t, :frozen? + assert_equal W8, stride(t) + assert_operator ObjectSpace.memsize_of(t), :<=, n + 128 + assert_equal 255, t[255] + assert_equal n, t.length + assert_raise(FrozenError) { t << 1 } + assert_equal W8, stride(t), "a rejected write must not widen" + end + + # Census: of all-Fixnum arrays, 78.5% fit 0..255, 96.5% fit 0..65535, + # 99.8% fit signed 32-bit, none hold a bignum. Each band maps to a width. + def test_census_value_bands + assert_equal W8, narrow(Array.new(N) { |i| i % 256 }) # 78.5% + assert_equal W16, narrow(Array.new(N) { |i| (i * 331) % 65_536 }) # 96.5% + assert_equal W32, narrow(Array.new(N) { |i| (i * 7_919) - 500_000 }) # 99.8% (signed) + assert_equal WIDE, narrow(Array.new(N) { |i| i == 0 ? 2**40 : 0 }) # the 0.2% + assert_equal WIDE, narrow(Array.new(N) { |i| i == 0 ? 2**64 : 0 }) # bignum: never + end + + # Census: 80% of all-immediate arrays are length 1 or 2, 98% are <= 8, + # 98.9% are embedded. Narrowing only applies past the embed limit, and + # must kick in exactly there. + def test_embed_boundary + assert_equal WIDE, stride(Array.new(EMBED_MAX) { 1 }.freeze), "#{EMBED_MAX} elements embed in the largest slot" + assert_equal W8, stride(Array.new(EMBED_MAX + 1) { 1 }.freeze), "#{EMBED_MAX + 1} elements need a heap buffer" + [1, 2, 3, 8].each do |len| + a = Array.new(len) { |i| i }.freeze + assert_equal WIDE, stride(a), "length #{len}" + assert_equal 0, ObjectSpace.memsize_of(a) - ObjectSpace.memsize_of(Array.new(len) { |i| i }.freeze) + end + end + + # Census: SFR's 6,604 all-Fixnum arrays total 292 kB and are 79% length 2. + # A realistic small-array workload must be untouched: no narrowing, no + # extra allocation, identical behaviour. + def test_small_fixnum_arrays_are_untouched + pairs = Array.new(1_000) { |i| [i, i + 1].freeze } + assert pairs.all? { |p| stride(p) == WIDE } + assert pairs.all? { |p| ObjectSpace.memsize_of(p) == 40 } + triples = Array.new(500) { |i| [i % 256, (i * 3) % 256, (i * 7) % 256] } + triples.each(&:freeze) + assert triples.all? { |t| stride(t) == WIDE } + end + + # Census shapes that must NOT narrow, so nothing is spent on them. + def test_census_non_candidates + # Method#parameters pairs: 53% of SFR's all-immediate arrays, all Symbols + assert_equal WIDE, stride([:req, :name].freeze) + assert_equal WIDE, stride(Array.new(N) { |i| i.even? ? :req : :opt }.freeze) + # iseq pathobj tuples: [String, nil], 22% of every array in Core + assert_equal WIDE, stride(["/tmp/x.rb", nil].freeze) + # SPLIT_DELIVERY_INCOMPATIBLE_APP_IDS: frozen literal, 69,937 ids, some > 2**32 + ids = Array.new(69_937) { |i| i * 1_600_003 } + ids[1_234] = 111_900_950_529 + ids.freeze + assert_equal WIDE, stride(ids) + assert_include ids, 111_900_950_529 + # ActiveModel Type::Integer bounds: one bit past Fixnum + assert_equal WIDE, stride(Array.new(N) { |i| i.zero? ? 2**63 : 0 }.freeze) + end + + # The Ragel driver loop only ever does table[int]. That path must stay on + # the narrow buffer indefinitely: no widen after any number of lookups. + def test_index_lookups_never_widen + t = Array.new(50_000) { |i| (i * 7) % 300 }.freeze + assert_equal W16, stride(t) + sum = 0 + 200_000.times { |i| sum += t[i % 50_000] } + assert_equal W16, stride(t) + assert_equal (0...50_000).sum { |i| (i * 7) % 300 } * 4, sum + end + + def test_json_escape_table_shape + a = Array.new(3_072) { |i| i % 256 } + assert_equal W8, narrow(a) + assert_equal 3_072, a.length + assert_equal 255, a[255] + assert_equal 0, a[256] + end + + def test_annex29_table_shape + a = Array.new(13_773) { |i| i % 40_000 } + assert_equal W16, narrow(a) + assert_equal 13_773, a.length + assert_equal 13_772, a[13_772] + end + + def test_split_delivery_membership_shape + ids = Array.new(69_937) { |i| i * 7 } + assert_equal W32, narrow(ids) + installed = Array.new(30) { |i| i * 1_000_003 } + assert_equal installed.select { |id| ids.include?(id) }, + installed.select { |id| (id % 7).zero? && id <= 69_936 * 7 } + end + + # ------------------------------------------------------------ reads stay right + + def test_aref_after_narrowing + a = Array.new(5_000) { |i| i % 250 } + narrow(a) + 5_000.times { |i| assert_equal i % 250, a[i], "a[#{i}]" } + assert_nil a[5_000] + assert_equal 4_999 % 250, a[-1] + assert_equal 0, a[-5_000] + assert_nil a[-5_001] + end + + def test_negative_values_round_trip + a = Array.new(1_000) { |i| -(i % 128) } + assert_equal W8, narrow(a) + 1_000.times { |i| assert_equal(-(i % 128), a[i]) } + end + + def test_include_after_narrowing + a = Array.new(5_000) { |i| i % 250 } + assert_equal W8, narrow(a) + assert_includes a, 0 + assert_includes a, 249 + assert_not_includes a, 250 + assert_not_includes a, -1 + assert_not_includes a, 1_000_000 + assert_not_includes a, "0" + assert_not_includes a, nil + assert_not_includes a, :zero + assert_includes a, 249.0 # cross-type numeric equality must still hold + end + + def test_include_on_signed_narrow_array + a = Array.new(1_000) { |i| (i % 200) - 100 } + assert_equal W8, narrow(a) + assert_includes a,(-100) + assert_includes a, 99 + assert_not_includes a, 100 + assert_not_includes a,(-101) + end + + def test_include_respects_redefined_equality + a = Array.new(1_000) { |i| i % 100 } + narrow(a) + assert_includes a, 50 + assert_not_includes a, 1_000 + end + + def test_other_queries_after_narrowing + src = Array.new(5_000) { |i| i % 250 } + a = src.dup + narrow(a) + assert_equal src.index(123), a.index(123) + assert_equal src.sum, a.sum + assert_equal src.max, a.max + assert_equal src.min, a.min + assert_equal src.count(7), a.count(7) + assert_equal src.first(3), a.first(3) + assert_equal src.last(3), a.last(3) + end + + def test_enumeration_after_narrowing + src = Array.new(2_000) { |i| i % 200 } + a = src.dup + narrow(a) + assert_equal src, a.map { |x| x } + assert_equal src.select(&:even?), a.select(&:even?) + collected = [] + a.each { |x| collected << x } + assert_equal src, collected + assert_equal src.sort, a.sort + assert_equal src.uniq.sort, a.uniq.sort + assert_equal src.join(","), a.join(",") + end + + # ------------------------------------------------ reads do not un-narrow + # + # Core reads elements through a narrow-aware RARRAY_AREF, so ordinary + # read-only methods leave the packed buffer in place. Values surviving is + # not enough: a method that quietly widened would pass every value check + # while silently undoing the memory saving. + + def narrow_fixture + src = Array.new(5_000) { |i| i % 250 } + a = Array.new(5_000) { |i| i % 250 } + assert_equal W8, narrow(a) + [src, a] + end + + def assert_stays_narrow(a, what) + assert_equal W8, stride(a), "#{what} widened the array" + end + + def test_each_family_keeps_narrow + src, a = narrow_fixture + out = []; a.each { |x| out << x }; assert_equal src, out; assert_stays_narrow(a, "each") + out = []; a.reverse_each { |x| out << x }; assert_equal src.reverse, out; assert_stays_narrow(a, "reverse_each") + out = []; a.each_with_index { |x, i| out << [x, i] }; assert_equal src.each_with_index.to_a, out; assert_stays_narrow(a, "each_with_index") + assert_equal src.map { |x| x * 2 }, a.map { |x| x * 2 }; assert_stays_narrow(a, "map") + assert_equal src.select(&:even?), a.select(&:even?); assert_stays_narrow(a, "select") + assert_equal src.reject(&:even?), a.reject(&:even?); assert_stays_narrow(a, "reject") + assert_equal src.each_slice(7).to_a, a.each_slice(7).to_a; assert_stays_narrow(a, "each_slice") + end + + def test_aggregates_keep_narrow + src, a = narrow_fixture + assert_equal src.sum, a.sum; assert_stays_narrow(a, "sum") + assert_equal src.max, a.max; assert_stays_narrow(a, "max") + assert_equal src.min, a.min; assert_stays_narrow(a, "min") + assert_equal src.minmax, a.minmax; assert_stays_narrow(a, "minmax") + assert_equal src.count(7), a.count(7); assert_stays_narrow(a, "count(x)") + assert_equal src.count(&:zero?), a.count(&:zero?); assert_stays_narrow(a, "count {}") + assert_equal src.tally, a.tally; assert_stays_narrow(a, "tally") + assert_equal src.join(","), a.join(","); assert_stays_narrow(a, "join") + assert_equal src.inspect, a.inspect; assert_stays_narrow(a, "inspect") + end + + def test_searches_keep_narrow + src, a = narrow_fixture + assert_equal src.index(123), a.index(123); assert_stays_narrow(a, "index(x)") + assert_equal src.rindex(123), a.rindex(123); assert_stays_narrow(a, "rindex(x)") + assert_equal src.index { |x| x > 200 }, a.index { |x| x > 200 }; assert_stays_narrow(a, "index {}") + assert_equal src.find { |x| x == 42 }, a.find { |x| x == 42 }; assert_stays_narrow(a, "find") + assert_includes a, 100; assert_stays_narrow(a, "include?") + refute_includes a, 250; assert_stays_narrow(a, "include? miss") + refute_includes a, "100"; assert_stays_narrow(a, "include? non-fixnum") + assert_equal src.any? { |x| x > 248 }, a.any? { |x| x > 248 }; assert_stays_narrow(a, "any?") + assert_equal src.all? { |x| x < 250 }, a.all? { |x| x < 250 }; assert_stays_narrow(a, "all?") + end + + def test_equality_keeps_narrow + src, a = narrow_fixture + b = Array.new(5_000) { |i| i % 250 }; narrow(b) + assert_equal src, a; assert_stays_narrow(a, "narrow == wide") + assert_equal a, src; assert_stays_narrow(a, "wide == narrow (receiver)") + assert_equal a, b; assert_stays_narrow(a, "narrow == narrow"); assert_stays_narrow(b, "narrow == narrow (arg)") + refute_equal a, src + [1]; assert_stays_narrow(a, "== different length") + c = src.dup; c[2_500] = 7; refute_equal a, c; assert_stays_narrow(a, "== differing element") + end + + def test_equality_narrow_paths + _, a = narrow_fixture # W8 unsigned, 0..249 + # narrow vs narrow, same width/sign: memcmp + b = Array.new(5_000) { |i| i % 250 }; narrow(b) + assert_equal a, b + b2 = Array.new(5_000) { |i| i % 250 }; b2[4_999] = 1; narrow(b2) + refute_equal a, b2 + # narrow vs narrow, different width (unequal content, so widths differ) + w16 = Array.new(5_000) { |i| i == 0 ? 60_000 : i % 250 }; narrow(w16) + assert_equal W16, stride(w16) + refute_equal a, w16; refute_equal w16, a + assert_stays_narrow(a, "=="); assert_equal W16, stride(w16), "== widened the W16 array" + # narrow vs wide containing non-Fixnums that are == : must take the slow path + floats = Array.new(5_000) { |i| (i % 250).to_f } + assert_equal a, floats + assert_equal floats, a + assert_stays_narrow(a, "== with Floats") + # a mismatching Fixnum on the wide side is a definite inequality + wide = Array.new(5_000) { |i| i % 250 }; wide[123] = 251 + refute_equal a, wide; refute_equal wide, a + # recursion guard still works with a narrow array inside a self-referential one + r = [a]; r << r + s = [a.map { _1 }]; s << s + assert_equal r, s + assert_stays_narrow(a, "recursive ==") + end + + def test_equality_survives_mutation_during_compare + _, a = narrow_fixture + victim = Array.new(5_000) { |i| i % 250 } + # an element whose == mutates the other array mid-comparison + saboteur = Object.new + def saboteur.==(o) = (@hit ||= 0; @hit += 1; $__victim.clear if @hit == 1; true) + $__victim = victim + victim[10] = saboteur + # a == victim: index 10 → slow path → saboteur.== clears victim → lengths differ → false + refute_equal victim, a + assert_stays_narrow(a, "== with mutating element") + ensure + $__victim = nil + end + + def test_aref_variants_keep_narrow + src, a = narrow_fixture + assert_equal src[10], a[10]; assert_stays_narrow(a, "[i]") + assert_equal src[-1], a[-1]; assert_stays_narrow(a, "[-i]") + assert_equal src.at(99), a.at(99); assert_stays_narrow(a, "at") + assert_equal src.fetch(99), a.fetch(99); assert_stays_narrow(a, "fetch") + assert_equal src.dig(99), a.dig(99); assert_stays_narrow(a, "dig") + assert_equal src.values_at(1, 5, 9), a.values_at(1, 5, 9); assert_stays_narrow(a, "values_at") + assert_equal src.first, a.first; assert_stays_narrow(a, "first") + assert_equal src.last, a.last; assert_stays_narrow(a, "last") + assert_equal 5_000, a.length; assert_stays_narrow(a, "length") + refute_predicate a, :empty?; assert_stays_narrow(a, "empty?") + end + + # Buffer-level operations (slicing shares the buffer; pack/splat/Marshal + # need real VALUEs) are expected to widen. Pinned here so a change is + # deliberate. + def test_buffer_level_reads_widen + _, a = narrow_fixture; a.first(3); assert_equal WIDE, stride(a), "first(n) shares, so widens" + _, a = narrow_fixture; a[10, 20]; assert_equal WIDE, stride(a), "slice shares, so widens" + _, a = narrow_fixture; a.dup; assert_equal WIDE, stride(a), "Array#dup shares (rb_ary_replace), so widens" + end + + # hash / reverse / sort / splat used to read the raw VALUE buffer and so + # widened the source; they now read element-wise and leave it packed. + def test_hash_keeps_narrow_and_matches_wide + src, a = narrow_fixture + assert_equal src.hash, a.hash + assert_stays_narrow(a, "hash") + assert_equal src.eql?(a), true + h = { src => :v } + assert_equal :v, h[a], "a narrow array must be usable as the same Hash key" + assert_stays_narrow(a, "hash lookup") + end + + def test_reverse_sort_keep_narrow + src, a = narrow_fixture + assert_equal src.reverse, a.reverse; assert_stays_narrow(a, "reverse") + assert_equal src.sort, a.sort; assert_stays_narrow(a, "sort") + assert_equal src.sort { |x, y| y <=> x }, a.sort { |x, y| y <=> x }; assert_stays_narrow(a, "sort {}") + assert_equal src.sort_by { -_1 }, a.sort_by { -_1 }; assert_stays_narrow(a, "sort_by") + assert_equal src.minmax, a.minmax; assert_stays_narrow(a, "minmax") + assert_equal src.uniq, a.uniq; assert_stays_narrow(a, "uniq") + assert_equal src.to_a, a.to_a; assert_stays_narrow(a, "to_a") + end + + def test_splat_keeps_narrow + src, a = narrow_fixture + assert_equal 5_000, (->(*args) { args.length }).call(*a); assert_stays_narrow(a, "*a (dup splat)") + def self.__take_all(*xs) = xs.length + assert_equal 5_000, __take_all(*a); assert_stays_narrow(a, "f(*a)") + def self.__take_kw(*xs, **kw) = [xs.length, kw] + assert_equal [5_000, { k: 1 }], __take_kw(*a, k: 1); assert_stays_narrow(a, "f(*a, **kw)") + def self.__take_lead(x, y, *rest) = [x, y, rest.length] + assert_equal [src[0], src[1], 4_998], __take_lead(*a); assert_stays_narrow(a, "f(*a) with lead params") + assert_equal src.first(3), [*a].first(3); assert_stays_narrow(a, "[*a]") + assert_equal src.sum, [*a, 0].sum; assert_stays_narrow(a, "[*a, x]") + ensure + singleton_class.send(:remove_method, :__take_all, :__take_kw, :__take_lead) rescue nil + end + + # --------------------------------------------- narrow fast paths (hoisted) + # + # include?/index/rindex/count(x)/sum/min/max run a width-specialised loop + # on the packed buffer instead of decoding per element. They must agree + # with the generic implementation in every case, including the ones where + # the fast path must step aside. + + def fast_path_fixtures + u8 = Array.new(5_000) { |i| i % 250 }; narrow(u8) # unsigned W8 + s8 = Array.new(5_000) { |i| (i % 200) - 100 }; narrow(s8) # signed W8 + u16 = Array.new(5_000) { |i| (i * 13) % 60_000 }; narrow(u16) # unsigned W16 + s32 = Array.new(5_000) { |i| (i * 7_919) - 2_000_000 }; narrow(s32) # signed W32 + assert_equal [W8, W8, W16, W32], [u8, s8, u16, s32].map { stride(_1) } + [u8, s8, u16, s32] + end + + def test_fast_paths_agree_with_generic + fast_path_fixtures.each do |a| + ref = a.map { |x| x } # same values, wide + assert_equal WIDE, stride(ref) + [a[17], a[-1], 0, -1, -100, 249, 250, 255, 256, 65_535, 65_536, 2**31 - 1, -2**31, 2**40, -2**40].each do |needle| + assert_equal ref.include?(needle), a.include?(needle), "include?(#{needle})" + assert_equal ref.index(needle), a.index(needle), "index(#{needle})" + assert_equal ref.rindex(needle), a.rindex(needle), "rindex(#{needle})" + assert_equal ref.count(needle), a.count(needle), "count(#{needle})" + end + assert_equal ref.sum, a.sum + assert_equal ref.sum(10), a.sum(10) + assert_equal ref.sum(2**70), a.sum(2**70) + assert_equal ref.sum(1.5), a.sum(1.5) + assert_equal ref.sum(Rational(1, 3)), a.sum(Rational(1, 3)) + assert_equal ref.sum { |x| x * 2 }, a.sum { |x| x * 2 } + assert_equal ref.max, a.max + assert_equal ref.min, a.min + assert_equal ref.max(3), a.max(3) + assert_equal ref.min(3), a.min(3) + assert_equal ref.max { |x, y| y <=> x }, a.max { |x, y| y <=> x } + assert_equal ref.minmax, a.minmax + assert_equal [W8, W8, W16, W32].include?(stride(a)), true, "fast paths must not widen" + end + end + + def test_fast_paths_unrepresentable_needle_is_a_miss_without_scanning + u8, s8, = fast_path_fixtures + refute_includes u8, -1 # negative in an unsigned buffer + refute_includes u8, 256 # above uint8 range + refute_includes s8, 200 # above int8 range, though 200 > 127 is a valid Fixnum + refute_includes s8, -129 + assert_nil u8.index(300) + assert_equal 0, s8.count(1_000) + assert_equal W8, stride(u8) + end + + def test_fast_paths_non_fixnum_needle_falls_back + u8, = fast_path_fixtures + refute_includes u8, "7" + refute_includes u8, 7.0 + 0.5 + assert_includes u8, 7.0 # Integer#== 7.0 is true, generic path must handle it + assert_equal 7, u8.index(7.0) + assert_equal u8.map { _1 }.count(7.0), u8.count(7.0) + assert_nil u8.index(nil) + assert_equal W8, stride(u8) + end + + def test_fast_paths_respect_redefined_operators + u8, = fast_path_fixtures + ref = u8.map { |x| x } + # With Integer#== redefined the fast path must step aside and agree with + # the generic path. rb_equal short-circuits on identical VALUEs, so a + # same-Fixnum needle is still found (as on a wide array); a Float needle + # is where the redefinition becomes observable. Results are captured + # while redefined and asserted afterwards, since assert_equal uses ==. + got = begin + Integer.class_eval { alias_method :__orig_eq, :==; def ==(o) = false } + [ref.include?(7), u8.include?(7), ref.index(7), u8.index(7), ref.count(7), u8.count(7), + ref.include?(7.0), u8.include?(7.0)] + ensure + Integer.class_eval { alias_method :==, :__orig_eq; remove_method :__orig_eq } + end + assert_equal [true, true, 7, 7, 20, 20, false, false], got # 5000 / 250 = 20 sevens + begin + Integer.class_eval { alias_method :__orig_cmp, :<=>; def <=>(o) = __orig_cmp(o)&.-@ } # invert ordering + assert_equal 0, u8.max, "redefined Integer#<=> must be honoured" + assert_equal 249, u8.min + ensure + Integer.class_eval { alias_method :<=>, :__orig_cmp; remove_method :__orig_cmp } + end + assert_equal W8, stride(u8) + end + + def test_fast_paths_edge_sizes + one = Array.new(EMBED_MAX + 1) { 42 }; narrow(one); assert_equal W8, stride(one) + assert_equal 42, one.max + assert_equal 42, one.min + assert_equal 42 * (EMBED_MAX + 1), one.sum + assert_equal EMBED_MAX + 1, one.count(42) + assert_equal 0, one.index(42) + assert_equal EMBED_MAX, one.rindex(42) + big = Array.new(300_000) { |i| i % 65_536 }; narrow(big); assert_equal W16, stride(big) + assert_equal (0...300_000).sum { |i| i % 65_536 }, big.sum + assert_equal 65_535, big.max + assert_equal 299_999, big.rindex(299_999 % 65_536) + end + + # ------------------------------------------------------- widening on contact + + def test_write_of_wider_value_widens + a = Array.new(1_000) { |i| i % 100 } + assert_equal W8, narrow(a) + a[0] = 70_000 + assert_equal WIDE, stride(a) + assert_equal 70_000, a[0] + assert_equal 1 % 100, a[1] + assert_equal 1_000, a.length + end + + def test_write_of_non_fixnum_widens + a = Array.new(1_000) { |i| i % 100 } + narrow(a) + a[5] = "x" + assert_equal WIDE, stride(a) + assert_equal "x", a[5] + assert_equal 6 % 100, a[6] + end + + def test_push_after_narrowing + a = Array.new(1_000) { |i| i % 100 } + narrow(a) + a << 42 + assert_equal WIDE, stride(a) + assert_equal 1_001, a.length + assert_equal 42, a[-1] + assert_equal 0, a[0] + end + + + def test_pack_keeps_narrow + # pack reads element by element through the narrow-aware accessor + a = Array.new(200) { |i| i } + narrow(a) + assert_equal 800, a.pack("l*").bytesize + assert_equal Array.new(200) { |i| i }.pack("C*"), a.pack("C*") + assert_equal W8, stride(a) + end + + def test_marshal_round_trip_keeps_narrow + src = Array.new(2_000) { |i| i % 250 } + a = Array.new(2_000) { |i| i % 250 } + narrow(a) + dumped = Marshal.dump(a) + assert_equal W8, stride(a) + assert_equal Marshal.dump(src), dumped, "narrow and wide arrays must marshal identically" + assert_equal src, Marshal.load(dumped) + end + + def test_dup_clone_and_slice + src = Array.new(2_000) { |i| i % 250 } + a = src.dup + narrow(a) + assert_equal src, a.dup + assert_equal src, a.clone + assert_equal src, a[0..-1] + assert_equal src[10, 20], a[10, 20] + assert_equal src.reverse, a.reverse + end + + def test_frozen_narrow_array_reads + a = Array.new(5_000) { |i| i % 250 }.freeze + assert_equal W8, narrow(a) + assert_predicate a, :frozen? + assert_equal 249, a[249] + assert_includes a, 100 + assert_raise(FrozenError) { a << 1 } + end + + def test_equality_between_narrow_and_wide + src = Array.new(1_000) { |i| i % 250 } + a = src.dup + narrow(a) + assert_equal src, a + assert_equal a, src + assert_equal src.hash, a.hash + assert_operator a, :eql?, src + end + + # ----------------------------------------------------------------- the GC + + def test_survives_major_gc + src = Array.new(50_000) { |i| i % 250 } + a = src.dup + narrow(a) + 4.times { GC.start } + assert_equal 50_000, a.length + assert_equal W8, stride(a) + (0...50_000).step(997) { |i| assert_equal src[i], a[i], "a[#{i}] after GC" } + assert_equal src.sum, a.sum + end + + def test_survives_compaction + omit "compaction not supported" unless GC.respond_to?(:compact) + src = Array.new(5_000) { |i| i % 250 } + kept = Array.new(20) { src.dup } + kept.each { |a| assert_equal W8, narrow(a) } + begin + GC.verify_compaction_references(expand_heap: true, toward: :empty) + rescue NotImplementedError, NoMethodError + GC.compact + end + kept.each do |a| + assert_equal 5_000, a.length + assert_equal 249, a[249] + assert_equal src.sum, a.sum + end + end + + def test_narrow_array_has_no_traceable_elements + a = Array.new(10_000) { |i| i % 250 } + narrow(a) + refs = ObjectSpace.reachable_objects_from(a) + assert_empty refs.reject { |o| o.is_a?(Class) || o.is_a?(Module) } + end + + # ------------------------------------------------------------- memory effect + + def test_narrowing_shrinks_memsize + a = Array.new(100_000) { |i| i % 250 } + before = ObjectSpace.memsize_of(a) + assert_equal W8, narrow(a) + after = ObjectSpace.memsize_of(a) + assert_operator after, :<, before + # one byte per element plus the object itself + assert_operator after, :<=, 100_000 + 128 + assert_operator before, :>=, 100_000 * 8 + end + + def test_widening_restores_memsize + a = Array.new(100_000) { |i| i % 250 } + narrow(a) + small = ObjectSpace.memsize_of(a) + a.first(3) # shares the buffer, so it widens + assert_equal WIDE, stride(a) + assert_operator ObjectSpace.memsize_of(a), :>=, 100_000 * 8 + assert_operator ObjectSpace.memsize_of(a), :>, small + assert_equal 249, a[249] + end + + # ------------------------------------------------------- trigger: freeze + # + # Array#freeze is the first automatic trigger. The census found homogeneous + # arrays of >= 1024 elements are 93.6% (Core) / 99.3% (SFR) frozen, and a + # frozen array never reaches rb_ary_modify, so it never pays to widen back. + + def test_freeze_narrows_all_fixnum_heap_array + a = Array.new(N) { |i| i % 256 } + assert_equal WIDE, stride(a) + a.freeze + assert_equal W8, stride(a) + assert_predicate a, :frozen? + N.times { |i| assert_equal i % 256, a[i] } + end + + def test_freeze_picks_width_from_contents + assert_equal W8, stride(Array.new(N) { |i| i % 256 }.freeze) + assert_equal W16, stride(Array.new(N) { |i| i * 300 }.freeze) + assert_equal W32, stride(Array.new(N) { |i| i * 70_000 }.freeze) + assert_equal W8, stride(Array.new(N) { |i| -(i % 128) }.freeze) + assert_equal WIDE, stride(Array.new(N) { |i| i == 0 ? 2**40 : 0 }.freeze) + end + + def test_freeze_leaves_non_candidates_alone + assert_equal WIDE, stride(Array.new(N) { |i| :"s#{i % 5}" }.freeze) + assert_equal WIDE, stride(Array.new(N) { |i| i.to_f }.freeze) + assert_equal WIDE, stride(Array.new(N) { |i| i.even? ? i : nil }.freeze) + assert_equal WIDE, stride([1, 2, 3].freeze) # embedded + assert_equal WIDE, stride(Array.new(EMBED_MAX) { 1 }.freeze) # embedded + assert_equal WIDE, stride([].freeze) + end + + def test_freeze_does_not_copy_a_shared_array + # Freezing must not change memory behaviour for arrays that borrow their + # buffer: a slice of a big array is left shared, not copied and packed. + src = Array.new(N * 10) { |i| i % 256 } + a = src[1..] + before = ObjectSpace.memsize_of(a) + a.freeze + assert_equal WIDE, stride(a) + assert_equal before, ObjectSpace.memsize_of(a) + assert_equal src[1..], a + end + + def test_dup_then_freeze_stays_shared + # Array#dup of a large array shares the buffer (rb_ary_replace), so the + # common `x.dup.freeze` idiom is a shared array at freeze time and is not + # narrowed. Documented here so a change in that trade-off is deliberate. + src = Array.new(N * 10) { |i| i % 256 } + a = src.dup.freeze + assert_equal WIDE, stride(a) + assert_equal src, a + end + + def test_freeze_shrinks_memsize + a = [] + (N * 10).times { |i| a << i % 256 } # grown by push, so capa > len + before = ObjectSpace.memsize_of(a) + a.freeze + assert_equal W8, stride(a) + # one byte per element plus the object slot itself + assert_operator ObjectSpace.memsize_of(a), :<=, N * 10 + 128 + assert_operator ObjectSpace.memsize_of(a), :<, before / 4 + end + + def test_frozen_literal_narrows + # `[...].freeze` compiles to opt_ary_freeze, which returns the compile-time + # array itself. That object never passes through Array#freeze, so the + # compiler narrows it directly. This is how frozen constant tables are + # usually written. + a = eval("[#{(0...N).map { |i| i % 256 }.join(',')}].freeze") + assert_equal W8, stride(a) + assert_equal N - 1, a[N - 1] + assert_predicate a, :frozen? + assert_equal eval("[#{(0...N).map { |i| i % 256 }.join(',')}]"), a + end + + def test_frozen_literal_is_the_same_object_each_evaluation + src = "def __stride_lit; [#{(0...N).map { |i| i % 256 }.join(',')}].freeze; end" + Object.class_eval(src) + x = __stride_lit + y = __stride_lit + assert_same x, y + assert_equal W8, stride(x) + assert_equal 199, x[199] + ensure + Object.send(:remove_method, :__stride_lit) if Object.method_defined?(:__stride_lit) + end + + def test_plain_literal_is_not_narrowed + # A non-frozen literal is a template that duparray shares on every + # evaluation. Narrowing it would force a copy per evaluation (O(n) + # where sharing is O(1)), so it is deliberately left wide; only the + # `.freeze` form (opt_ary_freeze) is narrowed at compile time. + a = eval("[#{(0...N).map { |i| i % 256 }.join(',')}]") + assert_equal WIDE, stride(a) + assert_equal 40, ObjectSpace.memsize_of(a), "shared view of the template" + a << 1 + assert_equal N + 1, a.length + end + + # A frozen array becomes its own shared root when it is dup'd or sliced, + # and the borrower copies the raw buffer pointer. A narrow buffer must + # never be handed out that way. + + def test_dup_of_frozen_narrow_array + src = Array.new(N * 10) { |i| i % 256 } + a = Array.new(N * 10) { |i| i % 256 }.freeze + assert_equal W8, stride(a) + b = a.dup + assert_equal src, b + assert_equal src, a + refute_predicate b, :frozen? + b[0] = 999 + assert_equal 0, a[0] + end + + def test_slice_of_frozen_narrow_array + src = Array.new(N * 10) { |i| i % 256 } + a = Array.new(N * 10) { |i| i % 256 }.freeze + assert_equal W8, stride(a) + assert_equal src[100..], a[100..] + assert_equal src[5, 500], a[5, 500] + assert_equal src.last(300), a.last(300) + assert_equal src.drop(7), a.drop(7) + end + + def test_replace_from_frozen_narrow_array + src = Array.new(N * 10) { |i| i % 256 } + a = Array.new(N * 10) { |i| i % 256 }.freeze + b = [:x] + b.replace(a) + assert_equal src, b + b << 1 + assert_equal src + [1], b + assert_equal src, a + end + + def test_flatten_bang_on_fixnum_array + # flatten! freezes an internal temporary and then rb_ary_replace()s it in + a = Array.new(N * 10) { |i| [i % 256] } + expected = a.flatten + a.flatten! + assert_equal expected, a + a << 7 + assert_equal expected + [7], a + end + + def test_frozen_narrow_array_survives_gc_and_compaction + kept = Array.new(10) { Array.new(N * 5) { |i| i % 256 }.freeze } + kept.each { |a| assert_equal W8, stride(a) } + GC.start + if GC.respond_to?(:compact) + begin + GC.verify_compaction_references(expand_heap: true, toward: :empty) + rescue NotImplementedError, NoMethodError + GC.compact + end + end + kept.each do |a| + assert_equal W8, stride(a) + assert_equal 255, a[255] + assert_equal N * 5, a.length + end + end + + def test_ractor_shareable_narrow_array_parallel_reads + omit "Ractor not available" unless defined?(Ractor) + src = Array.new(N * 10) { |i| i % 256 } + a = Ractor.make_shareable(Array.new(N * 10) { |i| i % 256 }) + assert_equal W8, stride(a) + assert_predicate a, :frozen? + # Every reader walks the array through a widening accessor at the same + # time; exactly one widen must happen. + rs = 4.times.map do + Ractor.new(a) { |ary| [ary.sum, ary.each_slice(100).count, ary.index(255)] } + end + results = rs.map { |r| r.respond_to?(:value) ? r.value : r.take } + results.each do |sum, slices, idx| + assert_equal src.sum, sum + assert_equal src.each_slice(100).count, slices + assert_equal 255, idx + end + assert_equal src, a + end +end diff --git a/vm_args.c b/vm_args.c index 960443d5ca1465..b58a1b12231b85 100644 --- a/vm_args.c +++ b/vm_args.c @@ -45,6 +45,18 @@ arg_rest_dup(struct args_info *args) } } +/* The setup code below reads the splat array through RARRAY_CONST_PTR, which + * would widen a narrowed array in place. Take a private (wide) copy instead, + * so f(*TABLE) leaves TABLE packed. rb_ary_dup decodes element-wise. */ +static inline void +arg_rest_unnarrow(struct args_info *args) +{ + if (RB_UNLIKELY(rb_array_stride(args->rest) != RARRAY_STRIDE_VALUE)) { + args->rest = rb_ary_dup(args->rest); + args->rest_dupped = TRUE; + } +} + static inline int args_argc(struct args_info *args) { @@ -677,6 +689,7 @@ setup_parameters_complex(rb_execution_context_t * const ec, const rb_iseq_t * co args->rest_index = 0; keyword_hash = locals[--args->argc]; args->rest = locals[--args->argc]; + arg_rest_unnarrow(args); if (ignore_keyword_hash_p(keyword_hash, iseq, &kw_flag, &converted_keyword_hash)) { keyword_hash = Qnil; @@ -728,6 +741,7 @@ setup_parameters_complex(rb_execution_context_t * const ec, const rb_iseq_t * co // f(*a) args->rest_index = 0; args->rest = locals[--args->argc]; + arg_rest_unnarrow(args); int len = RARRAY_LENINT(args->rest); given_argc += len - 1;