diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index e99a9e6dca6b..23e2e0d96e13 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -1262,6 +1262,7 @@ def try_specialize_str_index_compare( str_adjust_index_op, str_range_check_op, str_get_item_unsafe_as_int_op, + "string index out of range", ) if codepoint is None: return None diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 5555e38c78a7..5e89823851d9 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -609,8 +609,21 @@ def coerce( return tmp return src - def coerce_int_to_fixed_width(self, src: Value, target_type: RType, line: int) -> Value: + def coerce_int_to_fixed_width( + self, + src: Value, + target_type: RType, + line: int, + *, + slow_path_op: CFunctionDescription | None = None, + ) -> Value: + """Coerce a tagged int to a fixed-width native int. + + If slow_path_op is given, use it instead of the default primitive to convert + an int that isn't a short int (only supported for i64 targets). + """ assert is_fixed_width_rtype(target_type), target_type + assert slow_path_op is None or is_int64_rprimitive(target_type), target_type assert isinstance(target_type, RPrimitive), target_type fast, slow, overflow, end = BasicBlock(), BasicBlock(), BasicBlock(), BasicBlock() @@ -660,7 +673,9 @@ def coerce_int_to_fixed_width(self, src: Value, target_type: RType, line: int) - ) ptr2 = Register(c_pointer_rprimitive) self.add(Assign(ptr2, ptr)) - if is_int64_rprimitive(target_type): + if slow_path_op is not None: + conv_op = slow_path_op + elif is_int64_rprimitive(target_type): conv_op = int_to_int64_op elif is_uint64_rprimitive(target_type): conv_op = int_to_uint64_op diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 5baca9cc0bd0..b7807525170e 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -150,6 +150,7 @@ int_abs_op, int_to_big_endian_op, int_to_bytes_op, + int_to_index_op, int_to_little_endian_op, isinstance_int, ) @@ -1614,6 +1615,7 @@ def translate_ord(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value str_adjust_index_op, str_range_check_op, str_get_item_unsafe_as_int_op, + "string index out of range", ) return result @@ -1771,6 +1773,17 @@ def specialize_int_to_bytes(builder: IRBuilder, expr: CallExpr, callee: RefExpr) ) +def coerce_int_index_to_int64(builder: IRBuilder, index: Value, line: int) -> Value: + """Coerce an int index to i64 for the specialized item access primitives. + + Like CPython, raise IndexError (instead of ValueError, as a regular + coercion would) if the index doesn't fit in Py_ssize_t. + """ + return builder.builder.coerce_int_to_fixed_width( + index, int64_rprimitive, line, slow_path_op=int_to_index_op + ) + + def translate_getitem_with_bounds_check( builder: IRBuilder, base_expr: Expression, @@ -1779,6 +1792,7 @@ def translate_getitem_with_bounds_check( adjust_index_op: PrimitiveDescription, range_check_op: PrimitiveDescription, get_item_unsafe_op: PrimitiveDescription, + index_error_msg: str = "index out of range", ) -> Value | None: """Shared helper for optimized __getitem__ with bounds checking. @@ -1809,6 +1823,8 @@ def translate_getitem_with_bounds_check( # Get the index argument index = builder.accept(args[0]) + if is_int_rprimitive(index.type): + index = coerce_int_index_to_int64(builder, index, ctx_expr.line) # Adjust the index (handle negative indices) adjusted_index = builder.primitive_op(adjust_index_op, [obj, index], ctx_expr.line) @@ -1824,9 +1840,7 @@ def translate_getitem_with_bounds_check( # Handle invalid index - raise IndexError builder.activate_block(invalid_block) - builder.add( - RaiseStandardError(RaiseStandardError.INDEX_ERROR, "index out of range", ctx_expr.line) - ) + builder.add(RaiseStandardError(RaiseStandardError.INDEX_ERROR, index_error_msg, ctx_expr.line)) builder.add(Unreachable()) # Handle valid index - get the item @@ -1849,6 +1863,7 @@ def translate_bytes_writer_get_item( bytes_writer_adjust_index_op, bytes_writer_range_check_op, bytes_writer_get_item_unsafe_op, + "BytesWriter index out of range", ) @@ -1867,6 +1882,8 @@ def translate_bytes_writer_set_item( # Get the index and value arguments index = builder.accept(args[0]) value = builder.accept(args[1]) + if is_int_rprimitive(index.type): + index = coerce_int_index_to_int64(builder, index, ctx_expr.line) # Adjust the index (handle negative indices) adjusted_index = builder.primitive_op( @@ -1887,7 +1904,9 @@ def translate_bytes_writer_set_item( # Handle invalid index - raise IndexError builder.activate_block(invalid_block) builder.add( - RaiseStandardError(RaiseStandardError.INDEX_ERROR, "index out of range", ctx_expr.line) + RaiseStandardError( + RaiseStandardError.INDEX_ERROR, "BytesWriter index out of range", ctx_expr.line + ) ) builder.add(Unreachable()) @@ -1913,6 +1932,7 @@ def translate_string_writer_get_item( string_writer_adjust_index_op, string_writer_range_check_op, string_writer_get_item_unsafe_op, + "StringWriter index out of range", ) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 139e16925723..48f04c9d845d 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -197,6 +197,7 @@ CPyTagged CPyTagged_FromUInt32(uint32_t value); PyObject *CPyTagged_AsObject(CPyTagged x); PyObject *CPyTagged_StealAsObject(CPyTagged x); Py_ssize_t CPyTagged_AsSsize_t(CPyTagged x); +void CPy_LargeIndexError(CPyTagged index, const char *msg); void CPyTagged_IncRef(CPyTagged x); void CPyTagged_DecRef(CPyTagged x); void CPyTagged_XDecRef(CPyTagged x); @@ -227,6 +228,7 @@ PyObject *CPyLong_FromStrWithBase(PyObject *o, CPyTagged base); PyObject *CPyLong_FromStr(PyObject *o); PyObject *CPyBool_Str(bool b); int64_t CPyLong_AsInt64_(PyObject *o); +int64_t CPyLong_AsIndex(PyObject *o); int64_t CPyInt64_Divide(int64_t x, int64_t y); int64_t CPyInt64_Remainder(int64_t x, int64_t y); int32_t CPyLong_AsInt32_(PyObject *o); diff --git a/mypyc/lib-rt/bytes_ops.c b/mypyc/lib-rt/bytes_ops.c index 03fc41a6ad83..e86f08f4b48d 100644 --- a/mypyc/lib-rt/bytes_ops.c +++ b/mypyc/lib-rt/bytes_ops.c @@ -44,7 +44,7 @@ CPyTagged CPyBytes_GetItem(PyObject *o, CPyTagged index) { : ((PyByteArrayObject *)o)->ob_bytes[n]; return num << 1; } else { - PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + CPy_LargeIndexError(index, "index out of range"); return CPY_INT_TAG; } } diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c index 9e85519fae33..e454efd198f4 100644 --- a/mypyc/lib-rt/int_ops.c +++ b/mypyc/lib-rt/int_ops.c @@ -115,6 +115,28 @@ Py_ssize_t CPyTagged_AsSsize_t(CPyTagged x) { } } +// Set the exception for a sequence index that isn't a short int. Like CPython, +// raise IndexError("cannot fit 'int' into an index-sized integer") if it +// doesn't fit in Py_ssize_t. Otherwise raise IndexError(msg), since an index +// this large is always out of range (no list or tuple can be this long, and +// neither can a str or bytes object on a 64-bit platform). +void CPy_LargeIndexError(CPyTagged index, const char *msg) { + Py_ssize_t n = PyNumber_AsSsize_t(CPyTagged_LongAsObject(index), PyExc_IndexError); + if (n != -1 || !PyErr_Occurred()) { + PyErr_SetString(PyExc_IndexError, msg); + } +} + +// Convert an int used as a sequence index to i64. Like CPython, raise +// IndexError if the value doesn't fit in Py_ssize_t. +int64_t CPyLong_AsIndex(PyObject *o) { + Py_ssize_t n = PyNumber_AsSsize_t(o, PyExc_IndexError); + if (n == -1 && PyErr_Occurred()) { + return CPY_LL_INT_ERROR; + } + return n; +} + CPy_NOINLINE void CPyTagged_IncRef(CPyTagged x) { if (unlikely(CPyTagged_CheckLong(x))) { diff --git a/mypyc/lib-rt/list_ops.c b/mypyc/lib-rt/list_ops.c index 69205e811d70..b881a64ad82f 100644 --- a/mypyc/lib-rt/list_ops.c +++ b/mypyc/lib-rt/list_ops.c @@ -121,7 +121,7 @@ PyObject *CPyList_GetItem(PyObject *list, CPyTagged index) { Py_INCREF(result); return result; } else { - PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + CPy_LargeIndexError(index, "list index out of range"); return NULL; } } @@ -173,7 +173,7 @@ bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value) { return true; } else { Py_DECREF(value); - PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + CPy_LargeIndexError(index, "list assignment index out of range"); return false; } } @@ -235,7 +235,7 @@ PyObject *CPyList_GetItem_(PyObject *list, CPyTagged index) { } return PyList_GetItemRef(list, n); } else { - PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + CPy_LargeIndexError(index, "list index out of range"); return NULL; } } @@ -250,7 +250,7 @@ bool CPyList_SetItem_(PyObject *list, CPyTagged index, PyObject *value) { return PyList_SetItem(list, n, value) >= 0; } else { Py_DECREF(value); - PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + CPy_LargeIndexError(index, "list assignment index out of range"); return false; } } @@ -293,7 +293,7 @@ PyObject *CPyList_GetItemBorrow(PyObject *list, CPyTagged index) { } return PyList_GET_ITEM(list, n); } else { - PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + CPy_LargeIndexError(index, "list index out of range"); return NULL; } } @@ -377,18 +377,23 @@ PyObject *CPyList_PopLast(PyObject *list) PyObject *CPyList_Pop(PyObject *obj, CPyTagged index) { + Py_ssize_t n; if (CPyTagged_CheckShort(index)) { - Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + n = CPyTagged_ShortAsSsize_t(index); + } else { + // Like CPython, raise OverflowError only if the index doesn't fit in + // Py_ssize_t. Other large indexes raise IndexError below. + n = PyLong_AsSsize_t(CPyTagged_LongAsObject(index)); + if (n == -1 && PyErr_Occurred()) { + return NULL; + } + } #ifdef Py_GIL_DISABLED - // We must use a slower implementation on free-threaded builds. - return list_pop_index(obj, n); + // We must use a slower implementation on free-threaded builds. + return list_pop_index(obj, n); #else - return list_pop_impl((PyListObject *)obj, n); + return list_pop_impl((PyListObject *)obj, n); #endif - } else { - PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); - return NULL; - } } CPyTagged CPyList_Count(PyObject *obj, PyObject *value) @@ -398,14 +403,18 @@ CPyTagged CPyList_Count(PyObject *obj, PyObject *value) int CPyList_Insert(PyObject *list, CPyTagged index, PyObject *value) { + Py_ssize_t n; if (CPyTagged_CheckShort(index)) { - Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); - return PyList_Insert(list, n, value); + n = CPyTagged_ShortAsSsize_t(index); + } else { + // Like CPython, raise OverflowError only if the index doesn't fit in + // Py_ssize_t. Other large indexes are clamped to the list bounds. + n = PyLong_AsSsize_t(CPyTagged_LongAsObject(index)); + if (n == -1 && PyErr_Occurred()) { + return -1; + } } - // The max range doesn't exactly coincide with ssize_t, but we still - // want to keep the error message compatible with CPython. - PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); - return -1; + return PyList_Insert(list, n, value); } PyObject *CPyList_Extend(PyObject *o1, PyObject *o2) { diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c index c9552b6b87bf..530f03459b6c 100644 --- a/mypyc/lib-rt/str_ops.c +++ b/mypyc/lib-rt/str_ops.c @@ -126,7 +126,7 @@ PyObject *CPyStr_GetItem(PyObject *str, CPyTagged index) { } return unicode; } else { - PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + CPy_LargeIndexError(index, "string index out of range"); return NULL; } } else { diff --git a/mypyc/lib-rt/tuple_ops.c b/mypyc/lib-rt/tuple_ops.c index a544b49eadb6..754f5752ee1d 100644 --- a/mypyc/lib-rt/tuple_ops.c +++ b/mypyc/lib-rt/tuple_ops.c @@ -20,7 +20,7 @@ PyObject *CPySequenceTuple_GetItem_(PyObject *tuple, CPyTagged index) { Py_INCREF(result); return result; } else { - PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + CPy_LargeIndexError(index, "tuple index out of range"); return NULL; } } diff --git a/mypyc/primitives/int_ops.py b/mypyc/primitives/int_ops.py index 2bed5f1a5a91..124bf743c40e 100644 --- a/mypyc/primitives/int_ops.py +++ b/mypyc/primitives/int_ops.py @@ -310,6 +310,15 @@ def int_unary_op(name: str, c_function_name: str) -> PrimitiveDescription: error_kind=ERR_MAGIC_OVERLAPPING, ) +# Convert tagged int (as PyObject *) used as a sequence index to i64 +# (raise IndexError if it doesn't fit in Py_ssize_t, like CPython) +int_to_index_op = custom_op( + arg_types=[object_rprimitive], + return_type=int64_rprimitive, + c_function_name="CPyLong_AsIndex", + error_kind=ERR_MAGIC_OVERLAPPING, +) + ssize_t_to_int_op = custom_op( arg_types=[c_pyssize_t_rprimitive], return_type=int_rprimitive, diff --git a/mypyc/test-data/fixtures/testutil.py b/mypyc/test-data/fixtures/testutil.py index 587625145b01..db529c42de80 100644 --- a/mypyc/test-data/fixtures/testutil.py +++ b/mypyc/test-data/fixtures/testutil.py @@ -12,6 +12,11 @@ FLOAT_MAGIC: Final = -113.0 +# Largest Py_ssize_t value (sys.maxsize isn't in the sys stub used by tests) +SSIZE_T_MAX: Final[int] = getattr(sys, "maxsize") +# Smallest positive int that isn't a short int, but fits in Py_ssize_t +SSIZE_T_LONG: Final[int] = SSIZE_T_MAX // 2 + 1 + # Various different float values float_vals = [ float(n) * 0.25 for n in range(-10, 10) diff --git a/mypyc/test-data/irbuild-librt-strings.test b/mypyc/test-data/irbuild-librt-strings.test index 85f6295abde0..218a2ecc67e4 100644 --- a/mypyc/test-data/irbuild-librt-strings.test +++ b/mypyc/test-data/irbuild-librt-strings.test @@ -72,7 +72,7 @@ L0: r1 = CPyBytesWriter_RangeCheck(b, r0) if r1 goto L2 else goto L1 :: bool L1: - r2 = raise IndexError('index out of range') + r2 = raise IndexError('BytesWriter index out of range') unreachable L2: r3 = CPyBytesWriter_GetItem(b, r0) @@ -88,7 +88,7 @@ L0: r1 = CPyBytesWriter_RangeCheck(b, r0) if r1 goto L2 else goto L1 :: bool L1: - r2 = raise IndexError('index out of range') + r2 = raise IndexError('BytesWriter index out of range') unreachable L2: CPyBytesWriter_SetItem(b, r0, x) @@ -356,7 +356,7 @@ L0: r1 = CPyStringWriter_RangeCheck(s, r0) if r1 goto L2 else goto L1 :: bool L1: - r2 = raise IndexError('index out of range') + r2 = raise IndexError('StringWriter index out of range') unreachable L2: r3 = CPyStringWriter_GetItem(s, r0) @@ -408,7 +408,7 @@ L1: L2: r4 = i ^ 1 r5 = r4 - r6 = CPyLong_AsInt64(r5) + r6 = CPyLong_AsIndex(r5) r3 = r6 keep_alive i L3: @@ -416,7 +416,7 @@ L3: r8 = CPyStr_RangeCheck(s, r7) if r8 goto L5 else goto L4 :: bool L4: - r9 = raise IndexError('index out of range') + r9 = raise IndexError('string index out of range') unreachable L5: r10 = CPyStr_GetItemUnsafeAsInt(s, r7) diff --git a/mypyc/test-data/irbuild-str.test b/mypyc/test-data/irbuild-str.test index af006abe72c8..c0d3559b5d18 100644 --- a/mypyc/test-data/irbuild-str.test +++ b/mypyc/test-data/irbuild-str.test @@ -567,7 +567,7 @@ L1: L2: r4 = i ^ 1 r5 = r4 - r6 = CPyLong_AsInt64(r5) + r6 = CPyLong_AsIndex(r5) r3 = r6 keep_alive i L3: @@ -575,7 +575,7 @@ L3: r8 = CPyStr_RangeCheck(s, r7) if r8 goto L5 else goto L4 :: bool L4: - r9 = raise IndexError('index out of range') + r9 = raise IndexError('string index out of range') unreachable L5: r10 = CPyStr_GetItemUnsafeAsInt(s, r7) @@ -590,7 +590,7 @@ L0: r1 = CPyStr_RangeCheck(s, r0) if r1 goto L2 else goto L1 :: bool L1: - r2 = raise IndexError('index out of range') + r2 = raise IndexError('string index out of range') unreachable L2: r3 = CPyStr_GetItemUnsafeAsInt(s, r0) @@ -608,7 +608,7 @@ L0: r1 = CPyStr_RangeCheck(s, r0) if r1 goto L2 else goto L1 :: bool L1: - r2 = raise IndexError('index out of range') + r2 = raise IndexError('string index out of range') unreachable L2: r3 = CPyStr_GetItemUnsafeAsInt(s, r0) @@ -640,7 +640,7 @@ L0: r1 = CPyStr_RangeCheck(s, r0) if r1 goto L2 else goto L1 :: bool L1: - r2 = raise IndexError('index out of range') + r2 = raise IndexError('string index out of range') unreachable L2: r3 = CPyStr_GetItemUnsafeAsInt(s, r0) @@ -672,7 +672,7 @@ L0: r1 = CPyStr_RangeCheck(s, r0) if r1 goto L2 else goto L1 :: bool L1: - r2 = raise IndexError('index out of range') + r2 = raise IndexError('string index out of range') unreachable L2: r3 = CPyStr_GetItemUnsafeAsInt(s, r0) @@ -702,7 +702,7 @@ L0: r1 = CPyStr_RangeCheck(s, r0) if r1 goto L2 else goto L1 :: bool L1: - r2 = raise IndexError('index out of range') + r2 = raise IndexError('string index out of range') unreachable L2: r3 = CPyStr_GetItemUnsafeAsInt(s, r0) @@ -1218,7 +1218,7 @@ L1: L2: r4 = i ^ 1 r5 = r4 - r6 = CPyLong_AsInt64(r5) + r6 = CPyLong_AsIndex(r5) r3 = r6 keep_alive i L3: @@ -1226,7 +1226,7 @@ L3: r8 = CPyStr_RangeCheck(s, r7) if r8 goto L5 else goto L4 :: bool L4: - r9 = raise IndexError('index out of range') + r9 = raise IndexError('string index out of range') unreachable L5: r10 = CPyStr_GetItemUnsafeAsInt(s, r7) @@ -1255,7 +1255,7 @@ L1: L2: r4 = i ^ 1 r5 = r4 - r6 = CPyLong_AsInt64(r5) + r6 = CPyLong_AsIndex(r5) r3 = r6 keep_alive i L3: @@ -1263,7 +1263,7 @@ L3: r8 = CPyStr_RangeCheck(s, r7) if r8 goto L5 else goto L4 :: bool L4: - r9 = raise IndexError('index out of range') + r9 = raise IndexError('string index out of range') unreachable L5: r10 = CPyStr_GetItemUnsafeAsInt(s, r7) @@ -1292,7 +1292,7 @@ L1: L2: r4 = i ^ 1 r5 = r4 - r6 = CPyLong_AsInt64(r5) + r6 = CPyLong_AsIndex(r5) r3 = r6 keep_alive i L3: @@ -1300,7 +1300,7 @@ L3: r8 = CPyStr_RangeCheck(s, r7) if r8 goto L5 else goto L4 :: bool L4: - r9 = raise IndexError('index out of range') + r9 = raise IndexError('string index out of range') unreachable L5: r10 = CPyStr_GetItemUnsafeAsInt(s, r7) diff --git a/mypyc/test-data/run-bytes.test b/mypyc/test-data/run-bytes.test index a847fa5f7ecb..4c3fed4b4cab 100644 --- a/mypyc/test-data/run-bytes.test +++ b/mypyc/test-data/run-bytes.test @@ -45,7 +45,7 @@ def test_bytes_init() -> None: pass [case testBytesOps] -from testutil import assertRaises +from testutil import assertRaises, SSIZE_T_LONG, SSIZE_T_MAX def test_indexing() -> None: # Use bytes() to avoid constant folding @@ -66,6 +66,16 @@ def test_indexing() -> None: b[-5] with assertRaises(IndexError, "index out of range"): b[2**26] + for n in (SSIZE_T_LONG, -SSIZE_T_LONG - 1, SSIZE_T_MAX, -SSIZE_T_MAX - 1): + with assertRaises(IndexError, "index out of range"): + b[n + int()] + for n in (2**63, -2**63 - 1, 2**100, -2**100): + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + b[n + int()] + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + b[2**63] + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + b[-2**63 - 1] def test_concat() -> None: b1 = b'123' + bytes() diff --git a/mypyc/test-data/run-librt-strings.test b/mypyc/test-data/run-librt-strings.test index e48f50601140..aff237bf83b2 100644 --- a/mypyc/test-data/run-librt-strings.test +++ b/mypyc/test-data/run-librt-strings.test @@ -7,7 +7,7 @@ import struct from librt.strings import BytesWriter, StringWriter, write_i16_le, write_i16_be, read_i16_le, read_i16_be, write_i32_le, write_i32_be, read_i32_le, read_i32_be, write_i64_le, write_i64_be, read_i64_le, read_i64_be, write_f64_le, read_f64_le, write_f64_be, read_f64_be, write_f32_le, write_f32_be, read_f32_le, read_f32_be -from testutil import assertRaises +from testutil import assertRaises, SSIZE_T_LONG, SSIZE_T_MAX # Test values for i16 write/read operations, organized by byte size I16_TEST_VALUES: list[int] = [ @@ -100,6 +100,14 @@ def test_bytes_writer_get_item() -> None: w[1 << 50] with assertRaises(IndexError): w[-(1 << 50)] + for n in (SSIZE_T_LONG, -SSIZE_T_LONG - 1): + with assertRaises(IndexError, "BytesWriter index out of range"): + w[n + int()] + for n in (1 << 63, -(1 << 63) - 1, 1 << 100): + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + w[n + int()] + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + w[1 << 63] def test_bytes_writer_set_item() -> None: w = BytesWriter() @@ -118,6 +126,14 @@ def test_bytes_writer_set_item() -> None: w[1 << 50] = 0 with assertRaises(IndexError): w[-(1 << 50)] = 0 + for n in (SSIZE_T_LONG, -SSIZE_T_LONG - 1): + with assertRaises(IndexError, "BytesWriter index out of range"): + w[n + int()] = 0 + for n in (1 << 63, -(1 << 63) - 1, 1 << 100): + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + w[n + int()] = 0 + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + w[1 << 63] = 0 with assertRaises(ValueError): w[0] = int() - 1 @@ -1146,6 +1162,12 @@ def test_string_writer_get_item() -> None: w[1 << 50] with assertRaises(IndexError): w[-(1 << 50)] + for n in (SSIZE_T_LONG, -SSIZE_T_LONG - 1): + with assertRaises(IndexError, "StringWriter index out of range"): + w[n + int()] + for n in (1 << 63, -(1 << 63) - 1, 1 << 100): + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + w[n + int()] # Kind 2 (UCS-2) w2 = StringWriter() diff --git a/mypyc/test-data/run-lists.test b/mypyc/test-data/run-lists.test index a4dc37ab0906..15b589a70d65 100644 --- a/mypyc/test-data/run-lists.test +++ b/mypyc/test-data/run-lists.test @@ -151,7 +151,7 @@ print(primes(13)) \[0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1] [case testListPrimitives] -from testutil import assertRaises +from testutil import assertRaises, SSIZE_T_LONG, SSIZE_T_MAX def test_list_build() -> None: # Currently LIST_BUILDING_EXPANSION_THRESHOLD equals to 10 @@ -189,11 +189,12 @@ def test_get_item() -> None: a[zero + 3] with assertRaises(IndexError): a[zero - 4] - # TODO: Raise IndexError? - with assertRaises(OverflowError): - a[zero + 2**90] - with assertRaises(OverflowError): - a[zero - 2**90] + for n in (SSIZE_T_LONG, -SSIZE_T_LONG - 1, SSIZE_T_MAX, -SSIZE_T_MAX - 1): + with assertRaises(IndexError, "list index out of range"): + a[zero + n] + for n in (2**63, -2**63 - 1, 2**90, -2**90): + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + a[zero + n] def test_get_item_literal() -> None: assert a[0].s == "a" @@ -206,10 +207,15 @@ def test_get_item_literal() -> None: a[3] with assertRaises(IndexError): a[-4] - # TODO: Raise IndexError? - with assertRaises(OverflowError): + with assertRaises(IndexError): + a[2**62] # The message depends on the platform + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + a[2**63] + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + a[-2**63 - 1] + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): a[2**90] - with assertRaises(OverflowError): + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): a[-2**90] def test_set_item() -> None: @@ -228,10 +234,12 @@ def test_set_item() -> None: a[zero + 3] = C("z") with assertRaises(IndexError): a[zero - 4] = C("z") - with assertRaises(OverflowError): - a[zero + 2**90] = C("z") - with assertRaises(OverflowError): - a[zero - 2**90] = C("z") + for n in (SSIZE_T_LONG, -SSIZE_T_LONG - 1, SSIZE_T_MAX, -SSIZE_T_MAX - 1): + with assertRaises(IndexError, "list assignment index out of range"): + a[zero + n] = C("z") + for n in (2**63, -2**63 - 1, 2**90, -2**90): + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + a[zero + n] = C("z") def test_append() -> None: l = [1, 2] @@ -284,10 +292,16 @@ def test_pop_index_errors() -> None: l.pop() with assertRaises(IndexError, "pop from empty list"): l.pop(0) - with assertRaises(OverflowError): - l.pop(1 << 100) - with assertRaises(OverflowError): - l.pop(-(1 << 100)) + with assertRaises(IndexError, "pop from empty list"): + l.pop(SSIZE_T_LONG) + l.append(int() + 1000) + for n in (SSIZE_T_LONG, -SSIZE_T_LONG - 1, SSIZE_T_MAX, -SSIZE_T_MAX - 1): + with assertRaises(IndexError, "pop index out of range"): + l.pop(n) + for n in (1 << 63, -(1 << 63) - 1, 1 << 100, -(1 << 100)): + with assertRaises(OverflowError, "Python int too large to convert to C ssize_t"): + l.pop(n) + assert l == [1000] def test_count() -> None: l = [1, 3] @@ -309,7 +323,14 @@ def test_insert() -> None: assert l == [1, 3, 5] l.insert(-100, 6) assert l == [6, 1, 3, 5] - for long_int in 1 << 100, -(1 << 100): + l.insert(SSIZE_T_LONG, 7) + assert l == [6, 1, 3, 5, 7] + l.insert(-SSIZE_T_LONG - 1, 8) + assert l == [8, 6, 1, 3, 5, 7] + l.insert(SSIZE_T_MAX, 9) + l.insert(-SSIZE_T_MAX - 1, 10) + assert l == [10, 8, 6, 1, 3, 5, 7, 9] + for long_int in 1 << 63, -(1 << 63) - 1, 1 << 100, -(1 << 100): try: l.insert(long_int, 5) except Exception as e: @@ -783,6 +804,8 @@ def test_next() -> None: [case testListGetItemWithBorrow] from typing import List +from testutil import assertRaises, SSIZE_T_LONG, SSIZE_T_MAX + class D: def __init__(self, n: int) -> None: self.n = n @@ -804,6 +827,18 @@ def test_index_with_literal() -> None: d = a[-2].d assert d is d1 +def get_n(a: List[C], i: int) -> int: + return a[i].d.n + +def test_large_index() -> None: + a = [C(D(1))] + for n in (SSIZE_T_LONG, -SSIZE_T_LONG - 1): + with assertRaises(IndexError, "list index out of range"): + get_n(a, n) + for n in (2**63, -2**63 - 1, 2**100): + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + get_n(a, n) + [case testSorted] from typing import List @@ -872,12 +907,12 @@ from testutil import assertRaises class Sentinel: pass -for index in (1, -2, 2**100, -2**100): +for index in (1, -2, 2**62, 2**63, 2**100, -2**100): value = Sentinel() ref = weakref.ref(value) try: assign([None], index, value) - except (IndexError, OverflowError): + except IndexError: pass else: assert False, index diff --git a/mypyc/test-data/run-strings.test b/mypyc/test-data/run-strings.test index aaf2cb39b127..b807e863db4a 100644 --- a/mypyc/test-data/run-strings.test +++ b/mypyc/test-data/run-strings.test @@ -158,7 +158,7 @@ def test_unicode() -> None: [case testStringOps] from typing import Any, List, Optional, Tuple -from testutil import assertRaises +from testutil import assertRaises, SSIZE_T_LONG, SSIZE_T_MAX def do_split(s: str, sep: Optional[str] = None, max_split: Optional[int] = None) -> List[str]: if sep is not None: @@ -283,6 +283,12 @@ def test_getitem() -> None: getitem(s, 4) with assertRaises(IndexError, "string index out of range"): getitem(s, -4) + for n in (SSIZE_T_LONG, -SSIZE_T_LONG - 1, SSIZE_T_MAX, -SSIZE_T_MAX - 1): + with assertRaises(IndexError, "string index out of range"): + getitem(s, n) + for n in (2**63, -2**63 - 1, 2**100, -2**100): + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + getitem(s, n) def test_getitem_unicode() -> None: # ASCII - cached by CPython's Latin-1 table @@ -980,7 +986,7 @@ def test_chr_errors() -> None: chr(n + int()) [case testOrd] -from testutil import assertRaises +from testutil import assertRaises, SSIZE_T_LONG, SSIZE_T_MAX from mypy_extensions import i64, i32, i16, u8 def test_ord() -> None: @@ -1039,21 +1045,32 @@ def test_ord_str_index() -> None: assert ord(s5[-2 + int()]) == 128512 assert ord(s5[-3 + int()]) == 97 - with assertRaises(IndexError, "index out of range"): + with assertRaises(IndexError, "string index out of range"): ord(s1[5 + int()]) - with assertRaises(IndexError, "index out of range"): + with assertRaises(IndexError, "string index out of range"): ord(s1[100 + int()]) - with assertRaises(IndexError, "index out of range"): + with assertRaises(IndexError, "string index out of range"): ord(s1[-6 + int()]) - with assertRaises(IndexError, "index out of range"): + with assertRaises(IndexError, "string index out of range"): ord(s1[-100 + int()]) s_empty = "" - with assertRaises(IndexError, "index out of range"): + with assertRaises(IndexError, "string index out of range"): ord(s_empty[0 + int()]) - with assertRaises(IndexError, "index out of range"): + with assertRaises(IndexError, "string index out of range"): ord(s_empty[-1 + int()]) + for n in (SSIZE_T_LONG, -SSIZE_T_LONG - 1, SSIZE_T_MAX, -SSIZE_T_MAX - 1): + with assertRaises(IndexError, "string index out of range"): + ord(s1[n + int()]) + for n in (2**63, -2**63 - 1, 2**100, -2**100): + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + ord(s1[n + int()]) + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + ord(s1[2**63]) + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + ord(s1[-2**63 - 1]) + def test_ord_str_index_i64() -> None: s = "hello" @@ -1064,11 +1081,11 @@ def test_ord_str_index_i64() -> None: assert ord(s[idx_i64_neg]) == 111 # 'o' idx_overflow: i64 = 10 + int() - with assertRaises(IndexError, "index out of range"): + with assertRaises(IndexError, "string index out of range"): ord(s[idx_overflow]) idx_underflow: i64 = -10 + int() - with assertRaises(IndexError, "index out of range"): + with assertRaises(IndexError, "string index out of range"): ord(s[idx_underflow]) def test_ord_str_index_unicode_mix() -> None: @@ -1809,7 +1826,7 @@ class S(str): pass [case testStrIndexEqLiteralSpecialize] -from testutil import assertRaises +from testutil import assertRaises, SSIZE_T_LONG, SSIZE_T_MAX # The specializer fires on the AST shape `IndexExpr == StrLiteral` (or the # symmetric swap, and `!=`). The literal has to be a real source-level @@ -1868,6 +1885,12 @@ def test_out_of_range_raises_indexerror() -> None: eq_comma(s, 3) with assertRaises(IndexError): eq_comma(s, -4) + for n in (SSIZE_T_LONG, -SSIZE_T_LONG - 1, SSIZE_T_MAX, -SSIZE_T_MAX - 1): + with assertRaises(IndexError, "string index out of range"): + eq_comma(s, n) + for n in (2**63, -2**63 - 1, 2**100, -2**100): + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + eq_comma(s, n) [case testStrOpsOverridingSubclass] from strsub import StrOverride diff --git a/mypyc/test-data/run-tuples.test b/mypyc/test-data/run-tuples.test index d31b67eb62ba..d74015d67f80 100644 --- a/mypyc/test-data/run-tuples.test +++ b/mypyc/test-data/run-tuples.test @@ -168,7 +168,7 @@ assert Record.__annotations__ == { [case testTupleOps] from typing import Tuple, Final, List, Any, Optional, cast -from testutil import assertRaises +from testutil import assertRaises, SSIZE_T_LONG, SSIZE_T_MAX def f() -> Tuple[()]: return () @@ -203,6 +203,21 @@ def f5(x: List[int]) -> int: def test_sequence_tuple() -> None: assert f5([1,2,3,4]) == 2 +def tuple_get(t: Tuple[int, ...], i: int) -> int: + return t[i] + +def test_sequence_tuple_large_index() -> None: + t = (1, 2, 3) + assert tuple_get(t, -1) == 3 + for n in (SSIZE_T_LONG, -SSIZE_T_LONG - 1, SSIZE_T_MAX, -SSIZE_T_MAX - 1): + with assertRaises(IndexError, "tuple index out of range"): + tuple_get(t, n) + for n in (2**63, -2**63 - 1, 2**100, -2**100): + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + tuple_get(t, n) + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + t[2**63] + def f6(x: List[int]) -> int: return len(tuple(x)) diff --git a/mypyc/test-data/run-u64.test b/mypyc/test-data/run-u64.test index d5ed0a548922..8a8d6399f692 100644 --- a/mypyc/test-data/run-u64.test +++ b/mypyc/test-data/run-u64.test @@ -3,7 +3,7 @@ from typing import Any, Final, Tuple from mypy_extensions import u64, i16, i32, i64, u8 -from testutil import assertRaises +from testutil import assertRaises, SSIZE_T_LONG, SSIZE_T_MAX ERROR: Final = 239 @@ -419,10 +419,15 @@ def test_index_with_u64() -> None: with assertRaises(IndexError): str_get("abc", 3) # Index too large for a short int takes the slow conversion path - with assertRaises(OverflowError): - tuple_get((1, 2, 3), 2**63) - with assertRaises(OverflowError): - str_get("abc", 2**64 - 1) + with assertRaises(IndexError, "tuple index out of range"): + tuple_get((1, 2, 3), SSIZE_T_LONG) + with assertRaises(IndexError, "string index out of range"): + str_get("abc", SSIZE_T_LONG) + for n in (2**63, 2**64 - 1): + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + tuple_get((1, 2, 3), n) + with assertRaises(IndexError, "cannot fit 'int' into an index-sized integer"): + str_get("abc", n) def test_tuple_u64() -> None: a: u64 = 1