From 327cbf7149cf1d2f9b98c2651382214d758e8901 Mon Sep 17 00:00:00 2001 From: Jahongir Qurbonov Date: Sat, 14 Nov 2026 14:46:11 +0500 Subject: [PATCH 1/2] [mypyc] Cache single-character strings up to U+0FFF CPython only caches single-character Latin-1 strings, so iterating over or indexing text in other alphabetic scripts (Greek, Cyrillic, Arabic, Indic scripts, etc.) allocated and freed a new str object for every character. In compiled code the allocation dominated the cost of `for c in s`. Cache single-character strings for U+0100..U+0FFF in a table that is filled on first use and never evicted, and use it for str indexing, iteration and chr(). Larger code points such as CJK aren't cached: a small direct-mapped cache made random CJK text ~35% slower due to evictions. The cache is disabled on free-threaded builds. --- mypyc/lib-rt/str_ops.c | 66 ++++++++++++++++++++++++-------- mypyc/test-data/run-strings.test | 35 +++++++++++++++++ 2 files changed, 84 insertions(+), 17 deletions(-) diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c index 8d3965644f6a..d75e247098a9 100644 --- a/mypyc/lib-rt/str_ops.c +++ b/mypyc/lib-rt/str_ops.c @@ -95,6 +95,53 @@ char CPyStr_EqualLiteral(PyObject *str, PyObject *literal_str, Py_ssize_t litera return _CPyStr_Equal_NoIdentCheck(str, literal_str, literal_length); } +// CPython caches single-character Latin-1 strings, but iterating over or indexing +// text in most other alphabetic scripts allocates a new object for every character. +// Cache single-character strings for U+0100..U+0FFF as well (Latin Extended, Greek, +// Cyrillic, Hebrew, Arabic, Indic scripts, Thai, Georgian, etc.). Entries are created +// on first use and never evicted, so the cache costs nothing for text that doesn't use +// these characters. Larger code points (such as CJK) would need a much bigger cache. +// +// Free-threaded builds don't use the cache, since the slots would need atomic access. +// Compiled modules don't support multiple interpreters, so a static cache is fine. +#define CPY_CHAR_CACHE_LIMIT 0x1000 +#ifndef Py_GIL_DISABLED +static PyObject *CPyStr_CharCache[CPY_CHAR_CACHE_LIMIT - 256]; +#endif + +static PyObject *CPyStr_FromCodePoint(Py_UCS4 ch) { + if (ch < 256) { + // Latin-1 single-char strings are cached by CPython, so + // PyUnicode_FromOrdinal returns the cached object (with a + // new reference) instead of allocating a new string each time. + return PyUnicode_FromOrdinal(ch); + } +#ifndef Py_GIL_DISABLED + PyObject **slot = NULL; + if (ch < CPY_CHAR_CACHE_LIMIT) { + slot = &CPyStr_CharCache[ch - 256]; + if (*slot != NULL) { + return Py_NewRef(*slot); + } + } +#endif + PyObject *unicode = PyUnicode_New(1, ch); + if (unicode == NULL) + return NULL; + if (PyUnicode_KIND(unicode) == PyUnicode_2BYTE_KIND) { + PyUnicode_2BYTE_DATA(unicode)[0] = (Py_UCS2)ch; + } else { + assert(PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND); + PyUnicode_4BYTE_DATA(unicode)[0] = ch; + } +#ifndef Py_GIL_DISABLED + if (slot != NULL) { + *slot = Py_NewRef(unicode); + } +#endif + return unicode; +} + PyObject *CPyStr_GetItem(PyObject *str, CPyTagged index) { if (PyUnicode_READY(str) != -1) { if (CPyTagged_CheckShort(index)) { @@ -109,22 +156,7 @@ PyObject *CPyStr_GetItem(PyObject *str, CPyTagged index) { enum PyUnicode_Kind kind = (enum PyUnicode_Kind)PyUnicode_KIND(str); void *data = PyUnicode_DATA(str); Py_UCS4 ch = PyUnicode_READ(kind, data, n); - if (ch < 256) { - // Latin-1 single-char strings are cached by CPython, so - // PyUnicode_FromOrdinal returns the cached object (with a - // new reference) instead of allocating a new string each time. - return PyUnicode_FromOrdinal(ch); - } - PyObject *unicode = PyUnicode_New(1, ch); - if (unicode == NULL) - return NULL; - if (PyUnicode_KIND(unicode) == PyUnicode_2BYTE_KIND) { - PyUnicode_2BYTE_DATA(unicode)[0] = (Py_UCS2)ch; - } else { - assert(PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND); - PyUnicode_4BYTE_DATA(unicode)[0] = ch; - } - return unicode; + return CPyStr_FromCodePoint(ch); } else { PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); return NULL; @@ -640,7 +672,7 @@ PyObject *CPyStr_Chr(CPyTagged n) { if (likely(CPyTagged_CheckShort(n))) { Py_ssize_t v = CPyTagged_ShortAsSsize_t(n); if (likely(v >= 0 && v <= 0x10ffff)) { - return PyUnicode_FromOrdinal((int)v); + return CPyStr_FromCodePoint((Py_UCS4)v); } } #if CPY_3_13_FEATURES diff --git a/mypyc/test-data/run-strings.test b/mypyc/test-data/run-strings.test index a31ef0110f24..7c9f6123db7e 100644 --- a/mypyc/test-data/run-strings.test +++ b/mypyc/test-data/run-strings.test @@ -894,6 +894,41 @@ def test_chr_errors() -> None: with assertRaises(expected_type, expected_msg): chr(n + int()) +[case testStrSingleCharItems] +def test_index_and_iterate() -> None: + # Cover Latin-1, the range of cached single-character strings (U+0100..U+0FFF) + # and some code points beyond it + cps = [*range(0x1100), 0xFFFF, 0x10000, 0x10FFFF] + s = "".join([chr(cp) for cp in cps]) + # The second round reuses cached strings + for _ in range(2): + i = 0 + for c in s: + assert len(c) == 1 + assert ord(c) == cps[i] + x = s[i] + assert ord(x) == cps[i] + assert x == c == chr(cps[i]) + y = s[i - len(s)] + assert y == c + i += 1 + assert i == len(cps) + +def test_cached_char_not_modified() -> None: + s = "\u0400\u0fff" + c = s[0] + c += "x" + assert c == "\u0400x" + assert s[0] == "\u0400" + d = chr(0xFFF + int()) + d += "y" + assert d == "\u0fffy" + assert chr(0xFFF + int()) == "\u0fff" + for e in s: + e += "z" + assert len(e) == 2 + assert [e for e in s] == ["\u0400", "\u0fff"] + [case testOrd] from testutil import assertRaises from mypy_extensions import i64, i32, i16, u8 From 5b46fa9305a91282bb68e0ccd57db75728ef71ff Mon Sep 17 00:00:00 2001 From: Jahongir Qurbonov Date: Mon, 16 Nov 2026 23:25:45 +0500 Subject: [PATCH 2/2] [mypyc] Move the single-character string cache to librt.strings Add librt.strings.cached_chr and char_at, which cache single-character strings for U+0100..U+0FFF, instead of using the cache in the built-in chr(), str indexing and iteration, since the cache uses up to ~230 kB of memory. The cache is shared by all compiled modules via a new capsule API function. --- mypy/typeshed/stubs/librt/librt/strings.pyi | 5 + mypyc/doc/librt_strings.rst | 39 ++++++++ mypyc/lib-rt/str_ops.c | 66 ++++--------- mypyc/lib-rt/strings/librt_strings.c | 100 ++++++++++++++++++++ mypyc/lib-rt/strings/librt_strings.h | 10 +- mypyc/lib-rt/strings/librt_strings_api.h | 32 +++++++ mypyc/primitives/librt_strings_ops.py | 20 ++++ mypyc/test-data/irbuild-librt-strings.test | 29 ++++++ mypyc/test-data/run-librt-strings.test | 90 ++++++++++++++++++ mypyc/test-data/run-strings.test | 35 ------- 10 files changed, 340 insertions(+), 86 deletions(-) diff --git a/mypy/typeshed/stubs/librt/librt/strings.pyi b/mypy/typeshed/stubs/librt/librt/strings.pyi index b06ae319be42..7afc992ea690 100644 --- a/mypy/typeshed/stubs/librt/librt/strings.pyi +++ b/mypy/typeshed/stubs/librt/librt/strings.pyi @@ -73,3 +73,8 @@ def isidentifier(c: i32, /) -> bool: ... # unchanged. def toupper(c: i32, /) -> i32: ... def tolower(c: i32, /) -> i32: ... + +# Like chr(c) and s[i], but the single-character strings for code points in +# U+0100..U+0FFF are cached. This uses up to ~230 kB of memory. +def cached_chr(c: i64, /) -> str: ... +def char_at(s: str, i: i64, /) -> str: ... diff --git a/mypyc/doc/librt_strings.rst b/mypyc/doc/librt_strings.rst index 7333c8681f9a..682815f919cc 100644 --- a/mypyc/doc/librt_strings.rst +++ b/mypyc/doc/librt_strings.rst @@ -419,3 +419,42 @@ maximum Unicode code point ``0x10FFFF``) are returned unchanged. Return the lowercase of the code point, or the input unchanged if the lowercase does not consist of exactly one code point. + +Cached single-character strings +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +CPython caches one-character strings only for Latin-1 characters (code points below 256). +Getting a character in most other alphabetic scripts using :py:func:`chr`, ``s[i]`` or +``for c in s`` allocates a new string object, and in compiled code this is often most of the +cost of a loop that processes text one character at a time. + +These functions behave like :py:func:`chr` and ``s[i]``, but they also cache the +one-character strings for code points in the range U+0100 to U+0FFF (this includes Latin +Extended, Greek, Cyrillic, Armenian, Hebrew, Arabic, Indic scripts, Thai and Georgian). This +can make processing text in these scripts several times faster. Characters outside this +range are not cached, and are as fast as with :py:func:`chr` and ``s[i]``. + +The tradeoff is memory use: the cache takes about 30 kB, plus about 60 bytes for each +distinct cached character (up to about 230 kB in total). Cached strings are never freed. +The cache isn't used on free-threaded Python builds or in subinterpreters. + +This example counts occurrences of a character in a string, without allocating a new string +for each Cyrillic character:: + + from librt.strings import char_at + + def count_char(s: str, ch: str) -> int: + n = 0 + for i in range(len(s)): + if char_at(s, i) == ch: + n += 1 + return n + +.. function:: cached_chr(c: i64, /) -> str + + Return a one-character string for the code point. Equivalent to ``chr(c)``. + +.. function:: char_at(s: str, i: i64, /) -> str + + Return the character at index ``i`` (negative indices count from the end). Equivalent + to ``s[i]``. diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c index af2376500073..c9552b6b87bf 100644 --- a/mypyc/lib-rt/str_ops.c +++ b/mypyc/lib-rt/str_ops.c @@ -95,53 +95,6 @@ char CPyStr_EqualLiteral(PyObject *str, PyObject *literal_str, Py_ssize_t litera return _CPyStr_Equal_NoIdentCheck(str, literal_str, literal_length); } -// CPython caches single-character Latin-1 strings, but iterating over or indexing -// text in most other alphabetic scripts allocates a new object for every character. -// Cache single-character strings for U+0100..U+0FFF as well (Latin Extended, Greek, -// Cyrillic, Hebrew, Arabic, Indic scripts, Thai, Georgian, etc.). Entries are created -// on first use and never evicted, so the cache costs nothing for text that doesn't use -// these characters. Larger code points (such as CJK) would need a much bigger cache. -// -// Free-threaded builds don't use the cache, since the slots would need atomic access. -// Compiled modules don't support multiple interpreters, so a static cache is fine. -#define CPY_CHAR_CACHE_LIMIT 0x1000 -#ifndef Py_GIL_DISABLED -static PyObject *CPyStr_CharCache[CPY_CHAR_CACHE_LIMIT - 256]; -#endif - -static PyObject *CPyStr_FromCodePoint(Py_UCS4 ch) { - if (ch < 256) { - // Latin-1 single-char strings are cached by CPython, so - // PyUnicode_FromOrdinal returns the cached object (with a - // new reference) instead of allocating a new string each time. - return PyUnicode_FromOrdinal(ch); - } -#ifndef Py_GIL_DISABLED - PyObject **slot = NULL; - if (ch < CPY_CHAR_CACHE_LIMIT) { - slot = &CPyStr_CharCache[ch - 256]; - if (*slot != NULL) { - return Py_NewRef(*slot); - } - } -#endif - PyObject *unicode = PyUnicode_New(1, ch); - if (unicode == NULL) - return NULL; - if (PyUnicode_KIND(unicode) == PyUnicode_2BYTE_KIND) { - PyUnicode_2BYTE_DATA(unicode)[0] = (Py_UCS2)ch; - } else { - assert(PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND); - PyUnicode_4BYTE_DATA(unicode)[0] = ch; - } -#ifndef Py_GIL_DISABLED - if (slot != NULL) { - *slot = Py_NewRef(unicode); - } -#endif - return unicode; -} - PyObject *CPyStr_GetItem(PyObject *str, CPyTagged index) { if (PyUnicode_READY(str) != -1) { if (CPyTagged_CheckShort(index)) { @@ -156,7 +109,22 @@ PyObject *CPyStr_GetItem(PyObject *str, CPyTagged index) { enum PyUnicode_Kind kind = (enum PyUnicode_Kind)PyUnicode_KIND(str); void *data = PyUnicode_DATA(str); Py_UCS4 ch = PyUnicode_READ(kind, data, n); - return CPyStr_FromCodePoint(ch); + if (ch < 256) { + // Latin-1 single-char strings are cached by CPython, so + // PyUnicode_FromOrdinal returns the cached object (with a + // new reference) instead of allocating a new string each time. + return PyUnicode_FromOrdinal(ch); + } + PyObject *unicode = PyUnicode_New(1, ch); + if (unicode == NULL) + return NULL; + if (PyUnicode_KIND(unicode) == PyUnicode_2BYTE_KIND) { + PyUnicode_2BYTE_DATA(unicode)[0] = (Py_UCS2)ch; + } else { + assert(PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND); + PyUnicode_4BYTE_DATA(unicode)[0] = ch; + } + return unicode; } else { PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); return NULL; @@ -687,7 +655,7 @@ PyObject *CPyStr_Chr(CPyTagged n) { if (likely(CPyTagged_CheckShort(n))) { Py_ssize_t v = CPyTagged_ShortAsSsize_t(n); if (likely(v >= 0 && v <= 0x10ffff)) { - return CPyStr_FromCodePoint((Py_UCS4)v); + return PyUnicode_FromOrdinal((int)v); } } #if CPY_3_13_FEATURES diff --git a/mypyc/lib-rt/strings/librt_strings.c b/mypyc/lib-rt/strings/librt_strings.c index ef0df125faa1..e412d2f511ec 100644 --- a/mypyc/lib-rt/strings/librt_strings.c +++ b/mypyc/lib-rt/strings/librt_strings.c @@ -1365,6 +1365,99 @@ DEFINE_CP_BOOL_WRAPPER(isidentifier, LibRTStrings_IsIdentifier) DEFINE_CP_I32_WRAPPER(toupper, LibRTStrings_ToUpper) DEFINE_CP_I32_WRAPPER(tolower, LibRTStrings_ToLower) +// Cache of single-character strings for code points in U+0100..U+0FFF, used by +// cached_chr and char_at. CPython only caches single-character Latin-1 strings, +// so getting a character in most other alphabetic scripts (Latin Extended, Greek, +// Cyrillic, Hebrew, Arabic, Indic scripts, Thai, Georgian, etc.) allocates a new +// string. Entries are created on first use and never freed. The table takes 30 kB, +// and each cached string about 60 bytes more (about 230 kB if all are used). +// +// Free-threaded builds don't use the cache, since the slots would need atomic +// access. The cache is only used in the main interpreter, since objects can't be +// shared between interpreters (mypyc-compiled modules only support the main +// interpreter, so this only matters for calls from interpreted code). +#ifndef Py_GIL_DISABLED +static PyObject *char_cache[LIBRT_STRINGS_CHAR_CACHE_END - 256]; +#endif + +// Return a new reference to a single-character string for a valid code point, +// or NULL with an exception set. +static PyObject * +str_from_code_point(int32_t c, bool use_cache) { + if (c < 256) { + // CPython caches single-character Latin-1 strings + return PyUnicode_FromOrdinal(c); + } +#ifndef Py_GIL_DISABLED + PyObject **slot = NULL; + if (c < LIBRT_STRINGS_CHAR_CACHE_END && use_cache) { + slot = &char_cache[c - 256]; + if (*slot != NULL) { + return Py_NewRef(*slot); + } + } +#endif + PyObject *result = PyUnicode_FromOrdinal(c); +#ifndef Py_GIL_DISABLED + if (result != NULL && slot != NULL) { + *slot = Py_NewRef(result); + } +#endif + return result; +} + +// Variant for compiled code, which is always run in the main interpreter +static PyObject * +str_from_code_point_internal(int32_t c) { + return str_from_code_point(c, true); +} + +static inline bool +can_use_char_cache(void) { + return PyInterpreterState_Get() == PyInterpreterState_Main(); +} + +static PyObject* +cached_chr(PyObject *module, PyObject *arg) { + int64_t c = CPyLong_AsInt64(arg); + if (unlikely(c == CPY_LL_INT_ERROR && PyErr_Occurred())) { + return NULL; + } + if (unlikely(c < 0 || c > 0x10ffff)) { + PyErr_SetString(PyExc_ValueError, LIBRT_STRINGS_CACHED_CHR_ERRMSG); + return NULL; + } + return str_from_code_point((int32_t)c, can_use_char_cache()); +} + +static PyObject* +char_at(PyObject *module, PyObject *const *args, size_t nargs) { + if (unlikely(nargs != 2)) { + PyErr_Format(PyExc_TypeError, + "char_at() takes exactly 2 arguments (%zu given)", nargs); + return NULL; + } + PyObject *s = args[0]; + if (unlikely(!PyUnicode_Check(s))) { + PyErr_SetString(PyExc_TypeError, "char_at() argument 1 must be str"); + return NULL; + } + int64_t i = CPyLong_AsInt64(args[1]); + if (unlikely(i == CPY_LL_INT_ERROR && PyErr_Occurred())) { + return NULL; + } + Py_ssize_t len = PyUnicode_GET_LENGTH(s); + if (i < 0) { + i += len; + } + if (unlikely(i < 0 || i >= len)) { + PyErr_SetString(PyExc_IndexError, "string index out of range"); + return NULL; + } + Py_UCS4 c = PyUnicode_READ_CHAR(s, (Py_ssize_t)i); + return str_from_code_point((int32_t)c, can_use_char_cache()); +} + static PyMethodDef librt_strings_module_methods[] = { {"write_i8", (PyCFunction) write_i8, METH_FASTCALL, PyDoc_STR("Write an 8-bit signed integer to BytesWriter as a single byte") @@ -1489,6 +1582,12 @@ static PyMethodDef librt_strings_module_methods[] = { {"tolower", cp_tolower, METH_O, PyDoc_STR("Single-codepoint lowercase mapping for a codepoint (i32). Returns the input unchanged if the Unicode lowercase expands to multiple codepoints; use str.lower() for full Unicode case conversion.") }, + {"cached_chr", cached_chr, METH_O, + PyDoc_STR("Like chr(), but cache the result for code points in U+0100..U+0FFF.") + }, + {"char_at", (PyCFunction) char_at, METH_FASTCALL, + PyDoc_STR("Like s[i], but cache the result for code points in U+0100..U+0FFF.") + }, {NULL, NULL, 0, NULL} }; @@ -1534,6 +1633,7 @@ librt_strings_module_exec(PyObject *m) (void *)StringWriter_type_internal, (void *)StringWriter_write_internal, (void *)grow_string_buffer, + (void *)str_from_code_point_internal, }; PyObject *c_api_object = PyCapsule_New((void *)librt_strings_api, "librt.strings._C_API", NULL); if (PyModule_Add(m, "_C_API", c_api_object) < 0) { diff --git a/mypyc/lib-rt/strings/librt_strings.h b/mypyc/lib-rt/strings/librt_strings.h index 33af57dc2a2a..2f21e289346c 100644 --- a/mypyc/lib-rt/strings/librt_strings.h +++ b/mypyc/lib-rt/strings/librt_strings.h @@ -15,11 +15,17 @@ // API version -- more recent versions must maintain backward compatibility, i.e. // we can add new features but not remove or change existing features (unless // ABI version is changed, but see the comment above). -#define LIBRT_STRINGS_API_VERSION 4 +#define LIBRT_STRINGS_API_VERSION 5 // Number of functions in the capsule API. If you add a new function, also increase // LIBRT_STRINGS_API_VERSION. -#define LIBRT_STRINGS_API_LEN 14 +#define LIBRT_STRINGS_API_LEN 15 + +// Single-character strings for code points below this are cached by cached_chr and +// char_at (CPython caches those below 256) +#define LIBRT_STRINGS_CHAR_CACHE_END 0x1000 + +#define LIBRT_STRINGS_CACHED_CHR_ERRMSG "cached_chr() arg not in range(0x110000)" typedef struct { PyObject_HEAD diff --git a/mypyc/lib-rt/strings/librt_strings_api.h b/mypyc/lib-rt/strings/librt_strings_api.h index 536b90ad7f21..b9452b7b9f8e 100644 --- a/mypyc/lib-rt/strings/librt_strings_api.h +++ b/mypyc/lib-rt/strings/librt_strings_api.h @@ -24,6 +24,7 @@ extern void *LibRTStrings_API[LIBRT_STRINGS_API_LEN]; #define LibRTStrings_StringWriter_type_internal (*(PyTypeObject* (*)(void)) LibRTStrings_API[11]) #define LibRTStrings_StringWriter_write_internal (*(char (*)(PyObject *source, PyObject *value)) LibRTStrings_API[12]) #define LibRTStrings_grow_string_buffer (*(bool (*)(StringWriterObject *obj, Py_ssize_t n)) LibRTStrings_API[13]) +#define LibRTStrings_str_from_code_point_internal (*(PyObject* (*)(int32_t c)) LibRTStrings_API[14]) static inline bool CPyBytesWriter_Check(PyObject *obj) { @@ -34,4 +35,35 @@ static inline bool CPyStringWriter_Check(PyObject *obj) { return Py_TYPE(obj) == LibRTStrings_StringWriter_type_internal(); } +// Return a new reference to a single-character string for a code point, or NULL +// with an exception set. Strings for code points below +// LIBRT_STRINGS_CHAR_CACHE_END are cached. +static inline PyObject *LibRTStrings_StrFromCodePoint(Py_UCS4 c) { + if (c < 256) { + // CPython caches single-character Latin-1 strings + return PyUnicode_FromOrdinal((int)c); + } + return LibRTStrings_str_from_code_point_internal((int32_t)c); +} + +static inline PyObject *LibRTStrings_CachedChr(int64_t c) { + if (unlikely(c < 0 || c > 0x10ffff)) { + PyErr_SetString(PyExc_ValueError, LIBRT_STRINGS_CACHED_CHR_ERRMSG); + return NULL; + } + return LibRTStrings_StrFromCodePoint((Py_UCS4)c); +} + +static inline PyObject *LibRTStrings_CharAt(PyObject *s, int64_t i) { + Py_ssize_t len = PyUnicode_GET_LENGTH(s); + if (i < 0) { + i += len; + } + if (unlikely(i < 0 || i >= len)) { + PyErr_SetString(PyExc_IndexError, "string index out of range"); + return NULL; + } + return LibRTStrings_StrFromCodePoint(PyUnicode_READ_CHAR(s, (Py_ssize_t)i)); +} + #endif // LIBRT_STRINGS_API_H diff --git a/mypyc/primitives/librt_strings_ops.py b/mypyc/primitives/librt_strings_ops.py index 47dd769dd754..fdd4403c32f4 100644 --- a/mypyc/primitives/librt_strings_ops.py +++ b/mypyc/primitives/librt_strings_ops.py @@ -599,3 +599,23 @@ error_kind=ERR_NEVER, dependencies=[LIBRT_STRINGS], ) + +# Like chr() and s[i], but single-character strings for code points in +# U+0100..U+0FFF are cached (CPython only caches Latin-1 characters) +function_op( + name="librt.strings.cached_chr", + arg_types=[int64_rprimitive], + return_type=str_rprimitive, + c_function_name="LibRTStrings_CachedChr", + error_kind=ERR_MAGIC, + dependencies=[LIBRT_STRINGS], +) + +function_op( + name="librt.strings.char_at", + arg_types=[str_rprimitive, int64_rprimitive], + return_type=str_rprimitive, + c_function_name="LibRTStrings_CharAt", + error_kind=ERR_MAGIC, + dependencies=[LIBRT_STRINGS], +) diff --git a/mypyc/test-data/irbuild-librt-strings.test b/mypyc/test-data/irbuild-librt-strings.test index 85f6295abde0..b947fd0c91db 100644 --- a/mypyc/test-data/irbuild-librt-strings.test +++ b/mypyc/test-data/irbuild-librt-strings.test @@ -518,3 +518,32 @@ def lo(c): L0: r0 = LibRTStrings_ToLower(c) return r0 + +[case testLibrtStringsCachedChrIR] +from librt.strings import cached_chr +from mypy_extensions import i64 + +def f(c: i64) -> str: + return cached_chr(c) +[out] +def f(c): + c :: i64 + r0 :: str +L0: + r0 = LibRTStrings_CachedChr(c) + return r0 + +[case testLibrtStringsCharAtIR] +from librt.strings import char_at +from mypy_extensions import i64 + +def f(s: str, i: i64) -> str: + return char_at(s, i) +[out] +def f(s, i): + s :: str + i :: i64 + r0 :: str +L0: + r0 = LibRTStrings_CharAt(s, i) + return r0 diff --git a/mypyc/test-data/run-librt-strings.test b/mypyc/test-data/run-librt-strings.test index e48f50601140..7ce90dbadb74 100644 --- a/mypyc/test-data/run-librt-strings.test +++ b/mypyc/test-data/run-librt-strings.test @@ -1813,3 +1813,93 @@ def test_codepoint_case_conversion_via_any() -> None: f(1 << 40) with assertRaises(OverflowError, "codepoint out of i32 range"): f(-(1 << 40)) + +[case testLibrtStringsCachedChars_librt] +import sysconfig +from typing import Any +from librt.strings import cached_chr, char_at + +from testutil import assertRaises + +# The cache isn't used on free-threaded builds +CACHE_ENABLED = not sysconfig.get_config_var("Py_GIL_DISABLED") + +# Latin-1, the range of cached characters (U+0100..U+0FFF) and some code points beyond it +CODE_POINTS = [*range(0x1100), 0xFFFF, 0x10000, 0x10FFFF] + +def test_cached_chr() -> None: + # The second round uses cached strings + for _ in range(2): + for c in CODE_POINTS: + s = cached_chr(c) + assert len(s) == 1 + assert s == chr(c) + +def test_cached_chr_errors() -> None: + for c in (-1, 0x110000, -(1 << 62), 1 << 62): + with assertRaises(ValueError, "cached_chr() arg not in range(0x110000)"): + cached_chr(c) + +def test_char_at() -> None: + s = "".join([chr(c) for c in CODE_POINTS]) + n = len(s) + # The second round uses cached strings + for _ in range(2): + for i in range(n): + x = char_at(s, i) + assert len(x) == 1 + assert x == s[i] + assert char_at(s, i - n) == s[i] + +def test_char_at_errors() -> None: + for s in ("", "a", "ЀЁ", "\U00010000"): + for i in (len(s), -len(s) - 1, 1 << 62, -(1 << 62)): + with assertRaises(IndexError, "string index out of range"): + char_at(s, i) + +def test_cache_is_used() -> None: + if CACHE_ENABLED: + assert cached_chr(0x400) is char_at("xЀ", 1) + assert cached_chr(0xFFF) is cached_chr(0xFFF) + assert char_at("Ā", 0) is char_at("Ā", -1) + +def test_cached_string_not_modified() -> None: + c = cached_chr(0x400) + c += "x" + assert c == "Ѐx" + assert cached_chr(0x400) == "Ѐ" + d = char_at("࿿", 0) + d += "y" + assert d == "࿿y" + assert char_at("࿿", 0) == "࿿" + +def test_via_any() -> None: + f: Any = cached_chr + g: Any = char_at + s = "".join([chr(c) for c in CODE_POINTS]) + for _ in range(2): + for i, c in enumerate(CODE_POINTS): + assert f(c) == chr(c) + assert g(s, i) == s[i] + assert g(s, i - len(s)) == s[i] + if CACHE_ENABLED: + assert f(0x401) is cached_chr(0x401) + assert g("Ђ", 0) is cached_chr(0x402) + with assertRaises(ValueError, "cached_chr() arg not in range(0x110000)"): + f(-1) + with assertRaises(ValueError, "cached_chr() arg not in range(0x110000)"): + f(0x110000) + with assertRaises(ValueError, "int too large to convert to i64"): + f(1 << 70) + with assertRaises(TypeError): + f("a") + with assertRaises(IndexError, "string index out of range"): + g("ab", 2) + with assertRaises(IndexError, "string index out of range"): + g("ab", -3) + with assertRaises(ValueError, "int too large to convert to i64"): + g("ab", 1 << 70) + with assertRaises(TypeError, "char_at() argument 1 must be str"): + g(b"ab", 0) + with assertRaises(TypeError, "char_at() takes exactly 2 arguments (1 given)"): + g("ab") diff --git a/mypyc/test-data/run-strings.test b/mypyc/test-data/run-strings.test index cc0cd32fa429..aaf2cb39b127 100644 --- a/mypyc/test-data/run-strings.test +++ b/mypyc/test-data/run-strings.test @@ -979,41 +979,6 @@ def test_chr_errors() -> None: with assertRaises(expected_type, expected_msg): chr(n + int()) -[case testStrSingleCharItems] -def test_index_and_iterate() -> None: - # Cover Latin-1, the range of cached single-character strings (U+0100..U+0FFF) - # and some code points beyond it - cps = [*range(0x1100), 0xFFFF, 0x10000, 0x10FFFF] - s = "".join([chr(cp) for cp in cps]) - # The second round reuses cached strings - for _ in range(2): - i = 0 - for c in s: - assert len(c) == 1 - assert ord(c) == cps[i] - x = s[i] - assert ord(x) == cps[i] - assert x == c == chr(cps[i]) - y = s[i - len(s)] - assert y == c - i += 1 - assert i == len(cps) - -def test_cached_char_not_modified() -> None: - s = "\u0400\u0fff" - c = s[0] - c += "x" - assert c == "\u0400x" - assert s[0] == "\u0400" - d = chr(0xFFF + int()) - d += "y" - assert d == "\u0fffy" - assert chr(0xFFF + int()) == "\u0fff" - for e in s: - e += "z" - assert len(e) == 2 - assert [e for e in s] == ["\u0400", "\u0fff"] - [case testOrd] from testutil import assertRaises from mypy_extensions import i64, i32, i16, u8