Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions mypy/typeshed/stubs/librt/librt/strings.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -73,3 +73,8 @@ def isidentifier(c: i32, /) -> bool: ...
# unchanged.
def toupper(c: i32, /) -> i32: ...
def tolower(c: i32, /) -> i32: ...

# Like chr(c) and s[i], but the single-character strings for code points in
# U+0100..U+0FFF are cached. This uses up to ~230 kB of memory.
def cached_chr(c: i64, /) -> str: ...
def char_at(s: str, i: i64, /) -> str: ...
39 changes: 39 additions & 0 deletions mypyc/doc/librt_strings.rst
Original file line number Diff line number Diff line change
Expand Up @@ -419,3 +419,42 @@ maximum Unicode code point ``0x10FFFF``) are returned unchanged.

Return the lowercase of the code point, or the input unchanged if the lowercase does not
consist of exactly one code point.

Cached single-character strings
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

CPython caches one-character strings only for Latin-1 characters (code points below 256).
Getting a character in most other alphabetic scripts using :py:func:`chr`, ``s[i]`` or
``for c in s`` allocates a new string object, and in compiled code this is often most of the
cost of a loop that processes text one character at a time.

These functions behave like :py:func:`chr` and ``s[i]``, but they also cache the
one-character strings for code points in the range U+0100 to U+0FFF (this includes Latin
Extended, Greek, Cyrillic, Armenian, Hebrew, Arabic, Indic scripts, Thai and Georgian). This
can make processing text in these scripts several times faster. Characters outside this
range are not cached, and are as fast as with :py:func:`chr` and ``s[i]``.

The tradeoff is memory use: the cache takes about 30 kB, plus about 60 bytes for each
distinct cached character (up to about 230 kB in total). Cached strings are never freed.
The cache isn't used on free-threaded Python builds or in subinterpreters.

This example counts occurrences of a character in a string, without allocating a new string
for each Cyrillic character::

from librt.strings import char_at

def count_char(s: str, ch: str) -> int:
n = 0
for i in range(len(s)):
if char_at(s, i) == ch:
n += 1
return n

.. function:: cached_chr(c: i64, /) -> str

Return a one-character string for the code point. Equivalent to ``chr(c)``.

.. function:: char_at(s: str, i: i64, /) -> str

Return the character at index ``i`` (negative indices count from the end). Equivalent
to ``s[i]``.
100 changes: 100 additions & 0 deletions mypyc/lib-rt/strings/librt_strings.c
Original file line number Diff line number Diff line change
Expand Up @@ -1365,6 +1365,99 @@ DEFINE_CP_BOOL_WRAPPER(isidentifier, LibRTStrings_IsIdentifier)
DEFINE_CP_I32_WRAPPER(toupper, LibRTStrings_ToUpper)
DEFINE_CP_I32_WRAPPER(tolower, LibRTStrings_ToLower)

// Cache of single-character strings for code points in U+0100..U+0FFF, used by
// cached_chr and char_at. CPython only caches single-character Latin-1 strings,
// so getting a character in most other alphabetic scripts (Latin Extended, Greek,
// Cyrillic, Hebrew, Arabic, Indic scripts, Thai, Georgian, etc.) allocates a new
// string. Entries are created on first use and never freed. The table takes 30 kB,
// and each cached string about 60 bytes more (about 230 kB if all are used).
//
// Free-threaded builds don't use the cache, since the slots would need atomic
// access. The cache is only used in the main interpreter, since objects can't be
// shared between interpreters (mypyc-compiled modules only support the main
// interpreter, so this only matters for calls from interpreted code).
#ifndef Py_GIL_DISABLED
static PyObject *char_cache[LIBRT_STRINGS_CHAR_CACHE_END - 256];
#endif

// Return a new reference to a single-character string for a valid code point,
// or NULL with an exception set.
static PyObject *
str_from_code_point(int32_t c, bool use_cache) {
if (c < 256) {
// CPython caches single-character Latin-1 strings
return PyUnicode_FromOrdinal(c);
}
#ifndef Py_GIL_DISABLED
PyObject **slot = NULL;
if (c < LIBRT_STRINGS_CHAR_CACHE_END && use_cache) {
slot = &char_cache[c - 256];
if (*slot != NULL) {
return Py_NewRef(*slot);
}
}
#endif
PyObject *result = PyUnicode_FromOrdinal(c);
#ifndef Py_GIL_DISABLED
if (result != NULL && slot != NULL) {
*slot = Py_NewRef(result);
}
#endif
return result;
}

// Variant for compiled code, which is always run in the main interpreter
static PyObject *
str_from_code_point_internal(int32_t c) {
return str_from_code_point(c, true);
}

static inline bool
can_use_char_cache(void) {
return PyInterpreterState_Get() == PyInterpreterState_Main();
}

static PyObject*
cached_chr(PyObject *module, PyObject *arg) {
int64_t c = CPyLong_AsInt64(arg);
if (unlikely(c == CPY_LL_INT_ERROR && PyErr_Occurred())) {
return NULL;
}
if (unlikely(c < 0 || c > 0x10ffff)) {
PyErr_SetString(PyExc_ValueError, LIBRT_STRINGS_CACHED_CHR_ERRMSG);
return NULL;
}
return str_from_code_point((int32_t)c, can_use_char_cache());
}

static PyObject*
char_at(PyObject *module, PyObject *const *args, size_t nargs) {
if (unlikely(nargs != 2)) {
PyErr_Format(PyExc_TypeError,
"char_at() takes exactly 2 arguments (%zu given)", nargs);
return NULL;
}
PyObject *s = args[0];
if (unlikely(!PyUnicode_Check(s))) {
PyErr_SetString(PyExc_TypeError, "char_at() argument 1 must be str");
return NULL;
}
int64_t i = CPyLong_AsInt64(args[1]);
if (unlikely(i == CPY_LL_INT_ERROR && PyErr_Occurred())) {
return NULL;
}
Py_ssize_t len = PyUnicode_GET_LENGTH(s);
if (i < 0) {
i += len;
}
if (unlikely(i < 0 || i >= len)) {
PyErr_SetString(PyExc_IndexError, "string index out of range");
return NULL;
}
Py_UCS4 c = PyUnicode_READ_CHAR(s, (Py_ssize_t)i);
return str_from_code_point((int32_t)c, can_use_char_cache());
}

static PyMethodDef librt_strings_module_methods[] = {
{"write_i8", (PyCFunction) write_i8, METH_FASTCALL,
PyDoc_STR("Write an 8-bit signed integer to BytesWriter as a single byte")
Expand Down Expand Up @@ -1489,6 +1582,12 @@ static PyMethodDef librt_strings_module_methods[] = {
{"tolower", cp_tolower, METH_O,
PyDoc_STR("Single-codepoint lowercase mapping for a codepoint (i32). Returns the input unchanged if the Unicode lowercase expands to multiple codepoints; use str.lower() for full Unicode case conversion.")
},
{"cached_chr", cached_chr, METH_O,
PyDoc_STR("Like chr(), but cache the result for code points in U+0100..U+0FFF.")
},
{"char_at", (PyCFunction) char_at, METH_FASTCALL,
PyDoc_STR("Like s[i], but cache the result for code points in U+0100..U+0FFF.")
},
{NULL, NULL, 0, NULL}
};

Expand Down Expand Up @@ -1534,6 +1633,7 @@ librt_strings_module_exec(PyObject *m)
(void *)StringWriter_type_internal,
(void *)StringWriter_write_internal,
(void *)grow_string_buffer,
(void *)str_from_code_point_internal,
};
PyObject *c_api_object = PyCapsule_New((void *)librt_strings_api, "librt.strings._C_API", NULL);
if (PyModule_Add(m, "_C_API", c_api_object) < 0) {
Expand Down
10 changes: 8 additions & 2 deletions mypyc/lib-rt/strings/librt_strings.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,17 @@
// API version -- more recent versions must maintain backward compatibility, i.e.
// we can add new features but not remove or change existing features (unless
// ABI version is changed, but see the comment above).
#define LIBRT_STRINGS_API_VERSION 4
#define LIBRT_STRINGS_API_VERSION 5

// Number of functions in the capsule API. If you add a new function, also increase
// LIBRT_STRINGS_API_VERSION.
#define LIBRT_STRINGS_API_LEN 14
#define LIBRT_STRINGS_API_LEN 15

// Single-character strings for code points below this are cached by cached_chr and
// char_at (CPython caches those below 256)
#define LIBRT_STRINGS_CHAR_CACHE_END 0x1000

#define LIBRT_STRINGS_CACHED_CHR_ERRMSG "cached_chr() arg not in range(0x110000)"

typedef struct {
PyObject_HEAD
Expand Down
32 changes: 32 additions & 0 deletions mypyc/lib-rt/strings/librt_strings_api.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ extern void *LibRTStrings_API[LIBRT_STRINGS_API_LEN];
#define LibRTStrings_StringWriter_type_internal (*(PyTypeObject* (*)(void)) LibRTStrings_API[11])
#define LibRTStrings_StringWriter_write_internal (*(char (*)(PyObject *source, PyObject *value)) LibRTStrings_API[12])
#define LibRTStrings_grow_string_buffer (*(bool (*)(StringWriterObject *obj, Py_ssize_t n)) LibRTStrings_API[13])
#define LibRTStrings_str_from_code_point_internal (*(PyObject* (*)(int32_t c)) LibRTStrings_API[14])


static inline bool CPyBytesWriter_Check(PyObject *obj) {
Expand All @@ -34,4 +35,35 @@ static inline bool CPyStringWriter_Check(PyObject *obj) {
return Py_TYPE(obj) == LibRTStrings_StringWriter_type_internal();
}

// Return a new reference to a single-character string for a code point, or NULL
// with an exception set. Strings for code points below
// LIBRT_STRINGS_CHAR_CACHE_END are cached.
static inline PyObject *LibRTStrings_StrFromCodePoint(Py_UCS4 c) {
if (c < 256) {
// CPython caches single-character Latin-1 strings
return PyUnicode_FromOrdinal((int)c);
}
return LibRTStrings_str_from_code_point_internal((int32_t)c);
}

static inline PyObject *LibRTStrings_CachedChr(int64_t c) {
if (unlikely(c < 0 || c > 0x10ffff)) {
PyErr_SetString(PyExc_ValueError, LIBRT_STRINGS_CACHED_CHR_ERRMSG);
return NULL;
}
return LibRTStrings_StrFromCodePoint((Py_UCS4)c);
}

static inline PyObject *LibRTStrings_CharAt(PyObject *s, int64_t i) {
Py_ssize_t len = PyUnicode_GET_LENGTH(s);
if (i < 0) {
i += len;
}
if (unlikely(i < 0 || i >= len)) {
PyErr_SetString(PyExc_IndexError, "string index out of range");
return NULL;
}
return LibRTStrings_StrFromCodePoint(PyUnicode_READ_CHAR(s, (Py_ssize_t)i));
}

#endif // LIBRT_STRINGS_API_H
20 changes: 20 additions & 0 deletions mypyc/primitives/librt_strings_ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -599,3 +599,23 @@
error_kind=ERR_NEVER,
dependencies=[LIBRT_STRINGS],
)

# Like chr() and s[i], but single-character strings for code points in
# U+0100..U+0FFF are cached (CPython only caches Latin-1 characters)
function_op(
name="librt.strings.cached_chr",
arg_types=[int64_rprimitive],
return_type=str_rprimitive,
c_function_name="LibRTStrings_CachedChr",
error_kind=ERR_MAGIC,
dependencies=[LIBRT_STRINGS],
)

function_op(
name="librt.strings.char_at",
arg_types=[str_rprimitive, int64_rprimitive],
return_type=str_rprimitive,
c_function_name="LibRTStrings_CharAt",
error_kind=ERR_MAGIC,
dependencies=[LIBRT_STRINGS],
)
29 changes: 29 additions & 0 deletions mypyc/test-data/irbuild-librt-strings.test
Original file line number Diff line number Diff line change
Expand Up @@ -518,3 +518,32 @@ def lo(c):
L0:
r0 = LibRTStrings_ToLower(c)
return r0

[case testLibrtStringsCachedChrIR]
from librt.strings import cached_chr
from mypy_extensions import i64

def f(c: i64) -> str:
return cached_chr(c)
[out]
def f(c):
c :: i64
r0 :: str
L0:
r0 = LibRTStrings_CachedChr(c)
return r0

[case testLibrtStringsCharAtIR]
from librt.strings import char_at
from mypy_extensions import i64

def f(s: str, i: i64) -> str:
return char_at(s, i)
[out]
def f(s, i):
s :: str
i :: i64
r0 :: str
L0:
r0 = LibRTStrings_CharAt(s, i)
return r0
90 changes: 90 additions & 0 deletions mypyc/test-data/run-librt-strings.test
Original file line number Diff line number Diff line change
Expand Up @@ -1813,3 +1813,93 @@ def test_codepoint_case_conversion_via_any() -> None:
f(1 << 40)
with assertRaises(OverflowError, "codepoint out of i32 range"):
f(-(1 << 40))

[case testLibrtStringsCachedChars_librt]
import sysconfig
from typing import Any
from librt.strings import cached_chr, char_at

from testutil import assertRaises

# The cache isn't used on free-threaded builds
CACHE_ENABLED = not sysconfig.get_config_var("Py_GIL_DISABLED")

# Latin-1, the range of cached characters (U+0100..U+0FFF) and some code points beyond it
CODE_POINTS = [*range(0x1100), 0xFFFF, 0x10000, 0x10FFFF]

def test_cached_chr() -> None:
# The second round uses cached strings
for _ in range(2):
for c in CODE_POINTS:
s = cached_chr(c)
assert len(s) == 1
assert s == chr(c)

def test_cached_chr_errors() -> None:
for c in (-1, 0x110000, -(1 << 62), 1 << 62):
with assertRaises(ValueError, "cached_chr() arg not in range(0x110000)"):
cached_chr(c)

def test_char_at() -> None:
s = "".join([chr(c) for c in CODE_POINTS])
n = len(s)
# The second round uses cached strings
for _ in range(2):
for i in range(n):
x = char_at(s, i)
assert len(x) == 1
assert x == s[i]
assert char_at(s, i - n) == s[i]

def test_char_at_errors() -> None:
for s in ("", "a", "ЀЁ", "\U00010000"):
for i in (len(s), -len(s) - 1, 1 << 62, -(1 << 62)):
with assertRaises(IndexError, "string index out of range"):
char_at(s, i)

def test_cache_is_used() -> None:
if CACHE_ENABLED:
assert cached_chr(0x400) is char_at("xЀ", 1)
assert cached_chr(0xFFF) is cached_chr(0xFFF)
assert char_at("Ā", 0) is char_at("Ā", -1)

def test_cached_string_not_modified() -> None:
c = cached_chr(0x400)
c += "x"
assert c == "Ѐx"
assert cached_chr(0x400) == "Ѐ"
d = char_at("࿿", 0)
d += "y"
assert d == "࿿y"
assert char_at("࿿", 0) == "࿿"

def test_via_any() -> None:
f: Any = cached_chr
g: Any = char_at
s = "".join([chr(c) for c in CODE_POINTS])
for _ in range(2):
for i, c in enumerate(CODE_POINTS):
assert f(c) == chr(c)
assert g(s, i) == s[i]
assert g(s, i - len(s)) == s[i]
if CACHE_ENABLED:
assert f(0x401) is cached_chr(0x401)
assert g("Ђ", 0) is cached_chr(0x402)
with assertRaises(ValueError, "cached_chr() arg not in range(0x110000)"):
f(-1)
with assertRaises(ValueError, "cached_chr() arg not in range(0x110000)"):
f(0x110000)
with assertRaises(ValueError, "int too large to convert to i64"):
f(1 << 70)
with assertRaises(TypeError):
f("a")
with assertRaises(IndexError, "string index out of range"):
g("ab", 2)
with assertRaises(IndexError, "string index out of range"):
g("ab", -3)
with assertRaises(ValueError, "int too large to convert to i64"):
g("ab", 1 << 70)
with assertRaises(TypeError, "char_at() argument 1 must be str"):
g(b"ab", 0)
with assertRaises(TypeError, "char_at() takes exactly 2 arguments (1 given)"):
g("ab")
Loading