pyo3/types/
string.rs

1#[cfg(not(Py_LIMITED_API))]
2use crate::exceptions::PyUnicodeDecodeError;
3use crate::ffi_ptr_ext::FfiPtrExt;
4use crate::instance::Borrowed;
5use crate::platform::prelude::*;
6use crate::py_result_ext::PyResultExt;
7use crate::types::bytes::PyBytesMethods;
8use crate::types::PyBytes;
9use crate::{ffi, Bound, Py, PyAny, PyResult, Python};
10#[cfg(RustPython)]
11use crate::{
12    sync::PyOnceLock,
13    types::{PyType, PyTypeMethods},
14};
15use alloc::borrow::Cow;
16use core::ffi::CStr;
17use core::{fmt, str};
18
19/// Represents raw data backing a Python `str`.
20///
21/// Python internally stores strings in various representations. This enumeration
22/// represents those variations.
23#[cfg(not(Py_LIMITED_API))]
24#[derive(Clone, Copy, Debug, PartialEq, Eq)]
25pub enum PyStringData<'a> {
26    /// UCS1 representation.
27    Ucs1(&'a [u8]),
28
29    /// UCS2 representation.
30    Ucs2(&'a [u16]),
31
32    /// UCS4 representation.
33    Ucs4(&'a [u32]),
34}
35
36#[cfg(not(Py_LIMITED_API))]
37impl<'a> PyStringData<'a> {
38    /// Obtain the raw bytes backing this instance as a [u8] slice.
39    pub fn as_bytes(&self) -> &[u8] {
40        match self {
41            Self::Ucs1(s) => s,
42            Self::Ucs2(s) => unsafe {
43                core::slice::from_raw_parts(s.as_ptr().cast(), s.len() * self.value_width_bytes())
44            },
45            Self::Ucs4(s) => unsafe {
46                core::slice::from_raw_parts(s.as_ptr().cast(), s.len() * self.value_width_bytes())
47            },
48        }
49    }
50
51    /// Size in bytes of each value/item in the underlying slice.
52    #[inline]
53    pub fn value_width_bytes(&self) -> usize {
54        match self {
55            Self::Ucs1(_) => 1,
56            Self::Ucs2(_) => 2,
57            Self::Ucs4(_) => 4,
58        }
59    }
60
61    /// Convert the raw data to a Rust string.
62    ///
63    /// For UCS-1 / UTF-8, returns a borrow into the original slice. For UCS-2 and UCS-4,
64    /// returns an owned string.
65    ///
66    /// Returns [PyUnicodeDecodeError] if the string data isn't valid in its purported
67    /// storage format. This should only occur for strings that were created via Python
68    /// C APIs that skip input validation (like `PyUnicode_FromKindAndData`) and should
69    /// never occur for strings that were created from Python code.
70    pub fn to_string(self, py: Python<'_>) -> PyResult<Cow<'a, str>> {
71        match self {
72            Self::Ucs1(data) => match str::from_utf8(data) {
73                Ok(s) => Ok(Cow::Borrowed(s)),
74                Err(e) => Err(PyUnicodeDecodeError::new_utf8(py, data, e)?.into()),
75            },
76            Self::Ucs2(data) => match String::from_utf16(data) {
77                Ok(s) => Ok(Cow::Owned(s)),
78                Err(e) => {
79                    let mut message = e.to_string().as_bytes().to_vec();
80                    message.push(0);
81
82                    Err(PyUnicodeDecodeError::new(
83                        py,
84                        c"utf-16",
85                        self.as_bytes(),
86                        0..self.as_bytes().len(),
87                        CStr::from_bytes_with_nul(&message).unwrap(),
88                    )?
89                    .into())
90                }
91            },
92            Self::Ucs4(data) => match data.iter().copied().map(char::from_u32).collect() {
93                Some(s) => Ok(Cow::Owned(s)),
94                None => Err(PyUnicodeDecodeError::new(
95                    py,
96                    c"utf-32",
97                    self.as_bytes(),
98                    0..self.as_bytes().len(),
99                    c"error converting utf-32",
100                )?
101                .into()),
102            },
103        }
104    }
105
106    /// Convert the raw data to a Rust string, possibly with data loss.
107    ///
108    /// Invalid code points will be replaced with `U+FFFD REPLACEMENT CHARACTER`.
109    ///
110    /// Returns a borrow into original data, when possible, or owned data otherwise.
111    ///
112    /// The return value of this function should only disagree with [Self::to_string]
113    /// when that method would error.
114    pub fn to_string_lossy(self) -> Cow<'a, str> {
115        match self {
116            Self::Ucs1(data) => String::from_utf8_lossy(data),
117            Self::Ucs2(data) => Cow::Owned(String::from_utf16_lossy(data)),
118            Self::Ucs4(data) => Cow::Owned(
119                data.iter()
120                    .map(|&c| char::from_u32(c).unwrap_or('\u{FFFD}'))
121                    .collect(),
122            ),
123        }
124    }
125}
126
127/// Represents a Python `string` (a Unicode string object).
128///
129/// Values of this type are accessed via PyO3's smart pointers, e.g. as
130/// [`Py<PyString>`][crate::Py] or [`Bound<'py, PyString>`][Bound].
131///
132/// For APIs available on `str` objects, see the [`PyStringMethods`] trait which is implemented for
133/// [`Bound<'py, PyString>`][Bound].
134///
135/// # Equality
136///
137/// For convenience, [`Bound<'py, PyString>`] implements [`PartialEq<str>`] to allow comparing the
138/// data in the Python string to a Rust UTF-8 string slice.
139///
140/// This is not always the most appropriate way to compare Python strings, as Python string
141/// subclasses may have different equality semantics. In situations where subclasses overriding
142/// equality might be relevant, use [`PyAnyMethods::eq`](crate::types::any::PyAnyMethods::eq), at
143/// cost of the additional overhead of a Python method call.
144///
145/// ```rust
146/// # use pyo3::prelude::*;
147/// use pyo3::types::PyString;
148///
149/// # Python::attach(|py| {
150/// let py_string = PyString::new(py, "foo");
151/// // via PartialEq<str>
152/// assert_eq!(py_string, "foo");
153///
154/// // via Python equality
155/// assert!(py_string.as_any().eq("foo").unwrap());
156/// # });
157/// ```
158#[repr(transparent)]
159pub struct PyString(PyAny);
160
161#[cfg(not(RustPython))]
162pyobject_native_type_core!(PyString, pyobject_native_static_type_object!(ffi::PyUnicode_Type), "builtins", "str", #checkfunction=ffi::PyUnicode_Check);
163
164#[cfg(RustPython)]
165pyobject_native_type_core!(
166    PyString,
167    |py| {
168        static TYPE: PyOnceLock<Py<PyType>> = PyOnceLock::new();
169        TYPE.import(py, "builtins", "str").unwrap().as_type_ptr()
170    },
171    "builtins",
172    "str",
173    #checkfunction=ffi::PyUnicode_Check
174);
175
176impl PyString {
177    /// Creates a new Python string object.
178    ///
179    /// Panics if out of memory.
180    pub fn new<'py>(py: Python<'py>, s: &str) -> Bound<'py, PyString> {
181        let ptr = s.as_ptr().cast();
182        let len = s.len() as ffi::Py_ssize_t;
183        unsafe {
184            ffi::PyUnicode_FromStringAndSize(ptr, len)
185                .assume_owned(py)
186                .cast_into_unchecked()
187        }
188    }
189
190    /// Creates a new Python string object from bytes.
191    ///
192    /// Returns PyMemoryError if out of memory.
193    /// Returns [PyUnicodeDecodeError] if the slice is not a valid UTF-8 string.
194    pub fn from_bytes<'py>(py: Python<'py>, s: &[u8]) -> PyResult<Bound<'py, PyString>> {
195        let ptr = s.as_ptr().cast();
196        let len = s.len() as ffi::Py_ssize_t;
197        unsafe {
198            ffi::PyUnicode_FromStringAndSize(ptr, len)
199                .assume_owned_or_err(py)
200                .cast_into_unchecked()
201        }
202    }
203
204    /// Intern the given string
205    ///
206    /// This will return a reference to the same Python string object if called repeatedly with the same string.
207    ///
208    /// Note that while this is more memory efficient than [`PyString::new`], it unconditionally allocates a
209    /// temporary Python string object and is thereby slower than [`PyString::new`].
210    ///
211    /// Panics if out of memory.
212    pub fn intern<'py>(py: Python<'py>, s: &str) -> Bound<'py, PyString> {
213        let ptr = s.as_ptr().cast();
214        let len = s.len() as ffi::Py_ssize_t;
215        unsafe {
216            let mut ob = ffi::PyUnicode_FromStringAndSize(ptr, len);
217            if !ob.is_null() {
218                ffi::PyUnicode_InternInPlace(&mut ob);
219            }
220            ob.assume_owned(py).cast_into_unchecked()
221        }
222    }
223
224    /// Attempts to create a Python string from a Python [bytes-like object].
225    ///
226    /// The `encoding` and `errors` parameters are optional:
227    /// - If `encoding` is `None`, the default encoding is used (UTF-8).
228    /// - If `errors` is `None`, the default error handling is used ("strict").
229    ///
230    /// See the [Python documentation on codecs] for more information.
231    ///
232    /// [bytes-like object]: (https://docs.python.org/3/glossary.html#term-bytes-like-object).
233    /// [Python documentation on codecs]: https://docs.python.org/3/library/codecs.html#standard-encodings
234    pub fn from_encoded_object<'py>(
235        src: &Bound<'py, PyAny>,
236        encoding: Option<&CStr>,
237        errors: Option<&CStr>,
238    ) -> PyResult<Bound<'py, PyString>> {
239        let encoding = encoding.map_or(core::ptr::null(), CStr::as_ptr);
240        let errors = errors.map_or(core::ptr::null(), CStr::as_ptr);
241        // Safety:
242        // - `src` is a valid Python object
243        // - `encoding` and `errors` are either null or valid C strings. `encoding` and `errors` are
244        //   documented as allowing null.
245        // - `ffi::PyUnicode_FromEncodedObject` returns a new `str` object, or sets an error.
246        unsafe {
247            ffi::PyUnicode_FromEncodedObject(src.as_ptr(), encoding, errors)
248                .assume_owned_or_err(src.py())
249                .cast_into_unchecked()
250        }
251    }
252
253    /// Creates a Python string using a format string.
254    ///
255    /// This function is similar to [`format!`], but it returns a Python string object instead of a Rust string.
256    #[inline]
257    pub fn from_fmt<'py>(
258        py: Python<'py>,
259        args: fmt::Arguments<'_>,
260    ) -> PyResult<Bound<'py, PyString>> {
261        if let Some(static_string) = args.as_str() {
262            return Ok(PyString::new(py, static_string));
263        };
264
265        #[cfg(all(Py_3_14, not(Py_LIMITED_API)))]
266        {
267            use crate::fmt::PyUnicodeWriter;
268            use core::fmt::Write as _;
269
270            let mut writer = PyUnicodeWriter::new(py)?;
271            writer
272                .write_fmt(args)
273                .map_err(|_| writer.take_error().expect("expected error"))?;
274            writer.into_py_string()
275        }
276
277        #[cfg(any(not(Py_3_14), Py_LIMITED_API))]
278        {
279            Ok(PyString::new(py, &format!("{args}")))
280        }
281    }
282}
283
284/// Implementation of functionality for [`PyString`].
285///
286/// These methods are defined for the `Bound<'py, PyString>` smart pointer, so to use method call
287/// syntax these methods are separated into a trait, because stable Rust does not yet support
288/// `arbitrary_self_types`.
289#[doc(alias = "PyString")]
290pub trait PyStringMethods<'py>: crate::sealed::Sealed {
291    /// Gets the Python string as a Rust UTF-8 string slice.
292    ///
293    /// Returns a `UnicodeEncodeError` if the input is not valid unicode
294    /// (containing unpaired surrogates).
295    #[cfg(any(Py_3_10, not(Py_LIMITED_API)))]
296    fn to_str(&self) -> PyResult<&str>;
297
298    /// Converts the `PyString` into a Rust string, avoiding copying when possible.
299    ///
300    /// Returns a `UnicodeEncodeError` if the input is not valid unicode
301    /// (containing unpaired surrogates).
302    fn to_cow(&self) -> PyResult<Cow<'_, str>>;
303
304    /// Converts the `PyString` into a Rust string.
305    ///
306    /// Unpaired surrogates invalid UTF-8 sequences are
307    /// replaced with `U+FFFD REPLACEMENT CHARACTER`.
308    fn to_string_lossy(&self) -> Cow<'_, str>;
309
310    /// Encodes this string as a Python `bytes` object, using UTF-8 encoding.
311    fn encode_utf8(&self) -> PyResult<Bound<'py, PyBytes>>;
312
313    /// Obtains the raw data backing the Python string.
314    ///
315    /// If the Python string object was created through legacy APIs, its internal storage format
316    /// will be canonicalized before data is returned.
317    ///
318    /// # Safety
319    ///
320    /// This function implementation relies on manually decoding a C bitfield. In practice, this
321    /// works well on common little-endian architectures such as x86_64, where the bitfield has a
322    /// common representation (even if it is not part of the C spec). The PyO3 CI tests this API on
323    /// x86_64 platforms.
324    ///
325    /// By using this API, you accept responsibility for testing that PyStringData behaves as
326    /// expected on the targets where you plan to distribute your software.
327    #[cfg(not(any(Py_LIMITED_API, GraalPy, PyPy)))]
328    unsafe fn data(&self) -> PyResult<PyStringData<'_>>;
329}
330
331impl<'py> PyStringMethods<'py> for Bound<'py, PyString> {
332    #[cfg(any(Py_3_10, not(Py_LIMITED_API)))]
333    fn to_str(&self) -> PyResult<&str> {
334        self.as_borrowed().to_str()
335    }
336
337    fn to_cow(&self) -> PyResult<Cow<'_, str>> {
338        self.as_borrowed().to_cow()
339    }
340
341    fn to_string_lossy(&self) -> Cow<'_, str> {
342        self.as_borrowed().to_string_lossy()
343    }
344
345    fn encode_utf8(&self) -> PyResult<Bound<'py, PyBytes>> {
346        unsafe {
347            ffi::PyUnicode_AsUTF8String(self.as_ptr())
348                .assume_owned_or_err(self.py())
349                .cast_into_unchecked::<PyBytes>()
350        }
351    }
352
353    #[cfg(not(any(Py_LIMITED_API, GraalPy, PyPy)))]
354    unsafe fn data(&self) -> PyResult<PyStringData<'_>> {
355        unsafe { self.as_borrowed().data() }
356    }
357}
358
359impl<'a> Borrowed<'a, '_, PyString> {
360    #[cfg(any(Py_3_10, not(Py_LIMITED_API)))]
361    pub(crate) fn to_str(self) -> PyResult<&'a str> {
362        // PyUnicode_AsUTF8AndSize only available on limited API starting with 3.10.
363        let mut size: ffi::Py_ssize_t = 0;
364        let data: *const u8 =
365            unsafe { ffi::PyUnicode_AsUTF8AndSize(self.as_ptr(), &mut size).cast() };
366        if data.is_null() {
367            Err(crate::PyErr::fetch(self.py()))
368        } else {
369            Ok(unsafe {
370                core::str::from_utf8_unchecked(core::slice::from_raw_parts(data, size as usize))
371            })
372        }
373    }
374
375    pub(crate) fn to_cow(self) -> PyResult<Cow<'a, str>> {
376        // TODO: this method can probably be deprecated once Python 3.9 support is dropped,
377        // because all versions then support the more efficient `to_str`.
378        #[cfg(any(Py_3_10, not(Py_LIMITED_API)))]
379        {
380            self.to_str().map(Cow::Borrowed)
381        }
382
383        #[cfg(not(any(Py_3_10, not(Py_LIMITED_API))))]
384        {
385            let bytes = self.encode_utf8()?;
386            Ok(Cow::Owned(
387                unsafe { str::from_utf8_unchecked(bytes.as_bytes()) }.to_owned(),
388            ))
389        }
390    }
391
392    fn to_string_lossy(self) -> Cow<'a, str> {
393        let ptr = self.as_ptr();
394        let py = self.py();
395
396        #[cfg(any(Py_3_10, not(Py_LIMITED_API)))]
397        if let Ok(s) = self.to_str() {
398            return Cow::Borrowed(s);
399        }
400
401        let bytes = unsafe {
402            ffi::PyUnicode_AsEncodedString(ptr, c"utf-8".as_ptr(), c"surrogatepass".as_ptr())
403                .assume_owned(py)
404                .cast_into_unchecked::<PyBytes>()
405        };
406        Cow::Owned(String::from_utf8_lossy(bytes.as_bytes()).into_owned())
407    }
408
409    #[cfg(not(any(Py_LIMITED_API, GraalPy, PyPy)))]
410    unsafe fn data(self) -> PyResult<PyStringData<'a>> {
411        unsafe {
412            let ptr = self.as_ptr();
413
414            #[cfg(not(Py_3_12))]
415            #[allow(deprecated)]
416            {
417                let ready = ffi::PyUnicode_READY(ptr);
418                if ready != 0 {
419                    // Exception was created on failure.
420                    return Err(crate::PyErr::fetch(self.py()));
421                }
422            }
423
424            // The string should be in its canonical form after calling `PyUnicode_READY()`.
425            // And non-canonical form not possible after Python 3.12. So it should be safe
426            // to call these APIs.
427            let length = ffi::PyUnicode_GET_LENGTH(ptr) as usize;
428            let raw_data = ffi::PyUnicode_DATA(ptr);
429            let kind = ffi::PyUnicode_KIND(ptr);
430
431            match kind {
432                ffi::PyUnicode_1BYTE_KIND => Ok(PyStringData::Ucs1(core::slice::from_raw_parts(
433                    raw_data as *const u8,
434                    length,
435                ))),
436                ffi::PyUnicode_2BYTE_KIND => Ok(PyStringData::Ucs2(core::slice::from_raw_parts(
437                    raw_data as *const u16,
438                    length,
439                ))),
440                ffi::PyUnicode_4BYTE_KIND => Ok(PyStringData::Ucs4(core::slice::from_raw_parts(
441                    raw_data as *const u32,
442                    length,
443                ))),
444                _ => unreachable!(),
445            }
446        }
447    }
448}
449
450impl Py<PyString> {
451    /// Gets the Python string as a Rust UTF-8 string slice.
452    ///
453    /// Returns a `UnicodeEncodeError` if the input is not valid unicode
454    /// (containing unpaired surrogates).
455    ///
456    /// Because `str` objects are immutable, the returned slice is independent of
457    /// the GIL lifetime.
458    #[cfg(any(Py_3_10, not(Py_LIMITED_API)))]
459    pub fn to_str<'a>(&'a self, py: Python<'_>) -> PyResult<&'a str> {
460        self.bind_borrowed(py).to_str()
461    }
462
463    /// Converts the `PyString` into a Rust string, avoiding copying when possible.
464    ///
465    /// Returns a `UnicodeEncodeError` if the input is not valid unicode
466    /// (containing unpaired surrogates).
467    ///
468    /// Because `str` objects are immutable, the returned slice is independent of
469    /// the GIL lifetime.
470    pub fn to_cow<'a>(&'a self, py: Python<'_>) -> PyResult<Cow<'a, str>> {
471        self.bind_borrowed(py).to_cow()
472    }
473
474    /// Converts the `PyString` into a Rust string.
475    ///
476    /// Unpaired surrogates invalid UTF-8 sequences are
477    /// replaced with `U+FFFD REPLACEMENT CHARACTER`.
478    ///
479    /// Because `str` objects are immutable, the returned slice is independent of
480    /// the GIL lifetime.
481    pub fn to_string_lossy<'a>(&'a self, py: Python<'_>) -> Cow<'a, str> {
482        self.bind_borrowed(py).to_string_lossy()
483    }
484}
485
486/// Compares whether the data in the Python string is equal to the given UTF8.
487///
488/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
489impl PartialEq<str> for Bound<'_, PyString> {
490    #[inline]
491    fn eq(&self, other: &str) -> bool {
492        self.as_borrowed() == *other
493    }
494}
495
496/// Compares whether the data in the Python string is equal to the given UTF8.
497///
498/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
499impl PartialEq<&'_ str> for Bound<'_, PyString> {
500    #[inline]
501    fn eq(&self, other: &&str) -> bool {
502        self.as_borrowed() == **other
503    }
504}
505
506/// Compares whether the data in the Python string is equal to the given UTF8.
507///
508/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
509impl PartialEq<Bound<'_, PyString>> for str {
510    #[inline]
511    fn eq(&self, other: &Bound<'_, PyString>) -> bool {
512        *self == other.as_borrowed()
513    }
514}
515
516/// Compares whether the data in the Python string is equal to the given UTF8.
517///
518/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
519impl PartialEq<&'_ Bound<'_, PyString>> for str {
520    #[inline]
521    fn eq(&self, other: &&Bound<'_, PyString>) -> bool {
522        *self == other.as_borrowed()
523    }
524}
525
526/// Compares whether the data in the Python string is equal to the given UTF8.
527///
528/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
529impl PartialEq<Bound<'_, PyString>> for &'_ str {
530    #[inline]
531    fn eq(&self, other: &Bound<'_, PyString>) -> bool {
532        **self == other.as_borrowed()
533    }
534}
535
536/// Compares whether the data in the Python string is equal to the given UTF8.
537///
538/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
539impl PartialEq<str> for &'_ Bound<'_, PyString> {
540    #[inline]
541    fn eq(&self, other: &str) -> bool {
542        self.as_borrowed() == other
543    }
544}
545
546/// Compares whether the data in the Python string is equal to the given UTF8.
547///
548/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
549impl PartialEq<str> for Borrowed<'_, '_, PyString> {
550    #[inline]
551    fn eq(&self, other: &str) -> bool {
552        #[cfg(not(Py_3_13))]
553        {
554            self.to_cow().is_ok_and(|s| s == other)
555        }
556
557        #[cfg(Py_3_13)]
558        unsafe {
559            ffi::PyUnicode_EqualToUTF8AndSize(
560                self.as_ptr(),
561                other.as_ptr().cast(),
562                other.len() as _,
563            ) == 1
564        }
565    }
566}
567
568/// Compares whether the data in the Python string is equal to the given UTF8.
569///
570/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
571impl PartialEq<&str> for Borrowed<'_, '_, PyString> {
572    #[inline]
573    fn eq(&self, other: &&str) -> bool {
574        *self == **other
575    }
576}
577
578/// Compares whether the data in the Python string is equal to the given UTF8.
579///
580/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
581impl PartialEq<Borrowed<'_, '_, PyString>> for str {
582    #[inline]
583    fn eq(&self, other: &Borrowed<'_, '_, PyString>) -> bool {
584        other == self
585    }
586}
587
588/// Compares whether the data in the Python string is equal to the given UTF8.
589///
590/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
591impl PartialEq<Borrowed<'_, '_, PyString>> for &'_ str {
592    #[inline]
593    fn eq(&self, other: &Borrowed<'_, '_, PyString>) -> bool {
594        other == self
595    }
596}
597
598#[cfg(test)]
599mod tests {
600    use super::*;
601    use crate::{exceptions::PyLookupError, types::PyAnyMethods as _, IntoPyObject};
602
603    #[test]
604    fn test_to_cow_utf8() {
605        Python::attach(|py| {
606            let s = "ascii 🐈";
607            let py_string = PyString::new(py, s);
608            assert_eq!(s, py_string.to_cow().unwrap());
609        })
610    }
611
612    #[test]
613    fn test_to_cow_surrogate() {
614        Python::attach(|py| {
615            let py_string = py
616                .eval(cr"'\ud800'", None, None)
617                .unwrap()
618                .cast_into::<PyString>()
619                .unwrap();
620            assert!(py_string.to_cow().is_err());
621        })
622    }
623
624    #[test]
625    fn test_to_cow_unicode() {
626        Python::attach(|py| {
627            let s = "哈哈🐈";
628            let py_string = PyString::new(py, s);
629            assert_eq!(s, py_string.to_cow().unwrap());
630        })
631    }
632
633    #[test]
634    fn test_encode_utf8_unicode() {
635        Python::attach(|py| {
636            let s = "哈哈🐈";
637            let obj = PyString::new(py, s);
638            assert_eq!(s.as_bytes(), obj.encode_utf8().unwrap().as_bytes());
639        })
640    }
641
642    #[test]
643    fn test_encode_utf8_surrogate() {
644        Python::attach(|py| {
645            let obj: Py<PyAny> = py.eval(cr"'\ud800'", None, None).unwrap().into();
646            assert!(obj
647                .bind(py)
648                .cast::<PyString>()
649                .unwrap()
650                .encode_utf8()
651                .is_err());
652        })
653    }
654
655    #[test]
656    fn test_to_string_lossy() {
657        Python::attach(|py| {
658            let py_string = py
659                .eval(cr"'🐈 Hello \ud800World'", None, None)
660                .unwrap()
661                .cast_into::<PyString>()
662                .unwrap();
663
664            assert_eq!(py_string.to_string_lossy(), "🐈 Hello ���World");
665        })
666    }
667
668    #[test]
669    fn test_debug_string() {
670        Python::attach(|py| {
671            let s = "Hello\n".into_pyobject(py).unwrap();
672            assert_eq!(format!("{s:?}"), "'Hello\\n'");
673        })
674    }
675
676    #[test]
677    fn test_display_string() {
678        Python::attach(|py| {
679            let s = "Hello\n".into_pyobject(py).unwrap();
680            assert_eq!(format!("{s}"), "Hello\n");
681        })
682    }
683
684    #[test]
685    fn test_string_from_encoded_object() {
686        Python::attach(|py| {
687            let py_bytes = PyBytes::new(py, b"ab\xFFcd");
688
689            // default encoding is utf-8, default error handler is strict
690            let py_string = PyString::from_encoded_object(&py_bytes, None, None).unwrap_err();
691            assert!(py_string
692                .get_type(py)
693                .is(py.get_type::<crate::exceptions::PyUnicodeDecodeError>()));
694
695            // with `ignore` error handler, the invalid byte is dropped
696            let py_string =
697                PyString::from_encoded_object(&py_bytes, None, Some(c"ignore")).unwrap();
698
699            let result = py_string.to_cow().unwrap();
700            assert_eq!(result, "abcd");
701        });
702    }
703
704    #[test]
705    fn test_string_from_encoded_object_with_invalid_encoding_errors() {
706        Python::attach(|py| {
707            let py_bytes = PyBytes::new(py, b"abcd");
708
709            // invalid encoding
710            let err = PyString::from_encoded_object(&py_bytes, Some(c"wat"), None).unwrap_err();
711            assert!(err.is_instance(py, &py.get_type::<PyLookupError>()));
712            assert_eq!(err.to_string(), "LookupError: unknown encoding: wat");
713
714            // invalid error handler
715            let err =
716                PyString::from_encoded_object(&PyBytes::new(py, b"ab\xFFcd"), None, Some(c"wat"))
717                    .unwrap_err();
718            assert!(err.is_instance(py, &py.get_type::<PyLookupError>()));
719            assert_eq!(
720                err.to_string(),
721                "LookupError: unknown error handler name 'wat'"
722            );
723        });
724    }
725
726    #[test]
727    #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))]
728    fn test_string_data_ucs1() {
729        Python::attach(|py| {
730            let s = PyString::new(py, "hello, world");
731            let data = unsafe { s.data().unwrap() };
732
733            assert_eq!(data, PyStringData::Ucs1(b"hello, world"));
734            assert_eq!(data.to_string(py).unwrap(), Cow::Borrowed("hello, world"));
735            assert_eq!(data.to_string_lossy(), Cow::Borrowed("hello, world"));
736        })
737    }
738
739    #[test]
740    #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))]
741    fn test_string_data_ucs1_invalid() {
742        Python::attach(|py| {
743            // 0xfe is not allowed in UTF-8.
744            let buffer = b"f\xfe\0";
745            let ptr = unsafe {
746                crate::ffi::PyUnicode_FromKindAndData(
747                    crate::ffi::PyUnicode_1BYTE_KIND as _,
748                    buffer.as_ptr().cast(),
749                    2,
750                )
751            };
752            assert!(!ptr.is_null());
753            let s = unsafe { ptr.assume_owned(py).cast_into_unchecked::<PyString>() };
754            let data = unsafe { s.data().unwrap() };
755            assert_eq!(data, PyStringData::Ucs1(b"f\xfe"));
756            let err = data.to_string(py).unwrap_err();
757            assert!(err.get_type(py).is(py.get_type::<PyUnicodeDecodeError>()));
758            assert!(err
759                .to_string()
760                .contains("'utf-8' codec can't decode byte 0xfe in position 1"));
761            assert_eq!(data.to_string_lossy(), Cow::Borrowed("f�"));
762        });
763    }
764
765    #[test]
766    #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))]
767    fn test_string_data_ucs2() {
768        Python::attach(|py| {
769            let s = py.eval(c"'foo\\ud800'", None, None).unwrap();
770            let py_string = s.cast::<PyString>().unwrap();
771            let data = unsafe { py_string.data().unwrap() };
772
773            assert_eq!(data, PyStringData::Ucs2(&[102, 111, 111, 0xd800]));
774            assert_eq!(
775                data.to_string_lossy(),
776                Cow::Owned::<str>("foo�".to_string())
777            );
778        })
779    }
780
781    #[test]
782    #[cfg(all(not(any(Py_LIMITED_API, PyPy, GraalPy)), target_endian = "little"))]
783    fn test_string_data_ucs2_invalid() {
784        Python::attach(|py| {
785            // U+FF22 (valid) & U+d800 (never valid)
786            let buffer = b"\x22\xff\x00\xd8\x00\x00";
787            let ptr = unsafe {
788                crate::ffi::PyUnicode_FromKindAndData(
789                    crate::ffi::PyUnicode_2BYTE_KIND as _,
790                    buffer.as_ptr().cast(),
791                    2,
792                )
793            };
794            assert!(!ptr.is_null());
795            let s = unsafe { ptr.assume_owned(py).cast_into_unchecked::<PyString>() };
796            let data = unsafe { s.data().unwrap() };
797            assert_eq!(data, PyStringData::Ucs2(&[0xff22, 0xd800]));
798            let err = data.to_string(py).unwrap_err();
799            assert!(err.get_type(py).is(py.get_type::<PyUnicodeDecodeError>()));
800            assert!(err
801                .to_string()
802                .contains("'utf-16' codec can't decode bytes in position 0-3"));
803            assert_eq!(data.to_string_lossy(), Cow::Owned::<str>("Ｂ�".into()));
804        });
805    }
806
807    #[test]
808    #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))]
809    fn test_string_data_ucs4() {
810        Python::attach(|py| {
811            let s = "哈哈🐈";
812            let py_string = PyString::new(py, s);
813            let data = unsafe { py_string.data().unwrap() };
814
815            assert_eq!(data, PyStringData::Ucs4(&[21704, 21704, 128008]));
816            assert_eq!(data.to_string_lossy(), Cow::Owned::<str>(s.to_string()));
817        })
818    }
819
820    #[test]
821    #[cfg(all(not(any(Py_LIMITED_API, PyPy, GraalPy)), target_endian = "little"))]
822    fn test_string_data_ucs4_invalid() {
823        Python::attach(|py| {
824            // U+20000 (valid) & U+d800 (never valid)
825            let buffer = b"\x00\x00\x02\x00\x00\xd8\x00\x00\x00\x00\x00\x00";
826            let ptr = unsafe {
827                crate::ffi::PyUnicode_FromKindAndData(
828                    crate::ffi::PyUnicode_4BYTE_KIND as _,
829                    buffer.as_ptr().cast(),
830                    2,
831                )
832            };
833            assert!(!ptr.is_null());
834            let s = unsafe { ptr.assume_owned(py).cast_into_unchecked::<PyString>() };
835            let data = unsafe { s.data().unwrap() };
836            assert_eq!(data, PyStringData::Ucs4(&[0x20000, 0xd800]));
837            let err = data.to_string(py).unwrap_err();
838            assert!(err.get_type(py).is(py.get_type::<PyUnicodeDecodeError>()));
839            assert!(err
840                .to_string()
841                .contains("'utf-32' codec can't decode bytes in position 0-7"));
842            assert_eq!(data.to_string_lossy(), Cow::Owned::<str>("𠀀�".into()));
843        });
844    }
845
846    #[test]
847    #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))]
848    fn test_pystring_from_bytes() {
849        Python::attach(|py| {
850            let result = PyString::from_bytes(py, "\u{2122}".as_bytes());
851            assert!(result.is_ok());
852            let result = PyString::from_bytes(py, b"\x80");
853            assert!(result
854                .unwrap_err()
855                .get_type(py)
856                .is(py.get_type::<PyUnicodeDecodeError>()));
857        });
858    }
859
860    #[test]
861    fn test_intern_string() {
862        Python::attach(|py| {
863            let py_string1 = PyString::intern(py, "foo");
864            assert_eq!(py_string1, "foo");
865
866            let py_string2 = PyString::intern(py, "foo");
867            assert_eq!(py_string2, "foo");
868
869            assert_eq!(py_string1.as_ptr(), py_string2.as_ptr());
870
871            let py_string3 = PyString::intern(py, "bar");
872            assert_eq!(py_string3, "bar");
873
874            assert_ne!(py_string1.as_ptr(), py_string3.as_ptr());
875        });
876    }
877
878    #[test]
879    fn test_py_to_str_utf8() {
880        Python::attach(|py| {
881            let s = "ascii 🐈";
882            let py_string = PyString::new(py, s).unbind();
883
884            #[cfg(any(Py_3_10, not(Py_LIMITED_API)))]
885            assert_eq!(s, py_string.to_str(py).unwrap());
886
887            assert_eq!(s, py_string.to_cow(py).unwrap());
888        })
889    }
890
891    #[test]
892    fn test_py_to_str_surrogate() {
893        Python::attach(|py| {
894            let py_string: Py<PyString> = py
895                .eval(cr"'\ud800'", None, None)
896                .unwrap()
897                .extract()
898                .unwrap();
899
900            #[cfg(any(Py_3_10, not(Py_LIMITED_API)))]
901            assert!(py_string.to_str(py).is_err());
902
903            assert!(py_string.to_cow(py).is_err());
904        })
905    }
906
907    #[test]
908    fn test_py_to_string_lossy() {
909        Python::attach(|py| {
910            let py_string: Py<PyString> = py
911                .eval(cr"'🐈 Hello \ud800World'", None, None)
912                .unwrap()
913                .extract()
914                .unwrap();
915            assert_eq!(py_string.to_string_lossy(py), "🐈 Hello ���World");
916        })
917    }
918
919    #[test]
920    fn test_comparisons() {
921        Python::attach(|py| {
922            let s = "hello, world";
923            let py_string = PyString::new(py, s);
924
925            assert_eq!(py_string, "hello, world");
926
927            assert_eq!(py_string, s);
928            assert_eq!(&py_string, s);
929            assert_eq!(s, py_string);
930            assert_eq!(s, &py_string);
931
932            assert_eq!(py_string, *s);
933            assert_eq!(&py_string, *s);
934            assert_eq!(*s, py_string);
935            assert_eq!(*s, &py_string);
936
937            let py_string = py_string.as_borrowed();
938
939            assert_eq!(py_string, s);
940            assert_eq!(&py_string, s);
941            assert_eq!(s, py_string);
942            assert_eq!(s, &py_string);
943
944            assert_eq!(py_string, *s);
945            assert_eq!(*s, py_string);
946        })
947    }
948}
pyo3/types/string.rs

pyo3/types/
string.rs
