refactor for pyo3 0.23.3; deny missing_docs; use output_type_func in …

…derive/tests/01

example/derive_expression/expression_lib/Cargo.toml

@@ -10,7 +10,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 polars = { workspace = true, features = ["fmt", "dtype-date", "timezones"], default-features = false }
-pyo3 = { version = "0.22", features = ["abi3-py38"] }
+pyo3 = { version = "0.23.3", features = ["abi3-py312"] }
 pyo3-polars = { version = "*", path = "../../../pyo3-polars", features = ["derive"] }
 rayon = "1.7.0"
 serde = { version = "1", features = ["derive"] }

example/derive_expression/expression_lib/pyproject.toml

@@ -3,8 +3,9 @@ requires = ["maturin>=1.0,<2.0"]
 build-backend = "maturin"
 
 [project]
+version = "0.0.1"
 name = "expression_lib"
-requires-python = ">=3.8"
+requires-python = ">=3.12"
 classifiers = [
   "Programming Language :: Rust",
   "Programming Language :: Python :: Implementation :: CPython",

example/extend_polars_python_dispatch/extend_polars/Cargo.toml

@@ -12,6 +12,6 @@ crate-type = ["cdylib"]
 polars = { workspace = true, features = ["fmt"] }
 polars-core = { workspace = true }
 polars-lazy = { workspace = true }
-pyo3 = { version = "0.22", features = ["extension-module"] }
+pyo3 = { version = "0.23", features = ["extension-module"] }
 pyo3-polars = { version = "*", path = "../../../pyo3-polars", features = ["lazy"] }
 rayon = "1.10"

example/extend_polars_python_dispatch/extend_polars/src/lib.rs

@@ -33,7 +33,7 @@ fn lazy_parallel_jaccard(pydf: PyLazyFrame, col_a: &str, col_b: &str) -> PyResul
 }
 
 /// A Python module implemented in Rust.
-#[pymodule]
+#[pymodule(name = "expression_lib")]
 fn extend_polars(_py: Python, m: &Bound<PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(parallel_jaccard, m)?)?;
     m.add_function(wrap_pyfunction!(lazy_parallel_jaccard, m)?)?;

example/io_plugin/io_plugin/Cargo.toml

@@ -10,6 +10,6 @@ crate-type = ["cdylib"]
 
 [dependencies]
 polars = { workspace = true, features = ["fmt", "dtype-date", "timezones", "lazy"], default-features = false }
-pyo3 = { version = "0.22", features = ["abi3-py38"] }
+pyo3 = { version = "0.23.3", features = ["abi3-py312"] }
 pyo3-polars = { version = "*", path = "../../../pyo3-polars", features = ["derive", "lazy"] }
 rand = { version = "0.8.5", features = [] }

pyo3-polars-derive/Cargo.toml

@@ -25,4 +25,5 @@ quote = "1.0"
 syn = { version = "2", features = ["full", "extra-traits"] }
 
 [dev-dependencies]
+pyo3-polars = { path = "../pyo3-polars" }
 trybuild = { version = "1", features = ["diff"] }

pyo3-polars-derive/tests/01.rs

@@ -1,17 +1,17 @@
 use polars_core::error::PolarsResult;
-use polars_core::prelude::{Field, Series};
+use polars_core::prelude::{CompatLevel, Field, Series};
 use polars_plan::dsl::FieldsMapper;
 use pyo3_polars_derive::polars_expr;
 
 fn horizontal_product_output(input_fields: &[Field]) -> PolarsResult<Field> {
     FieldsMapper::new(input_fields).map_to_supertype()
 }
 
-#[polars_expr(type_func=horizontal_product_output)]
-fn horizontal_product(series: &[Series], _kwargs: Option<&str>) -> PolarsResult<Series> {
+#[polars_expr(output_type_func=horizontal_product_output)]
+fn horizontal_product(series: &[Series]) -> PolarsResult<Series> {
     let mut acc = series[0].clone();
     for s in &series[1..] {
-        acc = &acc * s
+        acc = (&acc * s)?;
     }
     Ok(acc)
 }

pyo3-polars-derive/tests/02.rs

@@ -1,12 +1,12 @@
 use polars_core::error::PolarsResult;
-use polars_core::prelude::Series;
+use polars_core::prelude::{CompatLevel, Series};
 use pyo3_polars_derive::polars_expr;
 
 #[polars_expr(output_type=Int32)]
-fn horizontal_product(series: &[Series], _kwargs: Option<&str>) -> PolarsResult<Series> {
+fn horizontal_product(series: &[Series]) -> PolarsResult<Series> {
     let mut acc = series[0].clone();
     for s in &series[1..] {
-        acc = &acc * s
+        acc = (&acc * s)?;
     }
     Ok(acc)
 }

pyo3-polars/Cargo.toml

@@ -10,19 +10,18 @@ description = "Expression plugins and PyO3 types for polars"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ciborium = { version = "0.2.1", optional = true }
+ciborium = { version = "0.2", optional = true }
 libc = "0.2" # pyo3 depends on libc already, so this does not introduce an extra dependence.
-once_cell = "1"
 polars = { workspace = true, default-features = false }
 polars-core = { workspace = true, default-features = false }
 polars-ffi = { workspace = true, optional = true }
 polars-lazy = { workspace = true, optional = true }
 polars-plan = { workspace = true, optional = true }
-pyo3 = "0.22"
-pyo3-polars-derive = { version = "0.13.0", path = "../pyo3-polars-derive", optional = true }
+pyo3 = "0.23.3"
+pyo3-polars-derive = { version = "0.13", path = "../pyo3-polars-derive", optional = true }
 serde = { version = "1", optional = true }
 serde-pickle = { version = "1", optional = true }
-thiserror = "1"
+thiserror = "2"
 
 [features]
 lazy = ["polars/serde-lazy", "polars-plan", "polars-lazy/serde", "ciborium"]

pyo3-polars/src/alloc.rs

@@ -1,7 +1,8 @@
 use std::alloc::{GlobalAlloc, Layout, System};
 use std::ffi::c_char;
+use std::sync::OnceLock;
 
-use once_cell::race::OnceRef;
+// use once_cell::race::OnceRef;
 use pyo3::ffi::{PyCapsule_Import, Py_IsInitialized};
 use pyo3::Python;
 
@@ -60,7 +61,7 @@ static ALLOCATOR_CAPSULE_NAME: &[u8] = b"polars.polars._allocator\0";
 ///
 /// If the allocator capsule (`polars.polars._allocator`) is not available,
 /// this allocator fallbacks to [`std::alloc::System`].
-pub struct PolarsAllocator(OnceRef<'static, AllocatorCapsule>);
+pub struct PolarsAllocator(OnceLock<&'static AllocatorCapsule>);
 
 impl PolarsAllocator {
     fn get_allocator(&self) -> &'static AllocatorCapsule {
@@ -81,6 +82,8 @@ impl PolarsAllocator {
                 // Do not use eprintln; it may alloc.
                 let msg = b"failed to get allocator capsule\n";
                 // Message length type is platform-dependent.
+                // allow "useless" conversion to disable warning about platform-dependent try-into of msg_len
+                #[allow(clippy::useless_conversion)]
                 let msg_len = msg.len().try_into().unwrap();
                 unsafe { libc::write(2, msg.as_ptr() as *const libc::c_void, msg_len) };
             }
@@ -90,7 +93,7 @@ impl PolarsAllocator {
 
     /// Create a `PolarsAllocator`.
     pub const fn new() -> Self {
-        PolarsAllocator(OnceRef::new())
+        PolarsAllocator(OnceLock::new())
     }
 }
 

pyo3-polars/src/derive.rs

@@ -1,3 +1,10 @@
+//! Utilities to deriving and work with Polars expressions in a Python context.
+//!
+//! Includes functions to deserialize pickled kwargs, update error messages, and set panic messages.
+//!
+//! Provides FFI functions to get the last error message and the plugin version.
+//!
+//! Sets up a custom panic hook to only show output if `POLARS_VERBOSE` environment variable is "1".
 use polars::prelude::PolarsError;
 use polars_core::error::{to_compute_err, PolarsResult};
 pub use pyo3_polars_derive::polars_expr;
@@ -16,19 +23,22 @@ thread_local! {
     static LAST_ERROR: RefCell<CString> = RefCell::new(CString::default());
 }
 
+/// deserializes a pickled kwargs object
 pub fn _parse_kwargs<'a, T>(kwargs: &'a [u8]) -> PolarsResult<T>
 where
     T: Deserialize<'a>,
 {
     serde_pickle::from_slice(kwargs, Default::default()).map_err(to_compute_err)
 }
 
+/// sets the error message in the thread-local error object
 pub fn _update_last_error(err: PolarsError) {
-    let msg = format!("{}", err);
+    let msg = format!("{err}");
     let msg = CString::new(msg).unwrap();
     LAST_ERROR.with(|prev| *prev.borrow_mut() = msg)
 }
 
+/// sets a panic message in the thread-local error object
 pub fn _set_panic() {
     let msg = "PANIC";
     let msg = CString::new(msg).unwrap();
@@ -49,7 +59,7 @@ fn start_up_init() {
     std::panic::set_hook(Box::new(|info| {
         let show_message = std::env::var("POLARS_VERBOSE").as_deref().unwrap_or("") == "1";
         if show_message {
-            eprintln!("{}", info)
+            eprintln!("{info}")
         }
     }));
 }

pyo3-polars/src/error.rs

@@ -1,3 +1,5 @@
+#![allow(missing_docs)] // note - only for create_exception ... document the rest
+
 use std::fmt::{Debug, Formatter};
 
 use polars::prelude::PolarsError;
@@ -6,10 +8,13 @@ use pyo3::exceptions::{PyException, PyIOError, PyIndexError, PyRuntimeError, PyV
 use pyo3::prelude::*;
 use thiserror::Error;
 
+/// Error types for `pyo3-polars`
 #[derive(Error)]
 pub enum PyPolarsErr {
+    /// A wrapped error from the Polars Rust crate
     #[error(transparent)]
     Polars(#[from] PolarsError),
+    /// A non-polars error from `pyo3-polars`
     #[error("{0}")]
     Other(String),
 }

pyo3-polars/src/export.rs

@@ -1,3 +1,4 @@
+//! exports `polars_core`, `polars_ffi`, and `polars_plan`
 pub use polars_core;
 pub use polars_ffi;
 pub use polars_plan;

pyo3-polars/src/ffi/to_py.rs

@@ -6,9 +6,9 @@ use pyo3::prelude::*;
 /// Arrow array to Python.
 pub(crate) fn to_py_array(
     array: ArrayRef,
-    py: Python,
+    // py: Python<'py>, // unused ?
     pyarrow: Bound<'_, PyModule>,
-) -> PyResult<PyObject> {
+) -> PyResult<Bound<'_, PyAny>> {
     let schema = Box::new(ffi::export_field_to_c(&ArrowField::new(
         "".into(),
         array.dtype().clone(),
@@ -24,5 +24,5 @@ pub(crate) fn to_py_array(
         (array_ptr as Py_uintptr_t, schema_ptr as Py_uintptr_t),
     )?;
 
-    Ok(array.to_object(py))
+    Ok(array)
 }

pyo3-polars/src/ffi/to_rust.rs

@@ -4,6 +4,7 @@ use polars::prelude::*;
 use pyo3::ffi::Py_uintptr_t;
 use pyo3::prelude::*;
 
+/// attempt to convert a python arrow array ffi object to a rust arrow array reference
 pub fn array_to_rust(obj: &Bound<PyAny>) -> PyResult<ArrayRef> {
     // prepare a pointer to receive the Array struct
     let array = Box::new(ffi::ArrowArray::empty());

pyo3-polars/src/lib.rs

@@ -41,6 +41,7 @@
 //! })
 //! out_df = my_cool_function(df)
 //! ```
+#![deny(missing_docs)]
 mod alloc;
 #[cfg(feature = "derive")]
 pub mod derive;
@@ -50,14 +51,19 @@ pub mod export;
 mod ffi;
 mod types;
 
+use std::sync::LazyLock;
+
 pub use crate::alloc::PolarsAllocator;
-use once_cell::sync::Lazy;
+// use once_cell::sync::Lazy;
 use pyo3::prelude::*;
 pub use types::*;
 
-pub(crate) static POLARS: Lazy<PyObject> = Lazy::new(|| {
-    Python::with_gil(|py| PyModule::import_bound(py, "polars").unwrap().to_object(py))
+pub(crate) static POLARS: LazyLock<Py<PyModule>> = LazyLock::new(|| {
+    Python::with_gil(|py| {
+        let x = PyModule::import(py, "polars").unwrap().unbind();
+        x
+    })
 });
 
-pub(crate) static SERIES: Lazy<PyObject> =
-    Lazy::new(|| Python::with_gil(|py| POLARS.getattr(py, "Series").unwrap()));
+pub(crate) static SERIES: LazyLock<PyObject> =
+    LazyLock::new(|| Python::with_gil(|py| POLARS.getattr(py, "Series").unwrap()));