diff --git a/python/Cargo.toml b/python/Cargo.toml index b03f1e997..f4966167a 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -39,7 +39,7 @@ datafusion-python = { version = "42" } pyo3 = { version = "0.22", features = ["extension-module", "abi3", "abi3-py38"] } pyo3-log = "0.11.0" -tokio = { version = "1.35", features = ["macros", "rt", "rt-multi-thread", "sync"] } +tokio = { version = "1.41", features = ["macros", "rt", "rt-multi-thread", "sync"] } [lib] crate-type = ["cdylib"] diff --git a/python/examples/example.py b/python/examples/example.py index 61a9abbd2..dec5d683c 100644 --- a/python/examples/example.py +++ b/python/examples/example.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +# %% + from ballista import BallistaBuilder from datafusion.context import SessionContext @@ -22,11 +24,9 @@ # set config variables with `config` ctx: SessionContext = BallistaBuilder()\ .config("ballista.job.name", "example ballista")\ - .config("ballista.shuffle.partitions", "16")\ + .config("datafusion.execution.target_partitions", "4")\ .standalone() - -#ctx_remote: SessionContext = ballista.remote("remote_ip", 50050) # Select 1 to verify its working ctx.sql("SELECT 1").show() -#ctx_remote.sql("SELECT 2").show() \ No newline at end of file +# %% diff --git a/python/pyproject.toml b/python/pyproject.toml index 2d06b225d..820d78872 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -16,7 +16,7 @@ # under the License. [build-system] -requires = ["maturin>=0.15,<0.16"] +requires = ["maturin>=1.5.1,<1.6.0"] build-backend = "maturin" [project] @@ -24,7 +24,7 @@ name = "ballista" description = "Python client for Apache Arrow Ballista Distributed SQL Query Engine" readme = "README.md" license = {file = "LICENSE.txt"} -requires-python = ">=3.6" +requires-python = ">=3.7" keywords = ["ballista", "sql", "rust", "distributed"] classifier = [ "Development Status :: 2 - Pre-Alpha", diff --git a/python/requirements.txt b/python/requirements.txt index a03a8f8d2..e9d03ffb1 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,3 +1,4 @@ -datafusion==35.0.0 +datafusion==42.0.0 +maturin==1.5.1 pyarrow pytest \ No newline at end of file diff --git a/python/src/lib.rs b/python/src/lib.rs index 41b4b6d31..11ec0bed2 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -20,12 +20,9 @@ use datafusion::execution::SessionStateBuilder; use datafusion::prelude::*; use datafusion_python::context::PySessionContext; use datafusion_python::utils::wait_for_future; - -use std::collections::HashMap; - use pyo3::prelude::*; +use std::collections::HashMap; mod utils; -use utils::to_pyerr; #[pymodule] fn ballista_internal(_py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { @@ -66,7 +63,11 @@ impl PyBallistaBuilder { /// Construct the standalone instance from the SessionContext pub fn standalone(&self, py: Python) -> PyResult { // Build the config - let config: SessionConfig = SessionConfig::from_string_hash_map(&self.conf)?; + let mut config: SessionConfig = SessionConfig::new_with_ballista(); + for (key, value) in &self.conf { + let _ = config.options_mut().set(&key, value); + } + // Build the state let state = SessionStateBuilder::new() .with_config(config) @@ -85,7 +86,10 @@ impl PyBallistaBuilder { /// Construct the remote instance from the SessionContext pub fn remote(&self, url: &str, py: Python) -> PyResult { // Build the config - let config: SessionConfig = SessionConfig::from_string_hash_map(&self.conf)?; + let mut config: SessionConfig = SessionConfig::new_with_ballista(); + for (key, value) in &self.conf { + let _ = config.options_mut().set(&key, value); + } // Build the state let state = SessionStateBuilder::new() .with_config(config) diff --git a/python/src/utils.rs b/python/src/utils.rs index 10278537e..176565311 100644 --- a/python/src/utils.rs +++ b/python/src/utils.rs @@ -19,6 +19,7 @@ use ballista_core::error::BallistaError; use pyo3::exceptions::PyException; use pyo3::PyErr; +#[allow(dead_code)] pub(crate) fn to_pyerr(err: BallistaError) -> PyErr { PyException::new_err(err.to_string()) }