chore: Update python deps

- pip
- cargo

and fix few small issues.

python/Cargo.toml

@@ -39,7 +39,7 @@ datafusion-python = { version = "42" }
 
 pyo3 = { version = "0.22", features = ["extension-module", "abi3", "abi3-py38"] }
 pyo3-log = "0.11.0"
-tokio = { version = "1.35", features = ["macros", "rt", "rt-multi-thread", "sync"] }
+tokio = { version = "1.41", features = ["macros", "rt", "rt-multi-thread", "sync"] }
 
 [lib]
 crate-type = ["cdylib"]

python/examples/example.py

@@ -15,18 +15,18 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# %%
+
 from ballista import BallistaBuilder
 from datafusion.context import SessionContext
 
 # Ballista will initiate with an empty config
 # set config variables with `config`
 ctx: SessionContext = BallistaBuilder()\
     .config("ballista.job.name", "example ballista")\
-    .config("ballista.shuffle.partitions", "16")\
+    .config("datafusion.execution.target_partitions", "4")\
     .standalone()
-
-#ctx_remote: SessionContext = ballista.remote("remote_ip", 50050)
 
 # Select 1 to verify its working
 ctx.sql("SELECT 1").show()
-#ctx_remote.sql("SELECT 2").show()
+# %%

python/pyproject.toml

@@ -16,15 +16,15 @@
 # under the License.
 
 [build-system]
-requires = ["maturin>=0.15,<0.16"]
+requires = ["maturin>=1.5.1,<1.6.0"]
 build-backend = "maturin"
 
 [project]
 name = "ballista"
 description = "Python client for Apache Arrow Ballista Distributed SQL Query Engine"
 readme = "README.md"
 license = {file = "LICENSE.txt"}
-requires-python = ">=3.6"
+requires-python = ">=3.7"
 keywords = ["ballista", "sql", "rust", "distributed"]
 classifier = [
     "Development Status :: 2 - Pre-Alpha",

python/requirements.txt

@@ -1,3 +1,4 @@
-datafusion==35.0.0
+datafusion==42.0.0
+maturin==1.5.1
 pyarrow
 pytest

python/src/lib.rs

@@ -20,12 +20,9 @@ use datafusion::execution::SessionStateBuilder;
 use datafusion::prelude::*;
 use datafusion_python::context::PySessionContext;
 use datafusion_python::utils::wait_for_future;
-
-use std::collections::HashMap;
-
 use pyo3::prelude::*;
+use std::collections::HashMap;
 mod utils;
-use utils::to_pyerr;
 
 #[pymodule]
 fn ballista_internal(_py: Python, m: Bound<'_, PyModule>) -> PyResult<()> {
@@ -66,7 +63,11 @@ impl PyBallistaBuilder {
     /// Construct the standalone instance from the SessionContext
     pub fn standalone(&self, py: Python) -> PyResult<PySessionContext> {
         // Build the config
-        let config: SessionConfig = SessionConfig::from_string_hash_map(&self.conf)?;
+        let mut config: SessionConfig = SessionConfig::new_with_ballista();
+        for (key, value) in &self.conf {
+            let _ = config.options_mut().set(&key, value);
+        }
+
         // Build the state
         let state = SessionStateBuilder::new()
             .with_config(config)
@@ -85,7 +86,10 @@ impl PyBallistaBuilder {
     /// Construct the remote instance from the SessionContext
     pub fn remote(&self, url: &str, py: Python) -> PyResult<PySessionContext> {
         // Build the config
-        let config: SessionConfig = SessionConfig::from_string_hash_map(&self.conf)?;
+        let mut config: SessionConfig = SessionConfig::new_with_ballista();
+        for (key, value) in &self.conf {
+            let _ = config.options_mut().set(&key, value);
+        }
         // Build the state
         let state = SessionStateBuilder::new()
             .with_config(config)

python/src/utils.rs

@@ -19,6 +19,7 @@ use ballista_core::error::BallistaError;
 use pyo3::exceptions::PyException;
 use pyo3::PyErr;
 
+#[allow(dead_code)]
 pub(crate) fn to_pyerr(err: BallistaError) -> PyErr {
     PyException::new_err(err.to_string())
 }