From ee35695dd47f033afc4bdd2be6e8c7a45fb435d1 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 5 Dec 2024 16:58:54 -0700 Subject: [PATCH 1/6] replace pub with pub(crate) --- native/core/README.md | 26 ++++ native/core/benches/common.rs | 8 +- native/core/benches/parquet_read.rs | 2 +- native/core/benches/perf.rs | 4 +- native/core/benches/row_columnar.rs | 4 +- native/core/src/common/bit.rs | 115 +++++++++--------- native/core/src/common/buffer.rs | 28 ++--- native/core/src/common/mod.rs | 4 +- native/core/src/data_type.rs | 32 ++--- native/core/src/errors.rs | 44 +++---- .../execution/datafusion/expressions/avg.rs | 12 +- .../datafusion/expressions/avg_decimal.rs | 17 ++- .../datafusion/expressions/bitwise_not.rs | 8 +- .../expressions/bloom_filter_agg.rs | 4 +- .../expressions/bloom_filter_might_contain.rs | 8 +- .../datafusion/expressions/checkoverflow.rs | 16 ++- .../expressions/comet_scalar_funcs.rs | 2 +- .../datafusion/expressions/correlation.rs | 8 +- .../datafusion/expressions/covariance.rs | 16 +-- .../execution/datafusion/expressions/mod.rs | 36 +++--- .../datafusion/expressions/negative.rs | 8 +- .../datafusion/expressions/normalize_nan.rs | 8 +- .../datafusion/expressions/stddev.rs | 10 +- .../datafusion/expressions/strings.rs | 20 +-- .../datafusion/expressions/subquery.rs | 8 +- .../datafusion/expressions/sum_decimal.rs | 4 +- .../datafusion/expressions/unbound.rs | 6 +- .../datafusion/expressions/variance.rs | 14 +-- native/core/src/execution/datafusion/mod.rs | 6 +- .../execution/datafusion/operators/expand.rs | 8 +- .../src/execution/datafusion/operators/mod.rs | 2 +- .../core/src/execution/datafusion/planner.rs | 18 +-- .../execution/datafusion/shuffle_writer.rs | 10 +- .../core/src/execution/datafusion/util/mod.rs | 4 +- .../datafusion/util/spark_bit_array.rs | 24 ++-- .../datafusion/util/spark_bloom_filter.rs | 18 +-- native/core/src/execution/jni_api.rs | 36 +++--- native/core/src/execution/kernels/hash.rs | 2 +- native/core/src/execution/kernels/mod.rs | 1 - native/core/src/execution/kernels/strings.rs | 10 +- native/core/src/execution/memory_pool.rs | 4 +- native/core/src/execution/metrics/mod.rs | 2 +- native/core/src/execution/metrics/utils.rs | 2 +- native/core/src/execution/mod.rs | 15 ++- native/core/src/execution/operators/copy.rs | 6 +- native/core/src/execution/operators/filter.rs | 18 +-- native/core/src/execution/operators/mod.rs | 9 +- native/core/src/execution/operators/scan.rs | 26 ++-- native/core/src/execution/serde.rs | 10 +- native/core/src/execution/shuffle/list.rs | 6 +- native/core/src/execution/shuffle/map.rs | 6 +- native/core/src/execution/shuffle/mod.rs | 2 +- native/core/src/execution/shuffle/row.rs | 18 +-- native/core/src/execution/sort.rs | 4 +- native/core/src/execution/utils.rs | 4 +- native/core/src/jvm_bridge/batch_iterator.rs | 12 +- native/core/src/jvm_bridge/comet_exec.rs | 52 ++++---- .../core/src/jvm_bridge/comet_metric_node.rs | 16 +-- .../jvm_bridge/comet_task_memory_manager.rs | 16 +-- native/core/src/jvm_bridge/mod.rs | 42 +++---- native/core/src/lib.rs | 10 +- native/core/src/parquet/compression.rs | 14 +-- native/core/src/parquet/data_type.rs | 6 +- native/core/src/parquet/mod.rs | 58 ++++----- native/core/src/parquet/mutable_vector.rs | 24 ++-- native/core/src/parquet/read/column.rs | 73 +++++------ native/core/src/parquet/read/levels.rs | 10 +- native/core/src/parquet/read/mod.rs | 16 +-- native/core/src/parquet/read/values.rs | 16 +-- native/core/src/parquet/util/bit_packing.rs | 6 +- native/core/src/parquet/util/buffer.rs | 16 +-- native/core/src/parquet/util/jni.rs | 15 ++- native/core/src/parquet/util/jni_buffer.rs | 6 +- native/core/src/parquet/util/memory.rs | 72 +++++------ native/core/src/parquet/util/mod.rs | 7 +- .../src/parquet/util/test_common/file_util.rs | 4 +- .../core/src/parquet/util/test_common/mod.rs | 10 +- .../src/parquet/util/test_common/page_util.rs | 16 +-- .../src/parquet/util/test_common/rand_gen.rs | 8 +- 79 files changed, 665 insertions(+), 603 deletions(-) create mode 100644 native/core/README.md diff --git a/native/core/README.md b/native/core/README.md new file mode 100644 index 000000000..c855216f4 --- /dev/null +++ b/native/core/README.md @@ -0,0 +1,26 @@ + + +# Apache DataFusion Comet: Plugin Native Code + +This crate contains the native code that is invoked from the Comet's Spark plugin and is not intended to be +used as a dependency from other projects. Therefore, it does not provide a public API and is not published to crates.io. + +However, the `datafusion-comet-proto` and `datafusion-comet-spark-expr` crates do provide public APIs and are +published to crates.io. \ No newline at end of file diff --git a/native/core/benches/common.rs b/native/core/benches/common.rs index 15952b83c..1596e56b5 100644 --- a/native/core/benches/common.rs +++ b/native/core/benches/common.rs @@ -28,11 +28,11 @@ use rand::{ use std::sync::Arc; /// Returns fixed seedable RNG -pub fn seedable_rng() -> StdRng { +pub(crate) fn seedable_rng() -> StdRng { StdRng::seed_from_u64(42) } -pub fn create_int64_array(size: usize, null_density: f32, min: i64, max: i64) -> Int64Array { +pub(crate) fn create_int64_array(size: usize, null_density: f32, min: i64, max: i64) -> Int64Array { let mut rng = seedable_rng(); (0..size) .map(|_| { @@ -46,7 +46,7 @@ pub fn create_int64_array(size: usize, null_density: f32, min: i64, max: i64) -> } #[allow(dead_code)] -pub fn create_primitive_array(size: usize, null_density: f32) -> PrimitiveArray +pub(crate) fn create_primitive_array(size: usize, null_density: f32) -> PrimitiveArray where T: ArrowPrimitiveType, Standard: Distribution, @@ -66,7 +66,7 @@ where /// Creates a dictionary with random keys and values, with value type `T`. /// Note here the keys are the dictionary indices. #[allow(dead_code)] -pub fn create_dictionary_array( +pub(crate) fn create_dictionary_array( size: usize, value_size: usize, null_density: f32, diff --git a/native/core/benches/parquet_read.rs b/native/core/benches/parquet_read.rs index 1f8178cd2..c5eadf073 100644 --- a/native/core/benches/parquet_read.rs +++ b/native/core/benches/parquet_read.rs @@ -156,7 +156,7 @@ struct TestColumnReader { } impl TestColumnReader { - pub fn new( + pub(crate) fn new( cd: ColumnDescriptor, promotion_info: TypePromotionInfo, batch_size: usize, diff --git a/native/core/benches/perf.rs b/native/core/benches/perf.rs index f92ec0250..3c21782e1 100644 --- a/native/core/benches/perf.rs +++ b/native/core/benches/perf.rs @@ -25,13 +25,13 @@ use pprof::ProfilerGuard; /// Mostly followed this blog post: https://www.jibbow.com/posts/criterion-flamegraphs/ /// After `cargo bench --bench -- --profile-time=