diff --git a/Cargo.toml b/Cargo.toml index 81b0bf6c..1b2f5aa4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-orc" -version = "0.7.0" +version = "0.8.0" edition = "2021" homepage = "https://github.com/datafusion-contrib/datafusion-orc" repository = "https://github.com/datafusion-contrib/datafusion-orc" @@ -31,14 +31,16 @@ rust-version = "1.73" [package.metadata.docs.rs] all-features = true +[workspace] + [dependencies] async-trait = { version = "0.1.77" } bytes = "1.4" -datafusion = "52.0" +datafusion = "=54.0.0" futures = { version = "0.3", default-features = false, features = ["std"] } futures-util = { version = "0.3" } -object_store = { version = "0.12" } -orc-rust = { version = "0.7", features = ["async"] } +object_store = { version = "0.13" } +orc-rust = { version = "0.8", features = ["async"] } tokio = { version = "1.28", features = [ "io-util", "sync", diff --git a/src/file_format.rs b/src/file_format.rs index 8ecaecf1..e60fc28d 100644 --- a/src/file_format.rs +++ b/src/file_format.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::any::Any; use std::collections::HashMap; use std::fmt::Debug; use std::sync::Arc; @@ -58,10 +57,6 @@ pub struct OrcFormat; #[async_trait] impl FileFormat for OrcFormat { - fn as_any(&self) -> &dyn Any { - self - } - fn get_ext(&self) -> String { "orc".to_string() } diff --git a/src/file_source.rs b/src/file_source.rs index 023a1e82..f5460589 100644 --- a/src/file_source.rs +++ b/src/file_source.rs @@ -22,7 +22,6 @@ use datafusion::datasource::table_schema::TableSchema; use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; use datafusion::physical_plan::projection::ProjectionExprs; use object_store::ObjectStore; -use std::any::Any; use std::sync::Arc; #[derive(Debug, Clone)] @@ -65,10 +64,6 @@ impl FileSource for OrcSource { .map(|f| Arc::new(f) as Arc) } - fn as_any(&self) -> &dyn Any { - self - } - fn table_schema(&self) -> &TableSchema { &self.table_schema } diff --git a/src/lib.rs b/src/lib.rs index 53cef42e..074e925c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -127,7 +127,8 @@ impl SessionContextOrcExt for SessionContext { // SessionContext::_read_type let table_paths = table_paths.to_urls()?; let session_config = self.copied_config(); - let listing_options = ListingOptions::new(Arc::new(OrcFormat)).with_file_extension(".orc"); + let listing_options = + ListingOptions::new(Arc::new(OrcFormat)).with_file_extension(options.file_extension); let option_extension = listing_options.file_extension.clone(); @@ -185,23 +186,19 @@ mod tests { ) .await?; - let actual = ctx - .sql("select int16, utf8 from table1 limit 5") - .await? - .collect() + let dataframe = ctx + .sql("select utf8 from table1 where int16 = 1") .await?; + assert_eq!(dataframe.schema().field(0).name(), "utf8"); + let actual = dataframe.collect().await?; assert_batches_sorted_eq!( [ - "+-------+--------+", - "| int16 | utf8 |", - "+-------+--------+", - "| | |", - "| -1 | |", - "| 0 | |", - "| 1 | a |", - "| 32767 | encode |", - "+-------+--------+", + "+------+", + "| utf8 |", + "+------+", + "| a |", + "+------+", ], &actual ); diff --git a/src/object_store_reader.rs b/src/object_store_reader.rs index 925d227a..7cda8646 100644 --- a/src/object_store_reader.rs +++ b/src/object_store_reader.rs @@ -22,7 +22,7 @@ use futures::future::BoxFuture; use futures::{FutureExt, TryFutureExt}; use orc_rust::reader::AsyncChunkReader; -use object_store::{GetOptions, ObjectMeta, ObjectStore}; +use object_store::{GetOptions, ObjectMeta, ObjectStore, ObjectStoreExt}; /// Implements [`AsyncChunkReader`] to allow reading ORC files via `object_store` API. pub struct ObjectStoreReader { @@ -53,7 +53,7 @@ impl AsyncChunkReader for ObjectStoreReader { let range = offset_from_start..(offset_from_start + length); self.store .get_range(&self.file.location, range) - .map_err(|e| e.into()) + .map_err(std::io::Error::from) .boxed() } }