feat: Add datafusion-spark crate

Cargo.toml

@@ -47,6 +47,7 @@ members = [
     "datafusion/proto/gen",
     "datafusion/proto-common",
     "datafusion/proto-common/gen",
+    "datafusion/spark",
     "datafusion/sql",
     "datafusion/sqllogictest",
     "datafusion/substrait",
@@ -136,6 +137,7 @@ datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", versio
 datafusion-physical-plan = { path = "datafusion/physical-plan", version = "46.0.0" }
 datafusion-proto = { path = "datafusion/proto", version = "46.0.0" }
 datafusion-proto-common = { path = "datafusion/proto-common", version = "46.0.0" }
+datafusion-spark = { path = "datafusion/spark", version = "46.0.0" }
 datafusion-sql = { path = "datafusion/sql", version = "46.0.0" }
 doc-comment = "0.3"
 env_logger = "0.11"

NOTICE.txt

@@ -1,5 +1,5 @@
 Apache DataFusion
-Copyright 2019-2024 The Apache Software Foundation
+Copyright 2019-2025 The Apache Software Foundation
 
 This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
+The Apache Software Foundation (http://www.apache.org/).

datafusion/core/Cargo.toml

@@ -85,6 +85,7 @@ serde = [
     # statements in `arrow-schema` crate
     "arrow-schema/serde",
 ]
+spark = ["datafusion-spark"]
 string_expressions = ["datafusion-functions/string_expressions"]
 unicode_expressions = [
     "datafusion-sql/unicode_expressions",
@@ -123,6 +124,7 @@ datafusion-physical-expr = { workspace = true }
 datafusion-physical-expr-common = { workspace = true }
 datafusion-physical-optimizer = { workspace = true }
 datafusion-physical-plan = { workspace = true }
+datafusion-spark = { workspace = true, optional = true }
 datafusion-sql = { workspace = true }
 flate2 = { version = "1.1.0", optional = true }
 futures = { workspace = true }

datafusion/core/src/execution/session_state_defaults.rs

@@ -104,28 +104,55 @@ impl SessionStateDefaults {
 
     /// returns the list of default [`ScalarUDF']'s
     pub fn default_scalar_functions() -> Vec<Arc<ScalarUDF>> {
-        #[cfg_attr(not(feature = "nested_expressions"), allow(unused_mut))]
+        #[cfg_attr(
+            not(any(feature = "nested_expressions", feature = "spark")),
+            allow(unused_mut)
+        )]
         let mut functions: Vec<Arc<ScalarUDF>> = functions::all_default_functions();
 
         #[cfg(feature = "nested_expressions")]
         functions.append(&mut functions_nested::all_default_nested_functions());
 
+        #[cfg(feature = "spark")]
+        functions.append(&mut datafusion_spark::all_default_scalar_functions());
+
         functions
     }
 
     /// returns the list of default [`AggregateUDF']'s
     pub fn default_aggregate_functions() -> Vec<Arc<AggregateUDF>> {
-        functions_aggregate::all_default_aggregate_functions()
+        #[cfg_attr(not(feature = "spark"), allow(unused_mut))]
+        let mut functions: Vec<Arc<AggregateUDF>> =
+            functions_aggregate::all_default_aggregate_functions();
+
+        #[cfg(feature = "spark")]
+        functions.append(&mut datafusion_spark::all_default_aggregate_functions());
+
+        functions
     }
 
     /// returns the list of default [`WindowUDF']'s
     pub fn default_window_functions() -> Vec<Arc<WindowUDF>> {
-        functions_window::all_default_window_functions()
+        #[cfg_attr(not(feature = "spark"), allow(unused_mut))]
+        let mut functions: Vec<Arc<WindowUDF>> =
+            functions_window::all_default_window_functions();
+
+        #[cfg(feature = "spark")]
+        functions.append(&mut datafusion_spark::all_default_window_functions());
+
+        functions
     }
 
     /// returns the list of default [`TableFunction`]s
     pub fn default_table_functions() -> Vec<Arc<TableFunction>> {
-        functions_table::all_default_table_functions()
+        #[cfg_attr(not(feature = "spark"), allow(unused_mut))]
+        let mut functions: Vec<Arc<TableFunction>> =
+            functions_table::all_default_table_functions();
+
+        #[cfg(feature = "spark")]
+        functions.append(&mut datafusion_spark::all_default_table_functions());
+
+        functions
     }
 
     /// returns the list of default [`FileFormatFactory']'s
@@ -148,6 +175,7 @@ impl SessionStateDefaults {
         Self::register_scalar_functions(state);
         Self::register_array_functions(state);
         Self::register_aggregate_functions(state);
+        Self::register_spark_functions(state);
     }
 
     /// registers all the builtin scalar functions
@@ -170,6 +198,13 @@ impl SessionStateDefaults {
             .expect("can not register aggregate functions");
     }
 
+    /// registers all the builtin Spark functions
+    #[cfg_attr(not(feature = "spark"), allow(unused_variables))]
+    pub fn register_spark_functions(state: &mut SessionState) {
+        #[cfg(feature = "spark")]
+        datafusion_spark::register_all(state).expect("can not register Spark functions");
+    }
+
     /// registers the default schema
     pub fn register_default_schema(
         config: &SessionConfig,

datafusion/functions-aggregate/src/macros.rs

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#[macro_export]
 macro_rules! make_udaf_expr {
     ($EXPR_FN:ident, $($arg:ident)*, $DOC:expr, $AGGREGATE_UDF_FN:ident) => {
         // "fluent expr_fn" style function
@@ -34,6 +35,7 @@ macro_rules! make_udaf_expr {
     };
 }
 
+#[macro_export]
 macro_rules! make_udaf_expr_and_func {
     ($UDAF:ty, $EXPR_FN:ident, $($arg:ident)*, $DOC:expr, $AGGREGATE_UDF_FN:ident) => {
         make_udaf_expr!($EXPR_FN, $($arg)*, $DOC, $AGGREGATE_UDF_FN);
@@ -59,6 +61,7 @@ macro_rules! make_udaf_expr_and_func {
     };
 }
 
+#[macro_export]
 macro_rules! create_func {
     ($UDAF:ty, $AGGREGATE_UDF_FN:ident) => {
         create_func!($UDAF, $AGGREGATE_UDF_FN, <$UDAF>::default());

datafusion/functions/src/macros.rs

@@ -40,6 +40,7 @@
 /// Exported functions accept:
 /// - `Vec<Expr>` argument (single argument followed by a comma)
 /// - Variable number of `Expr` arguments (zero or more arguments, must be without commas)
+#[macro_export]
 macro_rules! export_functions {
     ($(($FUNC:ident, $DOC:expr, $($arg:tt)*)),*) => {
         $(
@@ -69,6 +70,7 @@ macro_rules! export_functions {
 /// named `$NAME` which returns that singleton.
 ///
 /// This is used to ensure creating the list of `ScalarUDF` only happens once.
+#[macro_export]
 macro_rules! make_udf_function {
     ($UDF:ty, $NAME:ident) => {
         #[doc = concat!("Return a [`ScalarUDF`](datafusion_expr::ScalarUDF) implementation of ", stringify!($NAME))]

datafusion/functions/src/utils.rs

@@ -75,7 +75,7 @@ get_optimal_return_type!(utf8_to_int_type, DataType::Int64, DataType::Int32);
 /// Creates a scalar function implementation for the given function.
 /// * `inner` - the function to be executed
 /// * `hints` - hints to be used when expanding scalars to arrays
-pub(super) fn make_scalar_function<F>(
+pub fn make_scalar_function<F>(
     inner: F,
     hints: Vec<Hint>,
 ) -> impl Fn(&[ColumnarValue]) -> Result<ColumnarValue>

datafusion/spark/Cargo.toml

@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "datafusion-spark"
+description = "DataFusion expressions that emulate Apache Spark's behavior"
+version = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+authors = { workspace = true }
+readme = { workspace = true }
+license = { workspace = true }
+edition = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[lints]
+workspace = true
+
+[lib]
+name = "datafusion_spark"
+
+[dependencies]
+arrow = { workspace = true }
+datafusion-catalog = { workspace = true }
+datafusion-common = { workspace = true }
+datafusion-doc = { workspace = true }
+datafusion-execution = { workspace = true }
+datafusion-expr = { workspace = true }
+datafusion-functions = { workspace = true }
+datafusion-functions-aggregate = { workspace = true }
+datafusion-functions-aggregate-common = { workspace = true }
+datafusion-functions-nested = { workspace = true }
+datafusion-functions-table = { workspace = true }
+datafusion-functions-window = { workspace = true }
+datafusion-functions-window-common = { workspace = true }
+datafusion-macros = { workspace = true }
+log = { workspace = true }

datafusion/spark/LICENSE.txt

@@ -0,0 +1 @@
+../../LICENSE.txt

datafusion/spark/NOTICE.txt

@@ -0,0 +1 @@
+../../NOTICE.txt

datafusion/spark/README.md

@@ -0,0 +1,38 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# datafusion-spark: Spark-compatible Expressions
+
+This crate provides Apache Spark-compatible expressions for use with DataFusion.
+
+## Testing Guide
+
+When testing functions by directly invoking them (e.g., `test_scalar_function!()`), input coercion (from the `signature`
+or `coerce_types`) is not applied.
+
+Therefore, direct invocation tests should only be used to verify that the function is correctly implemented.
+
+Please be sure to add additional tests beyond direct invocation.
+For more detailed testing guidelines, refer to
+the [Spark SQLLogicTest README](../sqllogictest/test_files/spark/README.md).
+
+## Implementation References
+
+When implementing Spark-compatible functions, you can check if there are existing implementations in
+the [Sail](https://github.com/lakehq/sail) or [Comet](https://github.com/apache/datafusion-comet) projects first.

datafusion/spark/src/function/aggregate/mod.rs

@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use datafusion_expr::AggregateUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<AggregateUDF>> {
+    vec![]
+}

datafusion/spark/src/function/array/mod.rs

@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}

datafusion/spark/src/function/bitwise/mod.rs

@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}