From 21228ee31f043daa66846e94440aaf7b46cf2440 Mon Sep 17 00:00:00 2001 From: Andrew Coleman Date: Tue, 27 Oct 2026 09:12:43 +0000 Subject: [PATCH] fix(spark): create QueryExecution via SessionState.executePlan Constructing `QueryExecution` directly binds to Apache Spark's exact constructor signature. Vendor builds such as the Databricks Runtime change that signature, so conversion fails there with `NoSuchMethodError`. `SessionState.executePlan` is the entry point Spark itself uses (`Dataset.ofRows`), and it leaves the constructor to the session. The change is applied to all three variants so vendor builds based on Spark 3.x are covered too. --- .../spark-3.4/io/substrait/spark/compat/SparkCompatImpl.scala | 4 +--- .../spark-3.5/io/substrait/spark/compat/SparkCompatImpl.scala | 4 +--- .../spark-4.0/io/substrait/spark/compat/SparkCompatImpl.scala | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/spark/src/main/spark-3.4/io/substrait/spark/compat/SparkCompatImpl.scala b/spark/src/main/spark-3.4/io/substrait/spark/compat/SparkCompatImpl.scala index 09bb0ca30..7c454bd24 100644 --- a/spark/src/main/spark-3.4/io/substrait/spark/compat/SparkCompatImpl.scala +++ b/spark/src/main/spark-3.4/io/substrait/spark/compat/SparkCompatImpl.scala @@ -40,9 +40,7 @@ class SparkCompatImpl extends SparkCompat { override def createQueryExecution( spark: AnyRef, plan: LogicalPlan): org.apache.spark.sql.execution.QueryExecution = { - new org.apache.spark.sql.execution.QueryExecution( - spark.asInstanceOf[org.apache.spark.sql.SparkSession], - plan) + spark.asInstanceOf[org.apache.spark.sql.SparkSession].sessionState.executePlan(plan) } override def getConf(spark: AnyRef, key: String): String = { diff --git a/spark/src/main/spark-3.5/io/substrait/spark/compat/SparkCompatImpl.scala b/spark/src/main/spark-3.5/io/substrait/spark/compat/SparkCompatImpl.scala index 750b93005..445abf47d 100644 --- a/spark/src/main/spark-3.5/io/substrait/spark/compat/SparkCompatImpl.scala +++ b/spark/src/main/spark-3.5/io/substrait/spark/compat/SparkCompatImpl.scala @@ -38,9 +38,7 @@ class SparkCompatImpl extends SparkCompat { override def createQueryExecution( spark: AnyRef, plan: LogicalPlan): org.apache.spark.sql.execution.QueryExecution = { - new org.apache.spark.sql.execution.QueryExecution( - spark.asInstanceOf[org.apache.spark.sql.SparkSession], - plan) + spark.asInstanceOf[org.apache.spark.sql.SparkSession].sessionState.executePlan(plan) } override def getConf(spark: AnyRef, key: String): String = { diff --git a/spark/src/main/spark-4.0/io/substrait/spark/compat/SparkCompatImpl.scala b/spark/src/main/spark-4.0/io/substrait/spark/compat/SparkCompatImpl.scala index 8380d0c39..af74bdaf5 100644 --- a/spark/src/main/spark-4.0/io/substrait/spark/compat/SparkCompatImpl.scala +++ b/spark/src/main/spark-4.0/io/substrait/spark/compat/SparkCompatImpl.scala @@ -36,9 +36,7 @@ class SparkCompatImpl extends SparkCompat { override def createQueryExecution( spark: AnyRef, plan: LogicalPlan): org.apache.spark.sql.execution.QueryExecution = { - new org.apache.spark.sql.execution.QueryExecution( - spark.asInstanceOf[org.apache.spark.sql.classic.SparkSession], - plan) + spark.asInstanceOf[org.apache.spark.sql.classic.SparkSession].sessionState.executePlan(plan) } override def getConf(spark: AnyRef, key: String): String = {