diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala index 06e9d91c0fa..f67446d846f 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala @@ -92,8 +92,7 @@ trait VeloxFormatWriterInjects extends GlutenFormatWriterInjectsBase { datasourceJniWrapper.close(dsHandle) } - // Do NOT add override keyword for compatibility on spark 3.1. - def path(): String = { + override def path(): String = { filePath } } diff --git a/gluten-arrow/src/main/scala/org/apache/gluten/vectorized/ArrowColumnarBatch.scala b/gluten-arrow/src/main/scala/org/apache/gluten/vectorized/ArrowColumnarBatch.scala index decd87e78f5..67e823452ef 100644 --- a/gluten-arrow/src/main/scala/org/apache/gluten/vectorized/ArrowColumnarBatch.scala +++ b/gluten-arrow/src/main/scala/org/apache/gluten/vectorized/ArrowColumnarBatch.scala @@ -20,8 +20,10 @@ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.vectorized.ColumnVector /** - * Because Spark-3.2 declares ColumnarBatch as final, so `ArrowColumnarBatch` can't extend - * `ColumnarBatch`. The code is mainly copied from Spark-3.2 + * A `ColumnarBatch`-like container of [[ArrowWritableColumnVector]]s. Originally forked from + * Spark's `ColumnarBatch` before Gluten could depend on that class directly; it is kept as a + * standalone class rather than extending `ColumnarBatch` to preserve the writable-column contract + * and existing call sites. * * @param writableColumns * the columns this class wraps diff --git a/gluten-celeborn/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java b/gluten-celeborn/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java index 31d82ce0c7d..0a82e327265 100644 --- a/gluten-celeborn/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java +++ b/gluten-celeborn/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java @@ -379,7 +379,6 @@ public ShuffleWriter getWriter( } } - // Added in SPARK-32055, for Spark 3.1 and above public ShuffleReader getReader( ShuffleHandle handle, int startMapIndex, diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala index 13689f1b28a..98593a3e12b 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala @@ -287,7 +287,8 @@ object WriteFilesExecTransformer { .toLowerCase(Locale.ROOT) } - // To be compatible with Spark3.2/3.3/3.4, we do cleanup spark internal metadata manually. + // To be compatible with Spark 3.3/3.4, we clean up Spark internal metadata manually. + // Spark 3.5 already strips these fields via SPARK-43123. // See https://github.com/apache/spark/pull/40776 private val INTERNAL_METADATA_KEYS = Seq( "__autoGeneratedAlias", diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/SparkJvmOptions.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/SparkJvmOptions.java index d2c2934eaa7..e29a7df6d12 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/SparkJvmOptions.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/SparkJvmOptions.java @@ -16,28 +16,12 @@ */ package org.apache.gluten.integration; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; +import org.apache.spark.launcher.JavaModuleOptions; public class SparkJvmOptions { - private static final String MODULE_OPTIONS_CLASS_NAME = - "org.apache.spark.launcher.JavaModuleOptions"; public static String read() { - try { - final Class clazz = Class.forName(MODULE_OPTIONS_CLASS_NAME); - final Method method = clazz.getMethod("defaultModuleOptions"); - return (String) method.invoke(null); - } catch (ClassNotFoundException - | NoSuchMethodException - | InvocationTargetException - | IllegalAccessException e) { - throw new RuntimeException( - "Failed to read Spark JVM module options via " - + MODULE_OPTIONS_CLASS_NAME - + "#defaultModuleOptions", - e); - } + return JavaModuleOptions.defaultModuleOptions(); } public static void main(String[] args) {