From cf3ca1fb9e428f8e4f6bbb1d8e17dbec1538a4c4 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sat, 18 Jul 2026 09:34:40 +0800 Subject: [PATCH 1/6] [DNM] Debug CI: only run spark-test-spark41 jobs with crash diagnostics --- .github/workflows/velox_backend_x86.yml | 1144 +---------------------- 1 file changed, 46 insertions(+), 1098 deletions(-) diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 0822223ffc6..13cbe4f8899 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -94,1104 +94,6 @@ jobs: path: ./cpp/build/ if-no-files-found: error - tpc-test-ubuntu: - needs: build-native-lib-centos-7 - strategy: - fail-fast: false - matrix: - os: [ "ubuntu:22.04" ] - spark: [ "spark-3.3", "spark-3.4", "spark-3.5", "spark-4.0", "spark-4.1" ] - java: [ "java-8", "java-11", "java-17", "java-21", "java-25" ] - # Spark supports JDK17 since 3.3. - exclude: - - spark: spark-3.3 - java: java-25 - - spark: spark-3.4 - java: java-25 - - spark: spark-3.5 - java: java-25 - - spark: spark-3.3 - java: java-21 - - spark: spark-3.4 - java: java-21 - - spark: spark-3.5 - java: java-21 - - spark: spark-3.4 - java: java-17 - - spark: spark-3.5 - java: java-17 - - spark: spark-3.3 - java: java-11 - - spark: spark-3.4 - java: java-11 - - spark: spark-4.0 - java: java-8 - - spark: spark-4.0 - java: java-11 - - spark: spark-4.1 - java: java-8 - - spark: spark-4.1 - java: java-11 - - runs-on: ubuntu-22.04 - container: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Download All Native Artifacts - uses: actions/download-artifact@v4 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/ - - name: Setup tzdata - run: | - sed -i 's|http://archive|http://eu.archive|g' /etc/apt/sources.list - if [ "${{ matrix.os }}" = "ubuntu:22.04" ]; then - apt-get update - TZ="Etc/GMT" DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata - fi - - name: Setup java - run: | - if [ "${{ matrix.java }}" = "java-17" ]; then - apt-get update && apt-get install -y openjdk-17-jdk wget - apt remove openjdk-11* -y - elif [ "${{ matrix.java }}" = "java-25" ]; then - apt-get update && apt-get install -y openjdk-25-jdk wget - elif [ "${{ matrix.java }}" = "java-21" ]; then - apt-get update && apt-get install -y openjdk-21-jdk wget - elif [ "${{ matrix.java }}" = "java-11" ]; then - apt-get update && apt-get install -y openjdk-11-jdk wget - else - apt-get update && apt-get install -y openjdk-8-jdk wget - apt remove openjdk-11* -y - fi - - name: Install Hadoop & Setup HDFS - if: matrix.os == 'ubuntu:22.04' && matrix.spark == 'spark-3.5' && matrix.java == 'java-8' - shell: bash - run: | - export JAVA_HOME=/usr/lib/jvm/${{ matrix.java }}-openjdk-amd64 - source .github/workflows/util/install-spark-deps.sh - install_hadoop - setup_hdfs - - name: Install MinIO - if: matrix.os == 'ubuntu:22.04' && matrix.spark == 'spark-3.5' && matrix.java == 'java-8' - shell: bash - run: | - export JAVA_HOME=/usr/lib/jvm/${{ matrix.java }}-openjdk-amd64 - source .github/workflows/util/install-spark-deps.sh - install_minio - - name: Build and run TPC-H / TPC-DS - shell: bash - run: | - cd $GITHUB_WORKSPACE/ - export JAVA_HOME=/usr/lib/jvm/${{ matrix.java }}-openjdk-amd64 - echo "JAVA_HOME: $JAVA_HOME" - case "${{ matrix.spark }}" in - spark-4.0|spark-4.1) - $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pscala-2.13 -Pbackends-velox -DskipTests - ;; - *) - $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pbackends-velox -DskipTests - ;; - esac - cd $GITHUB_WORKSPACE/tools/gluten-it - $GITHUB_WORKSPACE/$MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} - # Setup S3 JARs after gluten-it build - if [ "${{ matrix.os }}" = "ubuntu:22.04" ] && \ - [ "${{ matrix.spark }}" = "spark-3.5" ] && \ - [ "${{ matrix.java }}" = "java-8" ]; then - source $GITHUB_WORKSPACE/.github/workflows/util/install-spark-deps.sh - SPARK_VERSION=$(echo "${{ matrix.spark }}" | sed 's/spark-//') - setup_minio "$SPARK_VERSION" - fi - GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 - if [ "${{ matrix.os }}" = "ubuntu:22.04" ] && \ - [ "${{ matrix.spark }}" = "spark-3.5" ] && \ - [ "${{ matrix.java }}" = "java-8" ]; then - GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - --queries=q1 --data-dir="hdfs://localhost:9000/test" - GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - --queries=q1 --data-dir="s3a://gluten-it/test" \ - --extra-conf=spark.hadoop.fs.s3a.endpoint=http://localhost:9100 \ - --extra-conf=spark.hadoop.fs.s3a.access.key=admin \ - --extra-conf=spark.hadoop.fs.s3a.secret.key=admin123 \ - --extra-conf=spark.hadoop.fs.s3a.path.style.access=true \ - --extra-conf=spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem - fi - - tpc-test-centos8: - needs: build-native-lib-centos-7 - strategy: - fail-fast: false - matrix: - os: [ "centos:8" ] - spark: [ "spark-3.3", "spark-3.4", "spark-3.5", "spark-4.0", "spark-4.1" ] - java: [ "java-8", "java-11", "java-17" ] - # Spark supports JDK17 since 3.3. - exclude: - - spark: spark-3.4 - java: java-17 - - spark: spark-3.5 - java: java-17 - - spark: spark-3.3 - java: java-11 - - spark: spark-3.4 - java: java-11 - - spark: spark-4.0 - java: java-8 - - spark: spark-4.0 - java: java-11 - - spark: spark-4.1 - java: java-8 - - spark: spark-4.1 - java: java-11 - - runs-on: ubuntu-22.04 - container: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Download All Native Artifacts - uses: actions/download-artifact@v4 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/ - - name: Update mirror list - run: | - if [ "${{ matrix.os }}" = "centos:7" ] || [ "${{ matrix.os }}" = "centos:8" ]; then - sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true - sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true - fi - - name: Setup java - run: | - if [ "${{ matrix.java }}" = "java-17" ]; then - yum update -y && yum install -y java-17-openjdk-devel wget - elif [ "${{ matrix.java }}" = "java-11" ]; then - yum update -y && yum install -y java-11-openjdk-devel wget - else - yum update -y && yum install -y java-1.8.0-openjdk-devel wget - fi - - name: Set environment variables - run: | - if [ "${{ matrix.java }}" = "java-17" ]; then - echo "JAVA_HOME=/usr/lib/jvm/java-17-openjdk" >> $GITHUB_ENV - elif [ "${{ matrix.java }}" = "java-11" ]; then - echo "JAVA_HOME=/usr/lib/jvm/java-11-openjdk" >> $GITHUB_ENV - else - echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV - fi - - name: Build gluten-it - run: | - echo "JAVA_HOME: $JAVA_HOME" - cd $GITHUB_WORKSPACE/ - case "${{ matrix.spark }}" in - spark-4.0|spark-4.1) - $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pscala-2.13 -Pbackends-velox -DskipTests - ;; - *) - $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pbackends-velox -DskipTests - ;; - esac - cd $GITHUB_WORKSPACE/tools/gluten-it - $GITHUB_WORKSPACE/$MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} - - name: Run TPC-H / TPC-DS - run: | - echo "JAVA_HOME: $JAVA_HOME" - cd $GITHUB_WORKSPACE/tools/gluten-it - GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 - GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 - - tpc-test-centos7: - needs: build-native-lib-centos-7 - strategy: - fail-fast: false - matrix: - spark: [ "spark-3.5" ] - java: [ "java-8" ] - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - name: Download All Native Artifacts - uses: actions/download-artifact@v4 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/ - - name: Build and run TPCH/DS tests - run: | - docker pull centos:7 - docker run -v $GITHUB_WORKSPACE:/work -v /$GITHUB_WORKSPACE/.m2:/root/.m2/ -w /work \ - -e matrix.java=${{ matrix.java }} -e matrix.spark=${{ matrix.spark }} \ - centos:7 \ - bash -c " - set -e - sed -i -e 's|mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-* || true - sed -i -e 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true - - yum update -y && yum install -y java-1.8.0-openjdk-devel wget tzdata python3-pip - # Set environment variables - export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk - - # Build gluten - build/mvn -ntp clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pbackends-velox -DskipTests - # Build gluten-it - cd /work/tools/gluten-it - /work/build/mvn -ntp clean install -P${{ matrix.spark }} -P${{ matrix.java }} - - # Run TPC-H / TPC-DS - GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 - GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 - " - - tpc-test-ubuntu-oom: - needs: build-native-lib-centos-7 - strategy: - fail-fast: false - matrix: - spark: [ "spark-3.3" ] - runs-on: ubuntu-22.04 - steps: - - name: Maximize build disk space - shell: bash - run: | - df -h - set -euo pipefail - echo "Removing unwanted software... " - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - sudo rm -rf /opt/hostedtoolcache/CodeQL - sudo docker image prune --all --force > /dev/null - df -h - - uses: actions/checkout@v4 - - name: Download All Native Artifacts - uses: actions/download-artifact@v4 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/ - - name: Setup java - run: | - sudo apt-get update - sudo apt-get install -y openjdk-8-jdk wget - - name: Set environment variables - run: | - echo "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" >> $GITHUB_ENV - - name: Build for Spark ${{ matrix.spark }} - run: | - cd $GITHUB_WORKSPACE/ - $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -DskipTests - cd $GITHUB_WORKSPACE/tools/gluten-it - $GITHUB_WORKSPACE/$MVN_CMD clean install -P${{ matrix.spark }} - GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh data-gen-only --local --benchmark-type=ds -s=30.0 --threads=12 - - name: TPC-DS SF30.0 Parquet local spark3.3 Q67/Q95 low memory, memory isolation off - run: | - cd tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q67,q95 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \ - --data-gen=skip -m=OffHeapExecutionMemory \ - -d=ISOLATION:OFF,spark.gluten.memory.isolation=false \ - -d=OFFHEAP_SIZE:6g,spark.memory.offHeap.size=6g \ - -d=OFFHEAP_SIZE:4g,spark.memory.offHeap.size=4g \ - -d=OVER_ACQUIRE:0.3,spark.gluten.memory.overAcquiredMemoryRatio=0.3 \ - -d=OVER_ACQUIRE:0.5,spark.gluten.memory.overAcquiredMemoryRatio=0.5 \ - --excluded-dims=OFFHEAP_SIZE:4g - - name: TPC-DS SF30.0 Parquet local spark3.3 Q67 low memory, memory isolation on - run: | - cd tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q67 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \ - --data-gen=skip -m=OffHeapExecutionMemory \ - -d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1 \ - -d=OFFHEAP_SIZE:6g,spark.memory.offHeap.size=6g \ - -d=OFFHEAP_SIZE:4g,spark.memory.offHeap.size=4g \ - -d=OVER_ACQUIRE:0.3,spark.gluten.memory.overAcquiredMemoryRatio=0.3 \ - -d=OVER_ACQUIRE:0.5,spark.gluten.memory.overAcquiredMemoryRatio=0.5 - - name: TPC-DS SF30.0 Parquet local spark3.3 Q95 low memory, memory isolation on - run: | - cd tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q95 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \ - --data-gen=skip -m=OffHeapExecutionMemory \ - -d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1 \ - -d=OFFHEAP_SIZE:6g,spark.memory.offHeap.size=6g \ - -d=OFFHEAP_SIZE:4g,spark.memory.offHeap.size=4g \ - -d=OVER_ACQUIRE:0.3,spark.gluten.memory.overAcquiredMemoryRatio=0.3 \ - -d=OVER_ACQUIRE:0.5,spark.gluten.memory.overAcquiredMemoryRatio=0.5 - - name: TPC-DS SF30.0 Parquet local spark3.3 Q23A/Q23B low memory - run: | - cd tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q23a,q23b -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \ - --data-gen=skip -m=OffHeapExecutionMemory \ - -d=ISOLATION:OFF,spark.gluten.memory.isolation=false \ - -d=OFFHEAP_SIZE:2g,spark.memory.offHeap.size=2g \ - -d=FLUSH_MODE:DISABLED,spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \ - -d=FLUSH_MODE:ABANDONED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \ - -d=FLUSH_MODE:FLUSHED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=0.05,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=0.1,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 - - name: TPC-DS SF30.0 Parquet local spark3.3 Q23A/Q23B low memory, memory isolation on - run: | - cd tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q23a,q23b -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \ - --data-gen=skip -m=OffHeapExecutionMemory \ - -d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1 \ - -d=OFFHEAP_SIZE:2g,spark.memory.offHeap.size=2g \ - -d=FLUSH_MODE:DISABLED,spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \ - -d=FLUSH_MODE:ABANDONED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \ - -d=FLUSH_MODE:FLUSHED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=0.05,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=0.1,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 - - name: TPC-DS SF30.0 Parquet local spark3.3 Q97 low memory - run: | - cd tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q97 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \ - --data-gen=skip -m=OffHeapExecutionMemory \ - --extra-conf=spark.gluten.sql.columnar.backend.velox.IOThreads=0 \ - -d=ISOLATION:OFF,spark.gluten.memory.isolation=false \ - -d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1 \ - -d=OFFHEAP_SIZE:2g,spark.memory.offHeap.size=2g \ - -d=OFFHEAP_SIZE:1g,spark.memory.offHeap.size=1g \ - -d=IO_THREADS:12,spark.gluten.sql.columnar.backend.velox.IOThreads=12 \ - -d=IO_THREADS:0,spark.gluten.sql.columnar.backend.velox.IOThreads=0 - - tpc-test-ubuntu-randomkill: - needs: build-native-lib-centos-7 - strategy: - fail-fast: false - matrix: - spark: [ "spark-3.3" ] - runs-on: ubuntu-22.04 - steps: - - name: Maximize build disk space - shell: bash - run: | - df -h - set -euo pipefail - echo "Removing unwanted software... " - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - sudo rm -rf /opt/hostedtoolcache/CodeQL - sudo docker image prune --all --force > /dev/null - df -h - - uses: actions/checkout@v4 - - name: Download All Native Artifacts - uses: actions/download-artifact@v4 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/ - - name: Setup java - run: | - sudo apt-get update - sudo apt-get install -y openjdk-8-jdk wget - - name: Set environment variables - run: | - echo "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" >> $GITHUB_ENV - - name: Build for Spark ${{ matrix.spark }} - run: | - cd $GITHUB_WORKSPACE/ - $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -DskipTests - cd $GITHUB_WORKSPACE/tools/gluten-it - $GITHUB_WORKSPACE/$MVN_CMD clean install -P${{ matrix.spark }} - GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh data-gen-only --local --benchmark-type=ds -s=30.0 --threads=12 - - name: TPC-DS SF30.0 Parquet local spark3.3 random kill tasks - run: | - cd tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries \ - --local --preset=velox --benchmark-type=ds --error-on-memleak -s=30.0 --off-heap-size=8g --threads=12 --shuffle-partitions=72 --iterations=1 \ - --data-gen=skip --random-kill-tasks --no-session-reuse - - tpc-test-centos8-uniffle: - needs: build-native-lib-centos-7 - strategy: - fail-fast: false - matrix: - spark: [ "spark-3.3" ] - uniffle: [ "0.10.0" ] - hadoop: [ "2.10.2" ] - runs-on: ubuntu-22.04 - container: apache/gluten:centos-8-jdk8 - steps: - - uses: actions/checkout@v4 - - name: Download All Native Artifacts - uses: actions/download-artifact@v4 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/ - - name: Install Uniffle ${{ matrix.uniffle }} - run: | - export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk - if [ ! -e "/opt/apache-uniffle-${{ matrix.uniffle }}-bin.tar.gz" ]; then - ${WGET_CMD} https://www.apache.org/dyn/closer.lua/uniffle/${{ matrix.uniffle }}/apache-uniffle-${{ matrix.uniffle }}-bin.tar.gz?action=download -O /opt/apache-uniffle-${{ matrix.uniffle }}-bin.tar.gz - fi - if [ ! -e "/opt/hadoop-${{ matrix.hadoop }}.tar.gz" ]; then - ${WGET_CMD} https://www.apache.org/dyn/closer.lua/hadoop/common/hadoop-${{ matrix.hadoop }}/hadoop-${{ matrix.hadoop }}.tar.gz?action=download -O /opt/hadoop-${{ matrix.hadoop }}.tar.gz - fi - cd /opt && rm -rf shims && \ - mkdir /opt/uniffle && tar xzf apache-uniffle-${{ matrix.uniffle }}-bin.tar.gz -C /opt/uniffle --strip-components=1 && \ - tar xzf hadoop-${{ matrix.hadoop }}.tar.gz -C /opt/ && \ - cd /opt/uniffle && mkdir shuffle_data && \ - bash -c "echo -e 'XMX_SIZE=16g\nHADOOP_HOME=/opt/hadoop-${{ matrix.hadoop }}' > ./conf/rss-env.sh" && \ - bash -c "echo -e 'rss.coordinator.shuffle.nodes.max 1\nrss.rpc.server.port 19999' > ./conf/coordinator.conf" && \ - bash -c "echo -e 'rss.server.app.expired.withoutHeartbeat 7200000\nrss.server.heartbeat.delay 3000\nrss.rpc.server.port 19997\nrss.rpc.server.type GRPC_NETTY\nrss.jetty.http.port 19996\nrss.server.netty.port 19995\nrss.storage.basePath /opt/uniffle/shuffle_data\nrss.storage.type MEMORY_LOCALFILE\nrss.coordinator.quorum localhost:19999\nrss.server.flush.thread.alive 10\nrss.server.single.buffer.flush.threshold 64m' > ./conf/server.conf" && \ - bash ./bin/start-coordinator.sh && bash ./bin/start-shuffle-server.sh - - name: Build for Spark ${{ matrix.spark }} - run: | - cd $GITHUB_WORKSPACE/ && \ - $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -Puniffle -DskipTests - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.3 with uniffle-${{ matrix.uniffle }} - run: | - export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \ - cd $GITHUB_WORKSPACE/tools/gluten-it && \ - $GITHUB_WORKSPACE/$MVN_CMD clean install -P${{ matrix.spark }} -Puniffle && \ - GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox-with-uniffle --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 - - tpc-test-ubuntu-2204-celeborn: - needs: build-native-lib-centos-7 - strategy: - fail-fast: false - matrix: - spark: [ "spark-3.3" ] - celeborn: [ "celeborn-0.6.3", "celeborn-0.5.4"] - writer: ["sort", "hash"] - runs-on: ubuntu-22.04 - container: centos:8 - steps: - - uses: actions/checkout@v4 - - name: Download All Native Artifacts - uses: actions/download-artifact@v4 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/ - - name: Update mirror list - run: | - sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true - sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true - - name: Setup java and maven - run: | - yum update -y && yum install -y java-1.8.0-openjdk-devel wget - - name: Set environment variables - run: | - echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV - - name: Build for Spark ${{ matrix.spark }} - run: | - cd $GITHUB_WORKSPACE/ - $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -Pceleborn -DskipTests - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.3 with ${{ matrix.celeborn }} - run: | - EXTRA_PROFILE="" - if [ "${{ matrix.celeborn }}" = "celeborn-0.5.4" ]; then - EXTRA_PROFILE="-Pceleborn-0.5" - elif [ "${{ matrix.celeborn }}" = "celeborn-0.6.3" ]; then - EXTRA_PROFILE="-Pceleborn-0.6" - fi - echo "EXTRA_PROFILE: ${EXTRA_PROFILE}" - if [ ! -e "/opt/apache-${{ matrix.celeborn }}-bin.tgz" ]; then - echo "WARNING: please pre-install your required package in docker image since the downloading is throttled by this site." - wget -nv https://www.apache.org/dyn/closer.lua/celeborn/${{ matrix.celeborn }}/apache-${{ matrix.celeborn }}-bin.tgz?action=download -O /opt/apache-${{ matrix.celeborn }}-bin.tgz - fi - cd /opt && rm -rf shims && mkdir -p celeborn && \ - tar xzf apache-${{ matrix.celeborn }}-bin.tgz -C /opt/celeborn --strip-components=1 && cd celeborn && \ - mv ./conf/celeborn-env.sh.template ./conf/celeborn-env.sh && \ - bash -c "echo -e 'CELEBORN_MASTER_MEMORY=8g\nCELEBORN_WORKER_MEMORY=8g\nCELEBORN_WORKER_OFFHEAP_MEMORY=16g' > ./conf/celeborn-env.sh" && \ - bash -c "echo -e 'celeborn.worker.commitFiles.threads 32\nceleborn.worker.sortPartition.threads 16' > ./conf/celeborn-defaults.conf" && \ - bash ./sbin/start-master.sh && bash ./sbin/start-worker.sh && \ - cd $GITHUB_WORKSPACE/tools/gluten-it && $GITHUB_WORKSPACE/$MVN_CMD clean install -Pspark-3.3 -Pceleborn ${EXTRA_PROFILE} && \ - GLUTEN_IT_JVM_ARGS=-Xmx16G sbin/gluten-it.sh queries-compare \ - --extra-conf=spark.celeborn.client.spark.shuffle.writer=${{ matrix.writer }} \ - --extra-conf=spark.sql.shuffle.partitions=16 \ - --extra-conf=spark.celeborn.client.eagerlyCreateInputStream.threads=4 \ - --local --preset=velox-with-celeborn --benchmark-type=h --error-on-memleak --off-heap-size=16g -s=1.0 --threads=16 --iterations=1 && \ - GLUTEN_IT_JVM_ARGS=-Xmx16G sbin/gluten-it.sh queries-compare \ - --local --preset=velox-with-celeborn --extra-conf=spark.celeborn.client.spark.shuffle.writer=${{ matrix.writer }} \ - --extra-conf=spark.gluten.sql.columnar.shuffle.celeborn.useRssSort=true \ - --extra-conf=spark.sql.shuffle.partitions=16 \ - --extra-conf=spark.celeborn.client.eagerlyCreateInputStream.threads=4 \ - --benchmark-type=ds --error-on-memleak \ - --off-heap-size=16g -s=1.0 --threads=16 --iterations=1 - if [ "${{ matrix.writer }}" = "sort" ]; then - GLUTEN_IT_JVM_ARGS=-Xmx16G sbin/gluten-it.sh queries-compare \ - --local --preset=velox-with-celeborn --extra-conf=spark.celeborn.client.spark.shuffle.writer=${{ matrix.writer }} \ - --extra-conf=spark.gluten.sql.columnar.shuffle.celeborn.useRssSort=false \ - --extra-conf=spark.celeborn.client.eagerlyCreateInputStream.threads=4 \ - --extra-conf=spark.sql.shuffle.partitions=16 \ - --benchmark-type=ds --error-on-memleak \ - --off-heap-size=16g -s=1.0 --threads=16 --iterations=1 - fi - - spark-test-spark33: - needs: build-native-lib-centos-7 - runs-on: ubuntu-22.04 - env: - SPARK_TESTING: true - container: apache/gluten:centos-8-jdk8 - steps: - - uses: actions/checkout@v4 - - name: Download All Artifacts - uses: actions/download-artifact@v4 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/ - - name: Prepare - run: | - dnf module -y install python39 && \ - alternatives --set python3 /usr/bin/python3.9 && \ - pip3 install setuptools==77.0.3 && \ - pip3 install pyspark==3.3.1 cython && \ - pip3 install pandas==2.2.3 pyarrow==20.0.0 - - name: Build and Run unit test for Spark 3.3.1 (other tests) - run: | - cd $GITHUB_WORKSPACE/ - export SPARK_SCALA_VERSION=2.12 - yum install -y java-17-openjdk-devel - export JAVA_HOME=/usr/lib/jvm/java-17-openjdk - export PATH=$JAVA_HOME/bin:$PATH - java -version - $MVN_CMD clean test -Pspark-3.3 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark33/spark_home/" \ - -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest - - name: Upload test report - if: always() - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' - - name: Upload unit tests log files - if: ${{ !success() }} - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-test-log - path: | - **/target/*.log - **/gluten-ut/**/hs_err_*.log - **/gluten-ut/**/core.* - - name: Upload golden files - if: failure() - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-golden-files - path: /tmp/tpch-approved-plan/** - - spark-test-spark33-slow: - needs: build-native-lib-centos-7 - runs-on: ubuntu-22.04 - env: - SPARK_TESTING: true - LANG: C.UTF-8 #TODO remove after image update - LC_ALL: C.UTF-8 - container: apache/gluten:centos-8-jdk8 - steps: - - uses: actions/checkout@v4 - - name: Download All Artifacts - uses: actions/download-artifact@v4 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/ - - name: Prepare Spark Resources for Spark 3.3.1 #TODO remove after image update - run: | - rm -rf /opt/shims/spark33 - bash .github/workflows/util/install-spark-resources.sh 3.3 - - - name: Build and Run unit test for Spark 3.3.1 (slow tests) - run: | - cd $GITHUB_WORKSPACE/ - yum install -y java-17-openjdk-devel - export JAVA_HOME=/usr/lib/jvm/java-17-openjdk - export PATH=$JAVA_HOME/bin:$PATH - java -version - $MVN_CMD clean test -Pspark-3.3 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark33/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - $MVN_CMD clean test -Pspark-3.3 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark33/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.SlowHiveTest - - name: Upload test report - if: always() - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' - - name: Upload unit tests log files - if: ${{ !success() }} - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-test-log - path: | - **/target/*.log - **/gluten-ut/**/hs_err_*.log - **/gluten-ut/**/core.* - - spark-test-spark34: - needs: build-native-lib-centos-7 - runs-on: ubuntu-22.04 - container: apache/gluten:centos-8-jdk8 - steps: - - uses: actions/checkout@v4 - - name: Download All Artifacts - uses: actions/download-artifact@v4 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/ - - name: Prepare spark.test.home for Spark 3.4.4 (other tests) - run: | - dnf module -y install python39 && \ - alternatives --set python3 /usr/bin/python3.9 && \ - pip3 install setuptools==77.0.3 && \ - pip3 install pyspark==3.4.4 cython && \ - pip3 install pandas==2.2.3 pyarrow==20.0.0 - - name: Build and Run unit test for Spark 3.4.4 (other tests) - run: | - cd $GITHUB_WORKSPACE/ - export SPARK_SCALA_VERSION=2.12 - yum install -y java-17-openjdk-devel - export JAVA_HOME=/usr/lib/jvm/java-17-openjdk - export PATH=$JAVA_HOME/bin:$PATH - java -version - export SPARK_HOME=/opt/shims/spark34/spark_home/ - ls -l $SPARK_HOME - $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \ - -DargLine="-Dspark.test.home=$SPARK_HOME" - - name: Upload test report - if: always() - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' - - name: Upload unit tests log files - if: ${{ !success() }} - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-test-log - path: | - **/target/*.log - **/gluten-ut/**/hs_err_*.log - **/gluten-ut/**/core.* - - name: Upload golden files - if: failure() - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-golden-files - path: /tmp/tpch-approved-plan/** - - spark-test-spark34-slow: - needs: build-native-lib-centos-7 - runs-on: ubuntu-22.04 - env: #TODO remove after image update - LANG: C.UTF-8 - LC_ALL: C.UTF-8 - container: apache/gluten:centos-8-jdk8 - steps: - - uses: actions/checkout@v4 - - name: Download All Artifacts - uses: actions/download-artifact@v4 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/ - - name: Prepare Spark Resources for Spark 3.4.4 #TODO remove after image update - run: | - rm -rf /opt/shims/spark34 - bash .github/workflows/util/install-spark-resources.sh 3.4 - - name: Build and Run unit test for Spark 3.4.4 (slow tests) - run: | - cd $GITHUB_WORKSPACE/ - yum install -y java-17-openjdk-devel - export JAVA_HOME=/usr/lib/jvm/java-17-openjdk - export PATH=$JAVA_HOME/bin:$PATH - java -version - export SPARK_HOME=/opt/shims/spark34/spark_home/ - ls -l $SPARK_HOME - $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \ - -DargLine="-Dspark.test.home=$SPARK_HOME" - $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DtagsToInclude=org.apache.spark.tags.SlowHiveTest \ - -DargLine="-Dspark.test.home=$SPARK_HOME" - - name: Upload test report - if: always() - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' - - name: Upload unit tests log files - if: ${{ !success() }} - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-test-log - path: | - **/target/*.log - **/gluten-ut/**/hs_err_*.log - **/gluten-ut/**/core.* - - spark-test-spark35: - needs: build-native-lib-centos-7 - runs-on: ubuntu-22.04 - env: - SPARK_TESTING: true - container: apache/gluten:centos-8-jdk8 - steps: - - uses: actions/checkout@v4 - - name: Download All Artifacts - uses: actions/download-artifact@v4 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/ - - name: Prepare - run: | - dnf module -y install python39 && \ - alternatives --set python3 /usr/bin/python3.9 && \ - pip3 install setuptools==77.0.3 && \ - pip3 install pyspark==3.5.5 cython && \ - pip3 install pandas==2.2.3 pyarrow==20.0.0 - - name: Build and Run unit test for Spark 3.5.5 (other tests) - run: | - cd $GITHUB_WORKSPACE/ - export SPARK_SCALA_VERSION=2.12 - yum install -y java-17-openjdk-devel - export JAVA_HOME=/usr/lib/jvm/java-17-openjdk - export PATH=$JAVA_HOME/bin:$PATH - java -version - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ - -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest - - name: Upload test report - if: always() - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' - - name: Upload unit tests log files - if: ${{ !success() }} - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-test-log - path: | - **/target/*.log - **/gluten-ut/**/hs_err_*.log - **/gluten-ut/**/core.* - - name: Upload golden files - if: failure() - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-golden-files - path: /tmp/tpch-approved-plan/** - - spark-test-spark35-scala213: - needs: build-native-lib-centos-7 - runs-on: ubuntu-22.04 - env: - SPARK_TESTING: true - container: apache/gluten:centos-8-jdk8 - steps: - - uses: actions/checkout@v4 - - name: Download All Artifacts - uses: actions/download-artifact@v4 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/ - - name: Prepare - run: | - dnf module -y install python39 && \ - alternatives --set python3 /usr/bin/python3.9 && \ - pip3 install setuptools==77.0.3 && \ - pip3 install pyspark==3.5.5 cython && \ - pip3 install pandas==2.2.3 pyarrow==20.0.0 - - name: Build and Run unit test for Spark 3.5.5 with scala-2.13 (other tests) - run: | - cd $GITHUB_WORKSPACE/ - export SPARK_SCALA_VERSION=2.13 - yum install -y java-17-openjdk-devel - export JAVA_HOME=/usr/lib/jvm/java-17-openjdk - export PATH=$JAVA_HOME/bin:$PATH - java -version - $MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Piceberg \ - -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark35-scala-2.13/spark_home/" \ - -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest - - name: Upload test report - if: always() - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' - - name: Upload unit tests log files - if: ${{ !success() }} - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-test-log - path: | - **/target/*.log - **/gluten-ut/**/hs_err_*.log - **/gluten-ut/**/core.* - - spark-test-spark35-slow: - needs: build-native-lib-centos-7 - runs-on: ubuntu-22.04 - env: - SPARK_TESTING: true - LANG: C.UTF-8 #TODO remove after image update - LC_ALL: C.UTF-8 - container: apache/gluten:centos-8-jdk8 - steps: - - uses: actions/checkout@v4 - - name: Download All Artifacts - uses: actions/download-artifact@v4 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/ - - name: Prepare Spark Resources for Spark 3.5.5 #TODO remove after image update - run: | - rm -rf /opt/shims/spark35 - bash .github/workflows/util/install-spark-resources.sh 3.5 - - name: Build and Run unit test for Spark 3.5.5 (slow tests) - run: | - cd $GITHUB_WORKSPACE/ - yum install -y java-17-openjdk-devel - export JAVA_HOME=/usr/lib/jvm/java-17-openjdk - export PATH=$JAVA_HOME/bin:$PATH - java -version - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.SlowHiveTest - - name: Upload test report - if: always() - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' - - name: Upload unit tests log files - if: ${{ !success() }} - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-test-log - path: | - **/target/*.log - **/gluten-ut/**/hs_err_*.log - **/gluten-ut/**/core.* - - cpp-test-udf-test: - runs-on: ubuntu-22.04 - container: apache/gluten:centos-9-jdk8 - steps: - - uses: actions/checkout@v4 - - name: Get Ccache - uses: actions/cache/restore@v4 - with: - path: '${{ env.CCACHE_DIR }}' - key: ccache-centos9-release-shared-${{runner.arch}}-${{github.sha}} - restore-keys: | - ccache-centos9-release-shared-${{runner.arch}} - - name: Build Gluten native libraries - run: | - df -a - bash dev/ci-velox-buildshared-centos-9.sh - ccache -s - - name: Run CPP unit test - run: | - cd ./cpp/build && ctest -V - - name: Run CPP benchmark test - run: | - $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -pl backends-velox -am \ - -DtagsToInclude="org.apache.gluten.tags.GenerateExample" -Dtest=none -DfailIfNoTests=false -Dexec.skip - # This test depends on files generated by the above mvn test. - ./cpp/build/velox/benchmarks/generic_benchmark --with-shuffle --partitioning hash --threads 1 --iterations 1 \ - --conf $(realpath backends-velox/generated-native-benchmark/conf_12_0_*.ini) \ - --plan $(realpath backends-velox/generated-native-benchmark/plan_12_0_*.json) \ - --data $(realpath backends-velox/generated-native-benchmark/data_12_0_*_0.parquet),$(realpath backends-velox/generated-native-benchmark/data_12_0_*_1.parquet) - - name: Run UDF test - run: | - yum install -y java-17-openjdk-devel - export JAVA_HOME=/usr/lib/jvm/java-17-openjdk - export PATH=$JAVA_HOME/bin:$PATH - java -version - # Depends on --build_example=ON. - $MVN_CMD test -Pspark-3.5 -Pbackends-velox -Pjava-17 -Piceberg -Pdelta -Ppaimon -DtagsToExclude=org.apache.gluten.tags.EnhancedFeaturesTest \ - -DtagsToInclude=org.apache.gluten.tags.UDFTest \ - -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" - - name: Run CPP benchmark test for Spark 4.0 - run: | - export SPARK_SCALA_VERSION=2.13 - yum install -y java-17-openjdk-devel - export JAVA_HOME=/usr/lib/jvm/java-17-openjdk - export PATH=$JAVA_HOME/bin:$PATH - $MVN_CMD clean test -Pspark-4.0 -Pjava-17 -Pscala-2.13 -Pbackends-velox -pl backends-velox -am \ - -DtagsToInclude="org.apache.gluten.tags.GenerateExample" -Dtest=none -DfailIfNoTests=false -Dexec.skip - # This test depends on files generated by the above mvn test. - ./cpp/build/velox/benchmarks/generic_benchmark --with-shuffle --partitioning hash --threads 1 --iterations 1 \ - --conf $(realpath backends-velox/generated-native-benchmark/conf_12_0_*.ini) \ - --plan $(realpath backends-velox/generated-native-benchmark/plan_12_0_*.json) \ - --data $(realpath backends-velox/generated-native-benchmark/data_12_0_*_0.parquet),$(realpath backends-velox/generated-native-benchmark/data_12_0_*_1.parquet) - - name: Upload test report - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' - - name: Upload unit tests log files - if: ${{ !success() }} - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-test-log - path: "**/target/*.log" - - build-cudf-centos-9: - runs-on: ubuntu-22.04 - steps: - - name: "node-cleanup" # by default the free runner does not have enough disk space - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL - sudo docker image prune --all --force - sudo docker builder prune -a - - run: df -h | sort -k 5 -nr # check disk space for debug - - - uses: actions/checkout@v4 - - name: Get Ccache - uses: actions/cache/restore@v4 - with: - path: '${{ env.CCACHE_DIR }}' - key: ccache-centos9-cudf-release-shared-${{runner.arch}}-${{github.sha}} - restore-keys: | - ccache-centos9-cudf-release-shared-${{runner.arch}} - - name: Build Gluten native libraries - run: | - docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:centos-9-jdk8-cudf bash -c " - set -e - rm -rf /opt/rh/gcc-toolset-12 && ln -s /opt/rh/gcc-toolset-14 /opt/rh/gcc-toolset-12 # hack to use gcc 14, should upgrade in Velox build script later - df -a - dnf autoremove -y && dnf clean all - dnf remove -y cuda-toolkit-12* && dnf install -y cuda-toolkit-13-1 - ls -l /usr/local/ - source /opt/rh/gcc-toolset-12/enable - - export CMAKE_BUILD_PARALLEL_LEVEL=4 - export NUM_THREADS=4 - export CCACHE_DIR=/work/.ccache - export LD_LIBRARY_PATH=/work/ep/build-velox/build/velox_ep/_build/release/_deps/curl-build/lib:$LD_LIBRARY_PATH - mkdir -p /work/.ccache - - cd /work - bash dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --build_tests=ON --build_benchmarks=ON --enable_gpu=ON - rm -rf ep/build-velox/build/velox_ep - build/mvn clean package -Pbackends-velox -Pspark-3.4 -DskipTests - ccache -s - " - - build-fast-build-test: - runs-on: ubuntu-22.04 - container: apache/gluten:centos-8-jdk17 - steps: - - uses: actions/checkout@v4 - - name: Build with fast-build profile (Spark 4.0, Java 17) - run: | - cd $GITHUB_WORKSPACE/ - $MVN_CMD clean test-compile -Pspark-4.0 -Pscala-2.13 -Pbackends-velox -Pspark-ut -Piceberg,iceberg-test,delta,paimon -Pfast-build - - spark-test-spark40: - needs: build-native-lib-centos-7 - runs-on: ubuntu-22.04 - env: - SPARK_TESTING: true - container: apache/gluten:centos-8-jdk17 - steps: - - uses: actions/checkout@v4 - - name: Download All Artifacts - uses: actions/download-artifact@v4 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/ - - name: Prepare - run: | - dnf module -y install python39 && \ - alternatives --set python3 /usr/bin/python3.9 && \ - pip3 install setuptools==77.0.3 && \ - pip3 install pyspark==3.5.5 cython && \ - pip3 install pandas==2.2.3 pyarrow==20.0.0 - - name: Prepare Spark Resources for Spark 4.0.2 #TODO remove after image update - run: | - rm -rf /opt/shims/spark40 - bash .github/workflows/util/install-spark-resources.sh 4.0 - - name: Build and Run unit test for Spark 4.0.0 with scala-2.13 (other tests) - run: | - cd $GITHUB_WORKSPACE/ - export SPARK_SCALA_VERSION=2.13 - yum install -y java-17-openjdk-devel - export JAVA_HOME=/usr/lib/jvm/java-17-openjdk - export PATH=$JAVA_HOME/bin:$PATH - java -version - $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Ppaimon \ - -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ - -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest - - name: Upload test report - if: always() - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' - - name: Upload unit tests log files - if: ${{ !success() }} - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-test-log - path: | - **/target/*.log - **/gluten-ut/**/hs_err_*.log - **/gluten-ut/**/core.* - - spark-test-spark40-slow: - needs: build-native-lib-centos-7 - runs-on: ubuntu-22.04 - env: - SPARK_TESTING: true - LANG: C.UTF-8 #TODO remove after image update - LC_ALL: C.UTF-8 - container: apache/gluten:centos-8-jdk17 - steps: - - uses: actions/checkout@v4 - - name: Download All Artifacts - uses: actions/download-artifact@v4 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/ - - name: Prepare Spark Resources for Spark 4.0.2 #TODO remove after image update - run: | - rm -rf /opt/shims/spark40 - bash .github/workflows/util/install-spark-resources.sh 4.0 - - name: Build and Run unit test for Spark 4.0 (slow tests) - run: | - cd $GITHUB_WORKSPACE/ - yum install -y java-17-openjdk-devel - export JAVA_HOME=/usr/lib/jvm/java-17-openjdk - export PATH=$JAVA_HOME/bin:$PATH - java -version - $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.SlowHiveTest - - name: Upload test report - if: always() - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' - - name: Upload unit tests log files - if: ${{ !success() }} - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-test-log - path: | - **/target/*.log - **/gluten-ut/**/hs_err_*.log - **/gluten-ut/**/core.* - spark-test-spark41: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 @@ -1229,6 +131,29 @@ jobs: $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox \ -Pspark-ut -Pdelta -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest + - name: Print crash diagnostics + if: always() + shell: bash + run: | + echo "=== Kernel / CPU ===" + uname -a || true + grep -m1 'model name' /proc/cpuinfo || true + echo "=== JVM fatal-error logs ===" + find . /tmp -name 'hs_err_pid*.log' -type f 2>/dev/null \ + -exec sh -c 'echo "----- $1"; cat "$1"' _ {} \; || true + echo "=== Surefire dumpstream ===" + find . -path '*/surefire-reports/*' -name '*.dumpstream' -type f \ + -exec sh -c 'echo "----- $1"; cat "$1"' _ {} \; || true + - name: Upload crash diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}-crash-diagnostics + if-no-files-found: warn + path: | + **/hs_err_pid*.log + **/target/surefire-reports/*.dumpstream + **/core.* - name: Upload test report if: always() uses: actions/upload-artifact@v4 @@ -1277,6 +202,29 @@ jobs: $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \ -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \ -DtagsToInclude=org.apache.spark.tags.SlowHiveTest + - name: Print crash diagnostics + if: always() + shell: bash + run: | + echo "=== Kernel / CPU ===" + uname -a || true + grep -m1 'model name' /proc/cpuinfo || true + echo "=== JVM fatal-error logs ===" + find . /tmp -name 'hs_err_pid*.log' -type f 2>/dev/null \ + -exec sh -c 'echo "----- $1"; cat "$1"' _ {} \; || true + echo "=== Surefire dumpstream ===" + find . -path '*/surefire-reports/*' -name '*.dumpstream' -type f \ + -exec sh -c 'echo "----- $1"; cat "$1"' _ {} \; || true + - name: Upload crash diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}-crash-diagnostics + if-no-files-found: warn + path: | + **/hs_err_pid*.log + **/target/surefire-reports/*.dumpstream + **/core.* - name: Upload test report if: always() uses: actions/upload-artifact@v4 From 43d504b20ca626a0968381f2cfbec27f189bec4e Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sat, 18 Jul 2026 14:54:45 +0800 Subject: [PATCH 2/6] [DNM] Debug CI experiment A: run spark41 tests in centos-8-jdk17 container --- .github/workflows/velox_backend_x86.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 13cbe4f8899..78862a6e8e4 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -99,7 +99,7 @@ jobs: runs-on: ubuntu-22.04 env: SPARK_TESTING: true - container: apache/gluten:centos-9-jdk17 + container: apache/gluten:centos-8-jdk17 steps: - uses: actions/checkout@v4 - name: Download All Artifacts @@ -109,10 +109,8 @@ jobs: path: ./cpp/build/ - name: Prepare run: | - dnf install -y python3.11 python3.11-pip python3.11-devel && \ - ls -la /usr/bin/python3.11 && \ - alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ - alternatives --set python3 /usr/bin/python3.11 && \ + dnf module -y install python39 && \ + alternatives --set python3 /usr/bin/python3.9 && \ pip3 install setuptools==77.0.3 && \ pip3 install pyspark==3.5.5 cython && \ pip3 install pandas==2.2.3 pyarrow==20.0.0 @@ -177,7 +175,7 @@ jobs: SPARK_TESTING: true LANG: C.UTF-8 #TODO remove after image update LC_ALL: C.UTF-8 - container: apache/gluten:centos-9-jdk17 + container: apache/gluten:centos-8-jdk17 steps: - uses: actions/checkout@v4 - name: Download All Artifacts From 0a9e43a5ccd816e4425657772b0bdb98a611b622 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sat, 18 Jul 2026 19:08:24 +0800 Subject: [PATCH 3/6] [DNM] Debug CI experiment B: centos-8-jdk17 with python3.11 for spark41 --- .github/workflows/velox_backend_x86.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 78862a6e8e4..55d0513810a 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -109,8 +109,9 @@ jobs: path: ./cpp/build/ - name: Prepare run: | - dnf module -y install python39 && \ - alternatives --set python3 /usr/bin/python3.9 && \ + dnf install -y python3.11 python3.11-pip && \ + alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ + alternatives --set python3 /usr/bin/python3.11 && \ pip3 install setuptools==77.0.3 && \ pip3 install pyspark==3.5.5 cython && \ pip3 install pandas==2.2.3 pyarrow==20.0.0 From a025993a285b42b25590158186a7bc879b91b427 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sat, 18 Jul 2026 20:18:12 +0800 Subject: [PATCH 4/6] [DNM] Debug CI experiment C: centos-8-jdk17 + miniconda python3.11 for spark41 --- .github/workflows/velox_backend_x86.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 55d0513810a..eb70471e980 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -109,9 +109,13 @@ jobs: path: ./cpp/build/ - name: Prepare run: | - dnf install -y python3.11 python3.11-pip && \ - alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ - alternatives --set python3 /usr/bin/python3.11 && \ + # Spark 4.1 python UDF tests require Python >= 3.10 which is not packaged + # for CentOS 8; install it via miniconda instead. + curl -fsSL -o /tmp/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py311_24.7.1-0-Linux-x86_64.sh && \ + bash /tmp/miniconda.sh -b -p /opt/miniconda3 && \ + ln -sf /opt/miniconda3/bin/python3.11 /usr/bin/python3 && \ + ln -sf /opt/miniconda3/bin/pip3 /usr/bin/pip3 && \ + python3 --version && \ pip3 install setuptools==77.0.3 && \ pip3 install pyspark==3.5.5 cython && \ pip3 install pandas==2.2.3 pyarrow==20.0.0 From fea1fa9faed41c73e31d4e9e0a7374151d362ef3 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sun, 19 Jul 2026 01:46:18 +0800 Subject: [PATCH 5/6] [DNM] Debug experiment D: keep centos-9, fix libgluten with -Bsymbolic --- .github/workflows/velox_backend_x86.yml | 15 ++++++--------- cpp/core/CMakeLists.txt | 13 +++++++++++++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index eb70471e980..13cbe4f8899 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -99,7 +99,7 @@ jobs: runs-on: ubuntu-22.04 env: SPARK_TESTING: true - container: apache/gluten:centos-8-jdk17 + container: apache/gluten:centos-9-jdk17 steps: - uses: actions/checkout@v4 - name: Download All Artifacts @@ -109,13 +109,10 @@ jobs: path: ./cpp/build/ - name: Prepare run: | - # Spark 4.1 python UDF tests require Python >= 3.10 which is not packaged - # for CentOS 8; install it via miniconda instead. - curl -fsSL -o /tmp/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py311_24.7.1-0-Linux-x86_64.sh && \ - bash /tmp/miniconda.sh -b -p /opt/miniconda3 && \ - ln -sf /opt/miniconda3/bin/python3.11 /usr/bin/python3 && \ - ln -sf /opt/miniconda3/bin/pip3 /usr/bin/pip3 && \ - python3 --version && \ + dnf install -y python3.11 python3.11-pip python3.11-devel && \ + ls -la /usr/bin/python3.11 && \ + alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ + alternatives --set python3 /usr/bin/python3.11 && \ pip3 install setuptools==77.0.3 && \ pip3 install pyspark==3.5.5 cython && \ pip3 install pandas==2.2.3 pyarrow==20.0.0 @@ -180,7 +177,7 @@ jobs: SPARK_TESTING: true LANG: C.UTF-8 #TODO remove after image update LC_ALL: C.UTF-8 - container: apache/gluten:centos-8-jdk17 + container: apache/gluten:centos-9-jdk17 steps: - uses: actions/checkout@v4 - name: Download All Artifacts diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt index e4cde06cf7a..6d96c5f317d 100644 --- a/cpp/core/CMakeLists.txt +++ b/cpp/core/CMakeLists.txt @@ -163,6 +163,19 @@ add_dependencies(gluten jni_proto) if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin") target_link_options( gluten PRIVATE -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map) + # libgluten statically links libstdc++ (see dev/vcpkg/toolchain.cmake's + # -static-libstdc++). Without -Bsymbolic, the C++ runtime symbols libgluten + # exports at default visibility can be interposed at load time by another + # libstdc++ already present in the process (e.g. the system gcc-11 + # libstdc++.so.6 loaded inside the centos-9 image). In particular the + # std::locale::id data objects that index the facet table get interposed, so + # libgluten looks up a num_put/codecvt facet under the wrong id, dereferences a + # bogus facet pointer, and crashes with SIGSEGV during Velox backend + # initialization. -Bsymbolic (not just -functions: the id objects are data) + # binds libgluten's own references to its internal definitions, removing the + # cross-libstdc++ interposition and matching how libvelox is already protected + # by its "local: *" symbol map. + target_link_options(gluten PRIVATE -Wl,-Bsymbolic) endif() if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0) From e144d59378b3ca5203cce41065a22883e7b32eb9 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sun, 19 Jul 2026 14:16:25 +0800 Subject: [PATCH 6/6] [DNM] Debug experiment E: keep centos-9, hide libgluten bundled libstdc++ via --exclude-libs --- cpp/core/CMakeLists.txt | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt index 6d96c5f317d..d1613c7795b 100644 --- a/cpp/core/CMakeLists.txt +++ b/cpp/core/CMakeLists.txt @@ -163,19 +163,27 @@ add_dependencies(gluten jni_proto) if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin") target_link_options( gluten PRIVATE -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map) - # libgluten statically links libstdc++ (see dev/vcpkg/toolchain.cmake's - # -static-libstdc++). Without -Bsymbolic, the C++ runtime symbols libgluten - # exports at default visibility can be interposed at load time by another - # libstdc++ already present in the process (e.g. the system gcc-11 - # libstdc++.so.6 loaded inside the centos-9 image). In particular the - # std::locale::id data objects that index the facet table get interposed, so - # libgluten looks up a num_put/codecvt facet under the wrong id, dereferences a - # bogus facet pointer, and crashes with SIGSEGV during Velox backend - # initialization. -Bsymbolic (not just -functions: the id objects are data) - # binds libgluten's own references to its internal definitions, removing the - # cross-libstdc++ interposition and matching how libvelox is already protected - # by its "local: *" symbol map. - target_link_options(gluten PRIVATE -Wl,-Bsymbolic) + # libgluten statically links libstdc++/libgcc (see dev/vcpkg/toolchain.cmake's + # -static-libstdc++ -static-libgcc). The version script above has no "local: *" + # catch-all (it must keep Arrow/substrait/etc. exported for libvelox to resolve + # against), so by default the bundled gcc-13 C++ runtime symbols also stay in + # libgluten's dynamic symbol table -- and the std::locale::id objects that index + # the facet table are emitted as STB_GNU_UNIQUE. When another libstdc++ is + # already loaded in the process (the system gcc-11 libstdc++.so.6 pulled in by + # the JVM inside the centos-9 image), the loader collapses those unique ids into + # one process-wide definition; libgluten then indexes its gcc-13 facet table + # with the system id value, reads a bogus num_put/codecvt facet pointer, and + # crashes with SIGSEGV during Velox backend initialization (do_unshift, + # si_addr=0x30). -Bsymbolic does not help because STB_GNU_UNIQUE overrides it. + # --exclude-libs demotes the symbols coming from the named C++ runtime archives + # to STB_LOCAL, dropping them from the dynamic symbol table entirely (this also + # clears their STB_GNU_UNIQUE binding) so they can neither interpose nor be + # interposed -- the same isolation libvelox already gets from its "local: *" + # map. Symbols from the other static deps (Arrow, substrait, ...) come from + # different archives and stay exported. + target_link_options( + gluten PRIVATE -Wl,--exclude-libs,libstdc++.a -Wl,--exclude-libs,libgcc.a + -Wl,--exclude-libs,libgcc_eh.a) endif() if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)