feat(spark): add e2e test for s3a filesystem uri job submission (#595)

.github/workflows/test-spark-examples.yaml

@@ -18,39 +18,57 @@ on:
   pull_request:
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  example-validation:
-    runs-on: ubuntu-latest
+  spark-e2e-test:
+    name: Spark E2E Test
+    runs-on: cncf-ubuntu-16-64-x86
+
+    defaults:
+      run:
+        working-directory: . # Execute make from the root of the SDK repo
+
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-version: ["1.32.0"]
+        kubernetes-version: ["1.32.11", "1.33.7", "1.34.3", "1.35.0"]
+        spark-operator-ref: ["master"]
+
+    env:
+      SPARK_TEST_CLUSTER: spark-test
+      SPARK_TEST_NAMESPACE: spark-test
+      SPARK_OPERATOR_NAMESPACE: spark-operator
 
     steps:
-      - name: Free disk space (Ubuntu runner)
-        run: |
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /usr/local/share/boost
-          sudo apt-get clean
-          docker system prune -af || true
-          df -h
+      - name: Checkout Kubeflow SDK repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
 
-      - name: Checkout repository
+      - name: Checkout Kubeflow Spark Operator repository
         uses: actions/checkout@v6
+        with:
+          repository: kubeflow/spark-operator
+          ref: ${{ matrix.spark-operator-ref }}
+          path: spark-operator
+          persist-credentials: false
 
-      - name: Set up Python 3.11
+      - name: Setup Python
         uses: actions/setup-python@v7
         with:
-          python-version: '3.11'
+          python-version: 3.11
 
-      - name: Install uv and dependencies
+      - name: Install uv
         uses: astral-sh/setup-uv@v7
         with:
           enable-cache: true
+
       - name: Install Python dependencies
         run: |
           echo "Installing Kubeflow SDK from source with the Notebook dependencies"
@@ -59,87 +77,79 @@ jobs:
           # SDK's hack/e2e-run-notebook.sh calls `papermill` from PATH
           echo "$PWD/.venv/bin" >> "$GITHUB_PATH"
 
-      - name: Install Kind, kubectl, and Helm
-        run: |
-          curl -Lo ./kind "https://kind.sigs.k8s.io/dl/v0.26.0/kind-linux-amd64" && chmod +x ./kind && sudo mv ./kind /usr/local/bin/kind
-          curl -Lo ./kubectl "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && chmod +x ./kubectl && sudo mv ./kubectl /usr/local/bin/kubectl
-          curl -sSfL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
-
-      - name: Checkout spark-operator
-        uses: actions/checkout@v6
-        with:
-          repository: kubeflow/spark-operator
-          path: spark-operator
-          fetch-depth: 1
-
-      - name: Build Spark Operator image
-        run: |
-          docker build -t ghcr.io/kubeflow/spark-operator/controller:local ./spark-operator
-        timeout-minutes: 10
-
-      - name: Setup Kind cluster with Spark Operator
+      - name: Setup cluster
         run: |
           make test-e2e-setup-cluster K8S_VERSION=${{ matrix.kubernetes-version }}
         env:
           SPARK_TEST_CLUSTER: spark-test
           SPARK_TEST_NAMESPACE: spark-test
           SPARK_OPERATOR_IMAGE_TAG: local
-        timeout-minutes: 15
+          E2E_OBJECT_STORE: seaweedfs
+          E2E_S3_BUCKETS: e2e
+          E2E_S3_SEED: "e2e/spark/spark_job.py=examples/spark/spark_job.py"
+        timeout-minutes: 30
 
-      - name: Run E2E test for Spark example Notebooks
+      - name: Run e2e test for Spark example Notebooks
         run: |
-          mkdir -p ./artifacts/notebooks
+          mkdir -p artifacts/notebooks # Create the output directory
+          OUT=./artifacts/notebooks/${{ matrix.kubernetes-version }}
 
           make test-e2e-notebook \
             NOTEBOOK_INPUT=./examples/spark/batch_job_lifecycle.ipynb \
-            NOTEBOOK_OUTPUT=./artifacts/notebooks/batch_job_lifecycle.ipynb \
+            NOTEBOOK_OUTPUT=${OUT}_batch_job_lifecycle.ipynb \
             PAPERMILL_TIMEOUT=600
 
           make test-e2e-notebook \
             NOTEBOOK_INPUT=./examples/spark/batch_func_job_lifecycle.ipynb \
-            NOTEBOOK_OUTPUT=./artifacts/notebooks/batch_func_job_lifecycle.ipynb \
+            NOTEBOOK_OUTPUT=${OUT}_batch_func_job_lifecycle.ipynb \
             PAPERMILL_TIMEOUT=600
 
           make test-e2e-notebook \
             NOTEBOOK_INPUT=./examples/spark/batch_job_options.ipynb \
-            NOTEBOOK_OUTPUT=./artifacts/notebooks/batch_job_options.ipynb \
+            NOTEBOOK_OUTPUT=${OUT}_batch_job_options.ipynb \
             PAPERMILL_TIMEOUT=600
 
           make test-e2e-notebook \
             NOTEBOOK_INPUT=./examples/spark/batch_failed_job.ipynb \
-            NOTEBOOK_OUTPUT=./artifacts/notebooks/batch_failed_job.ipynb \
+            NOTEBOOK_OUTPUT=${OUT}_batch_failed_job.ipynb \
+            PAPERMILL_TIMEOUT=600
+
+          make test-e2e-notebook \
+            NOTEBOOK_INPUT=./examples/spark/batch_job_s3a.ipynb \
+            NOTEBOOK_OUTPUT=./artifacts/notebooks/batch_job_s3a.ipynb \
+            PAPERMILL_PARAMS="-p S3_ENDPOINT $S3_ENDPOINT -p S3_REGION $S3_REGION -p S3_BUCKET e2e -p S3_PREFIX spark -p S3_SECRET_NAME $S3_SECRET_NAME" \
             PAPERMILL_TIMEOUT=600
 
           make test-e2e-notebook \
             NOTEBOOK_INPUT=./examples/spark/connect_existing_session.ipynb \
-            NOTEBOOK_OUTPUT=./artifacts/notebooks/connect_existing_session.ipynb \
+            NOTEBOOK_OUTPUT=${OUT}_connect_existing_session.ipynb \
             PAPERMILL_TIMEOUT=600
 
           make test-e2e-notebook \
             NOTEBOOK_INPUT=./examples/spark/demo_existing_sparkconnect.ipynb \
-            NOTEBOOK_OUTPUT=./artifacts/notebooks/demo_existing_sparkconnect.ipynb \
+            NOTEBOOK_OUTPUT=${OUT}_demo_existing_sparkconnect.ipynb \
             PAPERMILL_TIMEOUT=600
 
           make test-e2e-notebook \
             NOTEBOOK_INPUT=./examples/spark/spark_advanced_options.ipynb \
-            NOTEBOOK_OUTPUT=./artifacts/notebooks/spark_advanced_options.ipynb \
+            NOTEBOOK_OUTPUT=${OUT}_spark_advanced_options.ipynb \
             PAPERMILL_PARAMS='-p DRIVER_CPU 500m -p DRIVER_MEMORY 512Mi -p EXECUTOR_CPU 500m -p EXECUTOR_MEMORY 512Mi -p NUM_EXECUTORS 1' \
             PAPERMILL_TIMEOUT=600
 
           make test-e2e-notebook \
             NOTEBOOK_INPUT=./examples/spark/spark_connect_simple.ipynb \
-            NOTEBOOK_OUTPUT=./artifacts/notebooks/spark_connect_simple.ipynb \
+            NOTEBOOK_OUTPUT=${OUT}_spark_connect_simple.ipynb \
             PAPERMILL_TIMEOUT=600
-        env:
-          SPARK_TEST_CLUSTER: spark-test
-          SPARK_TEST_NAMESPACE: spark-test
         timeout-minutes: 40
 
-
       - name: Upload Artifacts to GitHub
         uses: actions/upload-artifact@v6
         if: always() # Ensure artifacts are uploaded even if previous steps fail
         with:
-          name: ${{ matrix.kubernetes-version }}
+          name: spark-${{ matrix.kubernetes-version }}
           path: ./artifacts/notebooks/* # Path relative to the workspace root
-          retention-days: 1 #
+          retention-days: 1
+
+      - name: Delete cluster
+        if: always()
+        run: make test-e2e-delete-cluster

.gitignore

@@ -23,5 +23,5 @@ htmlcov/
 # Documentation build
 docs/_build/
 
-# Local E2E: cloned for building Spark Operator image (SPARK_OPERATOR_IMAGE_TAG=local)
+# Local E2E: Spark Operator checkout used by hack/e2e-setup-cluster.sh
 spark-operator/

Makefile

@@ -21,6 +21,24 @@ SHELL = /usr/bin/env bash -o pipefail
 PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
 VENV_DIR := $(PROJECT_DIR)/.venv
 
+# Location to install E2E tool binaries to
+LOCALBIN ?= $(PROJECT_DIR)/bin
+
+# Tool versions
+K8S_VERSION ?= 1.32.11
+KIND_VERSION ?= v0.31.0
+HELM_VERSION ?= v3.19.0
+
+# Tool binaries
+KIND ?= $(LOCALBIN)/kind
+KUBECTL ?= $(LOCALBIN)/kubectl
+HELM ?= $(LOCALBIN)/helm
+
+# Host platform in the form used by the Kind, kubectl and Helm release downloads.
+HOST_OS := $(shell uname -s | tr '[:upper:]' '[:lower:]')
+HOST_ARCH := $(shell uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/')
+SHA256SUM := $(shell command -v sha256sum >/dev/null 2>&1 && echo sha256sum || echo "shasum -a 256")
+
 # Setting SED for compatibility with macos
 ifeq ($(shell command -v gsed 2>/dev/null),)
     SED ?= $(shell command -v sed)
@@ -129,17 +147,66 @@ else
 	@uv run coverage html
 endif
 
+##@ Dependencies
+
+$(LOCALBIN):
+	@mkdir -p $(LOCALBIN)
+
+.PHONY: kind
+kind: $(LOCALBIN) ## Download Kind binary if required.
+	@if ! $(KIND) version 2>/dev/null | grep -q "kind $(KIND_VERSION) "; then \
+		set -eu; \
+		echo "Installing Kind $(KIND_VERSION) into $(LOCALBIN)"; \
+		tmp_dir=$$(mktemp -d); trap 'rm -rf "$$tmp_dir"' EXIT; \
+		url=https://kind.sigs.k8s.io/dl/$(KIND_VERSION)/kind-$(HOST_OS)-$(HOST_ARCH); \
+		curl -sSfLo "$$tmp_dir/kind" "$$url"; \
+		sum=$$(curl -sSfL "$$url.sha256sum" | awk '{print $$1}'); \
+		[ -n "$$sum" ] && echo "$$sum  $$tmp_dir/kind" | $(SHA256SUM) -c - >/dev/null; \
+		chmod +x "$$tmp_dir/kind"; mv "$$tmp_dir/kind" $(KIND); \
+	fi
+
+.PHONY: kubectl
+kubectl: $(LOCALBIN) ## Download kubectl binary matching K8S_VERSION if required.
+	@if ! $(KUBECTL) version --client -o yaml 2>/dev/null | grep -q "gitVersion: v$(K8S_VERSION)$$"; then \
+		set -eu; \
+		echo "Installing kubectl v$(K8S_VERSION) into $(LOCALBIN)"; \
+		tmp_dir=$$(mktemp -d); trap 'rm -rf "$$tmp_dir"' EXIT; \
+		url=https://dl.k8s.io/release/v$(K8S_VERSION)/bin/$(HOST_OS)/$(HOST_ARCH)/kubectl; \
+		curl -sSfLo "$$tmp_dir/kubectl" "$$url"; \
+		sum=$$(curl -sSfL "$$url.sha256"); \
+		[ -n "$$sum" ] && echo "$$sum  $$tmp_dir/kubectl" | $(SHA256SUM) -c - >/dev/null; \
+		chmod +x "$$tmp_dir/kubectl"; mv "$$tmp_dir/kubectl" $(KUBECTL); \
+	fi
+
+.PHONY: helm
+helm: $(LOCALBIN) ## Download Helm binary if required.
+	@if ! $(HELM) version --short 2>/dev/null | grep -q "^$(HELM_VERSION)+"; then \
+		set -eu; \
+		echo "Installing Helm $(HELM_VERSION) into $(LOCALBIN)"; \
+		tmp_dir=$$(mktemp -d); trap 'rm -rf "$$tmp_dir"' EXIT; \
+		archive=helm-$(HELM_VERSION)-$(HOST_OS)-$(HOST_ARCH).tar.gz; \
+		curl -sSfLo "$$tmp_dir/$$archive" "https://get.helm.sh/$$archive"; \
+		sum=$$(curl -sSfL "https://get.helm.sh/$$archive.sha256sum" | awk '{print $$1}'); \
+		[ -n "$$sum" ] && echo "$$sum  $$tmp_dir/$$archive" | $(SHA256SUM) -c - >/dev/null; \
+		tar -xzf "$$tmp_dir/$$archive" -C "$$tmp_dir"; \
+		mv "$$tmp_dir/$(HOST_OS)-$(HOST_ARCH)/helm" $(HELM); \
+	fi
+
 ##@ E2E Testing
 
 .PHONY: test-e2e-setup-cluster
-test-e2e-setup-cluster:  ## Setup Kind cluster for Spark E2E tests
+test-e2e-setup-cluster: kind kubectl helm ## Setup Kind cluster for Spark E2E tests
 	@echo "Setting up E2E test cluster..."
 	@K8S_VERSION=$(K8S_VERSION) \
 	 SPARK_TEST_CLUSTER=$(SPARK_TEST_CLUSTER) \
 	 SPARK_TEST_NAMESPACE=$(SPARK_TEST_NAMESPACE) \
-	 SPARK_OPERATOR_VERSION=$(SPARK_OPERATOR_VERSION) \
-	 KIND=$(KIND) \
+	 SPARK_OPERATOR_NAMESPACE=$(SPARK_OPERATOR_NAMESPACE) \
 	 ./hack/e2e-setup-cluster.sh
+
+.PHONY: test-e2e-delete-cluster
+test-e2e-delete-cluster: kind ## Delete the Kind cluster used for Spark E2E tests
+	@SPARK_TEST_CLUSTER=$(SPARK_TEST_CLUSTER) ./hack/e2e-setup-cluster.sh --delete
+
 .PHONY: test-scripts
 test-scripts: uv-venv  ## Run GitHub Actions script tests
 	@uv sync
@@ -174,5 +241,5 @@ NOTEBOOK_OUTPUT=$(PROJECT_DIR)/artifacts/notebooks/out.ipynb
 PAPERMILL_PARAMS=
 PAPERMILL_TIMEOUT=900
 .PHONY: test-e2e-notebook
-test-e2e-notebook: ## Run Jupyter Notebook with Papermill.
-	NOTEBOOK_INPUT=$(NOTEBOOK_INPUT) NOTEBOOK_OUTPUT=$(NOTEBOOK_OUTPUT) PAPERMILL_PARAMS="$(PAPERMILL_PARAMS)" PAPERMILL_TIMEOUT=$(PAPERMILL_TIMEOUT) ./hack/e2e-run-notebook.sh
+test-e2e-notebook: kubectl ## Run Jupyter Notebook with Papermill.
+	PATH="$(LOCALBIN):$$PATH" NOTEBOOK_INPUT=$(NOTEBOOK_INPUT) NOTEBOOK_OUTPUT=$(NOTEBOOK_OUTPUT) PAPERMILL_PARAMS="$(PAPERMILL_PARAMS)" PAPERMILL_TIMEOUT=$(PAPERMILL_TIMEOUT) ./hack/e2e-run-notebook.sh

docs/source/spark/batch-jobs.rst

@@ -81,14 +81,11 @@ The SDK passes the URI through to the Spark runtime, which is responsible for re
    client.submit_job(job=FileJob(file_source="https://raw.githubusercontent.com/<repo>/<branch>/etl.py"))
 
 .. note::
-   ``https://`` is the scheme with a working, tested example today (see
-   ``examples/spark`` in the ``kubeflow/sdk`` repo). ``s3a://``, ``gs://``, and
-   ``hdfs://`` are supported the same way in principle — the SDK just passes the
-   URI to Spark — but they haven't been verified end-to-end and may need
-   additional filesystem connector packages that aren't guaranteed to be
-   preinstalled in every Spark image. Until there's an example notebook
-   covering that setup, start with ``https://`` if you want something that's
-   known to work.
+   ``https://`` and ``s3a://`` are covered with tested examples (see
+   ``examples/spark`` in the ``kubeflow/sdk`` repo). ``gs://`` and ``hdfs://``
+   are supported the same way in principle — the SDK passes the URI to Spark —
+   though they may need additional filesystem connector packages configured
+   in the Spark runtime.
 
 **Local sources** — when ``file_source`` uses a local URI
 (``local:///opt/spark/app/etl.py``), the file must already be available to the

examples/spark/README.md

@@ -25,6 +25,7 @@ For the full documentation, see [Interactive Sessions](https://sdk.kubeflow.org/
 - **batch_failed_job.ipynb** - Submit a `FileJob` expected to fail and inspect the `FAILED` state
 - **batch_job_options.ipynb** - Submit a batch job with Kubernetes options (labels, annotations, node selector, tolerations, custom name)
   See the [Options Reference](https://sdk.kubeflow.org/en/latest/spark/options.html) for details on Kubernetes options.
+- **batch_job_s3a.ipynb** - Submit a `FileJob` referencing an application script stored in S3 (`s3a://` URI)
 - **spark_job.py** - A simple Spark application used as the remote `file_source` for the batch job examples
 
 ## Prerequisites