Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions components/control-plane/internal/gateway/console.go
Original file line number Diff line number Diff line change
Expand Up @@ -830,6 +830,35 @@ func reconcileConsoleExposure(ctx context.Context, dynamicClient dynamic.Interfa
inactiveKind = "OpenShift Route"
case IngressModeRoute:
desired = buildConsoleOpenShiftRoute(namespace, host)
// Give the console Route its own publicly-trusted certificate when
// GATEWAY_ROUTE_TLS_ISSUER names a cert-manager ClusterIssuer.
//
// Without this the Route rides the router default certificate, which is
// only valid for the cluster ingress domain. Clusters whose gateway hosts
// live under a CUSTOM domain (e.g. *.gwlb.<instance>.<zone> on IBM ROKS)
// therefore serve a name-mismatched certificate and every browser refuses
// the console. Gateway-API clusters do not hit this because a wildcard
// certificate on the shared Gateway listener already covers the host, and
// HTTP-01 cannot issue wildcards -- so a per-host certificate is the only
// option where DNS-01 is unavailable.
if issuer := routeTLSIssuer(); issuer != "" {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The console Route reuses GATEWAY_ROUTE_TLS_ISSUER (via routeTLSIssuer()) with edge termination, whereas that env var's doc comment states it is "only meaningful with reencrypt termination." The reuse is reasonable, but please update the routeTLSIssuer doc so the next reader is not misled about its scope.

annotateRouteForIssuer(desired, issuer)
// reconcileResource replaces the whole Route, and the spec built above
// intentionally omits certificate/key: openshift-routes co-owns this
// Route and injects them. Carry any injection forward, otherwise every
// reconcile strips the certificate and the console flaps.
if cert, key := readInjectedRouteCert(ctx, dynamicClient, namespace, consoleName); cert != "" {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice reuse of the gateway-Route carry-forward pattern. One caveat: readInjectedRouteCert treats a missing/absent Route the same as a transient Get failure (both yield empty strings), so on a transient error the freshly built spec (which intentionally omits certificate/key) is applied and the injected cert is dropped for that reconcile. Fine as the common case; see the note on readInjectedRouteCert.

tls, _, _ := unstructured.NestedMap(desired.Object, "spec", "tls")
if tls == nil {
tls = map[string]interface{}{}
}
tls["certificate"] = cert
if key != "" {
tls["key"] = key
}
_ = unstructured.SetNestedMap(desired.Object, tls, "spec", "tls")
}
}
inactiveGVR = consoleHTTPRouteGVR
inactiveKind = "HTTPRoute"
default:
Expand Down Expand Up @@ -1005,3 +1034,17 @@ func deleteConsole(ctx context.Context, dynamicClient dynamic.Interface, clients

return errors.Join(errs...)
}

// annotateRouteForIssuer marks a Route for the cert-manager openshift-routes
// controller, which mints a certificate from the named ClusterIssuer and
// injects it into spec.tls.{certificate,key}.
func annotateRouteForIssuer(route *unstructured.Unstructured, issuer string) {
annotations, _, _ := unstructured.NestedStringMap(route.Object, "metadata", "annotations")
if annotations == nil {
annotations = map[string]string{}
}
annotations["cert-manager.io/issuer-name"] = issuer
annotations["cert-manager.io/issuer-kind"] = "ClusterIssuer"
annotations["cert-manager.io/issuer-group"] = "cert-manager.io"

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor: annotateRouteForIssuer writes three annotations (issuer-name, issuer-kind, issuer-group), while the gateway path in reconcileRouteResources (reconciler.go:616-617) writes only issuer-name and issuer-kind. Behavior is equivalent since issuer-group defaults to cert-manager.io, but two spellings of the same intent will drift. Consider having the gateway path call this shared helper so there is one source of truth.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

annotateRouteForIssuer writes cert-manager.io/issuer-group in addition to issuer-name/issuer-kind, but the gateway Route path (reconciler.go:616-617) sets only the latter two. The PR describes the console path as mirroring the gateway path. issuer-group: cert-manager.io is the cert-manager default so this is harmless, but consider aligning the two Route-issuance paths (either add the group to the gateway path or drop it here) so the annotation contract is identical.

_ = unstructured.SetNestedStringMap(route.Object, annotations, "metadata", "annotations")
}
76 changes: 76 additions & 0 deletions components/control-plane/internal/gateway/console_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -640,3 +640,79 @@ func TestReconcileConsoleSecret_PreservesCookieOnUpdate(t *testing.T) {
t.Errorf("client-secret = %q, want refreshed %q", v, "client-v2")
}
}

// TestReconcileConsoleExposureTLSIssuer covers GATEWAY_ROUTE_TLS_ISSUER on the
// console Route. Without a per-host certificate the console rides the router
// default certificate, which only covers the cluster ingress domain -- so on a
// custom gateway domain every browser rejects the console.
func TestReconcileConsoleExposureTLSIssuer(t *testing.T) {
const (
namespace = "openshell-abc"
host = "console-openshell-abc.gwlb.hyp8.example.com"
)

t.Run("issuer annotates the console Route", func(t *testing.T) {
t.Setenv("GATEWAY_ROUTE_TLS_ISSUER", "letsencrypt-http01")
dc := newConsoleRouteDynamicClient()

if err := reconcileConsoleExposure(context.Background(), dc, namespace, host, IngressModeRoute); err != nil {
t.Fatalf("reconcileConsoleExposure: %v", err)
}

route, err := dc.Resource(consoleOpenShiftRouteGVR).Namespace(namespace).Get(context.Background(), consoleName, metav1.GetOptions{})
if err != nil {
t.Fatalf("get console Route: %v", err)
}
ann, _, _ := unstructured.NestedStringMap(route.Object, "metadata", "annotations")
if ann["cert-manager.io/issuer-name"] != "letsencrypt-http01" {
t.Errorf("issuer-name annotation = %q, want letsencrypt-http01", ann["cert-manager.io/issuer-name"])
}
if ann["cert-manager.io/issuer-kind"] != "ClusterIssuer" {
t.Errorf("issuer-kind annotation = %q, want ClusterIssuer", ann["cert-manager.io/issuer-kind"])
}
})

t.Run("no issuer leaves the Route unannotated", func(t *testing.T) {
t.Setenv("GATEWAY_ROUTE_TLS_ISSUER", "")
dc := newConsoleRouteDynamicClient()

if err := reconcileConsoleExposure(context.Background(), dc, namespace, host, IngressModeRoute); err != nil {
t.Fatalf("reconcileConsoleExposure: %v", err)
}

route, err := dc.Resource(consoleOpenShiftRouteGVR).Namespace(namespace).Get(context.Background(), consoleName, metav1.GetOptions{})
if err != nil {
t.Fatalf("get console Route: %v", err)
}
ann, _, _ := unstructured.NestedStringMap(route.Object, "metadata", "annotations")
if _, ok := ann["cert-manager.io/issuer-name"]; ok {
t.Errorf("unexpected issuer annotation: %v", ann)
}
})

// reconcileResource replaces the whole Route and the built spec omits
// certificate/key, which openshift-routes owns. Losing the injection on each
// reconcile would make the console flap between a valid and a default cert.
t.Run("carries forward an injected certificate", func(t *testing.T) {
t.Setenv("GATEWAY_ROUTE_TLS_ISSUER", "letsencrypt-http01")
existing := buildConsoleOpenShiftRoute(namespace, host)
_ = unstructured.SetNestedField(existing.Object, "CERT", "spec", "tls", "certificate")
_ = unstructured.SetNestedField(existing.Object, "KEY", "spec", "tls", "key")
existing.SetGroupVersionKind(consoleOpenShiftRouteGVR.GroupVersion().WithKind("Route"))
dc := newConsoleRouteDynamicClient(existing)

if err := reconcileConsoleExposure(context.Background(), dc, namespace, host, IngressModeRoute); err != nil {
t.Fatalf("reconcileConsoleExposure: %v", err)
}

route, err := dc.Resource(consoleOpenShiftRouteGVR).Namespace(namespace).Get(context.Background(), consoleName, metav1.GetOptions{})
if err != nil {
t.Fatalf("get console Route: %v", err)
}
cert, _, _ := unstructured.NestedString(route.Object, "spec", "tls", "certificate")
key, _, _ := unstructured.NestedString(route.Object, "spec", "tls", "key")
if cert != "CERT" || key != "KEY" {
t.Errorf("injected cert not carried forward: cert=%q key=%q", cert, key)
}
})
}
11 changes: 8 additions & 3 deletions components/control-plane/internal/gateway/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -621,7 +621,7 @@ func reconcileRouteResources(ctx context.Context, dynamicClient dynamic.Interfac
// co-owns this Route (we own termination + destinationCACertificate, it
// owns the edge cert), so carry forward any certificate/key it has already
// injected -- otherwise each reconcile strips the cert and flaps h2.
if cert, key := readInjectedRouteCert(ctx, dynamicClient, namespace); cert != "" {
if cert, key := readInjectedRouteCert(ctx, dynamicClient, namespace, gatewayRouteName); cert != "" {
tlsConfig["certificate"] = cert
if key != "" {
tlsConfig["key"] = key
Expand Down Expand Up @@ -1663,6 +1663,11 @@ const (
const (
RouteTerminationPassthrough = "passthrough"
RouteTerminationReencrypt = "reencrypt"

// gatewayRouteName is the OpenShift Route serving the gateway itself.
// openshift-routes injects the issued certificate into it, and each reconcile
// must carry that injection forward (see readInjectedRouteCert).
gatewayRouteName = "openshell-gateway"
)

// routeTermination resolves the Route TLS termination for the "route" ingress
Expand Down Expand Up @@ -1692,9 +1697,9 @@ func routeTLSIssuer() string {
// The route reconcile does a full replace, so reconcileRouteResources carries
// these forward to avoid clobbering the injected edge certificate (which would
// flap ALPN h2, and hence gRPC, on every reconcile).
func readInjectedRouteCert(ctx context.Context, dynamicClient dynamic.Interface, namespace string) (string, string) {
func readInjectedRouteCert(ctx context.Context, dynamicClient dynamic.Interface, namespace, routeName string) (string, string) {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now that readInjectedRouteCert is parameterized by routeName, the doc comment above it (still describing "the existing openshell-gateway Route") is stale. Reword it to reflect that it reads the named Route's injected spec.tls.{certificate,key}.

gvr := schema.GroupVersionResource{Group: "route.openshift.io", Version: "v1", Resource: "routes"}
existing, err := dynamicClient.Resource(gvr).Namespace(namespace).Get(ctx, "openshell-gateway", metav1.GetOptions{})
existing, err := dynamicClient.Resource(gvr).Namespace(namespace).Get(ctx, routeName, metav1.GetOptions{})

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Minor] The if err != nil below returns ("", "") for any Get error, so a transient API error during reconcile (not just a genuinely absent Route) skips the carry-forward and lets reconcileResource replace the Route without the injected cert - briefly re-introducing the flap this mechanism exists to prevent. Consider distinguishing k8serrors.IsNotFound (expected on first reconcile) from transient errors so the carry-forward is only skipped when the Route truly has no injected cert. This mirrors the pre-existing gateway-Route behavior, so it is not a regression from this PR.

if err != nil {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor/robustness: this returns "", "" for any Get error, so a transient API failure drops the carry-forward and the full-replace reconcile then strips the injected edge certificate - the exact flap this logic guards against. Pre-existing for the gateway Route, but this PR now routes the console path through the same helper. Consider distinguishing IsNotFound (safe to skip) from transient errors (where returning an error / skipping the replace is safer than silently stripping the cert).

return "", ""
}
Expand Down
46 changes: 35 additions & 11 deletions tests/e2e/drivers/kind.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,15 @@
# Kind uses locally issued certificates. Other drivers may override this seam
# while reusing the production OIDC and role-assignment behavior below.

# Auto-detect container engine (docker or podman) if not explicitly set.
if [[ -z "${CONTAINER_ENGINE:-}" ]]; then
if command -v podman &>/dev/null; then
export CONTAINER_ENGINE=podman
elif command -v docker &>/dev/null; then
export CONTAINER_ENGINE=docker
fi
fi

# Kind's self-signed CA is not in the system trust store. Instruct the
# openshell CLI to skip TLS verification for gateway connections. curl already
# uses -sk (insecure) for all driver requests; this extends the same treatment
Expand Down Expand Up @@ -76,18 +85,33 @@ _kind_start_gw_socat() {
_KINDCCM_GW_PORT="${socat_port}"
}
_driver_curl() {
_kind_discover_port
local connect_args=()
if [[ -n "${_KINDCCM_PORT}" && "${_KINDCCM_PORT}" != "443" ]]; then
connect_args+=(
--connect-to "api.hypershell.localhost:443:127.0.0.1:${_KINDCCM_PORT}"
--connect-to "keycloak.hypershell.localhost:443:127.0.0.1:${_KINDCCM_PORT}"
--connect-to "console.hypershell.localhost:443:127.0.0.1:${_KINDCCM_PORT}"
--connect-to "health.hypershell.localhost:443:127.0.0.1:${_KINDCCM_PORT}"
--connect-to "observability.hypershell.localhost:443:127.0.0.1:${_KINDCCM_PORT}"
)
# Try direct HTTPRoute access first. This works in most setups where the
# routes are directly accessible on port 443 (docker, podman, or iptables-
# forwarded 443). If direct access fails (connection refused/timeout), fall
# back to port-remapping via the ephemeral kindccm-gw port.
local output status
output=$(curl -sk --ipv4 "$@" 2>&1)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Capturing with 2>&1 folds curl's stderr into output, which is then echoed as the response. With -s this is normally empty, but on failure curl's error text becomes the returned "body" and can corrupt callers that JSON-parse it. Consider keeping stderr separate from the captured body.

status=$?

# If direct access failed due to connection error, try port-remapping
if [[ $status -eq 7 ]]; then

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The port-remap fallback only triggers on curl exit code 7 (connection refused), but the comment says "connection refused/timeout." A real timeout is exit 28 and will not fall back. Either also handle 28 or drop "timeout" from the comment to avoid a false expectation.

_kind_discover_port
local connect_args=()
if [[ -n "${_KINDCCM_PORT}" && "${_KINDCCM_PORT}" != "443" ]]; then
connect_args+=(
--connect-to "api.hypershell.localhost:443:127.0.0.1:${_KINDCCM_PORT}"
--connect-to "keycloak.hypershell.localhost:443:127.0.0.1:${_KINDCCM_PORT}"
--connect-to "console.hypershell.localhost:443:127.0.0.1:${_KINDCCM_PORT}"
--connect-to "health.hypershell.localhost:443:127.0.0.1:${_KINDCCM_PORT}"
--connect-to "observability.hypershell.localhost:443:127.0.0.1:${_KINDCCM_PORT}"
)
output=$(curl -sk --ipv4 "${connect_args[@]}" "$@" 2>&1)
status=$?
fi
fi
curl -sk --ipv4 "${connect_args[@]}" "$@"

echo "$output"
return $status
}

# discover_api_host - find the HyperShell API server base URL.
Expand Down
Loading