diff --git a/components/control-plane/internal/gateway/console.go b/components/control-plane/internal/gateway/console.go index a883fee1f..cc9689cd1 100644 --- a/components/control-plane/internal/gateway/console.go +++ b/components/control-plane/internal/gateway/console.go @@ -100,14 +100,32 @@ var consoleHTTPRouteGVR = schema.GroupVersionResource{ Resource: "httproutes", } -// ConsoleRouteReady reports whether the per-gateway console HTTPRoute is actually -// serving: at least one parent (the shared Gateway listener) must report both -// Accepted=True and ResolvedRefs=True. A Ready console Deployment does not prove -// the public route works -- a missing or misnamed HTTP listener leaves the -// HTTPRoute rejected while the Deployment stays Ready -- so the reconciler gates -// console_address on this too, to avoid publishing a dead "Open console" link. -// When not ready, reason carries the parent/listener rejection descriptor. -func ConsoleRouteReady(ctx context.Context, dynamicClient dynamic.Interface, namespace string) (ready bool, reason string, err error) { +// consoleOpenShiftRouteGVR is the GroupVersionResource for the console Route. +var consoleOpenShiftRouteGVR = schema.GroupVersionResource{ + Group: "route.openshift.io", + Version: "v1", + Resource: "routes", +} + +// ConsoleExposureReady reports the readiness of the console exposure for the +// selected ingress mode. A Gateway API HTTPRoute needs Accepted=True and +// ResolvedRefs=True on one parent. An OpenShift Route needs Admitted=True from +// one router. The reason describes a known rejection or an incomplete status. +func ConsoleExposureReady(ctx context.Context, dynamicClient dynamic.Interface, namespace, ingressMode string) (ready bool, reason string, err error) { + switch ingressMode { + case IngressModeGatewayAPI: + return consoleHTTPRouteReady(ctx, dynamicClient, namespace) + case IngressModeRoute: + return consoleOpenShiftRouteReady(ctx, dynamicClient, namespace) + case IngressModeNone: + return false, "no console ingress mode selected", nil + default: + return false, "", fmt.Errorf("unsupported console ingress mode %q", ingressMode) + } +} + +// consoleHTTPRouteReady reports the readiness of the console HTTPRoute. +func consoleHTTPRouteReady(ctx context.Context, dynamicClient dynamic.Interface, namespace string) (ready bool, reason string, err error) { route, err := dynamicClient.Resource(consoleHTTPRouteGVR).Namespace(namespace).Get(ctx, consoleName, metav1.GetOptions{}) if err != nil { if k8serrors.IsNotFound(err) { @@ -147,6 +165,60 @@ func ConsoleRouteReady(ctx context.Context, dynamicClient dynamic.Interface, nam return false, reason, nil } +// consoleOpenShiftRouteReady reports the admission state of the console Route. +func consoleOpenShiftRouteReady(ctx context.Context, dynamicClient dynamic.Interface, namespace string) (ready bool, reason string, err error) { + route, err := dynamicClient.Resource(consoleOpenShiftRouteGVR).Namespace(namespace).Get(ctx, consoleName, metav1.GetOptions{}) + if err != nil { + if k8serrors.IsNotFound(err) { + return false, "console OpenShift Route not found", nil + } + return false, "", fmt.Errorf("get console OpenShift Route %s/%s: %w", namespace, consoleName, err) + } + + ingress, found, err := unstructured.NestedSlice(route.Object, "status", "ingress") + if err != nil { + return false, "", fmt.Errorf("read console OpenShift Route %s/%s status: %w", namespace, consoleName, err) + } + if !found || len(ingress) == 0 { + return false, "console OpenShift Route has no router status yet", nil + } + + const noAdmissionReason = "console OpenShift Route has no Admitted condition" + reason = noAdmissionReason + for _, item := range ingress { + router, ok := item.(map[string]interface{}) + if !ok { + continue + } + conditions, _, nestedErr := unstructured.NestedSlice(router, "conditions") + if nestedErr != nil { + return false, "", fmt.Errorf("read console OpenShift Route %s/%s ingress conditions: %w", namespace, consoleName, nestedErr) + } + for _, item := range conditions { + condition, ok := item.(map[string]interface{}) + if !ok { + continue + } + conditionType, _, _ := unstructured.NestedString(condition, "type") + if conditionType != "Admitted" { + continue + } + status, _, _ := unstructured.NestedString(condition, "status") + if status == "True" { + return true, "", nil + } + rejectionReason, _, _ := unstructured.NestedString(condition, "reason") + if rejectionReason != "" { + reason = fmt.Sprintf("console OpenShift Route not admitted: %s", rejectionReason) + } else if reason == noAdmissionReason { + reason = "console OpenShift Route not admitted: router rejected the route" + } + } + } + + return false, reason, nil +} + // routeConditionState returns whether the named HTTPRoute parent condition is // True, along with its Reason when it is not True (empty when the condition is // absent). @@ -180,13 +252,13 @@ func consoleListenerName() string { } // ReconcileConsole idempotently reconciles the per-gateway console (Keycloak -// client, credential Secret, Deployment, Service, HTTPRoute, NetworkPolicies) -// for an already-provisioned routed gateway. It exists so the continuous health -// reconciler can self-heal the console independently of the provisioning phase -// gate: a console failure is deliberately non-fatal to the gateway, so once the -// gateway reaches Running the provisioning path never runs again and a transient -// console failure (or later drift, e.g. a deleted HTTPRoute) would otherwise -// never be retried. It is a thin, exported wrapper over reconcileConsole using +// client, credential Secret, Deployment, Service, selected exposure, and +// NetworkPolicies) for an already-provisioned routed gateway. It exists so the +// continuous health reconciler can self-heal the console independently of the +// provisioning phase gate. A console failure does not stop the gateway. Thus, +// the provisioning path does not run again after the gateway reaches Running. +// The health reconciler repairs a transient failure or later drift. This +// function is a thin, exported wrapper over reconcileConsole that uses // the default image set; callers pass the same ReconcileOpts fields the console // reads (Keycloak, GatewayName, GatewayID, IsOpenShift, SkipNetworkPolicies). func ReconcileConsole(ctx context.Context, dynamicClient dynamic.Interface, clientset *kubernetes.Clientset, nsConfig NamespaceConfig, opts ReconcileOpts) error { @@ -198,25 +270,21 @@ func ReconcileConsole(ctx context.Context, dynamicClient dynamic.Interface, clie } // DeleteConsole removes the per-gateway console (Keycloak client, credential -// Secret, Deployment, Service, HTTPRoute, NetworkPolicies) and clears the stored -// console_address via opts.UpdateConsoleAddress. It is the exported counterpart -// to ReconcileConsole, letting the continuous health reconciler reconcile the -// console's desired *absence* independently of the provisioning phase gate: when -// a Running gateway's route is removed the provisioning path never runs again -// (see the reconciler's phase gate), so without this the console and its -// Keycloak client would leak. Idempotent -- absent resources are ignored -- so it -// is safe to call on every health tick. It returns the joined errors of every -// deletion that failed (an empty console yields nil) so callers can retry until -// the console is actually absent rather than stopping on partial cleanup. +// Secret, Deployment, Service, both exposure kinds, and NetworkPolicies). It +// clears the stored console_address through opts.UpdateConsoleAddress. It is the +// exported counterpart to ReconcileConsole. The continuous health reconciler +// can remove the console independently of the provisioning phase gate. This +// prevents a console and its Keycloak client from remaining after route removal. +// The function ignores absent resources, so each health check can call it. It +// returns all deletion errors so callers can retry partial cleanup. func DeleteConsole(ctx context.Context, dynamicClient dynamic.Interface, clientset *kubernetes.Clientset, namespace string, opts ReconcileOpts) error { return deleteConsole(ctx, dynamicClient, clientset, namespace, opts) } // reconcileConsole deploys the per-gateway OpenShell dashboard and its -// oauth2-proxy sidecar. It runs only from the Gateway API pass (route enabled + -// Gateway API available), so console lifecycle follows the route. Missing -// prerequisites (Keycloak, base domain) are logged and skipped without failing -// the gateway reconciliation. +// oauth2-proxy sidecar. It uses the selected gateway ingress mode for the +// console exposure. Missing prerequisites are logged and skipped without a +// gateway reconciliation failure. // // The Keycloak work (client, mappers, secret) is treated as one atomic step: // ProvisionConsoleClient rolls the client back if mapper creation fails, and a @@ -225,6 +293,11 @@ func DeleteConsole(ctx context.Context, dynamicClient dynamic.Interface, clients func reconcileConsole(ctx context.Context, dynamicClient dynamic.Interface, clientset *kubernetes.Clientset, nsConfig NamespaceConfig, opts ReconcileOpts, images ImageDefaults) error { namespace := nsConfig.Name + ingressMode := gatewayIngressMode(opts) + if ingressMode == IngressModeNone { + log.Printf("WARN console: no ingress mode selected; skipping console for namespace %s", namespace) + return nil + } if opts.Keycloak == nil { log.Printf("WARN console: Keycloak not configured; skipping console for namespace %s", namespace) return nil @@ -322,8 +395,8 @@ func reconcileConsole(ctx context.Context, dynamicClient dynamic.Interface, clie return fmt.Errorf("reconcile console Service in %s: %w", namespace, err) } - if err := reconcileResource(ctx, dynamicClient, buildConsoleHTTPRoute(namespace, host)); err != nil { - return fmt.Errorf("reconcile console HTTPRoute in %s: %w", namespace, err) + if err := reconcileConsoleExposure(ctx, dynamicClient, namespace, host, ingressMode); err != nil { + return fmt.Errorf("reconcile console exposure in %s: %w", namespace, err) } if opts.SkipNetworkPolicies { @@ -705,6 +778,85 @@ func buildConsoleHTTPRoute(namespace, host string) *unstructured.Unstructured { } } +// buildConsoleOpenShiftRoute builds the edge-terminated Route for the console. +// The OpenShift router sends HTTP traffic to the named Service port. +func buildConsoleOpenShiftRoute(namespace, host string) *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "route.openshift.io/v1", + "kind": "Route", + "metadata": map[string]interface{}{ + "name": consoleName, + "namespace": namespace, + "labels": consoleLabelsAny(), + }, + "spec": map[string]interface{}{ + "host": host, + "to": map[string]interface{}{ + "kind": "Service", + "name": consoleName, + }, + "port": map[string]interface{}{ + "targetPort": "http", + }, + "tls": map[string]interface{}{ + "termination": "edge", + "insecureEdgeTerminationPolicy": "Redirect", + }, + }, + }, + } +} + +// reconcileConsoleExposure creates only the exposure for the selected mode. +// It first removes the inactive exposure to prevent two controllers from +// claiming the same host during a mode change. +func reconcileConsoleExposure(ctx context.Context, dynamicClient dynamic.Interface, namespace, host, ingressMode string) error { + var desired *unstructured.Unstructured + var inactiveGVR schema.GroupVersionResource + var inactiveKind string + + switch ingressMode { + case IngressModeGatewayAPI: + desired = buildConsoleHTTPRoute(namespace, host) + inactiveGVR = consoleOpenShiftRouteGVR + inactiveKind = "OpenShift Route" + case IngressModeRoute: + desired = buildConsoleOpenShiftRoute(namespace, host) + inactiveGVR = consoleHTTPRouteGVR + inactiveKind = "HTTPRoute" + default: + return fmt.Errorf("unsupported console ingress mode %q", ingressMode) + } + + if err := deleteConsoleExposureResource(ctx, dynamicClient, namespace, inactiveGVR, inactiveKind); err != nil { + return fmt.Errorf("remove inactive exposure: %w", err) + } + if err := reconcileResource(ctx, dynamicClient, desired); err != nil { + return fmt.Errorf("reconcile selected %s exposure: %w", ingressMode, err) + } + return nil +} + +// deleteConsoleExposureResource deletes one console exposure. An absent +// resource is already converged and does not cause an error. +func deleteConsoleExposureResource(ctx context.Context, dynamicClient dynamic.Interface, namespace string, gvr schema.GroupVersionResource, kind string) error { + err := dynamicClient.Resource(gvr).Namespace(namespace).Delete(ctx, consoleName, metav1.DeleteOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return fmt.Errorf("delete console %s in %s: %w", kind, namespace, err) + } + return nil +} + +// deleteConsoleExposures removes both exposure kinds. It attempts both deletes +// so cleanup can converge after an ingress mode change. +func deleteConsoleExposures(ctx context.Context, dynamicClient dynamic.Interface, namespace string) error { + return errors.Join( + deleteConsoleExposureResource(ctx, dynamicClient, namespace, consoleHTTPRouteGVR, "HTTPRoute"), + deleteConsoleExposureResource(ctx, dynamicClient, namespace, consoleOpenShiftRouteGVR, "OpenShift Route"), + ) +} + // buildConsoleNetworkPolicies builds the two console NetworkPolicies: ingress to // oauth2-proxy from the shared Gateway namespace, and ingress to the gateway pod // from the console pod. @@ -806,9 +958,8 @@ func deleteConsole(ctx context.Context, dynamicClient dynamic.Interface, clients errs = append(errs, fmt.Errorf("delete console Service in %s: %w", namespace, err)) } - httpRouteGVR := schema.GroupVersionResource{Group: "gateway.networking.k8s.io", Version: "v1", Resource: "httproutes"} - if err := dynamicClient.Resource(httpRouteGVR).Namespace(namespace).Delete(ctx, consoleName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) { - errs = append(errs, fmt.Errorf("delete console HTTPRoute in %s: %w", namespace, err)) + if err := deleteConsoleExposures(ctx, dynamicClient, namespace); err != nil { + errs = append(errs, err) } netpolGVR := schema.GroupVersionResource{Group: "networking.k8s.io", Version: "v1", Resource: "networkpolicies"} diff --git a/components/control-plane/internal/gateway/console_test.go b/components/control-plane/internal/gateway/console_test.go index 26e76fdc1..5904b8ebf 100644 --- a/components/control-plane/internal/gateway/console_test.go +++ b/components/control-plane/internal/gateway/console_test.go @@ -7,6 +7,7 @@ import ( "testing" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -158,9 +159,8 @@ func TestConsoleDeployment_OpenShiftOverrideStripsFsGroup(t *testing.T) { } } -// consoleRouteWithConditions builds a console HTTPRoute unstructured object whose -// single parent reports the given Accepted/ResolvedRefs condition statuses. -func consoleRouteWithConditions(namespace string, conditions []interface{}) *unstructured.Unstructured { +// consoleHTTPRouteWithConditions builds a console HTTPRoute with one parent. +func consoleHTTPRouteWithConditions(namespace string, conditions []interface{}) *unstructured.Unstructured { route := &unstructured.Unstructured{} route.SetGroupVersionKind(consoleHTTPRouteGVR.GroupVersion().WithKind("HTTPRoute")) route.SetNamespace(namespace) @@ -176,6 +176,23 @@ func consoleRouteWithConditions(namespace string, conditions []interface{}) *uns return route } +// consoleOpenShiftRouteWithConditions builds a console Route with one router. +func consoleOpenShiftRouteWithConditions(namespace string, conditions []interface{}) *unstructured.Unstructured { + route := &unstructured.Unstructured{} + route.SetGroupVersionKind(consoleOpenShiftRouteGVR.GroupVersion().WithKind("Route")) + route.SetNamespace(namespace) + route.SetName(consoleName) + if conditions != nil { + _ = unstructured.SetNestedSlice(route.Object, []interface{}{ + map[string]interface{}{ + "host": "console.example.com", + "conditions": conditions, + }, + }, "status", "ingress") + } + return route +} + func routeCondition(condType, status, reason string) map[string]interface{} { return map[string]interface{}{ "type": condType, @@ -187,7 +204,8 @@ func routeCondition(condType, status, reason string) map[string]interface{} { func newConsoleRouteDynamicClient(objs ...runtime.Object) *dynamicfake.FakeDynamicClient { scheme := runtime.NewScheme() gvrToListKind := map[schema.GroupVersionResource]string{ - consoleHTTPRouteGVR: "HTTPRouteList", + consoleHTTPRouteGVR: "HTTPRouteList", + consoleOpenShiftRouteGVR: "RouteList", } return dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, objs...) } @@ -196,18 +214,18 @@ func newConsoleRouteDynamicClient(objs ...runtime.Object) *dynamicfake.FakeDynam // missing or misnamed HTTP listener on the shared Gateway). Publishing // console_address then yields a dead "Open console" link, so readiness must // require the route to be Accepted AND its backend refs resolved. -func TestConsoleRouteReady(t *testing.T) { +func TestConsoleExposureReadyHTTPRoute(t *testing.T) { ctx := context.Background() const ns = "openshell-abc" t.Run("accepted and resolved is ready", func(t *testing.T) { - client := newConsoleRouteDynamicClient(consoleRouteWithConditions(ns, []interface{}{ + client := newConsoleRouteDynamicClient(consoleHTTPRouteWithConditions(ns, []interface{}{ routeCondition("Accepted", "True", "Accepted"), routeCondition("ResolvedRefs", "True", "ResolvedRefs"), })) - ready, reason, err := ConsoleRouteReady(ctx, client, ns) + ready, reason, err := ConsoleExposureReady(ctx, client, ns, IngressModeGatewayAPI) if err != nil { - t.Fatalf("ConsoleRouteReady: %v", err) + t.Fatalf("ConsoleExposureReady: %v", err) } if !ready { t.Errorf("expected ready; reason=%q", reason) @@ -215,13 +233,13 @@ func TestConsoleRouteReady(t *testing.T) { }) t.Run("rejected listener is not ready and reports reason", func(t *testing.T) { - client := newConsoleRouteDynamicClient(consoleRouteWithConditions(ns, []interface{}{ + client := newConsoleRouteDynamicClient(consoleHTTPRouteWithConditions(ns, []interface{}{ routeCondition("Accepted", "False", "NoMatchingListenerHostname"), routeCondition("ResolvedRefs", "True", "ResolvedRefs"), })) - ready, reason, err := ConsoleRouteReady(ctx, client, ns) + ready, reason, err := ConsoleExposureReady(ctx, client, ns, IngressModeGatewayAPI) if err != nil { - t.Fatalf("ConsoleRouteReady: %v", err) + t.Fatalf("ConsoleExposureReady: %v", err) } if ready { t.Error("expected not ready when the route is not Accepted") @@ -232,13 +250,13 @@ func TestConsoleRouteReady(t *testing.T) { }) t.Run("unresolved backend refs is not ready", func(t *testing.T) { - client := newConsoleRouteDynamicClient(consoleRouteWithConditions(ns, []interface{}{ + client := newConsoleRouteDynamicClient(consoleHTTPRouteWithConditions(ns, []interface{}{ routeCondition("Accepted", "True", "Accepted"), routeCondition("ResolvedRefs", "False", "BackendNotFound"), })) - ready, reason, err := ConsoleRouteReady(ctx, client, ns) + ready, reason, err := ConsoleExposureReady(ctx, client, ns, IngressModeGatewayAPI) if err != nil { - t.Fatalf("ConsoleRouteReady: %v", err) + t.Fatalf("ConsoleExposureReady: %v", err) } if ready { t.Error("expected not ready when backend refs are unresolved") @@ -249,10 +267,10 @@ func TestConsoleRouteReady(t *testing.T) { }) t.Run("no parent status yet is not ready", func(t *testing.T) { - client := newConsoleRouteDynamicClient(consoleRouteWithConditions(ns, nil)) - ready, _, err := ConsoleRouteReady(ctx, client, ns) + client := newConsoleRouteDynamicClient(consoleHTTPRouteWithConditions(ns, nil)) + ready, _, err := ConsoleExposureReady(ctx, client, ns, IngressModeGatewayAPI) if err != nil { - t.Fatalf("ConsoleRouteReady: %v", err) + t.Fatalf("ConsoleExposureReady: %v", err) } if ready { t.Error("expected not ready when the route has no parent status") @@ -261,9 +279,9 @@ func TestConsoleRouteReady(t *testing.T) { t.Run("missing route is not ready without error", func(t *testing.T) { client := newConsoleRouteDynamicClient() - ready, reason, err := ConsoleRouteReady(ctx, client, ns) + ready, reason, err := ConsoleExposureReady(ctx, client, ns, IngressModeGatewayAPI) if err != nil { - t.Fatalf("ConsoleRouteReady on missing route should not error: %v", err) + t.Fatalf("ConsoleExposureReady on missing route should not error: %v", err) } if ready { t.Error("expected not ready when the console HTTPRoute is absent") @@ -274,6 +292,221 @@ func TestConsoleRouteReady(t *testing.T) { }) } +func TestConsoleExposureReadyOpenShiftRoute(t *testing.T) { + ctx := context.Background() + const ns = "openshell-abc" + + t.Run("admitted route is ready", func(t *testing.T) { + client := newConsoleRouteDynamicClient(consoleOpenShiftRouteWithConditions(ns, []interface{}{ + routeCondition("Admitted", "True", "RouteAdmitted"), + })) + ready, reason, err := ConsoleExposureReady(ctx, client, ns, IngressModeRoute) + if err != nil { + t.Fatalf("ConsoleExposureReady: %v", err) + } + if !ready { + t.Errorf("expected ready; reason=%q", reason) + } + }) + + t.Run("rejected route reports the admission reason", func(t *testing.T) { + client := newConsoleRouteDynamicClient(consoleOpenShiftRouteWithConditions(ns, []interface{}{ + routeCondition("Admitted", "False", "HostAlreadyClaimed"), + })) + ready, reason, err := ConsoleExposureReady(ctx, client, ns, IngressModeRoute) + if err != nil { + t.Fatalf("ConsoleExposureReady: %v", err) + } + if ready { + t.Fatal("expected a rejected Route to be not ready") + } + if !strings.Contains(reason, "HostAlreadyClaimed") { + t.Errorf("reason = %q, want the admission reason", reason) + } + }) + + t.Run("one admitted router makes the route ready", func(t *testing.T) { + route := consoleOpenShiftRouteWithConditions(ns, []interface{}{ + routeCondition("Admitted", "False", "HostAlreadyClaimed"), + }) + _ = unstructured.SetNestedSlice(route.Object, []interface{}{ + map[string]interface{}{ + "host": "console.example.com", + "conditions": []interface{}{ + routeCondition("Admitted", "False", "HostAlreadyClaimed"), + }, + }, + map[string]interface{}{ + "host": "console.example.com", + "conditions": []interface{}{ + routeCondition("Admitted", "True", "RouteAdmitted"), + }, + }, + }, "status", "ingress") + + client := newConsoleRouteDynamicClient(route) + ready, reason, err := ConsoleExposureReady(ctx, client, ns, IngressModeRoute) + if err != nil { + t.Fatalf("ConsoleExposureReady: %v", err) + } + if !ready { + t.Errorf("expected ready; reason=%q", reason) + } + }) + + t.Run("a generic rejection keeps an earlier specific reason", func(t *testing.T) { + route := consoleOpenShiftRouteWithConditions(ns, nil) + _ = unstructured.SetNestedSlice(route.Object, []interface{}{ + map[string]interface{}{ + "conditions": []interface{}{ + routeCondition("Admitted", "False", "HostAlreadyClaimed"), + }, + }, + map[string]interface{}{ + "conditions": []interface{}{ + routeCondition("Admitted", "False", ""), + }, + }, + }, "status", "ingress") + + client := newConsoleRouteDynamicClient(route) + ready, reason, err := ConsoleExposureReady(ctx, client, ns, IngressModeRoute) + if err != nil { + t.Fatalf("ConsoleExposureReady: %v", err) + } + if ready { + t.Fatal("expected a rejected Route to be not ready") + } + if !strings.Contains(reason, "HostAlreadyClaimed") { + t.Errorf("reason = %q, want the specific admission reason", reason) + } + }) + + t.Run("route without router status is not ready", func(t *testing.T) { + client := newConsoleRouteDynamicClient(consoleOpenShiftRouteWithConditions(ns, nil)) + ready, reason, err := ConsoleExposureReady(ctx, client, ns, IngressModeRoute) + if err != nil { + t.Fatalf("ConsoleExposureReady: %v", err) + } + if ready || reason == "" { + t.Errorf("ready = %v, reason = %q; want not ready with a reason", ready, reason) + } + }) + + t.Run("missing route is not ready", func(t *testing.T) { + client := newConsoleRouteDynamicClient() + ready, reason, err := ConsoleExposureReady(ctx, client, ns, IngressModeRoute) + if err != nil { + t.Fatalf("ConsoleExposureReady: %v", err) + } + if ready || reason == "" { + t.Errorf("ready = %v, reason = %q; want not ready with a reason", ready, reason) + } + }) +} + +func TestBuildConsoleOpenShiftRoute(t *testing.T) { + const ( + namespace = "openshell-abc" + host = "console-openshell-abc.apps.example.com" + ) + route := buildConsoleOpenShiftRoute(namespace, host) + + if route.GetAPIVersion() != "route.openshift.io/v1" || route.GetKind() != "Route" { + t.Errorf("route GVK = %s %s, want route.openshift.io/v1 Route", route.GetAPIVersion(), route.GetKind()) + } + if route.GetName() != consoleName || route.GetNamespace() != namespace { + t.Errorf("route identity = %s/%s, want %s/%s", route.GetNamespace(), route.GetName(), namespace, consoleName) + } + if got := route.GetLabels()["app.kubernetes.io/component"]; got != "console" { + t.Errorf("component label = %q, want console", got) + } + + checks := []struct { + name string + path []string + want string + }{ + {name: "host", path: []string{"spec", "host"}, want: host}, + {name: "service kind", path: []string{"spec", "to", "kind"}, want: "Service"}, + {name: "service name", path: []string{"spec", "to", "name"}, want: consoleName}, + {name: "target port", path: []string{"spec", "port", "targetPort"}, want: "http"}, + {name: "TLS termination", path: []string{"spec", "tls", "termination"}, want: "edge"}, + {name: "insecure policy", path: []string{"spec", "tls", "insecureEdgeTerminationPolicy"}, want: "Redirect"}, + } + for _, check := range checks { + got, found, err := unstructured.NestedString(route.Object, check.path...) + if err != nil || !found || got != check.want { + t.Errorf("%s = %q, found=%v, err=%v; want %q", check.name, got, found, err, check.want) + } + } +} + +func TestReconcileConsoleExposureSelectsOneMode(t *testing.T) { + const ( + namespace = "openshell-abc" + host = "console-openshell-abc.apps.example.com" + ) + + tests := []struct { + name string + mode string + desiredGVR schema.GroupVersionResource + inactiveGVR schema.GroupVersionResource + inactive runtime.Object + }{ + { + name: "Gateway API mode", + mode: IngressModeGatewayAPI, + desiredGVR: consoleHTTPRouteGVR, + inactiveGVR: consoleOpenShiftRouteGVR, + inactive: buildConsoleOpenShiftRoute(namespace, host), + }, + { + name: "OpenShift Route mode", + mode: IngressModeRoute, + desiredGVR: consoleOpenShiftRouteGVR, + inactiveGVR: consoleHTTPRouteGVR, + inactive: buildConsoleHTTPRoute(namespace, host), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := newConsoleRouteDynamicClient(test.inactive) + for pass := 1; pass <= 2; pass++ { + if err := reconcileConsoleExposure(context.Background(), client, namespace, host, test.mode); err != nil { + t.Fatalf("reconcileConsoleExposure pass %d: %v", pass, err) + } + } + + if _, err := client.Resource(test.desiredGVR).Namespace(namespace).Get(context.Background(), consoleName, metav1.GetOptions{}); err != nil { + t.Errorf("selected exposure is absent: %v", err) + } + if _, err := client.Resource(test.inactiveGVR).Namespace(namespace).Get(context.Background(), consoleName, metav1.GetOptions{}); !k8serrors.IsNotFound(err) { + t.Errorf("inactive exposure still exists: %v", err) + } + }) + } +} + +func TestDeleteConsoleExposuresDeletesBothKinds(t *testing.T) { + const namespace = "openshell-abc" + client := newConsoleRouteDynamicClient( + buildConsoleHTTPRoute(namespace, "console.example.com"), + buildConsoleOpenShiftRoute(namespace, "console.example.com"), + ) + + if err := deleteConsoleExposures(context.Background(), client, namespace); err != nil { + t.Fatalf("deleteConsoleExposures: %v", err) + } + for _, gvr := range []schema.GroupVersionResource{consoleHTTPRouteGVR, consoleOpenShiftRouteGVR} { + if _, err := client.Resource(gvr).Namespace(namespace).Get(context.Background(), consoleName, metav1.GetOptions{}); !k8serrors.IsNotFound(err) { + t.Errorf("%s exposure still exists: %v", gvr.Resource, err) + } + } +} + // In production the issuer is publicly trusted and the trusted-CA ConfigMap is // absent, so the sidecar must not reference a CA file or mount that would fail // to bind. diff --git a/components/control-plane/internal/gateway/reconciler.go b/components/control-plane/internal/gateway/reconciler.go index 46a60137d..5a441b06a 100644 --- a/components/control-plane/internal/gateway/reconciler.go +++ b/components/control-plane/internal/gateway/reconciler.go @@ -57,6 +57,7 @@ func ReconcileGateway( if images == nil { images = StaticImageDefaults{} } + ingressMode := gatewayIngressMode(opts) if !namespaceExists(ctx, clientset, nsConfig.Name) { if err := createNamespace(ctx, clientset, nsConfig.Name); err != nil { @@ -76,7 +77,7 @@ func ReconcileGateway( // hostname as a SAN or clients fail verification. The controller derives // that hostname, so it -- not the operator -- injects it into the cert SANs // here, before cert-manager mints the certificate below. - if mode := gatewayIngressMode(opts); mode != IngressModeNone && nsConfig.Gateway.Route.Enabled { + if ingressMode != IngressModeNone && nsConfig.Gateway.Route.Enabled { hostname, err := deriveGatewayHostname(nsConfig) if err != nil { log.Printf("WARN cannot add ingress hostname to gateway certificate SANs in %s: %v", nsConfig.Name, err) @@ -156,7 +157,7 @@ func ReconcileGateway( // Tenant ingress is environment-adaptive: Gateway API where available, // OpenShift Routes where it is not. See gatewayIngressMode. - switch mode := gatewayIngressMode(opts); mode { + switch ingressMode { case IngressModeGatewayAPI: if nsConfig.Gateway.Route.Enabled { // Propagate this error rather than logging and swallowing it: the only @@ -181,8 +182,14 @@ func ReconcileGateway( if err := reconcileRouteResources(ctx, dynamicClient, nsConfig, opts); err != nil { log.Printf("WARN failed to reconcile Route resources in %s: %v", nsConfig.Name, err) } + // The console uses the same selected ingress mode as the gateway. A + // console error must not fail gateway provisioning. The health loop + // retries the console until it can serve. + if err := ReconcileConsole(ctx, dynamicClient, clientset, nsConfig, opts); err != nil { + log.Printf("WARN failed to reconcile console in %s: %v", nsConfig.Name, err) + } } else { - if err := deleteRouteResources(ctx, dynamicClient, nsConfig.Name, opts); err != nil { + if err := DeleteRouteResources(ctx, dynamicClient, clientset, nsConfig.Name, opts); err != nil { log.Printf("WARN failed to remove Route resources in %s: %v", nsConfig.Name, err) } } @@ -427,14 +434,11 @@ type ConsoleClientChecker interface { ConsoleClientExists(ctx context.Context, consoleClientID string) (bool, error) } -// RouteResourcesAbsent reports whether every route- and console-owned resource -// this control plane creates for a routed gateway is absent -- both the -// namespaced Kubernetes objects and the external Keycloak console client. It -// lets the health loop's route teardown converge on the gateway's actual -// observed state rather than trusting a cached completion marker: a stale -// provisioning pass can recreate these resources after a teardown believed -// itself finished, and cleared address fields do not prove the resources are -// gone. Unknown state must never be read as absence. +// RouteResourcesAbsent reports whether the resources for the selected gateway +// ingress mode and all console resources are absent. It checks both console +// exposure kinds because cleanup removes an inactive exposure after a mode +// change. It also checks the external Keycloak console client. Unknown state +// must never be read as absence. // // It returns (true, nil) only when every probed resource is confirmed absent. // The first resource found present short-circuits to (false, nil). Any probe @@ -442,24 +446,42 @@ type ConsoleClientChecker interface { // so the caller treats absence as unconfirmed (and re-runs teardown) rather than // trusting an unknown state. The Keycloak client probe is skipped when // consoleClient is nil or consoleClientID is empty (no Keycloak configured). -func RouteResourcesAbsent(ctx context.Context, dynamicClient dynamic.Interface, clientset kubernetes.Interface, namespace string, consoleClient ConsoleClientChecker, consoleClientID string) (bool, error) { +func RouteResourcesAbsent(ctx context.Context, dynamicClient dynamic.Interface, clientset kubernetes.Interface, namespace, ingressMode string, consoleClient ConsoleClientChecker, consoleClientID string) (bool, error) { grpcRouteGVR := schema.GroupVersionResource{Group: "gateway.networking.k8s.io", Version: "v1", Resource: "grpcroutes"} btlsGVR := schema.GroupVersionResource{Group: "gateway.networking.k8s.io", Version: "v1", Resource: "backendtlspolicies"} httpRouteGVR := schema.GroupVersionResource{Group: "gateway.networking.k8s.io", Version: "v1", Resource: "httproutes"} + openShiftRouteGVR := schema.GroupVersionResource{Group: "route.openshift.io", Version: "v1", Resource: "routes"} netpolGVR := schema.GroupVersionResource{Group: "networking.k8s.io", Version: "v1", Resource: "networkpolicies"} - dynamicProbes := []struct { + type dynamicProbe struct { gvr schema.GroupVersionResource name string - }{ - {grpcRouteGVR, "openshell-gateway"}, - {btlsGVR, "openshell-gateway"}, - {netpolGVR, "openshell-gateway-allow-router"}, - {httpRouteGVR, consoleName}, - {schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}, consoleName}, - {netpolGVR, "openshell-console-allow-router"}, - {netpolGVR, "openshell-gateway-allow-console"}, } + var dynamicProbes []dynamicProbe + switch ingressMode { + case IngressModeGatewayAPI: + dynamicProbes = append(dynamicProbes, + dynamicProbe{grpcRouteGVR, "openshell-gateway"}, + dynamicProbe{btlsGVR, "openshell-gateway"}, + dynamicProbe{netpolGVR, "openshell-gateway-allow-router"}, + ) + case IngressModeRoute: + dynamicProbes = append(dynamicProbes, + dynamicProbe{openShiftRouteGVR, "openshell-gateway"}, + dynamicProbe{netpolGVR, "openshell-gateway-allow-router"}, + ) + case IngressModeNone: + // There is no selected gateway exposure to probe. + default: + return false, fmt.Errorf("unsupported ingress mode %q for resource absence probe", ingressMode) + } + dynamicProbes = append(dynamicProbes, + dynamicProbe{httpRouteGVR, consoleName}, + dynamicProbe{openShiftRouteGVR, consoleName}, + dynamicProbe{schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}, consoleName}, + dynamicProbe{netpolGVR, "openshell-console-allow-router"}, + dynamicProbe{netpolGVR, "openshell-gateway-allow-console"}, + ) for _, p := range dynamicProbes { if _, err := dynamicClient.Resource(p.gvr).Namespace(namespace).Get(ctx, p.name, metav1.GetOptions{}); err == nil { return false, nil @@ -468,10 +490,12 @@ func RouteResourcesAbsent(ctx context.Context, dynamicClient dynamic.Interface, } } - if _, err := clientset.CoreV1().ConfigMaps(namespace).Get(ctx, "openshell-backend-ca", metav1.GetOptions{}); err == nil { - return false, nil - } else if !k8serrors.IsNotFound(err) { - return false, fmt.Errorf("probe configmap openshell-backend-ca in %s: %w", namespace, err) + if ingressMode == IngressModeGatewayAPI { + if _, err := clientset.CoreV1().ConfigMaps(namespace).Get(ctx, "openshell-backend-ca", metav1.GetOptions{}); err == nil { + return false, nil + } else if !k8serrors.IsNotFound(err) { + return false, fmt.Errorf("probe configmap openshell-backend-ca in %s: %w", namespace, err) + } } if _, err := clientset.CoreV1().Services(namespace).Get(ctx, consoleName, metav1.GetOptions{}); err == nil { return false, nil @@ -622,14 +646,20 @@ func reconcileRouteResources(ctx context.Context, dynamicClient dynamic.Interfac return nil } -func deleteRouteResources(ctx context.Context, dynamicClient dynamic.Interface, namespace string, opts ReconcileOpts) error { +// DeleteRouteResources removes the OpenShift gateway Route, the router +// NetworkPolicy, and all console resources. It also clears the stored route +// address. It attempts all operations and returns all errors so the health loop +// can retry incomplete cleanup. +func DeleteRouteResources(ctx context.Context, dynamicClient dynamic.Interface, clientset *kubernetes.Clientset, namespace string, opts ReconcileOpts) error { + var errs []error + routeGVR := schema.GroupVersionResource{ Group: "route.openshift.io", Version: "v1", Resource: "routes", } if err := dynamicClient.Resource(routeGVR).Namespace(namespace).Delete(ctx, "openshell-gateway", metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) { - log.Printf("WARN failed to delete Route: %v", err) + errs = append(errs, fmt.Errorf("delete gateway Route in %s: %w", namespace, err)) } netpolGVR := schema.GroupVersionResource{ @@ -638,19 +668,25 @@ func deleteRouteResources(ctx context.Context, dynamicClient dynamic.Interface, Resource: "networkpolicies", } if err := dynamicClient.Resource(netpolGVR).Namespace(namespace).Delete(ctx, "openshell-gateway-allow-router", metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) { - log.Printf("WARN failed to delete router NetworkPolicy: %v", err) + errs = append(errs, fmt.Errorf("delete router NetworkPolicy in %s: %w", namespace, err)) + } + + if err := DeleteConsole(ctx, dynamicClient, clientset, namespace, opts); err != nil { + errs = append(errs, err) } if opts.UpdateRouteAddress != nil { if err := opts.UpdateRouteAddress(ctx, ""); err != nil { - log.Printf("WARN failed to clear routeAddress for gateway in %s: %v", namespace, err) + errs = append(errs, fmt.Errorf("clear routeAddress in %s: %w", namespace, err)) } else { log.Printf("INFO cleared routeAddress for gateway in %s", namespace) } } - log.Printf("INFO Route resources removed from namespace %s", namespace) - return nil + if len(errs) == 0 { + log.Printf("INFO Route resources removed from namespace %s", namespace) + } + return errors.Join(errs...) } func NamespaceExists(ctx context.Context, clientset kubernetes.Interface, namespace string) bool { diff --git a/components/control-plane/internal/gateway/reconciler_test.go b/components/control-plane/internal/gateway/reconciler_test.go index b68d3d9dc..cb1dbdf97 100644 --- a/components/control-plane/internal/gateway/reconciler_test.go +++ b/components/control-plane/internal/gateway/reconciler_test.go @@ -24,6 +24,7 @@ func routeResourceListKinds() map[schema.GroupVersionResource]string { {Group: "gateway.networking.k8s.io", Version: "v1", Resource: "grpcroutes"}: "GRPCRouteList", {Group: "gateway.networking.k8s.io", Version: "v1", Resource: "backendtlspolicies"}: "BackendTLSPolicyList", {Group: "gateway.networking.k8s.io", Version: "v1", Resource: "httproutes"}: "HTTPRouteList", + {Group: "route.openshift.io", Version: "v1", Resource: "routes"}: "RouteList", {Group: "networking.k8s.io", Version: "v1", Resource: "networkpolicies"}: "NetworkPolicyList", {Group: "apps", Version: "v1", Resource: "deployments"}: "DeploymentList", } @@ -39,7 +40,7 @@ func TestRouteResourcesAbsent(t *testing.T) { t.Run("all absent returns true", func(t *testing.T) { dc := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), routeResourceListKinds()) cs := k8sfake.NewSimpleClientset() - absent, err := RouteResourcesAbsent(context.Background(), dc, cs, ns, nil, "") + absent, err := RouteResourcesAbsent(context.Background(), dc, cs, ns, IngressModeGatewayAPI, nil, "") if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -48,12 +49,24 @@ func TestRouteResourcesAbsent(t *testing.T) { } }) + t.Run("all absent in route mode returns true", func(t *testing.T) { + dc := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), routeResourceListKinds()) + cs := k8sfake.NewSimpleClientset() + absent, err := RouteResourcesAbsent(context.Background(), dc, cs, ns, IngressModeRoute, nil, "") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !absent { + t.Fatal("want absent=true when no Route-mode resources exist") + } + }) + t.Run("a resurrected dynamic resource returns false", func(t *testing.T) { dc := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), routeResourceListKinds(), labeledResource("gateway.networking.k8s.io/v1", "GRPCRoute", ns, "openshell-gateway", true), ) cs := k8sfake.NewSimpleClientset() - absent, err := RouteResourcesAbsent(context.Background(), dc, cs, ns, nil, "") + absent, err := RouteResourcesAbsent(context.Background(), dc, cs, ns, IngressModeGatewayAPI, nil, "") if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -62,12 +75,60 @@ func TestRouteResourcesAbsent(t *testing.T) { } }) + t.Run("route mode probes the gateway Route", func(t *testing.T) { + dc := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), routeResourceListKinds(), + labeledResource("route.openshift.io/v1", "Route", ns, "openshell-gateway", true), + ) + cs := k8sfake.NewSimpleClientset() + absent, err := RouteResourcesAbsent(context.Background(), dc, cs, ns, IngressModeRoute, nil, "") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if absent { + t.Fatal("want absent=false when the gateway Route reappeared") + } + }) + + for _, tc := range []struct { + name string + mode string + apiVersion string + kind string + }{ + { + name: "gateway api mode probes an inactive console Route", + mode: IngressModeGatewayAPI, + apiVersion: "route.openshift.io/v1", + kind: "Route", + }, + { + name: "route mode probes an inactive console HTTPRoute", + mode: IngressModeRoute, + apiVersion: "gateway.networking.k8s.io/v1", + kind: "HTTPRoute", + }, + } { + t.Run(tc.name, func(t *testing.T) { + dc := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), routeResourceListKinds(), + labeledResource(tc.apiVersion, tc.kind, ns, consoleName, true), + ) + cs := k8sfake.NewSimpleClientset() + absent, err := RouteResourcesAbsent(context.Background(), dc, cs, ns, tc.mode, nil, "") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if absent { + t.Fatalf("want absent=false when the inactive console %s reappeared", tc.kind) + } + }) + } + t.Run("a resurrected typed resource returns false", func(t *testing.T) { dc := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), routeResourceListKinds()) cs := k8sfake.NewSimpleClientset(&corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "openshell-backend-ca"}, }) - absent, err := RouteResourcesAbsent(context.Background(), dc, cs, ns, nil, "") + absent, err := RouteResourcesAbsent(context.Background(), dc, cs, ns, IngressModeGatewayAPI, nil, "") if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -82,7 +143,7 @@ func TestRouteResourcesAbsent(t *testing.T) { cs.PrependReactor("get", "configmaps", func(k8stesting.Action) (bool, runtime.Object, error) { return true, nil, fmt.Errorf("apiserver unavailable") }) - absent, err := RouteResourcesAbsent(context.Background(), dc, cs, ns, nil, "") + absent, err := RouteResourcesAbsent(context.Background(), dc, cs, ns, IngressModeGatewayAPI, nil, "") if err == nil { t.Fatal("want an error when a probe cannot observe the resource") } @@ -98,7 +159,7 @@ func TestRouteResourcesAbsent(t *testing.T) { dc := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), routeResourceListKinds()) cs := k8sfake.NewSimpleClientset() checker := &fakeConsoleClientChecker{exists: true} - absent, err := RouteResourcesAbsent(context.Background(), dc, cs, ns, checker, "gw-1-console") + absent, err := RouteResourcesAbsent(context.Background(), dc, cs, ns, IngressModeGatewayAPI, checker, "gw-1-console") if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -113,7 +174,7 @@ func TestRouteResourcesAbsent(t *testing.T) { dc := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), routeResourceListKinds()) cs := k8sfake.NewSimpleClientset() checker := &fakeConsoleClientChecker{err: fmt.Errorf("keycloak unreachable")} - absent, err := RouteResourcesAbsent(context.Background(), dc, cs, ns, checker, "gw-1-console") + absent, err := RouteResourcesAbsent(context.Background(), dc, cs, ns, IngressModeGatewayAPI, checker, "gw-1-console") if err == nil { t.Fatal("want an error when the Keycloak client cannot be observed") } @@ -128,7 +189,7 @@ func TestRouteResourcesAbsent(t *testing.T) { dc := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), routeResourceListKinds()) cs := k8sfake.NewSimpleClientset() checker := &fakeConsoleClientChecker{exists: false} - absent, err := RouteResourcesAbsent(context.Background(), dc, cs, ns, checker, "gw-1-console") + absent, err := RouteResourcesAbsent(context.Background(), dc, cs, ns, IngressModeGatewayAPI, checker, "gw-1-console") if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/components/control-plane/internal/reconciler/health.go b/components/control-plane/internal/reconciler/health.go index 6eea92c5e..58fec61ca 100644 --- a/components/control-plane/internal/reconciler/health.go +++ b/components/control-plane/internal/reconciler/health.go @@ -2,6 +2,7 @@ package reconciler import ( "context" + "errors" "fmt" "log" "os" @@ -35,17 +36,9 @@ const defaultListGatewaysPageSize = 100 // routeVerifyInterval is the minimum time between residual route/console // absence re-checks for a settled (torn-down, addressless) gateway. // -// Verification is deliberately NOT bounded to a fixed window after teardown. -// A stale in-flight provisioning pass creates the GRPCRoute before its -// TLS-secret wait and fail-closed route-intent re-check, so elapsed wall-clock -// time is not proof that every stale writer has drained -- a late pass can -// resurrect resources after any wall-clock window would have expired. Instead -// the health loop keeps re-verifying absence indefinitely, but at most once per -// interval, so a settled non-routed gateway costs one cheap absence probe per -// interval (not per tick). That bounds steady-state Keycloak/apiserver traffic -// at fleet scale while never trusting a wall-clock guess that resources stay -// gone. Comfortably larger than the provisioning path's 60s TLS wait plus -// reconcile time, so the steady-state cost is low. +// Verification does not stop after a fixed time. A stale provisioning pass can +// create resources after cleanup. The health loop continues to verify absence, +// but it does this at most once per interval. This limit reduces API traffic. const routeVerifyInterval = 5 * time.Minute // GatewayHealthReconciler continuously observes the health of provisioned @@ -64,6 +57,8 @@ type GatewayHealthReconciler struct { routeReadyTimeout time.Duration keycloakConfig *gateway.KeycloakConfig isOpenShift bool + hasGatewayAPI bool + ingressMode string skipNetworkPolicies bool // consoleClientChecker is a single, long-lived Keycloak client reused across @@ -116,8 +111,11 @@ func NewGatewayHealthReconciler(clientset *kubernetes.Clientset, dynamicClient d keycloakConfig.ClientSecret, ) } - // Mirror GatewayReconciler's environment detection so the health loop's - // console self-heal produces the same resources the provisioning path would. + // Use the same cluster capabilities and mode resolver as the provisioning + // reconciler. This keeps console repair and cleanup on the selected ingress. + isOpenShift := gateway.DetectOpenShift(clientset) + hasGatewayAPI := gateway.DetectGatewayAPI(clientset) + ingressMode := gateway.IngressMode(hasGatewayAPI, isOpenShift) return &GatewayHealthReconciler{ clientset: clientset, dynamicClient: dynamicClient, @@ -127,7 +125,9 @@ func NewGatewayHealthReconciler(clientset *kubernetes.Clientset, dynamicClient d routeReadyTimeout: routeReadyTimeout(), keycloakConfig: keycloakConfig, consoleClientChecker: consoleClientChecker, - isOpenShift: gateway.DetectOpenShift(clientset), + isOpenShift: isOpenShift, + hasGatewayAPI: hasGatewayAPI, + ingressMode: ingressMode, skipNetworkPolicies: os.Getenv("GATEWAY_SKIP_NETWORK_POLICIES") == "true", now: time.Now, routeNotReadySince: make(map[string]time.Time), @@ -150,7 +150,7 @@ func routeReadyTimeout() time.Duration { // Run drives the health reconciliation loop until the context is cancelled. func (h *GatewayHealthReconciler) Run(ctx context.Context) error { - log.Printf("INFO gateway health reconciler started (interval=%s routeReadyTimeout=%s)", h.interval, h.routeReadyTimeout) + log.Printf("INFO gateway health reconciler started (interval=%s routeReadyTimeout=%s ingressMode=%s)", h.interval, h.routeReadyTimeout, h.ingressMode) ticker := time.NewTicker(h.interval) defer ticker.Stop() @@ -231,7 +231,7 @@ func (h *GatewayHealthReconciler) reconcileGatewayHealth(ctx context.Context, cl // Keep the console_address in sync with the console pod's readiness so the web // UI's console button only appears once the console can serve (and disappears // if it later goes unready). Independent of the gateway workload's own phase. - consoleServable := syncConsoleAddress(ctx, h.clientset, h.dynamicClient, client, gatewayID, gw, h.exposure != nil) + consoleServable := syncConsoleAddress(ctx, h.clientset, h.dynamicClient, client, gatewayID, gw, h.ingressMode) // Self-heal the console. A console failure is deliberately non-fatal to the // gateway, so once the gateway reaches Running the provisioning path never runs @@ -244,16 +244,9 @@ func (h *GatewayHealthReconciler) reconcileGatewayHealth(ctx context.Context, cl h.selfHealConsole(ctx, gatewayID, gw) } - // Reconcile the route's desired absence. A gateway whose route was removed - // keeps its route resources (GRPCRoute, BackendTLSPolicy, backend-CA - // ConfigMap, router NetworkPolicy) and its console (Deployment, Service, - // HTTPRoute, Keycloak client), plus the published route_address and - // console_address, until torn down. syncConsoleAddress and selfHealConsole - // both no-op for a non-routed gateway, and the provisioning path never runs - // again for a gateway the health loop owns (phase gate), so this is the only - // place an un-routed gateway's route and console are cleaned up. Clearing the - // torn-down marker while routed lets a later un-routing trigger a fresh - // teardown. + // Remove ingress and console resources when the gateway route is disabled. + // The health loop owns this work after the provisioning phase gate closes. + // Clear the cleanup marker when the route is enabled again. if isRoutedGateway(gw) { h.clearRouteTornDown(gatewayID) } else { @@ -317,11 +310,11 @@ func (h *GatewayHealthReconciler) reconcileGatewayHealth(ctx context.Context, cl // selfHealConsole re-reconciles the per-gateway console when it is observed not // servable, so a console that failed to provision or has since drifted is // recreated without a gateway spec change. It is a no-op unless the gateway is -// routed (its console lifecycle follows the route) and Keycloak is configured. +// routed, an ingress mode is selected, and Keycloak is configured. // The reconcile is idempotent and its failures are logged, never propagated: // they must not perturb the gateway's own health phase. func (h *GatewayHealthReconciler) selfHealConsole(ctx context.Context, gatewayID string, gw *pb.Gateway) { - if h.exposure == nil || !isRoutedGateway(gw) || h.keycloakConfig == nil { + if h.ingressMode == gateway.IngressModeNone || !isRoutedGateway(gw) || h.keycloakConfig == nil { return } namespace, err := gatewayNamespace(gw) @@ -331,6 +324,7 @@ func (h *GatewayHealthReconciler) selfHealConsole(ctx context.Context, gatewayID } opts := gateway.ReconcileOpts{ IsOpenShift: h.isOpenShift, + HasGatewayAPI: h.hasGatewayAPI, SkipNetworkPolicies: h.skipNetworkPolicies, Keycloak: h.keycloakConfig, GatewayID: gatewayID, @@ -343,21 +337,10 @@ func (h *GatewayHealthReconciler) selfHealConsole(ctx context.Context, gatewayID log.Printf("INFO console self-heal reconciled in %s", namespace) } -// teardownRoute reconciles the desired absence of the route -- and the console -// that follows it -- for a gateway the health loop owns that is no longer routed, -// mirroring the provisioning path's route-disabled branch (which the phase gate -// prevents from running again once the gateway is Running). It removes the -// GRPCRoute, BackendTLSPolicy, backend-CA ConfigMap, router NetworkPolicy and all -// console resources + Keycloak client, and clears the published route_address and -// console_address. -// -// It keeps retrying every tick until DeleteGatewayAPIResources reports a clean -// pass (no residual resources, no address left to clear), then records the -// gateway as torn down so subsequent ticks add no delete or Keycloak traffic. A -// teardown that fails part-way is deliberately left unmarked so the next tick -// retries -- teardown must converge on full absence, not stop on partial cleanup. -// Failures are logged, never propagated: route teardown must not perturb the -// gateway's own health phase. +// teardownRoute removes the exposure for the selected ingress mode and removes +// all console resources. It also clears the published addresses. It retries on +// each health tick until cleanup succeeds. A cleanup error does not change the +// gateway health phase. func (h *GatewayHealthReconciler) teardownRoute(ctx context.Context, client pb.GatewayServiceClient, gatewayID string, gw *pb.Gateway) { namespace, err := gatewayNamespace(gw) if err != nil { @@ -373,12 +356,8 @@ func (h *GatewayHealthReconciler) teardownRoute(ctx context.Context, client pb.G // actually absent; if any reappeared -- or absence cannot be confirmed -- // drop the marker and re-run teardown so cleanup converges on real absence. if h.teardownSettled(gatewayID, gw) { - // Re-verify absence at a low, indefinite cadence rather than only within a - // wall-clock window after teardown: a stale provisioning pass can create the - // GRPCRoute before its TLS wait and fail-closed route-intent re-check, so no - // elapsed time proves the resurrection race has drained. Between verifications - // trust the completion marker so a settled non-routed gateway costs at most one - // probe per routeVerifyInterval -- not one per tick -- at fleet scale. + // Continue to verify absence at a low rate. A stale provisioning pass can + // recreate an exposure after cleanup. if !h.dueForVerify(gatewayID) { return } @@ -392,7 +371,7 @@ func (h *GatewayHealthReconciler) teardownRoute(ctx context.Context, client pb.G if h.consoleClientChecker != nil && gw.GetName() != "" && gatewayID != "" { consoleClientID = fmt.Sprintf("%s-%s-console", gw.GetName(), gatewayID) } - absent, perr := gateway.RouteResourcesAbsent(ctx, h.dynamicClient, h.clientset, namespace, h.consoleClientChecker, consoleClientID) + absent, perr := gateway.RouteResourcesAbsent(ctx, h.dynamicClient, h.clientset, namespace, h.ingressMode, h.consoleClientChecker, consoleClientID) switch { case perr != nil: // Leave the verification timestamp stale so the next tick re-probes rather @@ -410,6 +389,7 @@ func (h *GatewayHealthReconciler) teardownRoute(ctx context.Context, client pb.G } opts := gateway.ReconcileOpts{ IsOpenShift: h.isOpenShift, + HasGatewayAPI: h.hasGatewayAPI, SkipNetworkPolicies: h.skipNetworkPolicies, Keycloak: h.keycloakConfig, GatewayID: gatewayID, @@ -435,10 +415,28 @@ func (h *GatewayHealthReconciler) teardownRoute(ctx context.Context, client pb.G return uerr } } - if err := gateway.DeleteGatewayAPIResources(ctx, h.dynamicClient, h.clientset, namespace, opts); err != nil { + var teardownErr error + switch h.ingressMode { + case gateway.IngressModeGatewayAPI: + teardownErr = gateway.DeleteGatewayAPIResources(ctx, h.dynamicClient, h.clientset, namespace, opts) + case gateway.IngressModeRoute: + teardownErr = gateway.DeleteRouteResources(ctx, h.dynamicClient, h.clientset, namespace, opts) + case gateway.IngressModeNone: + // No gateway exposure is active. Remove remaining console resources and + // clear a stored route address from an earlier configuration. + teardownErr = gateway.DeleteConsole(ctx, h.dynamicClient, h.clientset, namespace, opts) + if opts.UpdateRouteAddress != nil { + if err := opts.UpdateRouteAddress(ctx, ""); err != nil { + teardownErr = errors.Join(teardownErr, fmt.Errorf("clear route address in %s: %w", namespace, err)) + } + } + default: + teardownErr = fmt.Errorf("unsupported gateway ingress mode %q", h.ingressMode) + } + if teardownErr != nil { // Leave the gateway unmarked so the next tick retries until every // route-owned resource and stored address is gone. - log.Printf("WARN route teardown in %s (gateway no longer routed): %v", namespace, err) + log.Printf("WARN route teardown in %s (gateway no longer routed): %v", namespace, teardownErr) return } h.markRouteTornDown(gatewayID) diff --git a/components/control-plane/internal/reconciler/health_test.go b/components/control-plane/internal/reconciler/health_test.go index ebe4473de..60e6477e1 100644 --- a/components/control-plane/internal/reconciler/health_test.go +++ b/components/control-plane/internal/reconciler/health_test.go @@ -10,6 +10,13 @@ import ( "github.com/openshift-online/hypershell/components/control-plane/internal/exposure" "github.com/openshift-online/hypershell/components/control-plane/internal/gateway" "google.golang.org/grpc" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + k8sfake "k8s.io/client-go/kubernetes/fake" ) // fakeExposure is a stub Gateway Exposure port for driving the health @@ -57,19 +64,149 @@ func TestSelfHealConsole_NoOpWhenUnconfigured(t *testing.T) { ctx := context.Background() t.Run("no keycloak config", func(t *testing.T) { - h := &GatewayHealthReconciler{exposure: fakeExposure{}} // keycloakConfig nil + h := &GatewayHealthReconciler{ingressMode: gateway.IngressModeRoute} // keycloakConfig nil h.selfHealConsole(ctx, "gw-1", routedGateway("gw-1", "openshell-abc")) }) - t.Run("no exposure port", func(t *testing.T) { - h := &GatewayHealthReconciler{keycloakConfig: &gateway.KeycloakConfig{}} // exposure nil + t.Run("no selected ingress", func(t *testing.T) { + h := &GatewayHealthReconciler{keycloakConfig: &gateway.KeycloakConfig{}} h.selfHealConsole(ctx, "gw-1", routedGateway("gw-1", "openshell-abc")) }) t.Run("not a routed gateway", func(t *testing.T) { - h := &GatewayHealthReconciler{exposure: fakeExposure{}, keycloakConfig: &gateway.KeycloakConfig{}} + h := &GatewayHealthReconciler{ingressMode: gateway.IngressModeRoute, keycloakConfig: &gateway.KeycloakConfig{}} h.selfHealConsole(ctx, "gw-1", &pb.Gateway{Metadata: &pb.ObjectReference{Id: "gw-1"}, Namespace: "openshell-abc"}) }) + + t.Run("route explicitly disabled", func(t *testing.T) { + route := `{"enabled":false}` + h := &GatewayHealthReconciler{ingressMode: gateway.IngressModeRoute, keycloakConfig: &gateway.KeycloakConfig{}} + h.selfHealConsole(ctx, "gw-1", &pb.Gateway{ + Metadata: &pb.ObjectReference{Id: "gw-1"}, + Namespace: "openshell-abc", + Route: &route, + }) + }) +} + +func TestSyncConsoleAddressUsesSelectedRouteExposure(t *testing.T) { + t.Setenv("GATEWAY_API_BASE_DOMAIN", "apps.example.com") + const namespace = "openshell-abc" + + for _, tc := range []struct { + name string + admitted bool + current string + wantReady bool + wantAddress string + }{ + { + name: "admitted Route publishes the address", + admitted: true, + wantReady: true, + wantAddress: "https://console-openshell-abc.apps.example.com", + }, + { + name: "rejected Route clears the address", + current: "https://console-openshell-abc.apps.example.com", + wantAddress: "", + }, + } { + t.Run(tc.name, func(t *testing.T) { + replicas := int32(1) + clientset := k8sfake.NewSimpleClientset(&appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: gateway.ConsoleDeploymentName, Namespace: namespace}, + Spec: appsv1.DeploymentSpec{Replicas: &replicas}, + Status: appsv1.DeploymentStatus{ReadyReplicas: replicas}, + }) + + status := "False" + reason := "HostAlreadyClaimed" + if tc.admitted { + status = "True" + reason = "" + } + route := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "route.openshift.io/v1", + "kind": "Route", + "metadata": map[string]interface{}{ + "name": gateway.ConsoleDeploymentName, + "namespace": namespace, + }, + "status": map[string]interface{}{ + "ingress": []interface{}{ + map[string]interface{}{ + "conditions": []interface{}{ + map[string]interface{}{"type": "Admitted", "status": status, "reason": reason}, + }, + }, + }, + }, + }} + routeGVR := schema.GroupVersionResource{Group: "route.openshift.io", Version: "v1", Resource: "routes"} + dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds( + runtime.NewScheme(), + map[schema.GroupVersionResource]string{routeGVR: "RouteList"}, + route, + ) + + var gotAddress string + updates := 0 + client := &fakeGatewayClient{updateFn: func(_ context.Context, request *pb.UpdateGatewayRequest, _ ...grpc.CallOption) (*pb.UpdateGatewayResponse, error) { + updates++ + if request.ConsoleAddress == nil { + t.Fatal("console address update is nil") + } + gotAddress = *request.ConsoleAddress + return &pb.UpdateGatewayResponse{}, nil + }} + gw := routedGateway("gw-1", namespace) + gw.ConsoleAddress = &tc.current + + ready := syncConsoleAddress(context.Background(), clientset, dynamicClient, client, "gw-1", gw, gateway.IngressModeRoute) + if ready != tc.wantReady { + t.Fatalf("ready = %v, want %v", ready, tc.wantReady) + } + if gotAddress != tc.wantAddress { + t.Fatalf("console address = %q, want %q", gotAddress, tc.wantAddress) + } + if updates != 1 { + t.Fatalf("address updates = %d, want 1", updates) + } + }) + } +} + +func TestSyncConsoleAddressClearsAddressWithoutBaseDomain(t *testing.T) { + t.Setenv("GATEWAY_API_BASE_DOMAIN", "") + current := "https://console-openshell-abc.apps.example.com" + gw := routedGateway("gw-1", "openshell-abc") + gw.ConsoleAddress = ¤t + + var gotAddress string + client := &fakeGatewayClient{updateFn: func(_ context.Context, request *pb.UpdateGatewayRequest, _ ...grpc.CallOption) (*pb.UpdateGatewayResponse, error) { + if request.ConsoleAddress == nil { + t.Fatal("console address update is nil") + } + gotAddress = *request.ConsoleAddress + return &pb.UpdateGatewayResponse{}, nil + }} + + ready := syncConsoleAddress( + context.Background(), + k8sfake.NewSimpleClientset(), + nil, + client, + "gw-1", + gw, + gateway.IngressModeRoute, + ) + if ready { + t.Fatal("ready = true, want false without a base domain") + } + if gotAddress != "" { + t.Fatalf("console address = %q, want empty", gotAddress) + } } // teardownRoute reconciles the desired absence of the route and console for a @@ -298,6 +435,8 @@ func TestIsRoutedGateway(t *testing.T) { {"null route", str("null"), false}, {"empty object", str("{}"), true}, {"enabled route", str(`{"enabled":true}`), true}, + {"disabled route", str(`{"enabled":false}`), false}, + {"host-only route", str(`{"host":"gateway.example.com"}`), true}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -310,7 +449,8 @@ func TestIsRoutedGateway(t *testing.T) { type fakeGatewayClient struct { pb.GatewayServiceClient - listFn func(ctx context.Context, in *pb.ListGatewaysRequest, opts ...grpc.CallOption) (*pb.ListGatewaysResponse, error) + listFn func(ctx context.Context, in *pb.ListGatewaysRequest, opts ...grpc.CallOption) (*pb.ListGatewaysResponse, error) + updateFn func(ctx context.Context, in *pb.UpdateGatewayRequest, opts ...grpc.CallOption) (*pb.UpdateGatewayResponse, error) } func (f *fakeGatewayClient) ListGateways(ctx context.Context, in *pb.ListGatewaysRequest, opts ...grpc.CallOption) (*pb.ListGatewaysResponse, error) { @@ -320,6 +460,13 @@ func (f *fakeGatewayClient) ListGateways(ctx context.Context, in *pb.ListGateway return &pb.ListGatewaysResponse{}, nil } +func (f *fakeGatewayClient) UpdateGateway(ctx context.Context, in *pb.UpdateGatewayRequest, opts ...grpc.CallOption) (*pb.UpdateGatewayResponse, error) { + if f.updateFn != nil { + return f.updateFn(ctx, in, opts...) + } + return &pb.UpdateGatewayResponse{}, nil +} + func TestListAllGateways_Pagination(t *testing.T) { // 250 gateways distributed across 3 pages (100, 100, 50). total := 250 diff --git a/components/control-plane/internal/reconciler/reconciler.go b/components/control-plane/internal/reconciler/reconciler.go index dd89a82bd..7be9afcf9 100644 --- a/components/control-plane/internal/reconciler/reconciler.go +++ b/components/control-plane/internal/reconciler/reconciler.go @@ -1220,6 +1220,7 @@ type GatewayReconciler struct { isOpenShift bool hasCertManager bool hasGatewayAPI bool + ingressMode string skipNetworkPolicies bool hasCNPG bool manifestsDir string @@ -1246,6 +1247,7 @@ func NewGatewayReconciler( isOpenShift := gateway.DetectOpenShift(clientset) hasCertManager := gateway.DetectCertManager(clientset) hasGatewayAPI := gateway.DetectGatewayAPI(clientset) + ingressMode := gateway.IngressMode(hasGatewayAPI, isOpenShift) skipNetworkPolicies := os.Getenv("GATEWAY_SKIP_NETWORK_POLICIES") == "true" hasCNPG := gateway.DetectCNPG(clientset) @@ -1260,8 +1262,8 @@ func NewGatewayReconciler( log.Printf("INFO keycloak integration enabled: server=%s realm=%s", keycloakConfig.ServerURL, keycloakConfig.Realm) } - log.Printf("INFO gateway reconciler initialized: manifests=%d openshift=%v certmanager=%v gatewayapi=%v cnpg=%v keycloak=%v netpol=%v", - len(manifests), isOpenShift, hasCertManager, hasGatewayAPI, hasCNPG, kcClient != nil, !skipNetworkPolicies) + log.Printf("INFO gateway reconciler initialized: manifests=%d openshift=%v certmanager=%v gatewayapi=%v ingressMode=%s cnpg=%v keycloak=%v netpol=%v", + len(manifests), isOpenShift, hasCertManager, hasGatewayAPI, ingressMode, hasCNPG, kcClient != nil, !skipNetworkPolicies) return &GatewayReconciler{ active: make(map[string]struct{}), @@ -1272,6 +1274,7 @@ func NewGatewayReconciler( isOpenShift: isOpenShift, hasCertManager: hasCertManager, hasGatewayAPI: hasGatewayAPI, + ingressMode: ingressMode, skipNetworkPolicies: skipNetworkPolicies, hasCNPG: hasCNPG, manifestsDir: manifestsDir, @@ -1447,9 +1450,9 @@ func (r *GatewayReconciler) Handle(ctx context.Context, event watcher.Event[*pb. gwConfig.OIDC = oidcConfig } - if gw.Route != nil && *gw.Route != "" { - var routeConfig gateway.RouteConfig - if err := json.Unmarshal([]byte(*gw.Route), &routeConfig); err != nil { + if gw.Route != nil { + routeConfig, err := parseGatewayRouteConfig(*gw.Route) + if err != nil { reconcileErr = fmt.Errorf("invalid route config for gateway %s: %w", gw.Name, err) return reconcileErr } @@ -1522,7 +1525,8 @@ func (r *GatewayReconciler) Handle(ctx context.Context, event watcher.Event[*pb. // non-routed gateway - or any gateway on a cluster without the exposure port - // is Running on Deployment readiness alone. See // openshell-gateway-health.spec.md ยง Phase Reflects Workload and Route Readiness. - if r.exposure != nil && isRoutedGateway(gw) { + routed := isRoutedGateway(gw) + if r.exposure != nil && routed { if r.waitForRouteReady(ctx, namespace) { r.updateGatewayHealth(ctx, event.ResourceID, "Running", "Healthy") log.Printf("INFO gateway %s provisioned and route ready in namespace %s", gw.Name, namespace) @@ -1530,18 +1534,17 @@ func (r *GatewayReconciler) Handle(ctx context.Context, event watcher.Event[*pb. r.updateGatewayHealth(ctx, event.ResourceID, "Provisioning", "Deployment ready; awaiting route readiness") log.Printf("INFO gateway %s deployment ready in namespace %s; awaiting route readiness", gw.Name, namespace) } - // The console pod starts after the gateway is routed and typically becomes - // Ready seconds to a minute later. Poll its readiness on a tight cadence in - // the background and publish console_address as soon as it can serve, so the - // web UI's console button enables promptly instead of waiting for the next - // 30s health-reconciler tick. It runs in the background so a slow console - // image pull never blocks the (serial) gateway watch loop; the health - // reconciler remains the backstop that publishes and retracts the address. - go r.publishConsoleAddressWhenReady(ctx, event.ResourceID, gw) } else { r.updateGatewayHealth(ctx, event.ResourceID, "Running", "Healthy") log.Printf("INFO gateway %s provisioned and ready in namespace %s", gw.Name, namespace) } + + // The console can start after the gateway is ready. Poll the console in the + // background so the address appears without waiting for the next health tick. + // The health reconciler continues to publish and retract the address. + if routed && r.ingressMode != gateway.IngressModeNone { + go r.publishConsoleAddressWhenReady(ctx, event.ResourceID, gw) + } return nil } @@ -1629,7 +1632,7 @@ func (r *GatewayReconciler) publishConsoleAddressWhenReady(ctx context.Context, // End the poll rather than publishing against the stale snapshot. return true } - return syncConsoleAddress(ctx, r.clientset, r.dynamicClient, client, gatewayID, current, r.exposure != nil) + return syncConsoleAddress(ctx, r.clientset, r.dynamicClient, client, gatewayID, current, r.ingressMode) }) } @@ -1682,15 +1685,35 @@ func poll(ctx context.Context, interval, window time.Duration, attempt func() bo } } -// isRoutedGateway reports whether a Gateway declares external route exposure -// (a non-empty `route` configuration), and therefore requires its external -// exposure to be observed Ready before it can be reported Running. +// parseGatewayRouteConfig parses the route field. A route object is enabled by +// default for compatibility with existing empty and host-only route objects. +// An explicit enabled=false value disables it. An empty or null value has no +// route. +func parseGatewayRouteConfig(raw string) (gateway.RouteConfig, error) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" || trimmed == "null" { + return gateway.RouteConfig{}, nil + } + + config := gateway.RouteConfig{Enabled: true} + if err := json.Unmarshal([]byte(trimmed), &config); err != nil { + return gateway.RouteConfig{}, err + } + return config, nil +} + +// isRoutedGateway reports whether a Gateway enables external route exposure. func isRoutedGateway(gw *pb.Gateway) bool { if gw.Route == nil { return false } - route := strings.TrimSpace(*gw.Route) - return route != "" && route != "null" + routeConfig, err := parseGatewayRouteConfig(*gw.Route) + if err != nil { + // The provisioning path reports invalid configuration. Preserve the + // exposure until that path can resolve the invalid value. + return true + } + return routeConfig.Enabled } // gatewayNamespace returns the Kubernetes namespace a Gateway is deployed into. @@ -1780,17 +1803,12 @@ func consoleAddressFor(ready bool, url string) string { // syncConsoleAddress publishes the gateway's console_address once its console is // observed servable and clears it otherwise, so the web UI only offers the -// console button when the console can actually serve. "Servable" requires both -// the console Deployment to be Ready AND the console HTTPRoute to be accepted -// (Accepted + ResolvedRefs on the shared Gateway listener) -- a Ready Deployment -// alone does not prove the public route works, and publishing the address then -// would enable a dead link. It is a no-op for gateways without a console (no -// exposure port, or not routed) and when the base domain is unconfigured, and it -// leaves the address untouched on a transient readiness-observation error rather -// than flapping the button. It returns whether the console is currently servable, -// so a caller polling during provisioning can stop once the address is published. -func syncConsoleAddress(ctx context.Context, clientset *kubernetes.Clientset, dynamicClient dynamic.Interface, client pb.GatewayServiceClient, gatewayID string, gw *pb.Gateway, hasExposure bool) bool { - if gatewayID == "" || !hasExposure || !isRoutedGateway(gw) { +// console button when the console can serve. The console Deployment and the +// selected exposure resource must both be Ready. It does not publish an address +// without a selected ingress mode. It leaves the address unchanged after a +// temporary observation error. It returns whether the console can serve. +func syncConsoleAddress(ctx context.Context, clientset kubernetes.Interface, dynamicClient dynamic.Interface, client pb.GatewayServiceClient, gatewayID string, gw *pb.Gateway, ingressMode string) bool { + if gatewayID == "" || ingressMode == gateway.IngressModeNone || !isRoutedGateway(gw) { return false } namespace, err := gatewayNamespace(gw) @@ -1798,25 +1816,24 @@ func syncConsoleAddress(ctx context.Context, clientset *kubernetes.Clientset, dy log.Printf("WARN console address for %s: %v", gatewayID, err) return false } - url, ok := gateway.ConsoleURL(namespace) - if !ok { - return false - } - ready, _, err := gateway.DeploymentReadiness(ctx, clientset, namespace, gateway.ConsoleDeploymentName) - if err != nil { - log.Printf("WARN console readiness for %s: %v", namespace, err) - return false + url, hasURL := gateway.ConsoleURL(namespace) + ready := false + if hasURL { + ready, _, err = gateway.DeploymentReadiness(ctx, clientset, namespace, gateway.ConsoleDeploymentName) + if err != nil { + log.Printf("WARN console readiness for %s: %v", namespace, err) + return false + } } if ready { - // The Deployment is Ready; require the public route to be accepted too - // before publishing the address, logging the listener rejection reason - // otherwise so a misconfigured HTTP listener is diagnosable. - routeReady, reason, routeErr := gateway.ConsoleRouteReady(ctx, dynamicClient, namespace) - if routeErr != nil { - log.Printf("WARN console route readiness for %s: %v", namespace, routeErr) + // The selected public exposure must be Ready before the reconciler + // publishes the address. + exposureReady, reason, exposureErr := gateway.ConsoleExposureReady(ctx, dynamicClient, namespace, ingressMode) + if exposureErr != nil { + log.Printf("WARN console exposure readiness for %s: %v", namespace, exposureErr) return false } - if !routeReady { + if !exposureReady { log.Printf("INFO console for %s not servable yet: %s", namespace, reason) ready = false } diff --git a/deploy/base/controller-rbac.yaml b/deploy/base/controller-rbac.yaml index f33b32cd1..d37e431bb 100644 --- a/deploy/base/controller-rbac.yaml +++ b/deploy/base/controller-rbac.yaml @@ -28,8 +28,12 @@ rules: resources: ["gateways", "grpcroutes", "httproutes", "backendtlspolicies"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: ["route.openshift.io"] - resources: ["routes", "routes/custom-host"] + resources: ["routes"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + # OpenShift checks this subresource when the controller sets spec.host. + - apiGroups: ["route.openshift.io"] + resources: ["routes/custom-host"] + verbs: ["create", "update"] - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["get", "list"] diff --git a/skills/RECONCILE.md b/skills/RECONCILE.md index 1404d6a6b..02a3bdddf 100644 --- a/skills/RECONCILE.md +++ b/skills/RECONCILE.md @@ -45,9 +45,9 @@ skills/ ## Reconciliation State -**Last analyzed**: 2026-08-21 (HYPERSHELL-49 spec delta only) -**Spec corpus**: 40 spec files; the coverage table tracks 31 previously analyzed feature/spec groups after adding OpenShellGatewayServiceAccounts -**Codebase commit**: 2b5eaf4 (spec/machine-account-client-secret) +**Last analyzed**: 2026-08-27 (OpenShell Gateway Console GC-W1 complete) +**Spec corpus**: 40 spec files; the coverage table tracks 32 analyzed feature/spec groups after adding OpenShell Gateway Console +**Codebase commit**: 9984ed0 (fix/console OpenShift Route ingress) ### Coverage Summary @@ -60,6 +60,7 @@ skills/ | Platform - Gateway TLS | 1 | 7 | 3 | 2 | 2 | 0 | 57% | | Platform - Gateway OIDC | 1 | 9 | 6 | 1 | 2 | 0 | 72% | | Platform - Gateway Routing | 1 | 18 | 6 | 4 | 8 | 0 | 44% | +| Platform - Gateway Console | 1 | 9 | 9 | 0 | 0 | 0 | 100% | | Platform - Gateway Keycloak | 1 | 9 | 9 | 0 | 0 | 0 | 100% | | Platform - Gateway Service Accounts | 1 | 15 | 15 | 0 | 0 | 0 | 100% | | Platform - Gateway Secret Rotation | 1 | 8 | 5 | 0 | 1 | 2 | 63% | @@ -71,7 +72,7 @@ skills/ | Web Console - Architecture | 1 | 28 | 21 | 5 | 2 | 0 | 86% | | Security - RBAC Enforcement | 1 | 13 | 11 | 0 | 0 | 2 | 85% | | Standards | 13 | 0 | 0 | 0 | 0 | 0 | N/A | -| **TOTAL** | **31** | **215** | **166** | **18** | **26** | **5** | **81%** | +| **TOTAL** | **32** | **224** | **175** | **18** | **26** | **5** | **82%** | ### Spec Dependency Order @@ -84,6 +85,7 @@ Layer 4: openshell-gateway-oidc (depends on TLS for trusted CA) Layer 4.5: openshell-gateway-secret-rotation (depends on database, credentials, TLS) Layer 5: openshell-gateway-routing (depends on TLS for BackendTLSPolicy) Layer 5.5: openshell-gateway-keycloak (depends on oidc, rbac-enforcement) +Layer 5.6: openshell-gateway-console (depends on routing, keycloak) Layer 5.75: openshell-gateway-service-accounts (depends on keycloak, oidc, rbac-enforcement, security) Layer 6: local-development (depends on all platform specs) Layer 1.5: security/rbac-enforcement (depends on data-model) @@ -94,6 +96,26 @@ Layer 7: web-console/architecture (depends on data-model, security, UI ## Gap Table +### openshell-gateway-console.spec.md + +| # | Requirement | Status | Gap | Code Location | Wave | +|---|-------------|--------|-----|---------------|------| +| GC-1 | Console enablement follows the selected ingress mode | Present | - | `control-plane/internal/gateway/{reconciler.go,console.go}`, `reconciler/health.go` | GC-W1 | +| GC-2 | Confidential console Keycloak client | Present | - | `control-plane/internal/keycloak/client.go`, `gateway/console.go` | - | +| GC-3 | Stable console credential Secret | Present | - | `control-plane/internal/gateway/console.go` | - | +| GC-4 | Console Deployment | Present | - | `control-plane/internal/gateway/console.go` | - | +| GC-5 | Service and mode-selected HTTP exposure | Present | - | `control-plane/internal/gateway/console.go`, `deploy/base/controller-rbac.yaml` | GC-W1 | +| GC-6 | Console NetworkPolicies | Present | The policy source uses the configured ingress namespace and supports both ingress controllers. | `control-plane/internal/gateway/console.go` | - | +| GC-7 | Console lifecycle and cleanup | Present | - | `control-plane/internal/gateway/{console.go,reconciler.go}`, `reconciler/health.go` | GC-W1 | +| GC-8 | Provisioning atomicity and idempotency | Present | - | `control-plane/internal/gateway/console.go`, `internal/keycloak/client.go` | - | +| GC-9 | Console address discovery | Present | - | `control-plane/internal/reconciler/reconciler.go`, `gateway/console.go` | GC-W1 | + +**Scoped analysis notes:** + +- The API, data model, SDKs, CLI, Keycloak client, Secret, Deployment, Service, and NetworkPolicy contracts need no change. +- GC-W1 added the OpenShift Route adapter and made provisioning, readiness, cleanup, and health repair use the selected ingress mode. +- The base controller role permits Route CRUD and create and update access to `routes/custom-host` because the console Route sets `spec.host`. + ### openshell-gateway-service-accounts.spec.md | # | Requirement | Status | Gap | Code Location | Wave | @@ -418,6 +440,22 @@ Layer 7: web-console/architecture (depends on data-model, security, UI ## Wave Plan +### GC-W1: OpenShift Route support for the Gateway Console + +**Scope:** GC-1, GC-5, GC-7, GC-9 +**Dependency:** Existing Gateway Console and Route ingress implementations +**Status:** Complete + +1. Select console exposure from the effective gateway ingress mode. +2. Create an edge-terminated OpenShift Route for `openshell-console` in Route mode. +3. Observe HTTPRoute acceptance or OpenShift Route admission for address publication. +4. Reconcile initial provisioning, health repair, inactive exposure removal, and teardown for both modes. +5. Add `routes/custom-host` controller RBAC. +6. Add unit tests for resource shape, readiness, mode selection, and cleanup. +7. Run control-plane build, vet, test, alignment, and review checks. + +**GC-W1 summary:** Added an edge-terminated OpenShift Route for the console, selected readiness by ingress mode, removed inactive console exposures, aligned route-enable semantics, and added custom-host RBAC. The complete control-plane test suite, affected-package race tests, vet, lint, build, Kustomize renders, alignment scan, and independent review passed. + ### HYPERSHELL-49 OpenShellGatewayServiceAccount waves | Wave | Scope | Status | @@ -629,6 +667,8 @@ label-selected pod informer. | Date | Commit | Action | Coverage | Notes | |------|--------|--------|----------|-------| +| 2026-08-27 | 9984ed0 | Completed Gateway Console GC-W1 | 82% | Added mode-selected Route exposure, admission readiness, lifecycle cleanup, custom-host RBAC, and tests. All nine console requirements are present. | +| 2026-08-27 | 612b373 | Gateway Console scoped gap analysis | 81% | Added the console spec to the registry and found four partial requirements. Planned one control-plane wave for OpenShift Route exposure, readiness, cleanup, RBAC, and tests. | | 2026-08-03 | initial | Initial setup | 100% | Baseline with 6 Kinds fully implemented | | 2026-08-05 | working tree | Registered UI standards | 100% platform | UI standards are evaluated by `/ui-standards`, not counted as feature reconciliation requirements | | 2026-08-05 | working tree | Added PatternFly standard | 100% platform | PatternFly 6, canonical reuse, and duplicate-component prevention apply to the web console | diff --git a/specs/index.spec.md b/specs/index.spec.md index 441164e70..05bbd639c 100644 --- a/specs/index.spec.md +++ b/specs/index.spec.md @@ -38,6 +38,7 @@ Machine-readable index for autonomous reconciliation (`/reconcile` skill). | `platform/openshell-gateway-credentials.spec.md` | platform | Credential storage drivers, KEK conditional provisioning | CP | openshell-gateway, openshell-gateway-database | | `platform/openshell-gateway-secret-rotation.spec.md` | platform | Secret rotation: DB password, KEK, TLS certificates | CP | openshell-gateway-database, openshell-gateway-credentials, openshell-gateway-tls | | `platform/openshell-gateway-keycloak.spec.md` | platform | Keycloak OIDC client provisioning, per-gateway OIDC role bridge | CP | openshell-gateway, openshell-gateway-oidc, rbac-enforcement | +| `platform/openshell-gateway-console.spec.md` | platform | Per-gateway OpenShell dashboard, oauth2-proxy, HTTP ingress | CP | openshell-gateway, openshell-gateway-routing, openshell-gateway-keycloak | | `platform/openshell-gateway-service-accounts.spec.md` | platform | OpenShellGatewayServiceAccounts and Keycloak client-credentials lifecycle | API, CP, CLI, WEB, SDK | openshell-gateway-keycloak, openshell-gateway-oidc, rbac-enforcement, security, UI standards | | `platform/openshell-inference-routing.spec.md` | platform | Inference router, inference.local, credential-free sandbox model access, provider translation | CP | openshell-gateway, openshell-gateway-credentials | | `platform/global-architecture.spec.md` | platform | Global hub, multi-cloud, CNPG, Tekton, ArgoCD, Vault | CP, ALL | data-model, control-plane | diff --git a/specs/platform/openshell-gateway-console.spec.md b/specs/platform/openshell-gateway-console.spec.md index 7f3214df1..11a508454 100644 --- a/specs/platform/openshell-gateway-console.spec.md +++ b/specs/platform/openshell-gateway-console.spec.md @@ -1,9 +1,9 @@ # OpenShell Gateway Console Specification -**Date:** 2026-08-18 +**Date:** 2026-08-27 **Status:** Draft **Parent:** `openshell-gateway.spec.md` -**Related:** `openshell-gateway-keycloak.spec.md` (per-gateway client, OIDC Role Bridge); `openshell-gateway-routing.spec.md` (shared Gateway, hostnames, NetworkPolicy) +**Related:** `openshell-gateway-keycloak.spec.md` (per-gateway client, OIDC Role Bridge); `openshell-gateway-routing.spec.md` (Gateway API and OpenShift Route ingress, hostnames, NetworkPolicy) **Upstream:** [OpenShell Dashboard](https://github.com/Gkrumbach07/openshell-dashboard); [oauth2-proxy](https://oauth2-proxy.github.io/oauth2-proxy/) --- @@ -35,7 +35,7 @@ This spec covers Option 1: one oauth2-proxy for each gateway, real tokens with t The console client is a second Keycloak client. It is not the CLI client (`{name}-{id}`). Its mappers target the gateway client. The gateway accepts console tokens like CLI tokens. The CLI client stays unchanged. -A console needs a route. Browser traffic reaches the console through that route, and a routed gateway has `client_ca_path` removed on its externally-routed data path because the ingress proxy cannot present a client certificate (see `openshell-gateway-routing.spec.md`). The console's dashboard does not use that route to reach the gateway: it dials the in-cluster `openshell-gateway..svc.cluster.local:8080` admin API directly over gRPC with mutual TLS, presenting the `openshell-client` certificate and verifying the gateway server certificate against the openshell CA. +A console needs external ingress. Browser traffic reaches the console through an HTTPRoute in `gateway-api` mode or through an OpenShift Route in `route` mode. A routed gateway has `client_ca_path` removed on its external data path because the ingress proxy cannot present a client certificate (see `openshell-gateway-routing.spec.md`). The console dashboard does not use external ingress to reach the gateway. It connects to the in-cluster `openshell-gateway..svc.cluster.local:8080` admin API through gRPC with mutual TLS. It presents the `openshell-client` certificate and verifies the gateway server certificate against the OpenShell CA. --- @@ -43,21 +43,33 @@ A console needs a route. Browser traffic reaches the console through that route, ### Requirement: Console Enablement Tied to Routing -The reconciler must deploy the console when all of these conditions are true: +The reconciler SHALL deploy the console when all of these conditions are true: -- The gateway has a route. -- The cluster supports the Gateway API. +- The gateway has an enabled route. A present route object defaults to enabled when it omits `enabled`; an explicit `route.enabled = false` disables it. +- The selected ingress mode is `gateway-api` or `route`. - Keycloak is configured. -The console has no separate configuration field. The console follows the route lifecycle. +The console has no separate configuration field. The console SHALL use the same effective `GATEWAY_INGRESS_MODE` as the gateway. The console SHALL follow the route lifecycle. -#### Scenario: Routed gateway gets a console +#### Scenario: Gateway API mode creates a console -- GIVEN a gateway with a route, on a cluster with the Gateway API and Keycloak +- GIVEN a gateway with an enabled route +- AND the selected ingress mode is `gateway-api` +- AND Keycloak is configured - WHEN the reconciler reconciles the gateway - THEN it must create all console resources: the console client, the console Secret, the Deployment, the Service, the HTTPRoute, and the NetworkPolicies - AND the console must answer at `https://console-.` +#### Scenario: OpenShift Route mode creates a console + +- GIVEN a gateway with an enabled route +- AND the selected ingress mode is `route` +- AND Keycloak is configured +- WHEN the reconciler reconciles the gateway +- THEN it must create all console resources: the console client, the console Secret, the Deployment, the Service, the OpenShift Route, and the NetworkPolicies +- AND it must not create a console HTTPRoute +- AND the console must answer at `https://console-.` + #### Scenario: Non-routed gateway gets no console - GIVEN a gateway with no route @@ -65,9 +77,9 @@ The console has no separate configuration field. The console follows the route l - THEN it must not create console resources - AND it must not create a console client -#### Scenario: Prerequisites absent +#### Scenario: Prerequisite is absent -- GIVEN a routed gateway on a cluster with no Gateway API, or with no Keycloak +- GIVEN a routed gateway with no selected ingress mode or with no Keycloak configuration - WHEN the reconciler reconciles the gateway - THEN it must write a warning - AND it must skip the console @@ -206,9 +218,11 @@ oauth2-proxy must expose readiness and liveness probes on `/ready` and `/ping` ( --- -### Requirement: Console Service and HTTP Exposure +### Requirement: Console Service and Mode-Selected HTTP Exposure + +The reconciler SHALL create a `ClusterIP` Service named `openshell-console` on port `4180`. It SHALL create only the console exposure resource for the selected ingress mode. It SHALL remove an exposure resource from the inactive mode when one exists. -The reconciler must create a `ClusterIP` Service `openshell-console` on port `4180`. The reconciler must create an HTTPRoute that attaches to the shared Gateway. +In `gateway-api` mode, the reconciler SHALL create this HTTPRoute. The HTTPRoute attaches to the shared Gateway. ```yaml apiVersion: gateway.networking.k8s.io/v1 @@ -231,6 +245,28 @@ spec: The hostname `console-.` is a subdomain of ``. The shared Gateway wildcard certificate covers it, so the console needs no separate certificate. This hostname differs from the gateway gRPC hostname (`gw-.`), so the two attach to different listeners. +In `route` mode, the reconciler SHALL create this OpenShift Route. + +```yaml +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: openshell-console + namespace: +spec: + host: console-. + to: + kind: Service + name: openshell-console + port: + targetPort: http + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect +``` + +The OpenShift router SHALL terminate TLS. It SHALL send HTTP to oauth2-proxy on Service port `http`. The Route SHALL use the OpenShift router certificate when no Route certificate is set. + #### Scenario: HTTP listener absent on the shared Gateway - GIVEN a shared Gateway with no listener that matches `GATEWAY_API_HTTP_LISTENER_NAME` @@ -239,13 +275,28 @@ The hostname `console-.` is a subdomain of ``. The - AND the reconciler must write a warning that names the missing listener - AND it must not fail the gateway reconciliation +#### Scenario: OpenShift Route is admitted + +- GIVEN the selected ingress mode is `route` +- AND the console Deployment is Ready +- WHEN an OpenShift router reports `Admitted=True` for the console Route +- THEN the console exposure must be Ready + +#### Scenario: OpenShift Route is not admitted + +- GIVEN the selected ingress mode is `route` +- WHEN no OpenShift router reports `Admitted=True` for the console Route +- THEN the reconciler must not publish `consoleAddress` +- AND it must write the Route admission reason when one exists +- AND it must not fail the gateway reconciliation + --- ### Requirement: Console NetworkPolicies -The reconciler must create two NetworkPolicies. Existing gateway policies already select the gateway pod for ingress, so the namespace denies traffic by default from any other source. +The reconciler must create two NetworkPolicies. Existing gateway policies already select the gateway pod for ingress, so the namespace denies traffic by default from any other source. The ingress controller namespace is `GATEWAY_API_GATEWAY_NAMESPACE`, with the default value `openshift-ingress`. This namespace applies to the shared Gateway proxy and to the OpenShift router. -1. **`openshell-console-allow-router`** -- selects the console pod (`app.kubernetes.io/instance: openshell-console`); allows ingress on TCP `4180` from the shared Gateway namespace (`GATEWAY_API_GATEWAY_NAMESPACE`). +1. **`openshell-console-allow-router`** -- selects the console pod (`app.kubernetes.io/instance: openshell-console`); allows ingress on TCP `4180` from the selected ingress controller namespace. 2. **`openshell-gateway-allow-console`** -- selects the gateway pod (`app.kubernetes.io/instance: openshell-gateway`); allows ingress on TCP `8080` from the console pod in the same namespace. #### Scenario: Console reaches the gateway @@ -265,7 +316,7 @@ The reconciler must reconcile and remove the console together with the route and - GIVEN a gateway that had a route and a console - WHEN the route field is removed -- THEN the reconciler must delete all console resources: the Deployment, the Service, the HTTPRoute, the NetworkPolicies, and the `openshell-console-oauth2` Secret +- THEN the reconciler must delete all console resources: the Deployment, the Service, the HTTPRoute or OpenShift Route, the NetworkPolicies, and the `openshell-console-oauth2` Secret - AND it must delete the console client `{name}-{id}-console` - AND it must clear the `consoleAddress` field on the gateway @@ -306,6 +357,7 @@ The reconciler must PATCH the console URL into a read-only `consoleAddress` fiel - Format: `https://console-.`. - The reconciler sets it only once the console Deployment (dashboard + oauth2-proxy) is observed Ready, so the web-console's console button never appears before the console pod can serve. +- The reconciler sets it only when the exposure resource for the selected ingress mode is Ready. An HTTPRoute is Ready when one parent reports `Accepted=True` and `ResolvedRefs=True`. An OpenShift Route is Ready when one router reports `Admitted=True`. - The reconciler clears it when the console pod is not Ready, when the console is removed, or when the base domain is unknown (for example, `GATEWAY_API_BASE_DOMAIN` is unset). A console that later goes unready has its address retracted, hiding the button. - Readiness is observed both during provisioning (a prompt check once the gateway's route is ready) and continuously by the health reconciler, so the field self-heals as the console pod's readiness changes. @@ -313,7 +365,7 @@ The reconciler must PATCH the console URL into a read-only `consoleAddress` fiel - GIVEN a routed gateway whose console resources are applied but whose console pod is not yet Ready - THEN `consoleAddress` stays empty and the web-console does not offer the console button -- WHEN the console Deployment becomes Ready +- WHEN the console Deployment and its selected exposure resource become Ready - THEN the reconciler sets `consoleAddress` to `https://console-.` and the button appears --- @@ -332,7 +384,8 @@ The gateway has no user field for the console. The console follows the route. | Variable | Default | Description | |---|---|---| -| `GATEWAY_API_HTTP_LISTENER_NAME` | `https` | The `sectionName` of the shared Gateway HTTP listener for console HTTPRoutes | +| `GATEWAY_INGRESS_MODE` | auto-detect | Selects `gateway-api`, `route`, or no managed ingress for both the gateway and its console | +| `GATEWAY_API_HTTP_LISTENER_NAME` | `https` | The `sectionName` of the shared Gateway HTTP listener for console HTTPRoutes; it has no effect in `route` mode | | `HYPERSHELL_CONSOLE_IMAGE` | *(ImageDefaults default)* | The OpenShell dashboard image | | `HYPERSHELL_OAUTH2_PROXY_IMAGE` | *(ImageDefaults default)* | The oauth2-proxy image | @@ -358,7 +411,7 @@ The REST API and the gRPC API must not let a user set or update `consoleAddress` ## RBAC -The control-plane ServiceAccount already has create, update, patch, and delete on `services`, `secrets`, `deployments`, and `networkpolicies`. The console needs one more rule in gateway namespaces: +The control-plane ServiceAccount already has create, update, patch, and delete access to `services`, `secrets`, `deployments`, and `networkpolicies`. Gateway API mode needs this rule in gateway namespaces: ```yaml - apiGroups: ["gateway.networking.k8s.io"] @@ -366,13 +419,24 @@ The control-plane ServiceAccount already has create, update, patch, and delete o verbs: ["get", "list", "create", "update", "patch", "delete"] ``` +OpenShift Route mode needs these rules. The `routes/custom-host` access permits the controller to set the explicit `spec.host` value. + +```yaml +- apiGroups: ["route.openshift.io"] + resources: ["routes"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: ["route.openshift.io"] + resources: ["routes/custom-host"] + verbs: ["create", "update"] +``` + The console needs no new Keycloak permission. The `hypershell-keycloak-admin` account already manages clients, roles, and mappers. --- ## Prerequisites -1. **Shared Gateway HTTP listener.** The admin must add an HTTP listener to the shared Gateway (HTTPS/Terminate, port 443, wildcard `*.` certificate). The listener must accept HTTPRoutes from gateway namespaces. Its `sectionName` must match `GATEWAY_API_HTTP_LISTENER_NAME`. +1. **Ingress controller.** In `gateway-api` mode, the admin must add an HTTP listener to the shared Gateway (HTTPS/Terminate, port 443, wildcard `*.` certificate). The listener must accept HTTPRoutes from gateway namespaces. Its `sectionName` must match `GATEWAY_API_HTTP_LISTENER_NAME`. In `route` mode, an OpenShift ingress controller must admit Routes for `*.` and serve its router certificate. 2. **Dashboard image.** The upstream project ([Gkrumbach07/openshell-dashboard](https://github.com/Gkrumbach07/openshell-dashboard)) publishes the dashboard to `quay.io/gkrumbach07/openshell-dashboard` (per-commit `sha-` tags plus `latest`). The control plane's `ImageDefaults` pin it by digest, so clusters pull it directly (imagePullPolicy `IfNotPresent`) with no build-from-source step; Kind pulls the same public image. Production should mirror the pinned digest into the platform registry and override `HYPERSHELL_CONSOLE_IMAGE`. The image contract (`OPENSHELL_GATEWAY_URL` as a `grpcs://` URL, mutual TLS through `GATEWAY_CA_CERT` plus `GATEWAY_CLIENT_CERT`/`GATEWAY_CLIENT_KEY`, and the `X-Forwarded-Access-Token` relay) is an upstream dependency. 3. **Keycloak realm.** The realm prerequisites in `openshell-gateway-keycloak.spec.md` apply. The console adds no realm-level object. @@ -392,6 +456,7 @@ The console needs no new Keycloak permission. The `hypershell-keycloak-admin` ac - [oauth2-proxy](https://oauth2-proxy.github.io/oauth2-proxy/) -- OIDC provider, PKCE, `pass-access-token`, `reverse-proxy` - [oauth2-proxy #1714](https://github.com/oauth2-proxy/oauth2-proxy/issues/1714) -- a client secret is required with PKCE - [Gateway API HTTPRoute](https://gateway-api.sigs.k8s.io/api-types/httproute/) +- [OpenShift Route](https://docs.redhat.com/en/documentation/openshift_container_platform/latest/html/ingress_and_load_balancing/configuring-routes) - [Keycloak Admin REST API](https://www.keycloak.org/docs-api/latest/rest-api/) - `openshell-gateway-keycloak.spec.md` -- the per-gateway client and the OIDC Role Bridge - `openshell-gateway-routing.spec.md` -- the shared Gateway, hostnames, and NetworkPolicy pattern