Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions .forbidden-terms-whitelist.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,22 @@
},
{
"filename": "specs/platform/global-architecture.spec.md",
"line": 103,
"line": 104,
"rationale": "Mermaid diagram node id 'ACP' abbreviates 'Control Plane' in the reconciliation flow; it is a diagram identifier, not the discouraged term."
},
{
"filename": "specs/platform/global-architecture.spec.md",
"line": 104,
"line": 106,
"rationale": "Mermaid diagram node id 'ACP' abbreviates 'Control Plane' in the reconciliation flow; it is a diagram identifier, not the discouraged term."
},
{
"filename": "specs/platform/global-architecture.spec.md",
"line": 107,
"rationale": "Mermaid diagram node id 'ACP' abbreviates 'Control Plane' in the reconciliation flow; it is a diagram identifier, not the discouraged term."
},
{
"filename": "specs/platform/global-architecture.spec.md",
"line": 1190,
"line": 1529,
"rationale": "Example directory names 'vteam-stage'/'vteam-uat' in a GitOps directory-tree illustration reference real external cluster environments; they are example paths, not the discouraged term."
},
{
Expand Down
14 changes: 14 additions & 0 deletions components/api-server/plugins/managedClusters/dao.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ type ManagedClusterDao interface {
FindByIDs(ctx context.Context, ids []string) (ManagedClusterList, error)
All(ctx context.Context) (ManagedClusterList, error)
FindByOIDCSubject(ctx context.Context, subject string) (*ManagedCluster, error)
FindByNameNoOIDCSubject(ctx context.Context, name string) (*ManagedCluster, error)
InventorySnapshot(ctx context.Context, evaluationTime time.Time) (*ClusterInventorySnapshot, error)
}

Expand Down Expand Up @@ -94,6 +95,19 @@ func (d *sqlManagedClusterDao) FindByOIDCSubject(ctx context.Context, subject st
return &managedCluster, nil
}

// FindByNameNoOIDCSubject looks up the registration record keyed on name alone,
// i.e. the record created when the API server runs with authentication disabled
// (empty oidc_subject). It deliberately excludes records that carry an OIDC
// subject so the authenticated and unauthenticated identity spaces stay separate.
func (d *sqlManagedClusterDao) FindByNameNoOIDCSubject(ctx context.Context, name string) (*ManagedCluster, error) {
g2 := (*d.sessionFactory).New(ctx)
var managedCluster ManagedCluster
if err := g2.Take(&managedCluster, "name = ? AND (oidc_subject IS NULL OR oidc_subject = '')", name).Error; err != nil {
return nil, err
}
return &managedCluster, nil
}

func (d *sqlManagedClusterDao) InventorySnapshot(ctx context.Context, evaluationTime time.Time) (*ClusterInventorySnapshot, error) {
clusters, err := d.All(ctx)
if err != nil {
Expand Down
32 changes: 18 additions & 14 deletions components/api-server/plugins/managedClusters/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,20 +61,24 @@ func (h managedClusterHandler) Register(w http.ResponseWriter, r *http.Request)
}

ctx := r.Context()
token, tokenErr := auth.TokenFromContext(ctx)
if tokenErr != nil || token == nil {
handlers.HandleError(r.Context(), w, errors.Unauthenticated("missing identity"))
return
}
claims, ok := token.Claims.(jwt.MapClaims)
if !ok {
handlers.HandleError(r.Context(), w, errors.Unauthenticated("invalid token claims"))
return
}
oidcSubject, _ := claims["sub"].(string)
if oidcSubject == "" {
handlers.HandleError(r.Context(), w, errors.Unauthenticated("missing sub claim"))
return
// Registration is unconditional. When the API server has authentication
// enabled, the RBAC/JWT middleware has already validated the caller (and its
// managed-cluster-registrar role) before this handler runs, so a token is
// present here; its OIDC subject keys the record. When authentication is
// disabled (local development) no token is present, so the record is keyed on
// name alone with an empty subject.
oidcSubject := ""
if token, tokenErr := auth.TokenFromContext(ctx); tokenErr == nil && token != nil {
claims, ok := token.Claims.(jwt.MapClaims)
if !ok {
handlers.HandleError(r.Context(), w, errors.Unauthenticated("invalid token claims"))
return
}
oidcSubject, _ = claims["sub"].(string)
if oidcSubject == "" {
handlers.HandleError(r.Context(), w, errors.Unauthenticated("missing sub claim"))
return
}
}

description := ""
Expand Down
9 changes: 9 additions & 0 deletions components/api-server/plugins/managedClusters/mock_dao.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,15 @@ func (d *managedClusterDaoMock) FindByOIDCSubject(ctx context.Context, subject s
return nil, gorm.ErrRecordNotFound
}

func (d *managedClusterDaoMock) FindByNameNoOIDCSubject(ctx context.Context, name string) (*ManagedCluster, error) {
for _, mc := range d.managedClusters {
if mc.Name == name && mc.OIDCSubject == "" {
return mc, nil
}
}
return nil, gorm.ErrRecordNotFound
}

func (d *managedClusterDaoMock) InventorySnapshot(ctx context.Context, evaluationTime time.Time) (*ClusterInventorySnapshot, error) {
return buildClusterInventorySnapshot(d.managedClusters, evaluationTime), nil
}
24 changes: 21 additions & 3 deletions components/api-server/plugins/managedClusters/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ type ManagedClusterService interface {
All(ctx context.Context) (ManagedClusterList, *errors.ServiceError)

FindByIDs(ctx context.Context, ids []string) (ManagedClusterList, *errors.ServiceError)
// Register upserts a ManagedCluster by (oidcSubject, name). Returns the cluster,
// Register upserts a ManagedCluster keyed on the OIDC subject when oidcSubject
// is non-empty (authentication enabled), or on name alone when oidcSubject is
// empty (authentication disabled, e.g. local development). Returns the cluster,
// whether it was newly created (true=201, false=200), and any error.
Register(ctx context.Context, name, description, oidcSubject string) (*ManagedCluster, bool, *errors.ServiceError)

Expand Down Expand Up @@ -157,13 +159,29 @@ func (s *sqlManagedClusterService) All(ctx context.Context) (ManagedClusterList,
}

func (s *sqlManagedClusterService) Register(ctx context.Context, name, description, oidcSubject string) (*ManagedCluster, bool, *errors.ServiceError) {
lockOwnerID, lockErr := s.lockFactory.NewAdvisoryLock(ctx, oidcSubject, managedClustersLockType)
// The registration identity (and upsert key) is the OIDC subject when the API
// server has authentication enabled. When authentication is disabled the
// subject is empty and the record is keyed on name alone; the advisory lock
// keys on name in that case so concurrent first-time registrations for the same
// name serialize (an empty subject would otherwise collide across all
// unauthenticated clusters).
lockKey := oidcSubject
if lockKey == "" {
lockKey = "name:" + name
}
lockOwnerID, lockErr := s.lockFactory.NewAdvisoryLock(ctx, lockKey, managedClustersLockType)
if lockErr != nil {
return nil, false, errors.DatabaseAdvisoryLock(lockErr)
}
defer s.lockFactory.Unlock(ctx, lockOwnerID)

existing, err := s.managedClusterDao.FindByOIDCSubject(ctx, oidcSubject)
var existing *ManagedCluster
var err error
if oidcSubject != "" {
existing, err = s.managedClusterDao.FindByOIDCSubject(ctx, oidcSubject)
} else {
existing, err = s.managedClusterDao.FindByNameNoOIDCSubject(ctx, name)
}
if err != nil && !stderrors.Is(err, gorm.ErrRecordNotFound) {
return nil, false, errors.GeneralError("registration lookup failed: %s", err)
}
Expand Down
83 changes: 42 additions & 41 deletions components/control-plane/cmd/hypershell-controller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,9 @@ import (
const defaultManifestsDir = "/manifests/gateway"

// registerWithBackoff calls regClient.Register with exponential backoff until it
// succeeds. A 403 response is non-retryable: the spoke lacks the required Keycloak
// role, so it logs a fatal message and exits immediately.
// succeeds. A 403 response is non-retryable: the control plane lacks the required
// Keycloak role (only possible when the API server has authentication enabled),
// so it returns immediately so the caller can exit.
func registerWithBackoff(ctx context.Context, regClient *registration.Client) (string, error) {
backoff := time.Second
const maxBackoff = 60 * time.Second
Expand All @@ -54,7 +55,7 @@ func registerWithBackoff(ctx context.Context, regClient *registration.Client) (s
return "", fmt.Errorf("managed-cluster-registrar role not assigned in Keycloak; assign the role and restart: %w", err)
}

log.Printf("WARN spoke registration failed (retrying in %s): %v", backoff, err)
log.Printf("WARN registration failed (retrying in %s): %v", backoff, err)
select {
case <-ctx.Done():
return "", fmt.Errorf("registration cancelled: %w", ctx.Err())
Expand Down Expand Up @@ -84,11 +85,7 @@ func main() {

log.Printf("INFO hypershell-controller starting")
log.Printf("INFO grpc=%s api=%s namespace=%s database_provider=%s", cfg.GRPCServerAddr, cfg.APIServerURL, cfg.Namespace, cfg.DatabaseProvider)
if cfg.ClusterID != "" {
log.Printf("INFO managed-cluster mode: scoping gateway watch/seed/health to cluster_id=%s", cfg.ClusterID)
} else {
log.Printf("INFO single-cluster mode: handling all gateways (no cluster_id filter)")
}
log.Printf("INFO self-registration name=%s (cluster_id resolved at registration)", cfg.ManagedClusterName)

ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
Expand Down Expand Up @@ -127,44 +124,48 @@ func main() {
log.Printf("INFO OIDC authentication disabled for gRPC connections")
}

// Spoke self-registration: resolve cluster_id at runtime before any gRPC watch.
// Requires both HYPERSHELL_MANAGED_CLUSTER_NAME and OIDC credentials.
if cfg.ManagedClusterName != "" && tokenProvider != nil {
regClient := registration.NewClient(cfg.APIServerURL, cfg.ManagedClusterName, tokenProvider)
// Self-registration is unconditional: every control plane resolves its
// cluster_id at runtime before any gRPC watch, whether or not OIDC is
// configured, so the same startup path runs in local development and in
// production. When OIDC is unset (auth-disabled API server, e.g. local
// development) the token source is nil and the record is keyed on the cluster
// name alone; otherwise the OIDC subject keys the record.
var regTokens registration.TokenSource
if tokenProvider != nil {
regTokens = tokenProvider
}
regClient := registration.NewClient(cfg.APIServerURL, cfg.ManagedClusterName, regTokens)

clusterID, regErr := registerWithBackoff(ctx, regClient)
if regErr != nil {
log.Fatalf("FATAL spoke registration failed: %v", regErr)
}
cfg.ClusterID = clusterID
log.Printf("INFO spoke registered as cluster_id=%s (name=%s)", cfg.ClusterID, cfg.ManagedClusterName)

// Heartbeat: re-register every 60s to update last_seen_at on the hub.
go func() {
ticker := time.NewTicker(60 * time.Second)
defer ticker.Stop()
var consecutiveFailures int
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if _, err := regClient.Register(ctx); err != nil {
consecutiveFailures++
if consecutiveFailures >= 5 {
log.Printf("ERROR heartbeat has failed %d consecutive times; hub may be unreachable: %v", consecutiveFailures, err)
} else {
log.Printf("WARN heartbeat registration failed: %v", err)
}
clusterID, regErr := registerWithBackoff(ctx, regClient)
if regErr != nil {
log.Fatalf("FATAL registration failed: %v", regErr)
}
cfg.ClusterID = clusterID
log.Printf("INFO registered as cluster_id=%s (name=%s)", cfg.ClusterID, cfg.ManagedClusterName)

// Heartbeat: re-register every 60s to update last_seen_at on the API server.
go func() {
ticker := time.NewTicker(60 * time.Second)
defer ticker.Stop()
var consecutiveFailures int
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if _, err := regClient.Register(ctx); err != nil {
consecutiveFailures++
if consecutiveFailures >= 5 {
log.Printf("ERROR heartbeat has failed %d consecutive times; API server may be unreachable: %v", consecutiveFailures, err)
} else {
consecutiveFailures = 0
log.Printf("WARN heartbeat registration failed: %v", err)
}
} else {
consecutiveFailures = 0
}
}
}()
} else if cfg.ManagedClusterName != "" {
log.Printf("WARN HYPERSHELL_MANAGED_CLUSTER_NAME is set but OIDC is not configured; skipping self-registration")
}
}
}()

conn, err := grpc.NewClient(cfg.GRPCServerAddr, dialOpts...)
if err != nil {
Expand Down
33 changes: 22 additions & 11 deletions components/control-plane/internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,25 +31,36 @@ const (
// specs/platform/gateway-reconcile-concurrency.spec.md.
const DefaultGatewayReconcileWorkers = 4

// DefaultManagedClusterName is the fallback registration name when
// HYPERSHELL_MANAGED_CLUSTER_NAME is unset. Registration is unconditional, so a
// control plane always needs a stable name to register under. In local
// development (authentication disabled) this name is the record's identity, so it
// must be stable across restarts; "local" is a fixed literal rather than a
// per-pod value for that reason. Production overrides it with a unique name.
const DefaultManagedClusterName = "local"

type Config struct {
GRPCServerAddr string
APIServerURL string
Namespace string
LogLevel string

// ClusterID is this control-plane's managed-cluster identity (a Gateway
// cluster_id / KSUID). When set, the control-plane restricts the gateways it
// watches, seeds, and health-checks to those whose cluster_id matches, so a
// managed-cluster spoke only ever provisions its own gateways (the pull
// model). Empty preserves the single-cluster behaviour of handling every
// gateway. Sourced from HYPERSHELL_CLUSTER_ID; in production, resolved at
// runtime via spoke self-registration and should NOT be set in gitops.
// cluster_id / KSUID). The control-plane restricts the gateways it watches,
// seeds, and health-checks to those whose cluster_id matches, so it only ever
// provisions its own gateways (the pull model). It is resolved at runtime from
// the self-registration response and should NOT be set in gitops. Sourced from
// HYPERSHELL_CLUSTER_ID only as an escape hatch (e.g. tests); normally empty
// at load and populated by registration.
ClusterID string

// ManagedClusterName is the human-readable name of this spoke cluster, unique
// per fleet (e.g. hyp0-mc1). When set together with OIDC credentials, the
// control plane self-registers on startup, resolving ClusterID dynamically.
// Sourced from HYPERSHELL_MANAGED_CLUSTER_NAME.
// ManagedClusterName is the human-readable name of this control plane, used as
// its registration identity. Every control plane self-registers on startup
// using this name (see cmd/hypershell-controller): when OIDC is configured the
// registered record is keyed on the OIDC subject, and when the API server runs
// with authentication disabled (local development) it is keyed on this name
// alone. Sourced from HYPERSHELL_MANAGED_CLUSTER_NAME; defaults to
// DefaultManagedClusterName when unset so registration is always unconditional.
ManagedClusterName string

// ServiceAccountProvisionerAddress is the in-cluster bind address for the
Expand Down Expand Up @@ -98,7 +109,7 @@ func Load() (*Config, error) {
Namespace: getEnv("HYPERSHELL_NAMESPACE", "hypershell"),
LogLevel: strings.ToLower(getEnv("HYPERSHELL_LOG_LEVEL", "info")),
ClusterID: getEnv("HYPERSHELL_CLUSTER_ID", ""),
ManagedClusterName: getEnv("HYPERSHELL_MANAGED_CLUSTER_NAME", ""),
ManagedClusterName: getEnv("HYPERSHELL_MANAGED_CLUSTER_NAME", DefaultManagedClusterName),
ServiceAccountProvisionerAddress: getEnv("HYPERSHELL_SERVICE_ACCOUNT_PROVISIONER_BIND_ADDRESS", ""),

NamespaceGCEnabled: getEnvBool("GATEWAY_NAMESPACE_GC_ENABLED", true),
Expand Down
28 changes: 21 additions & 7 deletions components/control-plane/internal/registration/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,23 +13,29 @@ import (
)

// ErrForbidden is returned when the API server responds 403.
// This is non-retryable: the spoke lacks the required Keycloak role.
// This is non-retryable: the control plane lacks the required Keycloak role.
// It can only occur when the API server has authentication enabled.
var ErrForbidden = fmt.Errorf("registration denied: missing managed-cluster-registrar role in Keycloak")

// TokenSource can produce a bearer token.
type TokenSource interface {
Token() (string, error)
}

// Client registers a spoke control-plane with the hub API server.
// Client registers a control plane with the API server. Registration is
// unconditional: every control plane registers on startup. When tokens is nil
// (the API server runs with authentication disabled, e.g. local development) the
// request carries no Authorization header and the server keys the record on name
// alone; otherwise the bearer token's OIDC subject keys the record.
type Client struct {
apiServerURL string
clusterName string
tokens TokenSource
httpClient *http.Client
}

// NewClient creates a registration Client.
// NewClient creates a registration Client. tokens may be nil when the API server
// runs with authentication disabled; in that case no bearer token is sent.
func NewClient(apiServerURL, clusterName string, tokens TokenSource) *Client {
return &Client{
apiServerURL: strings.TrimRight(apiServerURL, "/"),
Expand All @@ -51,9 +57,15 @@ type registrationResponse struct {
// Returns (clusterID, nil) on success, (ErrForbidden, nil) on 403, or an
// error for transient failures that should be retried.
func (c *Client) Register(ctx context.Context) (string, error) {
token, err := c.tokens.Token()
if err != nil {
return "", fmt.Errorf("get OIDC token: %w", err)
// tokens is nil when the API server runs with authentication disabled; the
// request is then sent unauthenticated and the server keys the record on name.
var token string
if c.tokens != nil {
t, err := c.tokens.Token()
if err != nil {
return "", fmt.Errorf("get OIDC token: %w", err)
}
token = t
}

body, err := json.Marshal(registrationRequest{Name: c.clusterName})
Expand All @@ -67,7 +79,9 @@ func (c *Client) Register(ctx context.Context) (string, error) {
return "", fmt.Errorf("build registration request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+token)
if token != "" {
req.Header.Set("Authorization", "Bearer "+token)
}

resp, err := c.httpClient.Do(req)
if err != nil {
Expand Down
Loading
Loading