feat(api): revoke every key's delegations before removing a tenant

pkg/api/service/tenant/service.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"github.com/fil-forge/hilt/pkg/client/upload"
+	"github.com/fil-forge/hilt/pkg/grant"
 	"github.com/fil-forge/hilt/pkg/store"
 	"github.com/fil-forge/hilt/pkg/store/accesskey"
 	"github.com/fil-forge/hilt/pkg/store/bucket"
@@ -26,6 +27,7 @@ import (
 	"github.com/fil-forge/ucantone/did"
 	"github.com/fil-forge/ucantone/did/plc"
 	"github.com/fil-forge/ucantone/multikey/secp256k1"
+	"github.com/fil-forge/ucantone/ucan"
 	"go.uber.org/zap"
 )
 
@@ -48,6 +50,7 @@ type Service struct {
 	wrapKeys    wrapkeystore.Store
 	plcClient   *plc.DirectoryClient
 	upload      *upload.Client
+	revocations grant.RevocationPublisher
 }
 
 // New constructs the tenant service.
@@ -64,6 +67,7 @@ func New(
 	wrapKeys wrapkeystore.Store,
 	plcClient *plc.DirectoryClient,
 	upload *upload.Client,
+	revocations grant.RevocationPublisher,
 ) *Service {
 	return &Service{
 		logger:      logger,
@@ -78,6 +82,7 @@ func New(
 		plcClient:   plcClient,
 		wrapKeys:    wrapKeys,
 		upload:      upload,
+		revocations: revocations,
 	}
 }
 
@@ -291,18 +296,41 @@ func (s *Service) Delete(ctx context.Context, externalID string) error {
 
 	signingVaultKey := vault.TenantKeyPath(rec.ID)
 
-	// Deactivate the did:plc first — it requires the (still-present) tenant key.
-	// Aborting here leaves all local state intact for a retry.
-	if err := deactivateTenantDID(ctx, s.plcClient, s.secrets, signingVaultKey, rec.ID); err != nil {
-		log.Error("deactivating tenant DID", zap.Error(err))
-		return ErrDIDDeactivation
-	}
-
-	// Cascade: access keys (records + their delegations + vault keys).
 	keys, err := s.accessKeys.ListByTenant(ctx, rec.ID)
 	if err != nil {
 		return fmt.Errorf("listing access keys: %w", err)
 	}
+
+	// Revoke and remove every delegation the tenant's keys hold, in one
+	// request and one delegation-store write, before the did:plc goes: the
+	// tenant signs the revocations and Swarf verifies them against its DID. A
+	// warm key would otherwise keep serving from the gateway's cache until the
+	// next UTC midnight. A failed publish leaves everything in place for a
+	// retry. A DID an earlier attempt already deactivated skips both: that
+	// attempt published first, and Swarf could not verify a new publish against
+	// the tombstone anyway. A tenant tombstoned by a removal that predates the
+	// publish, and that then failed in its cascade, is the one case nothing
+	// here can revoke for: its keys' cached grants last until they expire.
+	last, err := s.plcClient.Last(ctx, rec.ID)
+	if _, deactivated := errors.AsType[*plc.DeactivatedDIDError](err); deactivated {
+		log.Info("tenant DID is already deactivated; skipping the revocation publish and the tombstone")
+	} else {
+		if err != nil {
+			log.Error("fetching the tenant DID's last operation", zap.Error(err))
+			return ErrDIDDeactivation
+		}
+		if err := s.revokeKeyDelegations(ctx, log, rec.ID, keys); err != nil {
+			return err
+		}
+		// Deactivate the did:plc next — it requires the (still-present) tenant
+		// key. Aborting here leaves all local state intact for a retry.
+		if err := deactivateTenantDID(ctx, s.plcClient, s.secrets, signingVaultKey, rec.ID, last); err != nil {
+			log.Error("deactivating tenant DID", zap.Error(err))
+			return ErrDIDDeactivation
+		}
+	}
+
+	// Cascade: access keys (records + any delegations left + vault keys).
 	for _, ak := range keys {
 		if err := s.delegations.DeleteByAudience(ctx, ak.ID); err != nil {
 			return fmt.Errorf("deleting access key delegations: %w", err)
@@ -316,8 +344,7 @@ func (s *Service) Delete(ctx context.Context, externalID string) error {
 	}
 
 	// Cascade: principals. They reference the tenant row (FK RESTRICT), so they
-	// go before it. Nothing is published: the tenant is disabled and its keys are
-	// gone, so no request can reach a principal's cached access.
+	// go before it. Their keys' delegations were revoked above.
 	if err := s.principals.DeleteByTenant(ctx, rec.ID); err != nil {
 		return fmt.Errorf("deleting principals: %w", err)
 	}
@@ -388,6 +415,38 @@ func (s *Service) Delete(ctx context.Context, externalID string) error {
 	return nil
 }
 
+// revokeKeyDelegations publishes, in one request signed by the tenant, a
+// revocation for every delegation the keys hold, and removes them in the same
+// delegation-store write, so a failed publish leaves every key's delegations
+// in place. Nothing is published when they hold none.
+func (s *Service) revokeKeyDelegations(ctx context.Context, log *zap.Logger, tenantID did.DID, keys []accesskey.Record) error {
+	if len(keys) == 0 {
+		return nil
+	}
+	audiences := make([]did.DID, len(keys))
+	for i, ak := range keys {
+		audiences[i] = ak.ID
+	}
+	issuer, err := vault.TenantIssuer(ctx, s.secrets, tenantID)
+	if err != nil {
+		return err
+	}
+	// The publish runs while the keys' delegations are locked, so the whole
+	// write is bounded at the batch deadline, as the rotator's is.
+	ctx, cancel := context.WithTimeout(ctx, grant.BatchTimeout)
+	defer cancel()
+	return s.delegations.Replace(ctx, audiences, func(ctx context.Context, current map[did.DID][]ucan.Delegation) (map[did.DID][]ucan.Delegation, error) {
+		var all []ucan.Delegation
+		for _, held := range current {
+			all = append(all, held...)
+		}
+		if err := grant.PublishRevocations(ctx, log, s.revocations, issuer, all); err != nil {
+			return nil, fmt.Errorf("revoking the tenant's key delegations: %w", err)
+		}
+		return nil, nil
+	})
+}
+
 // cleanupKey removes an orphaned tenant key after a provisioning failure. It runs
 // on a context detached from the request so a client disconnect — which cancels
 // ctx — cannot abort the cleanup partway.
@@ -397,17 +456,9 @@ func (s *Service) cleanupKey(ctx context.Context, log *zap.Logger, vaultKey stri
 	}
 }
 
-// deactivateTenantDID publishes a tombstone for the tenant's did:plc, signed with
-// its rotation key from the vault. If the DID is already deactivated it is a no-op.
-func deactivateTenantDID(ctx context.Context, plcClient *plc.DirectoryClient, secrets vault.Vault, vaultKey string, tenantID did.DID) error {
-	last, err := plcClient.Last(ctx, tenantID)
-	if err != nil {
-		if _, ok := errors.AsType[*plc.DeactivatedDIDError](err); ok {
-			return nil // already deactivated
-		}
-		return fmt.Errorf("fetching last operation: %w", err)
-	}
-
+// deactivateTenantDID publishes a tombstone for the tenant's did:plc after its
+// last operation, signed with its rotation key from the vault.
+func deactivateTenantDID(ctx context.Context, plcClient *plc.DirectoryClient, secrets vault.Vault, vaultKey string, tenantID did.DID, last *plc.SignedOperation) error {
 	keyBytes, err := secrets.Read(ctx, vaultKey)
 	if err != nil {
 		return fmt.Errorf("reading tenant key: %w", err)

pkg/api/service/tenant/service_test.go

@@ -2,17 +2,21 @@ package tenant_test
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"testing"
+	"time"
 
 	"github.com/fil-forge/hilt/internal/testutil"
 	tenantsvc "github.com/fil-forge/hilt/pkg/api/service/tenant"
 	"github.com/fil-forge/hilt/pkg/bucketpolicy"
 	"github.com/fil-forge/hilt/pkg/client/upload"
+	"github.com/fil-forge/hilt/pkg/grant"
 	"github.com/fil-forge/hilt/pkg/store"
+	accesskeystore "github.com/fil-forge/hilt/pkg/store/accesskey"
 	accesskeymemory "github.com/fil-forge/hilt/pkg/store/accesskey/memory"
 	bucketmemory "github.com/fil-forge/hilt/pkg/store/bucket/memory"
 	bucketpolicystore "github.com/fil-forge/hilt/pkg/store/bucketpolicy"
@@ -30,9 +34,12 @@ import (
 	"github.com/fil-forge/ucantone/binding"
 	"github.com/fil-forge/ucantone/did"
 	"github.com/fil-forge/ucantone/did/plc"
+	"github.com/fil-forge/ucantone/multikey"
 	"github.com/fil-forge/ucantone/multikey/secp256k1"
 	"github.com/fil-forge/ucantone/server"
+	"github.com/fil-forge/ucantone/ucan"
 	"github.com/fil-forge/ucantone/ucan/container"
+	"github.com/ipfs/go-cid"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
 )
@@ -87,7 +94,7 @@ func provisionSetup(t *testing.T, plcStatus int) provisionEnv {
 	require.NoError(t, err)
 
 	svc := tenantsvc.New(zap.NewNop(), tenantmemory.New(), providers, bucketmemory.New(),
-		accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), vaultmemory.New(), wrapkeysmemory.New(), plcClient, upload)
+		accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), vaultmemory.New(), wrapkeysmemory.New(), plcClient, upload, &testutil.FakeSwarf{})
 	return provisionEnv{svc: svc, providers: providers, sprueFailed: sprueFailed}
 }
 
@@ -148,7 +155,7 @@ func TestProvision(t *testing.T) {
 // clients — enough for Get and SetStatus, which never touch them.
 func simpleService(tenants tenant.Store) *tenantsvc.Service {
 	return tenantsvc.New(zap.NewNop(), tenants, providermemory.New(), bucketmemory.New(),
-		accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), vaultmemory.New(), wrapkeysmemory.New(), nil, nil)
+		accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), vaultmemory.New(), wrapkeysmemory.New(), nil, nil, &testutil.FakeSwarf{})
 }
 
 func TestGetAndSetStatus(t *testing.T) {
@@ -224,9 +231,17 @@ type deleteEnv struct {
 	buckets    *bucketmemory.Store
 	policies   *bucketpolicymemory.Store
 	principals *principalmemory.Store
-	bucketID   did.DID
-	tenantID   did.DID
-	directory  *plcDirectory
+	swarf      *testutil.FakeSwarf
+	// publisher is swarf as the service sees it, recording each publish's deadline.
+	publisher *deadlinePublisher
+	bucketID  did.DID
+	tenantID  did.DID
+	// grants are the delegations the tenant's one access key holds.
+	grants    []cid.Cid
+	directory *plcDirectory
+	// tombstone is the DID's signed tombstone, for a directory that already
+	// served a deactivation.
+	tombstone []byte
 }
 
 func deleteSetup(t *testing.T, status tenant.Status) deleteEnv {
@@ -244,6 +259,12 @@ func deleteSetup(t *testing.T, status tenant.Status) deleteEnv {
 
 	var genesisJSON bytes.Buffer
 	require.NoError(t, genesis.MarshalDagJSON(&genesisJSON))
+	tomb, err := plc.NewTombstoneFromPrevious(genesis)
+	require.NoError(t, err)
+	signedTomb, err := plc.SignTombstone(signer, tomb)
+	require.NoError(t, err)
+	var tombstoneJSON bytes.Buffer
+	require.NoError(t, signedTomb.MarshalDagJSON(&tombstoneJSON))
 	directory := &plcDirectory{logLast: genesisJSON.Bytes()}
 	dirServer := httptest.NewServer(directory)
 	t.Cleanup(dirServer.Close)
@@ -270,10 +291,42 @@ func deleteSetup(t *testing.T, status tenant.Status) deleteEnv {
 	}, nil)
 	require.NoError(t, err)
 
+	// One access key holding a grant over the bucket, so the revocations the
+	// removal publishes are observable.
+	accessKeys, delegations := accesskeymemory.New(), delegationmemory.New()
+	keyID := testutil.RandomDID(t)
+	require.NoError(t, accessKeys.Add(ctx, accesskeystore.Input{ID: keyID, Tenant: tenantID, Name: "k1", Permissions: []string{"s3:GetObject"}}))
+	held, err := grant.Issue(multikey.NewIssuer(tenantID, signer), keyID, []did.DID{bucketID}, []string{"s3:GetObject"}, nil)
+	require.NoError(t, err)
+	require.NoError(t, delegations.PutBatch(ctx, held))
+	var grants []cid.Cid
+	for _, d := range held {
+		grants = append(grants, d.Link())
+	}
+
+	swarf := &testutil.FakeSwarf{}
+	publisher := &deadlinePublisher{FakeSwarf: swarf}
 	svc := tenantsvc.New(zap.NewNop(), tenants, providermemory.New(), buckets,
-		accesskeymemory.New(), principals, policies, delegationmemory.New(), secrets, wrapkeysmemory.New(), plcClient, nil)
+		accessKeys, principals, policies, delegations, secrets, wrapkeysmemory.New(), plcClient, nil, publisher)
 	return deleteEnv{svc: svc, tenants: tenants, buckets: buckets, policies: policies,
-		principals: principals, bucketID: bucketID, tenantID: tenantID, directory: directory}
+		principals: principals, swarf: swarf, publisher: publisher, bucketID: bucketID, tenantID: tenantID, grants: grants, directory: directory,
+		tombstone: tombstoneJSON.Bytes()}
+}
+
+// deadlinePublisher records how long each publish had left on its deadline,
+// zero when it had none.
+type deadlinePublisher struct {
+	*testutil.FakeSwarf
+	remaining []time.Duration
+}
+
+func (p *deadlinePublisher) PublishBatch(ctx context.Context, revoker ucan.Issuer, revoked []ucan.Delegation) error {
+	var left time.Duration
+	if deadline, ok := ctx.Deadline(); ok {
+		left = time.Until(deadline)
+	}
+	p.remaining = append(p.remaining, left)
+	return p.FakeSwarf.PublishBatch(ctx, revoker, revoked)
 }
 
 func TestDelete(t *testing.T) {
@@ -290,6 +343,41 @@ func TestDelete(t *testing.T) {
 		require.Equal(t, 1, env.directory.deactivations)
 	})
 
+	t.Run("revokes every key's delegations in one request before deactivating the DID", func(t *testing.T) {
+		env := deleteSetup(t, tenant.Disabled)
+		require.NoError(t, env.svc.Delete(ctx, "tenant-1"))
+		require.Equal(t, 1, env.swarf.Calls())
+		require.ElementsMatch(t, env.grants, env.swarf.Revoked())
+		require.Equal(t, 1, env.directory.deactivations)
+	})
+
+	t.Run("bounds the revocation publish at the batch deadline", func(t *testing.T) {
+		env := deleteSetup(t, tenant.Disabled)
+		require.NoError(t, env.svc.Delete(context.Background(), "tenant-1"))
+		require.Len(t, env.publisher.remaining, 1)
+		require.Greater(t, env.publisher.remaining[0], time.Duration(0), "a stalled publish must not hold the keys' delegation locks past the batch deadline")
+		require.LessOrEqual(t, env.publisher.remaining[0], grant.BatchTimeout)
+	})
+
+	t.Run("a publish failure leaves the tenant and its DID in place", func(t *testing.T) {
+		env := deleteSetup(t, tenant.Disabled)
+		env.swarf.Err = errors.New("swarf unreachable")
+		require.ErrorContains(t, env.svc.Delete(ctx, "tenant-1"), "swarf unreachable")
+		_, err := env.tenants.GetByExternalID(ctx, "tenant-1")
+		require.NoError(t, err)
+		require.Zero(t, env.directory.deactivations)
+	})
+
+	t.Run("an already deactivated DID publishes nothing and finishes the cascade", func(t *testing.T) {
+		env := deleteSetup(t, tenant.Disabled)
+		env.directory.logLast = env.tombstone
+		require.NoError(t, env.svc.Delete(ctx, "tenant-1"))
+		require.Zero(t, env.swarf.Calls())
+		require.Zero(t, env.directory.deactivations)
+		_, err := env.tenants.GetByExternalID(ctx, "tenant-1")
+		require.ErrorIs(t, err, store.ErrRecordNotFound)
+	})
+
 	t.Run("deletes the tenant's buckets and their policies", func(t *testing.T) {
 		env := deleteSetup(t, tenant.Disabled)
 		require.NoError(t, env.svc.Delete(ctx, "tenant-1"))

pkg/api/tenants_test.go

@@ -230,7 +230,7 @@ func setupProvision(t *testing.T, cfg *setupConfig) (*echo.Echo, *provisionDeps)
 		upload.WithHTTPClient(&http.Client{Transport: srv}))
 	require.NoError(t, err)
 
-	svc := tenantsvc.New(zap.NewNop(), deps.tenants, deps.providers, bucketmemory.New(), accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), deps.secrets, deps.wrapKeys, plcClient, upload)
+	svc := tenantsvc.New(zap.NewNop(), deps.tenants, deps.providers, bucketmemory.New(), accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), deps.secrets, deps.wrapKeys, plcClient, upload, &testutil.FakeSwarf{})
 	route := api.NewProvisionTenantHandler(zap.NewNop(), svc)
 	e := echo.New()
 	e.Add(route.Method, route.Path, route.Handler)
@@ -513,7 +513,7 @@ func TestGetTenantHandler(t *testing.T) {
 	ctx := t.Context()
 	tenants := tenantmemory.New()
 	require.NoError(t, tenants.Add(ctx, testutil.RandomDID(t), "tenant-1", testutil.RandomDID(t), tenant.Active))
-	svc := tenantsvc.New(zap.NewNop(), tenants, providermemory.New(), bucketmemory.New(), accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), vaultmemory.New(), wrapkeymemory.New(), nil, nil)
+	svc := tenantsvc.New(zap.NewNop(), tenants, providermemory.New(), bucketmemory.New(), accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), vaultmemory.New(), wrapkeymemory.New(), nil, nil, &testutil.FakeSwarf{})
 	e := serve(api.NewGetTenantHandler(zap.NewNop(), svc))
 
 	t.Run("found", func(t *testing.T) {
@@ -541,7 +541,7 @@ func TestUpdateTenantStatusHandler(t *testing.T) {
 	tenants := tenantmemory.New()
 	id := testutil.RandomDID(t)
 	require.NoError(t, tenants.Add(ctx, id, "tenant-1", testutil.RandomDID(t), tenant.Active))
-	svc := tenantsvc.New(zap.NewNop(), tenants, providermemory.New(), bucketmemory.New(), accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), vaultmemory.New(), wrapkeymemory.New(), nil, nil)
+	svc := tenantsvc.New(zap.NewNop(), tenants, providermemory.New(), bucketmemory.New(), accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), vaultmemory.New(), wrapkeymemory.New(), nil, nil, &testutil.FakeSwarf{})
 	e := serve(api.NewUpdateTenantStatusHandler(zap.NewNop(), svc))
 
 	statusBody := func(s api.TenantStatus) []byte {
@@ -689,7 +689,7 @@ func setupDelete(t *testing.T, status tenant.Status) (*echo.Echo, *deleteDeps) {
 		VaultKey: wrapkeystore.VaultKey(tenantID, 1),
 	}))
 
-	svc := tenantsvc.New(zap.NewNop(), deps.tenants, providermemory.New(), deps.buckets, deps.accessKeys, deps.principals, bucketpolicymemory.New(), deps.delegations, deps.secrets, deps.wrapKeys, plcClient, nil)
+	svc := tenantsvc.New(zap.NewNop(), deps.tenants, providermemory.New(), deps.buckets, deps.accessKeys, deps.principals, bucketpolicymemory.New(), deps.delegations, deps.secrets, deps.wrapKeys, plcClient, nil, &testutil.FakeSwarf{})
 	route := api.NewDeleteTenantHandler(zap.NewNop(), svc)
 	return serve(route), deps
 }