diff --git a/include/user_settings.h b/include/user_settings.h
index 5c59ae3ed1..adcf405ac0 100644
--- a/include/user_settings.h
+++ b/include/user_settings.h
@@ -870,12 +870,14 @@ extern int tolower(int c);
      * SHA384/SHA512 on the secure side, WC_MAX_DIGEST_SIZE caps at
      * SHA256's 32 and wc_ecc_sign_hash (ecc.c:7281) rejects legitimately
      * oversized hashes (e.g. ECDSA truncation tests) with BAD_LENGTH_E. */
-#   ifndef WOLFSSL_SHA384
-#       define WOLFSSL_SHA384
-#   endif
-#   ifndef WOLFSSL_SHA512
-#       define WOLFSSL_SHA512
-#   endif
+#   ifdef WOLFCRYPT_TZ_WOLFHSM
+#       ifndef WOLFSSL_SHA384
+#           define WOLFSSL_SHA384
+#       endif
+#       ifndef WOLFSSL_SHA512
+#           define WOLFSSL_SHA512
+#       endif
+#   endif /* WOLFCRYPT_TZ_WOLFHSM */
     /* Match the keycache sizing the wolfHSM test suite is validated
      * against (test/config/wolfhsm_cfg.h: 9 regular + 3 big). The
      * library defaults (8 + 1) are one regular slot short of the
diff --git a/options.mk b/options.mk
index c3d6b75bd6..821ef966b4 100644
--- a/options.mk
+++ b/options.mk
@@ -1535,13 +1535,6 @@ ifeq ($(WOLFHSM_SERVER),1)
     $(WOLFBOOT_LIB_WOLFSSL)/wolfcrypt/src/cryptocb.o \
     $(WOLFBOOT_LIB_WOLFSSL)/wolfcrypt/src/coding.o \
     $(WOLFBOOT_LIB_WOLFSSL)/wolfcrypt/src/random.o
-  # SHA-384/512 are used by the wolfHSM crypto handlers (HKDF, larger
-  # ECDSA hash sizes, etc.). Always link sha512.o except when ED25519
-  # is the signature algorithm, which already pulls it in.
-  ifneq ($(SIGN),ED25519)
-    WOLFCRYPT_OBJS += $(WOLFBOOT_LIB_WOLFSSL)/wolfcrypt/src/sha512.o
-  endif
-
   ifeq ($(SIGN),ML_DSA)
     WOLFCRYPT_OBJS += $(MATH_OBJS)
     # Large enough to handle the largest ML-DSA key/signature