diff --git a/arch/xtensa/src/common/Make.defs b/arch/xtensa/src/common/Make.defs
index 31e2e61b6c..95644a9991 100644
--- a/arch/xtensa/src/common/Make.defs
+++ b/arch/xtensa/src/common/Make.defs
@@ -38,6 +38,10 @@ CMN_CSRCS += xtensa_schedsigaction.c xtensa_udelay.c xtensa_usestack.c
 
 # Configuration-dependent common Xtensa files
 
+ifeq ($(CONFIG_ARCH_USE_TEXT_HEAP),y)
+  CMN_ASRCS += xtensa_loadstore.S
+endif
+
 ifeq ($(CONFIG_DEBUG_TCBINFO),y)
   CMN_CSRCS += xtensa_tcbinfo.c
 endif
diff --git a/arch/xtensa/src/esp32/Make.defs b/arch/xtensa/src/esp32/Make.defs
index cab253b1f8..1a6f5fc91d 100644
--- a/arch/xtensa/src/esp32/Make.defs
+++ b/arch/xtensa/src/esp32/Make.defs
@@ -173,7 +173,6 @@ endif
 
 ifeq ($(CONFIG_ARCH_USE_TEXT_HEAP),y)
 CHIP_CSRCS += esp32_textheap.c
-CMN_ASRCS  += xtensa_loadstore.S
 endif
 
 ifeq ($(CONFIG_ESP32_RT_TIMER),y)