diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile
index 706e11c89935..f5237c0825d1 100644
--- a/arch/arm64/Makefile
+++ b/arch/arm64/Makefile
@@ -126,22 +126,26 @@ KBUILD_CPPFLAGS	+= -mbig-endian
 CHECKFLAGS	+= -D__AARCH64EB__
 AS		+= -EB
 LD		+= -EB
+ifeq ($(cc-name),clang)
 ifeq ($(ld-name),gold)
 LDFLAGS		+= -maarch64_elf64_be_vec
 else
 LDFLAGS		+= -maarch64elfb
 endif
+endif
 UTS_MACHINE	:= aarch64_be
 else
 KBUILD_CPPFLAGS	+= -mlittle-endian
 CHECKFLAGS	+= -D__AARCH64EL__
 AS		+= -EL
 LD		+= -EL
+ifeq ($(cc-name),clang)
 ifeq ($(ld-name),gold)
 LDFLAGS		+= -maarch64_elf64_le_vec
 else
 LDFLAGS		+= -maarch64elf
 endif
+endif
 UTS_MACHINE	:= aarch64
 endif