arm64: mm: Handle alignment faults

Allow kernel/user space code to perform unaligned accesses to memory
regions that do not normally support them (e.g. device mappings) by
trapping alignment faults on common load/store instructions and
breaking up the offending accesses into naturally aligned ones.

Signed-off-by: Mario Bălănică <mariobalanica02@gmail.com>

Author: mariobalanica

arch/arm64/Kconfig

@@ -1696,6 +1696,15 @@ config ARM64_TAGGED_ADDR_ABI
 	  to system calls as pointer arguments. For details, see
 	  Documentation/arch/arm64/tagged-address-abi.rst.
 
+config ARM64_ALIGNMENT_FIXUPS
+	bool "Fix up misaligned multi-word loads and stores in 64-bit kernel/user space"
+	default y
+	help
+	  This option enables kernel/user space code to perform unaligned accesses
+	  to memory regions that do not normally support them (e.g. device mappings)
+	  by trapping alignment faults on common load/store instructions and breaking
+	  up the offending accesses into properly aligned ones.
+
 menuconfig COMPAT
 	bool "Kernel support for 32-bit EL0"
 	depends on ARM64_4K_PAGES || EXPERT

arch/arm64/include/asm/exception.h

@@ -82,6 +82,7 @@ void do_sp_pc_abort(unsigned long addr, unsigned long esr, struct pt_regs *regs)
 void bad_el0_sync(struct pt_regs *regs, int reason, unsigned long esr);
 void do_el0_cp15(unsigned long esr, struct pt_regs *regs);
 int do_compat_alignment_fixup(unsigned long addr, struct pt_regs *regs);
+int do_alignment_fixup(unsigned long addr, unsigned int esr, struct pt_regs *regs);
 void do_el0_svc(struct pt_regs *regs);
 void do_el0_svc_compat(struct pt_regs *regs);
 void do_el0_fpac(struct pt_regs *regs, unsigned long esr);

arch/arm64/kernel/Makefile

@@ -36,6 +36,7 @@ obj-y			:= debug-monitors.o entry.o irq.o fpsimd.o		\
 			   syscall.o proton-pack.o idle.o patching.o pi/	\
 			   rsi.o jump_label.o
 
+obj-$(CONFIG_ARM64_ALIGNMENT_FIXUPS)	+= alignment.o
 obj-$(CONFIG_COMPAT)			+= sys32.o signal32.o			\
 					   sys_compat.o
 obj-$(CONFIG_COMPAT)			+= sigreturn32.o

arch/arm64/mm/fault.c

@@ -795,8 +795,10 @@ static int __kprobes do_translation_fault(unsigned long far,
 static int do_alignment_fault(unsigned long far, unsigned long esr,
 			      struct pt_regs *regs)
 {
-	if (IS_ENABLED(CONFIG_COMPAT_ALIGNMENT_FIXUPS) &&
-	    compat_user_mode(regs))
+	if (!compat_user_mode(regs)) {
+		if (IS_ENABLED(CONFIG_ARM64_ALIGNMENT_FIXUPS))
+			return do_alignment_fixup(far, esr, regs);
+	} else if (IS_ENABLED(CONFIG_COMPAT_ALIGNMENT_FIXUPS))
 		return do_compat_alignment_fixup(far, regs);
 	do_bad_area(far, esr, regs);
 	return 0;