[ Upstream commit 080b6f40763565f65ebb9540219c71ce885cf568 ]
Commit 3193c0836 ("bpf: Disable GCC -fgcse optimization for
___bpf_prog_run()") introduced a __no_fgcse macro that expands to a
function scope __attribute__((optimize("-fno-gcse"))), to disable a
GCC specific optimization that was causing trouble on x86 builds, and
was not expected to have any positive effect in the first place.
However, as the GCC manual documents, __attribute__((optimize))
is not for production use, and results in all other optimization
options to be forgotten for the function in question. This can
cause all kinds of trouble, but in one particular reported case,
it causes -fno-asynchronous-unwind-tables to be disregarded,
resulting in .eh_frame info to be emitted for the function.
This reverts commit 3193c0836, and instead, it disables the -fgcse
optimization for the entire source file, but only when building for
X86 using GCC with CONFIG_BPF_JIT_ALWAYS_ON disabled. Note that the
original commit states that CONFIG_RETPOLINE=n triggers the issue,
whereas CONFIG_RETPOLINE=y performs better without the optimization,
so it is kept disabled in both cases.
Fixes: 3193c0836f20 ("bpf: Disable GCC -fgcse optimization for ___bpf_prog_run()")
Change-Id: I513d26e71b1c3928ed5cb9530ce2afa392aa7038
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Tested-by: Geert Uytterhoeven <geert+renesas@glider.be>
Reviewed-by: Nick Desaulniers <ndesaulniers@google.com>
Link: https://lore.kernel.org/lkml/CAMuHMdUg0WJHEcq6to0-eODpXPOywLot6UD2=GFHpzoj_hCoBQ@mail.gmail.com/
Link: https://lore.kernel.org/bpf/20201028171506.15682-2-ardb@kernel.org
Signed-off-by: Sasha Levin <sashal@kernel.org>
Signed-off-by: Naveen <133593113+elohim-etz@users.noreply.github.com>
381 lines
13 KiB
C
381 lines
13 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef __LINUX_COMPILER_TYPES_H
|
|
#error "Please don't include <linux/compiler-gcc.h> directly, include <linux/compiler.h> instead."
|
|
#endif
|
|
|
|
/*
|
|
* Common definitions for all gcc versions go here.
|
|
*/
|
|
#define GCC_VERSION (__GNUC__ * 10000 \
|
|
+ __GNUC_MINOR__ * 100 \
|
|
+ __GNUC_PATCHLEVEL__)
|
|
|
|
/* Optimization barrier */
|
|
|
|
/* The "volatile" is due to gcc bugs */
|
|
#define barrier() __asm__ __volatile__("": : :"memory")
|
|
/*
|
|
* This version is i.e. to prevent dead stores elimination on @ptr
|
|
* where gcc and llvm may behave differently when otherwise using
|
|
* normal barrier(): while gcc behavior gets along with a normal
|
|
* barrier(), llvm needs an explicit input variable to be assumed
|
|
* clobbered. The issue is as follows: while the inline asm might
|
|
* access any memory it wants, the compiler could have fit all of
|
|
* @ptr into memory registers instead, and since @ptr never escaped
|
|
* from that, it proved that the inline asm wasn't touching any of
|
|
* it. This version works well with both compilers, i.e. we're telling
|
|
* the compiler that the inline asm absolutely may see the contents
|
|
* of @ptr. See also: https://llvm.org/bugs/show_bug.cgi?id=15495
|
|
*/
|
|
#define barrier_data(ptr) __asm__ __volatile__("": :"r"(ptr) :"memory")
|
|
|
|
/*
|
|
* This macro obfuscates arithmetic on a variable address so that gcc
|
|
* shouldn't recognize the original var, and make assumptions about it.
|
|
*
|
|
* This is needed because the C standard makes it undefined to do
|
|
* pointer arithmetic on "objects" outside their boundaries and the
|
|
* gcc optimizers assume this is the case. In particular they
|
|
* assume such arithmetic does not wrap.
|
|
*
|
|
* A miscompilation has been observed because of this on PPC.
|
|
* To work around it we hide the relationship of the pointer and the object
|
|
* using this macro.
|
|
*
|
|
* Versions of the ppc64 compiler before 4.1 had a bug where use of
|
|
* RELOC_HIDE could trash r30. The bug can be worked around by changing
|
|
* the inline assembly constraint from =g to =r, in this particular
|
|
* case either is valid.
|
|
*/
|
|
#define RELOC_HIDE(ptr, off) \
|
|
({ \
|
|
unsigned long __ptr; \
|
|
__asm__ ("" : "=r"(__ptr) : "0"(ptr)); \
|
|
(typeof(ptr)) (__ptr + (off)); \
|
|
})
|
|
|
|
/* Make the optimizer believe the variable can be manipulated arbitrarily. */
|
|
#define OPTIMIZER_HIDE_VAR(var) \
|
|
__asm__ ("" : "=r" (var) : "0" (var))
|
|
|
|
#ifdef __CHECKER__
|
|
#define __must_be_array(a) 0
|
|
#else
|
|
/* &a[0] degrades to a pointer: a different type from an array */
|
|
#define __must_be_array(a) BUILD_BUG_ON_ZERO(__same_type((a), &(a)[0]))
|
|
#endif
|
|
|
|
/*
|
|
* Feature detection for gnu_inline (gnu89 extern inline semantics). Either
|
|
* __GNUC_STDC_INLINE__ is defined (not using gnu89 extern inline semantics,
|
|
* and we opt in to the gnu89 semantics), or __GNUC_STDC_INLINE__ is not
|
|
* defined so the gnu89 semantics are the default.
|
|
*/
|
|
#ifdef __GNUC_STDC_INLINE__
|
|
# define __gnu_inline __attribute__((gnu_inline))
|
|
#else
|
|
# define __gnu_inline
|
|
#endif
|
|
|
|
/*
|
|
* Force always-inline if the user requests it so via the .config,
|
|
* or if gcc is too old.
|
|
* GCC does not warn about unused static inline functions for
|
|
* -Wunused-function. This turns out to avoid the need for complex #ifdef
|
|
* directives. Suppress the warning in clang as well by using "unused"
|
|
* function attribute, which is redundant but not harmful for gcc.
|
|
* Prefer gnu_inline, so that extern inline functions do not emit an
|
|
* externally visible function. This makes extern inline behave as per gnu89
|
|
* semantics rather than c99. This prevents multiple symbol definition errors
|
|
* of extern inline functions at link time.
|
|
* A lot of inline functions can cause havoc with function tracing.
|
|
*/
|
|
#if !defined(CONFIG_ARCH_SUPPORTS_OPTIMIZED_INLINING) || \
|
|
!defined(CONFIG_OPTIMIZE_INLINING) || (__GNUC__ < 4)
|
|
#define inline \
|
|
inline __attribute__((always_inline, unused)) notrace __gnu_inline
|
|
#else
|
|
#define inline inline __attribute__((unused)) notrace __gnu_inline
|
|
#endif
|
|
|
|
#define __inline__ inline
|
|
#define __inline inline
|
|
#define __always_inline inline __attribute__((always_inline))
|
|
#define noinline __attribute__((noinline))
|
|
|
|
#define __deprecated __attribute__((deprecated))
|
|
#define __packed __attribute__((packed))
|
|
#define __weak __attribute__((weak))
|
|
#define __alias(symbol) __attribute__((alias(#symbol)))
|
|
|
|
#ifdef CONFIG_RETPOLINE
|
|
#define __noretpoline __attribute__((indirect_branch("keep")))
|
|
#endif
|
|
|
|
/*
|
|
* it doesn't make sense on ARM (currently the only user of __naked)
|
|
* to trace naked functions because then mcount is called without
|
|
* stack and frame pointer being set up and there is no chance to
|
|
* restore the lr register to the value before mcount was called.
|
|
*
|
|
* The asm() bodies of naked functions often depend on standard calling
|
|
* conventions, therefore they must be noinline and noclone.
|
|
*
|
|
* GCC 4.[56] currently fail to enforce this, so we must do so ourselves.
|
|
* See GCC PR44290.
|
|
*/
|
|
#define __naked __attribute__((naked)) noinline __noclone notrace
|
|
|
|
#define __noreturn __attribute__((noreturn))
|
|
|
|
/*
|
|
* From the GCC manual:
|
|
*
|
|
* Many functions have no effects except the return value and their
|
|
* return value depends only on the parameters and/or global
|
|
* variables. Such a function can be subject to common subexpression
|
|
* elimination and loop optimization just as an arithmetic operator
|
|
* would be.
|
|
* [...]
|
|
*/
|
|
#define __pure __attribute__((pure))
|
|
#define __aligned(x) __attribute__((aligned(x)))
|
|
#define __aligned_largest __attribute__((aligned))
|
|
#define __printf(a, b) __attribute__((format(printf, a, b)))
|
|
#define __scanf(a, b) __attribute__((format(scanf, a, b)))
|
|
#define __attribute_const__ __attribute__((__const__))
|
|
#define __maybe_unused __attribute__((unused))
|
|
#define __always_unused __attribute__((unused))
|
|
#define __mode(x) __attribute__((mode(x)))
|
|
|
|
/* gcc version specific checks */
|
|
|
|
#if GCC_VERSION < 30200
|
|
# error Sorry, your compiler is too old - please upgrade it.
|
|
#elif defined(CONFIG_ARM64) && GCC_VERSION < 50100 && !defined(__clang__)
|
|
/*
|
|
* https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63293
|
|
* https://lore.kernel.org/r/20210107111841.GN1551@shell.armlinux.org.uk
|
|
*/
|
|
# error Sorry, your version of GCC is too old - please use 5.1 or newer.
|
|
#endif
|
|
|
|
#if GCC_VERSION < 30300
|
|
# define __used __attribute__((__unused__))
|
|
#else
|
|
# define __used __attribute__((__used__))
|
|
#endif
|
|
|
|
#ifdef CONFIG_GCOV_KERNEL
|
|
# if GCC_VERSION < 30400
|
|
# error "GCOV profiling support for gcc versions below 3.4 not included"
|
|
# endif /* __GNUC_MINOR__ */
|
|
#endif /* CONFIG_GCOV_KERNEL */
|
|
|
|
#if GCC_VERSION >= 30400
|
|
#define __must_check __attribute__((warn_unused_result))
|
|
#define __malloc __attribute__((__malloc__))
|
|
#endif
|
|
|
|
#if GCC_VERSION >= 40000
|
|
|
|
/* GCC 4.1.[01] miscompiles __weak */
|
|
#ifdef __KERNEL__
|
|
# if GCC_VERSION >= 40100 && GCC_VERSION <= 40101
|
|
# error Your version of gcc miscompiles the __weak directive
|
|
# endif
|
|
#endif
|
|
|
|
#define __used __attribute__((__used__))
|
|
#define __compiler_offsetof(a, b) \
|
|
__builtin_offsetof(a, b)
|
|
|
|
#if GCC_VERSION >= 40100
|
|
# define __compiletime_object_size(obj) __builtin_object_size(obj, 0)
|
|
#endif
|
|
|
|
#if GCC_VERSION >= 40300
|
|
/* Mark functions as cold. gcc will assume any path leading to a call
|
|
* to them will be unlikely. This means a lot of manual unlikely()s
|
|
* are unnecessary now for any paths leading to the usual suspects
|
|
* like BUG(), printk(), panic() etc. [but let's keep them for now for
|
|
* older compilers]
|
|
*
|
|
* Early snapshots of gcc 4.3 don't support this and we can't detect this
|
|
* in the preprocessor, but we can live with this because they're unreleased.
|
|
* Maketime probing would be overkill here.
|
|
*
|
|
* gcc also has a __attribute__((__hot__)) to move hot functions into
|
|
* a special section, but I don't see any sense in this right now in
|
|
* the kernel context
|
|
*/
|
|
#define __cold __attribute__((__cold__))
|
|
|
|
#define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __COUNTER__)
|
|
|
|
#ifndef __CHECKER__
|
|
# define __compiletime_warning(message) __attribute__((warning(message)))
|
|
# define __compiletime_error(message) __attribute__((error(message)))
|
|
#endif /* __CHECKER__ */
|
|
#endif /* GCC_VERSION >= 40300 */
|
|
|
|
#if GCC_VERSION >= 40400
|
|
#define __optimize(level) __attribute__((__optimize__(level)))
|
|
#define __nostackprotector __optimize("no-stack-protector")
|
|
#endif /* GCC_VERSION >= 40400 */
|
|
|
|
#if GCC_VERSION >= 40500
|
|
|
|
#ifndef __CHECKER__
|
|
#ifdef LATENT_ENTROPY_PLUGIN
|
|
#define __latent_entropy __attribute__((latent_entropy))
|
|
#endif
|
|
#endif
|
|
|
|
/*
|
|
* calling noreturn functions, __builtin_unreachable() and __builtin_trap()
|
|
* confuse the stack allocation in gcc, leading to overly large stack
|
|
* frames, see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82365
|
|
*
|
|
* Adding an empty inline assembly before it works around the problem
|
|
*/
|
|
#define barrier_before_unreachable() asm volatile("")
|
|
|
|
/*
|
|
* Mark a position in code as unreachable. This can be used to
|
|
* suppress control flow warnings after asm blocks that transfer
|
|
* control elsewhere.
|
|
*
|
|
* Early snapshots of gcc 4.5 don't support this and we can't detect
|
|
* this in the preprocessor, but we can live with this because they're
|
|
* unreleased. Really, we need to have autoconf for the kernel.
|
|
*/
|
|
#define unreachable() \
|
|
do { \
|
|
annotate_unreachable(); \
|
|
barrier_before_unreachable(); \
|
|
__builtin_unreachable(); \
|
|
} while (0)
|
|
|
|
/* Mark a function definition as prohibited from being cloned. */
|
|
#define __noclone __attribute__((__noclone__, __optimize__("no-tracer")))
|
|
|
|
#ifdef RANDSTRUCT_PLUGIN
|
|
#define __randomize_layout __attribute__((randomize_layout))
|
|
#define __no_randomize_layout __attribute__((no_randomize_layout))
|
|
/* This anon struct can add padding, so only enable it under randstruct. */
|
|
#define randomized_struct_fields_start struct {
|
|
#define randomized_struct_fields_end } __randomize_layout;
|
|
#endif
|
|
|
|
#endif /* GCC_VERSION >= 40500 */
|
|
|
|
#if GCC_VERSION >= 40600
|
|
|
|
/*
|
|
* When used with Link Time Optimization, gcc can optimize away C functions or
|
|
* variables which are referenced only from assembly code. __visible tells the
|
|
* optimizer that something else uses this function or variable, thus preventing
|
|
* this.
|
|
*/
|
|
#define __visible __attribute__((externally_visible))
|
|
|
|
#endif /* GCC_VERSION >= 40600 */
|
|
|
|
|
|
#if GCC_VERSION >= 40900 && !defined(__CHECKER__)
|
|
/*
|
|
* __assume_aligned(n, k): Tell the optimizer that the returned
|
|
* pointer can be assumed to be k modulo n. The second argument is
|
|
* optional (default 0), so we use a variadic macro to make the
|
|
* shorthand.
|
|
*
|
|
* Beware: Do not apply this to functions which may return
|
|
* ERR_PTRs. Also, it is probably unwise to apply it to functions
|
|
* returning extra information in the low bits (but in that case the
|
|
* compiler should see some alignment anyway, when the return value is
|
|
* massaged by 'flags = ptr & 3; ptr &= ~3;').
|
|
*/
|
|
#define __assume_aligned(a, ...) __attribute__((__assume_aligned__(a, ## __VA_ARGS__)))
|
|
#endif
|
|
|
|
/*
|
|
* GCC 'asm goto' miscompiles certain code sequences:
|
|
*
|
|
* http://gcc.gnu.org/bugzilla/show_bug.cgi?id=58670
|
|
*
|
|
* Work it around via a compiler barrier quirk suggested by Jakub Jelinek.
|
|
*
|
|
* (asm goto is automatically volatile - the naming reflects this.)
|
|
*/
|
|
#define asm_volatile_goto(x...) do { asm goto(x); asm (""); } while (0)
|
|
|
|
/*
|
|
* sparse (__CHECKER__) pretends to be gcc, but can't do constant
|
|
* folding in __builtin_bswap*() (yet), so don't set these for it.
|
|
*/
|
|
#if defined(CONFIG_ARCH_USE_BUILTIN_BSWAP) && !defined(__CHECKER__)
|
|
#if GCC_VERSION >= 40400
|
|
#define __HAVE_BUILTIN_BSWAP32__
|
|
#define __HAVE_BUILTIN_BSWAP64__
|
|
#endif
|
|
#if GCC_VERSION >= 40800
|
|
#define __HAVE_BUILTIN_BSWAP16__
|
|
#endif
|
|
#endif /* CONFIG_ARCH_USE_BUILTIN_BSWAP && !__CHECKER__ */
|
|
|
|
#if GCC_VERSION >= 70000
|
|
#define KASAN_ABI_VERSION 5
|
|
#elif GCC_VERSION >= 50000
|
|
#define KASAN_ABI_VERSION 4
|
|
#elif GCC_VERSION >= 40902
|
|
#define KASAN_ABI_VERSION 3
|
|
#endif
|
|
|
|
/*
|
|
* Older GCCs (< 5) don't support __has_attribute, so instead of checking
|
|
* __has_attribute(__no_sanitize_address__) do a GCC version check.
|
|
*/
|
|
#ifndef __has_attribute
|
|
# define __has_attribute(x) __GCC4_has_attribute_##x
|
|
# define __GCC4_has_attribute___no_sanitize_address__ (__GNUC_MINOR__ >= 8)
|
|
#endif
|
|
|
|
#if __has_attribute(__no_sanitize_address__)
|
|
#define __no_sanitize_address __attribute__((no_sanitize_address))
|
|
#else
|
|
#define __no_sanitize_address
|
|
#endif
|
|
|
|
#if GCC_VERSION >= 50100
|
|
/*
|
|
* Mark structures as requiring designated initializers.
|
|
* https://gcc.gnu.org/onlinedocs/gcc/Designated-Inits.html
|
|
*/
|
|
#define __designated_init __attribute__((designated_init))
|
|
#endif
|
|
|
|
#if GCC_VERSION >= 90100
|
|
#define __copy(symbol) __attribute__((__copy__(symbol)))
|
|
#endif
|
|
|
|
#endif /* gcc version >= 40000 specific checks */
|
|
|
|
#if !defined(__noclone)
|
|
#define __noclone /* not needed */
|
|
#endif
|
|
|
|
#if !defined(__no_sanitize_address)
|
|
#define __no_sanitize_address
|
|
#endif
|
|
|
|
/*
|
|
* A trick to suppress uninitialized variable warning without generating any
|
|
* code
|
|
*/
|
|
#define uninitialized_var(x) x = x
|
|
|
|
#if GCC_VERSION >= 50100
|
|
#define COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW 1
|
|
#endif
|