20743a0645
Event tracing is moving to SRCU in order to take advantage of the fact that SRCU may be safely used from idle and even offline CPUs. However, event tracing can invoke call_srcu() very early in the boot process, even before workqueue_init_early() is invoked (let alone rcu_init()). Therefore, call_srcu()'s attempts to queue work fail miserably. This commit therefore detects this situation, and refrains from attempting to queue work before rcu_init() time, but does everything else that it would have done, and in addition, adds the srcu_struct to a global list. The rcu_init() function now invokes a new srcu_init() function, which is empty if CONFIG_SRCU=n. Otherwise, srcu_init() queues work for each srcu_struct on the list. This all happens early enough in boot that there is but a single CPU with interrupts disabled, which allows synchronization to be dispensed with. Of course, the queued work won't actually be invoked until after workqueue_init() is invoked, which happens shortly after the scheduler is up and running. This means that although call_srcu() may be invoked any time after per-CPU variables have been set up, there is still a very narrow window when synchronize_srcu() won't work, and this window extends from the time that the scheduler starts until the time that workqueue_init() returns. This can be fixed in a manner similar to the fix for synchronize_rcu_expedited() and friends, but until someone actually needs to use synchronize_srcu() during this window, this fix is added churn for no benefit. Finally, note that Tree SRCU's new srcu_init() function invokes queue_work() rather than the queue_delayed_work() function that is invoked post-boot. The reason is that queue_delayed_work() will (as you would expect) post a timer, and timers have not yet been initialized. So use of queue_work() avoids the complaints about use of uninitialized spinlocks that would otherwise result. Besides, some delay is already provide by the aforementioned fact that the queued work won't actually be invoked until after the scheduler is up and running. Requested-by: Steven Rostedt <rostedt@goodmis.org> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Tested-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
106 lines
3.3 KiB
C
106 lines
3.3 KiB
C
/*
|
|
* Sleepable Read-Copy Update mechanism for mutual exclusion,
|
|
* tiny variant.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program; if not, you can access it online at
|
|
* http://www.gnu.org/licenses/gpl-2.0.html.
|
|
*
|
|
* Copyright (C) IBM Corporation, 2017
|
|
*
|
|
* Author: Paul McKenney <paulmck@us.ibm.com>
|
|
*/
|
|
|
|
#ifndef _LINUX_SRCU_TINY_H
|
|
#define _LINUX_SRCU_TINY_H
|
|
|
|
#include <linux/swait.h>
|
|
|
|
struct srcu_struct {
|
|
short srcu_lock_nesting[2]; /* srcu_read_lock() nesting depth. */
|
|
short srcu_idx; /* Current reader array element. */
|
|
u8 srcu_gp_running; /* GP workqueue running? */
|
|
u8 srcu_gp_waiting; /* GP waiting for readers? */
|
|
struct swait_queue_head srcu_wq;
|
|
/* Last srcu_read_unlock() wakes GP. */
|
|
struct rcu_head *srcu_cb_head; /* Pending callbacks: Head. */
|
|
struct rcu_head **srcu_cb_tail; /* Pending callbacks: Tail. */
|
|
struct work_struct srcu_work; /* For driving grace periods. */
|
|
struct list_head srcu_boot_entry; /* Early-boot callbacks. */
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
struct lockdep_map dep_map;
|
|
#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
|
|
};
|
|
|
|
void srcu_drive_gp(struct work_struct *wp);
|
|
|
|
#define __SRCU_STRUCT_INIT(name, __ignored) \
|
|
{ \
|
|
.srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq), \
|
|
.srcu_cb_tail = &name.srcu_cb_head, \
|
|
.srcu_work = __WORK_INITIALIZER(name.srcu_work, srcu_drive_gp), \
|
|
.srcu_boot_entry = LIST_HEAD_INIT(name.srcu_boot_entry), \
|
|
__SRCU_DEP_MAP_INIT(name) \
|
|
}
|
|
|
|
/*
|
|
* This odd _STATIC_ arrangement is needed for API compatibility with
|
|
* Tree SRCU, which needs some per-CPU data.
|
|
*/
|
|
#define DEFINE_SRCU(name) \
|
|
struct srcu_struct name = __SRCU_STRUCT_INIT(name, name)
|
|
#define DEFINE_STATIC_SRCU(name) \
|
|
static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name)
|
|
|
|
void synchronize_srcu(struct srcu_struct *sp);
|
|
|
|
/*
|
|
* Counts the new reader in the appropriate per-CPU element of the
|
|
* srcu_struct. Can be invoked from irq/bh handlers, but the matching
|
|
* __srcu_read_unlock() must be in the same handler instance. Returns an
|
|
* index that must be passed to the matching srcu_read_unlock().
|
|
*/
|
|
static inline int __srcu_read_lock(struct srcu_struct *sp)
|
|
{
|
|
int idx;
|
|
|
|
idx = READ_ONCE(sp->srcu_idx);
|
|
WRITE_ONCE(sp->srcu_lock_nesting[idx], sp->srcu_lock_nesting[idx] + 1);
|
|
return idx;
|
|
}
|
|
|
|
static inline void synchronize_srcu_expedited(struct srcu_struct *sp)
|
|
{
|
|
synchronize_srcu(sp);
|
|
}
|
|
|
|
static inline void srcu_barrier(struct srcu_struct *sp)
|
|
{
|
|
synchronize_srcu(sp);
|
|
}
|
|
|
|
/* Defined here to avoid size increase for non-torture kernels. */
|
|
static inline void srcu_torture_stats_print(struct srcu_struct *sp,
|
|
char *tt, char *tf)
|
|
{
|
|
int idx;
|
|
|
|
idx = READ_ONCE(sp->srcu_idx) & 0x1;
|
|
pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd)\n",
|
|
tt, tf, idx,
|
|
READ_ONCE(sp->srcu_lock_nesting[!idx]),
|
|
READ_ONCE(sp->srcu_lock_nesting[idx]));
|
|
}
|
|
|
|
#endif
|