Lines Matching refs:rcu_data
19 static inline int rcu_lockdep_is_held_nocb(struct rcu_data *rdp) in rcu_lockdep_is_held_nocb()
24 static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp) in rcu_current_is_nocb_kthread()
97 static void rcu_nocb_bypass_lock(struct rcu_data *rdp) in rcu_nocb_bypass_lock()
121 static void rcu_nocb_wait_contended(struct rcu_data *rdp) in rcu_nocb_wait_contended()
132 static bool rcu_nocb_bypass_trylock(struct rcu_data *rdp) in rcu_nocb_bypass_trylock()
141 static void rcu_nocb_bypass_unlock(struct rcu_data *rdp) in rcu_nocb_bypass_unlock()
152 static void rcu_nocb_lock(struct rcu_data *rdp) in rcu_nocb_lock()
164 static void rcu_nocb_unlock(struct rcu_data *rdp) in rcu_nocb_unlock()
176 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, in rcu_nocb_unlock_irqrestore()
188 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) in rcu_lockdep_assert_cblist_protected()
215 static bool __wake_nocb_gp(struct rcu_data *rdp_gp, in __wake_nocb_gp()
216 struct rcu_data *rdp, in __wake_nocb_gp()
250 static bool wake_nocb_gp(struct rcu_data *rdp, bool force) in wake_nocb_gp()
253 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in wake_nocb_gp()
263 static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype, in wake_nocb_gp_defer()
267 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in wake_nocb_gp_defer()
298 static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_do_flush_bypass()
328 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_flush_bypass()
342 static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j) in rcu_nocb_try_flush_bypass()
369 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_try_bypass()
490 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, in __call_rcu_nocb_wake()
549 static int nocb_gp_toggle_rdp(struct rcu_data *rdp, in nocb_gp_toggle_rdp()
587 static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) in nocb_gp_sleep()
599 static void nocb_gp_wait(struct rcu_data *my_rdp) in nocb_gp_wait()
611 struct rcu_data *rdp, *rdp_toggling = NULL; in nocb_gp_wait()
791 struct rcu_data *rdp = arg; in rcu_nocb_gp_kthread()
801 static inline bool nocb_cb_can_run(struct rcu_data *rdp) in nocb_cb_can_run()
808 static inline bool nocb_cb_wait_cond(struct rcu_data *rdp) in nocb_cb_wait_cond()
817 static void nocb_cb_wait(struct rcu_data *rdp) in nocb_cb_wait()
899 struct rcu_data *rdp = arg; in rcu_nocb_cb_kthread()
911 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level) in rcu_nocb_need_deferred_wakeup()
917 static bool do_nocb_deferred_wakeup_common(struct rcu_data *rdp_gp, in do_nocb_deferred_wakeup_common()
918 struct rcu_data *rdp, int level, in do_nocb_deferred_wakeup_common()
941 struct rcu_data *rdp = from_timer(rdp, t, nocb_timer); in do_nocb_deferred_wakeup_timer()
956 static bool do_nocb_deferred_wakeup(struct rcu_data *rdp) in do_nocb_deferred_wakeup()
959 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in do_nocb_deferred_wakeup()
970 do_nocb_deferred_wakeup(this_cpu_ptr(&rcu_data)); in rcu_nocb_flush_deferred_wakeup()
974 static int rdp_offload_toggle(struct rcu_data *rdp, in rdp_offload_toggle()
979 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in rdp_offload_toggle()
1008 struct rcu_data *rdp = arg; in rcu_nocb_rdp_deoffload()
1012 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in rcu_nocb_rdp_deoffload()
1103 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_nocb_cpu_deoffload()
1127 struct rcu_data *rdp = arg; in rcu_nocb_rdp_offload()
1131 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in rcu_nocb_rdp_offload()
1188 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_nocb_cpu_offload()
1215 struct rcu_data *rdp; in rcu_init_nohz()
1266 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_init_nohz()
1277 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) in rcu_boot_init_nocb_percpu_data()
1297 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_spawn_cpu_nocb_kthread()
1298 struct rcu_data *rdp_gp; in rcu_spawn_cpu_nocb_kthread()
1362 struct rcu_data *rdp; in rcu_organize_nocb_kthreads()
1363 struct rcu_data *rdp_gp = NULL; /* Suppress misguided gcc warn. */ in rcu_organize_nocb_kthreads()
1378 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_organize_nocb_kthreads()
1436 static void show_rcu_nocb_gp_state(struct rcu_data *rdp) in show_rcu_nocb_gp_state()
1460 static void show_rcu_nocb_state(struct rcu_data *rdp) in show_rcu_nocb_state()
1464 struct rcu_data *nocb_next_rdp; in show_rcu_nocb_state()
1521 static inline int rcu_lockdep_is_held_nocb(struct rcu_data *rdp) in rcu_lockdep_is_held_nocb()
1526 static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp) in rcu_current_is_nocb_kthread()
1532 static void rcu_nocb_lock(struct rcu_data *rdp) in rcu_nocb_lock()
1537 static void rcu_nocb_unlock(struct rcu_data *rdp) in rcu_nocb_unlock()
1542 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, in rcu_nocb_unlock_irqrestore()
1549 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) in rcu_lockdep_assert_cblist_protected()
1567 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_flush_bypass()
1573 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_try_bypass()
1579 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty, in __call_rcu_nocb_wake()
1585 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) in rcu_boot_init_nocb_percpu_data()
1589 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level) in rcu_nocb_need_deferred_wakeup()
1594 static bool do_nocb_deferred_wakeup(struct rcu_data *rdp) in do_nocb_deferred_wakeup()
1603 static void show_rcu_nocb_state(struct rcu_data *rdp) in show_rcu_nocb_state()