Lines Matching refs:rcu_data

16 static bool rcu_rdp_is_offloaded(struct rcu_data *rdp)  in rcu_rdp_is_offloaded()
31 (rdp == this_cpu_ptr(&rcu_data) && in rcu_rdp_is_offloaded()
151 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_preempt_ctxt_queue()
289 if (__this_cpu_read(rcu_data.cpu_no_qs.b.norm)) { in rcu_qs()
291 __this_cpu_read(rcu_data.gp_seq), in rcu_qs()
293 __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); in rcu_qs()
315 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_note_context_switch()
473 struct rcu_data *rdp; in rcu_preempt_deferred_qs_irqrestore()
483 rdp = this_cpu_ptr(&rcu_data); in rcu_preempt_deferred_qs_irqrestore()
588 return (__this_cpu_read(rcu_data.cpu_no_qs.b.exp) || in rcu_preempt_need_deferred_qs()
615 struct rcu_data *rdp; in rcu_preempt_deferred_qs_handler()
617 rdp = container_of(iwp, struct rcu_data, defer_qs_iw); in rcu_preempt_deferred_qs_handler()
641 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_read_unlock_special()
741 __this_cpu_read(rcu_data.core_needs_qs) && in rcu_flavor_sched_clock_irq()
742 __this_cpu_read(rcu_data.cpu_no_qs.b.norm) && in rcu_flavor_sched_clock_irq()
783 struct rcu_data *rdp; in dump_blkd_tasks()
805 rdp = per_cpu_ptr(&rcu_data, cpu); in dump_blkd_tasks()
822 struct rcu_data *rdp; in rcu_read_unlock_strict()
826 rdp = this_cpu_ptr(&rcu_data); in rcu_read_unlock_strict()
851 if (!__this_cpu_read(rcu_data.cpu_no_qs.s)) in rcu_qs()
854 __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs")); in rcu_qs()
855 __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); in rcu_qs()
856 if (__this_cpu_read(rcu_data.cpu_no_qs.b.exp)) in rcu_qs()
857 rcu_report_exp_rdp(this_cpu_ptr(&rcu_data)); in rcu_qs()
871 if (!raw_cpu_read(rcu_data.rcu_urgent_qs)) in rcu_all_qs()
875 if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) { in rcu_all_qs()
879 this_cpu_write(rcu_data.rcu_urgent_qs, false); in rcu_all_qs()
880 if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) { in rcu_all_qs()
898 if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) in rcu_note_context_switch()
900 this_cpu_write(rcu_data.rcu_urgent_qs, false); in rcu_note_context_switch()
901 if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) in rcu_note_context_switch()
944 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_preempt_deferred_qs()
1007 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_cpu_kthread_setup()
1018 static bool rcu_is_callbacks_nocb_kthread(struct rcu_data *rdp) in rcu_is_callbacks_nocb_kthread()
1031 static bool rcu_is_callbacks_kthread(struct rcu_data *rdp) in rcu_is_callbacks_kthread()