1 /* 2 * Specialised local-global spinlock. Can only be declared as global variables 3 * to avoid overhead and keep things simple (and we don't want to start using 4 * these inside dynamically allocated structures). 5 * 6 * "local/global locks" (lglocks) can be used to: 7 * 8 * - Provide fast exclusive access to per-CPU data, with exclusive access to 9 * another CPU's data allowed but possibly subject to contention, and to 10 * provide very slow exclusive access to all per-CPU data. 11 * - Or to provide very fast and scalable read serialisation, and to provide 12 * very slow exclusive serialisation of data (not necessarily per-CPU data). 13 * 14 * Brlocks are also implemented as a short-hand notation for the latter use 15 * case. 16 * 17 * Copyright 2009, 2010, Nick Piggin, Novell Inc. 18 */ 19 #ifndef __LINUX_LGLOCK_H 20 #define __LINUX_LGLOCK_H 21 22 #include <linux/spinlock.h> 23 #include <linux/lockdep.h> 24 #include <linux/percpu.h> 25 #include <linux/cpu.h> 26 27 /* can make br locks by using local lock for read side, global lock for write */ 28 #define br_lock_init(name) name##_lock_init() 29 #define br_read_lock(name) name##_local_lock() 30 #define br_read_unlock(name) name##_local_unlock() 31 #define br_write_lock(name) name##_global_lock_online() 32 #define br_write_unlock(name) name##_global_unlock_online() 33 34 #define DECLARE_BRLOCK(name) DECLARE_LGLOCK(name) 35 #define DEFINE_BRLOCK(name) DEFINE_LGLOCK(name) 36 37 38 #define lg_lock_init(name) name##_lock_init() 39 #define lg_local_lock(name) name##_local_lock() 40 #define lg_local_unlock(name) name##_local_unlock() 41 #define lg_local_lock_cpu(name, cpu) name##_local_lock_cpu(cpu) 42 #define lg_local_unlock_cpu(name, cpu) name##_local_unlock_cpu(cpu) 43 #define lg_global_lock(name) name##_global_lock() 44 #define lg_global_unlock(name) name##_global_unlock() 45 #define lg_global_lock_online(name) name##_global_lock_online() 46 #define lg_global_unlock_online(name) name##_global_unlock_online() 47 48 #ifdef CONFIG_DEBUG_LOCK_ALLOC 49 #define LOCKDEP_INIT_MAP lockdep_init_map 50 51 #define DEFINE_LGLOCK_LOCKDEP(name) \ 52 struct lock_class_key name##_lock_key; \ 53 struct lockdep_map name##_lock_dep_map; \ 54 EXPORT_SYMBOL(name##_lock_dep_map) 55 56 #else 57 #define LOCKDEP_INIT_MAP(a, b, c, d) 58 59 #define DEFINE_LGLOCK_LOCKDEP(name) 60 #endif 61 62 63 #define DECLARE_LGLOCK(name) \ 64 extern void name##_lock_init(void); \ 65 extern void name##_local_lock(void); \ 66 extern void name##_local_unlock(void); \ 67 extern void name##_local_lock_cpu(int cpu); \ 68 extern void name##_local_unlock_cpu(int cpu); \ 69 extern void name##_global_lock(void); \ 70 extern void name##_global_unlock(void); \ 71 extern void name##_global_lock_online(void); \ 72 extern void name##_global_unlock_online(void); \ 73 74 #define DEFINE_LGLOCK(name) \ 75 \ 76 DEFINE_SPINLOCK(name##_cpu_lock); \ 77 cpumask_t name##_cpus __read_mostly; \ 78 DEFINE_PER_CPU(arch_spinlock_t, name##_lock); \ 79 DEFINE_LGLOCK_LOCKDEP(name); \ 80 \ 81 static int \ 82 name##_lg_cpu_callback(struct notifier_block *nb, \ 83 unsigned long action, void *hcpu) \ 84 { \ 85 switch (action & ~CPU_TASKS_FROZEN) { \ 86 case CPU_UP_PREPARE: \ 87 spin_lock(&name##_cpu_lock); \ 88 cpu_set((unsigned long)hcpu, name##_cpus); \ 89 spin_unlock(&name##_cpu_lock); \ 90 break; \ 91 case CPU_UP_CANCELED: case CPU_DEAD: \ 92 spin_lock(&name##_cpu_lock); \ 93 cpu_clear((unsigned long)hcpu, name##_cpus); \ 94 spin_unlock(&name##_cpu_lock); \ 95 } \ 96 return NOTIFY_OK; \ 97 } \ 98 static struct notifier_block name##_lg_cpu_notifier = { \ 99 .notifier_call = name##_lg_cpu_callback, \ 100 }; \ 101 void name##_lock_init(void) { \ 102 int i; \ 103 LOCKDEP_INIT_MAP(&name##_lock_dep_map, #name, &name##_lock_key, 0); \ 104 for_each_possible_cpu(i) { \ 105 arch_spinlock_t *lock; \ 106 lock = &per_cpu(name##_lock, i); \ 107 *lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; \ 108 } \ 109 register_hotcpu_notifier(&name##_lg_cpu_notifier); \ 110 get_online_cpus(); \ 111 for_each_online_cpu(i) \ 112 cpu_set(i, name##_cpus); \ 113 put_online_cpus(); \ 114 } \ 115 EXPORT_SYMBOL(name##_lock_init); \ 116 \ 117 void name##_local_lock(void) { \ 118 arch_spinlock_t *lock; \ 119 preempt_disable(); \ 120 rwlock_acquire_read(&name##_lock_dep_map, 0, 0, _THIS_IP_); \ 121 lock = &__get_cpu_var(name##_lock); \ 122 arch_spin_lock(lock); \ 123 } \ 124 EXPORT_SYMBOL(name##_local_lock); \ 125 \ 126 void name##_local_unlock(void) { \ 127 arch_spinlock_t *lock; \ 128 rwlock_release(&name##_lock_dep_map, 1, _THIS_IP_); \ 129 lock = &__get_cpu_var(name##_lock); \ 130 arch_spin_unlock(lock); \ 131 preempt_enable(); \ 132 } \ 133 EXPORT_SYMBOL(name##_local_unlock); \ 134 \ 135 void name##_local_lock_cpu(int cpu) { \ 136 arch_spinlock_t *lock; \ 137 preempt_disable(); \ 138 rwlock_acquire_read(&name##_lock_dep_map, 0, 0, _THIS_IP_); \ 139 lock = &per_cpu(name##_lock, cpu); \ 140 arch_spin_lock(lock); \ 141 } \ 142 EXPORT_SYMBOL(name##_local_lock_cpu); \ 143 \ 144 void name##_local_unlock_cpu(int cpu) { \ 145 arch_spinlock_t *lock; \ 146 rwlock_release(&name##_lock_dep_map, 1, _THIS_IP_); \ 147 lock = &per_cpu(name##_lock, cpu); \ 148 arch_spin_unlock(lock); \ 149 preempt_enable(); \ 150 } \ 151 EXPORT_SYMBOL(name##_local_unlock_cpu); \ 152 \ 153 void name##_global_lock_online(void) { \ 154 int i; \ 155 spin_lock(&name##_cpu_lock); \ 156 rwlock_acquire(&name##_lock_dep_map, 0, 0, _RET_IP_); \ 157 for_each_cpu(i, &name##_cpus) { \ 158 arch_spinlock_t *lock; \ 159 lock = &per_cpu(name##_lock, i); \ 160 arch_spin_lock(lock); \ 161 } \ 162 } \ 163 EXPORT_SYMBOL(name##_global_lock_online); \ 164 \ 165 void name##_global_unlock_online(void) { \ 166 int i; \ 167 rwlock_release(&name##_lock_dep_map, 1, _RET_IP_); \ 168 for_each_cpu(i, &name##_cpus) { \ 169 arch_spinlock_t *lock; \ 170 lock = &per_cpu(name##_lock, i); \ 171 arch_spin_unlock(lock); \ 172 } \ 173 spin_unlock(&name##_cpu_lock); \ 174 } \ 175 EXPORT_SYMBOL(name##_global_unlock_online); \ 176 \ 177 void name##_global_lock(void) { \ 178 int i; \ 179 preempt_disable(); \ 180 rwlock_acquire(&name##_lock_dep_map, 0, 0, _RET_IP_); \ 181 for_each_possible_cpu(i) { \ 182 arch_spinlock_t *lock; \ 183 lock = &per_cpu(name##_lock, i); \ 184 arch_spin_lock(lock); \ 185 } \ 186 } \ 187 EXPORT_SYMBOL(name##_global_lock); \ 188 \ 189 void name##_global_unlock(void) { \ 190 int i; \ 191 rwlock_release(&name##_lock_dep_map, 1, _RET_IP_); \ 192 for_each_possible_cpu(i) { \ 193 arch_spinlock_t *lock; \ 194 lock = &per_cpu(name##_lock, i); \ 195 arch_spin_unlock(lock); \ 196 } \ 197 preempt_enable(); \ 198 } \ 199 EXPORT_SYMBOL(name##_global_unlock); 200 #endif 201