Lines Matching refs:sma
283 static void unmerge_queues(struct sem_array *sma) in unmerge_queues() argument
288 if (sma->complex_count) in unmerge_queues()
295 list_for_each_entry_safe(q, tq, &sma->pending_alter, list) { in unmerge_queues()
297 curr = &sma->sems[q->sops[0].sem_num]; in unmerge_queues()
301 INIT_LIST_HEAD(&sma->pending_alter); in unmerge_queues()
313 static void merge_queues(struct sem_array *sma) in merge_queues() argument
316 for (i = 0; i < sma->sem_nsems; i++) { in merge_queues()
317 struct sem *sem = &sma->sems[i]; in merge_queues()
319 list_splice_init(&sem->pending_alter, &sma->pending_alter); in merge_queues()
326 struct sem_array *sma = container_of(p, struct sem_array, sem_perm); in sem_rcu_free() local
328 security_sem_free(&sma->sem_perm); in sem_rcu_free()
329 kvfree(sma); in sem_rcu_free()
336 static void complexmode_enter(struct sem_array *sma) in complexmode_enter() argument
341 if (sma->use_global_lock > 0) { in complexmode_enter()
347 WRITE_ONCE(sma->use_global_lock, USE_GLOBAL_LOCK_HYSTERESIS); in complexmode_enter()
350 WRITE_ONCE(sma->use_global_lock, USE_GLOBAL_LOCK_HYSTERESIS); in complexmode_enter()
352 for (i = 0; i < sma->sem_nsems; i++) { in complexmode_enter()
353 sem = &sma->sems[i]; in complexmode_enter()
363 static void complexmode_tryleave(struct sem_array *sma) in complexmode_tryleave() argument
365 if (sma->complex_count) { in complexmode_tryleave()
371 if (sma->use_global_lock == 1) { in complexmode_tryleave()
374 smp_store_release(&sma->use_global_lock, 0); in complexmode_tryleave()
376 WRITE_ONCE(sma->use_global_lock, in complexmode_tryleave()
377 sma->use_global_lock-1); in complexmode_tryleave()
389 static inline int sem_lock(struct sem_array *sma, struct sembuf *sops, in sem_lock() argument
397 ipc_lock_object(&sma->sem_perm); in sem_lock()
400 complexmode_enter(sma); in sem_lock()
411 idx = array_index_nospec(sops->sem_num, sma->sem_nsems); in sem_lock()
412 sem = &sma->sems[idx]; in sem_lock()
418 if (!READ_ONCE(sma->use_global_lock)) { in sem_lock()
426 if (!smp_load_acquire(&sma->use_global_lock)) { in sem_lock()
434 ipc_lock_object(&sma->sem_perm); in sem_lock()
436 if (sma->use_global_lock == 0) { in sem_lock()
448 ipc_unlock_object(&sma->sem_perm); in sem_lock()
460 static inline void sem_unlock(struct sem_array *sma, int locknum) in sem_unlock() argument
463 unmerge_queues(sma); in sem_unlock()
464 complexmode_tryleave(sma); in sem_unlock()
465 ipc_unlock_object(&sma->sem_perm); in sem_unlock()
467 struct sem *sem = &sma->sems[locknum]; in sem_unlock()
499 static inline void sem_lock_and_putref(struct sem_array *sma) in sem_lock_and_putref() argument
501 sem_lock(sma, NULL, -1); in sem_lock_and_putref()
502 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in sem_lock_and_putref()
512 struct sem_array *sma; in sem_alloc() local
514 if (nsems > (INT_MAX - sizeof(*sma)) / sizeof(sma->sems[0])) in sem_alloc()
517 sma = kvzalloc(struct_size(sma, sems, nsems), GFP_KERNEL_ACCOUNT); in sem_alloc()
518 if (unlikely(!sma)) in sem_alloc()
521 return sma; in sem_alloc()
534 struct sem_array *sma; in newary() local
545 sma = sem_alloc(nsems); in newary()
546 if (!sma) in newary()
549 sma->sem_perm.mode = (semflg & S_IRWXUGO); in newary()
550 sma->sem_perm.key = key; in newary()
552 sma->sem_perm.security = NULL; in newary()
553 retval = security_sem_alloc(&sma->sem_perm); in newary()
555 kvfree(sma); in newary()
560 INIT_LIST_HEAD(&sma->sems[i].pending_alter); in newary()
561 INIT_LIST_HEAD(&sma->sems[i].pending_const); in newary()
562 spin_lock_init(&sma->sems[i].lock); in newary()
565 sma->complex_count = 0; in newary()
566 sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS; in newary()
567 INIT_LIST_HEAD(&sma->pending_alter); in newary()
568 INIT_LIST_HEAD(&sma->pending_const); in newary()
569 INIT_LIST_HEAD(&sma->list_id); in newary()
570 sma->sem_nsems = nsems; in newary()
571 sma->sem_ctime = ktime_get_real_seconds(); in newary()
574 retval = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni); in newary()
576 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in newary()
581 sem_unlock(sma, -1); in newary()
584 return sma->sem_perm.id; in newary()
593 struct sem_array *sma; in sem_more_checks() local
595 sma = container_of(ipcp, struct sem_array, sem_perm); in sem_more_checks()
596 if (params->u.nsems > sma->sem_nsems) in sem_more_checks()
646 static int perform_atomic_semop_slow(struct sem_array *sma, struct sem_queue *q) in perform_atomic_semop_slow() argument
660 int idx = array_index_nospec(sop->sem_num, sma->sem_nsems); in perform_atomic_semop_slow()
661 curr = &sma->sems[idx]; in perform_atomic_semop_slow()
688 ipc_update_pid(&sma->sems[sop->sem_num].sempid, pid); in perform_atomic_semop_slow()
710 sma->sems[sop->sem_num].semval -= sem_op; in perform_atomic_semop_slow()
719 static int perform_atomic_semop(struct sem_array *sma, struct sem_queue *q) in perform_atomic_semop() argument
732 return perform_atomic_semop_slow(sma, q); in perform_atomic_semop()
741 int idx = array_index_nospec(sop->sem_num, sma->sem_nsems); in perform_atomic_semop()
743 curr = &sma->sems[idx]; in perform_atomic_semop()
767 curr = &sma->sems[sop->sem_num]; in perform_atomic_semop()
799 static void unlink_queue(struct sem_array *sma, struct sem_queue *q) in unlink_queue() argument
803 sma->complex_count--; in unlink_queue()
816 static inline int check_restart(struct sem_array *sma, struct sem_queue *q) in check_restart() argument
819 if (!list_empty(&sma->pending_alter)) in check_restart()
854 static int wake_const_ops(struct sem_array *sma, int semnum, in wake_const_ops() argument
862 pending_list = &sma->pending_const; in wake_const_ops()
864 pending_list = &sma->sems[semnum].pending_const; in wake_const_ops()
867 int error = perform_atomic_semop(sma, q); in wake_const_ops()
872 unlink_queue(sma, q); in wake_const_ops()
893 static int do_smart_wakeup_zero(struct sem_array *sma, struct sembuf *sops, in do_smart_wakeup_zero() argument
905 if (sma->sems[num].semval == 0) { in do_smart_wakeup_zero()
907 semop_completed |= wake_const_ops(sma, num, wake_q); in do_smart_wakeup_zero()
915 for (i = 0; i < sma->sem_nsems; i++) { in do_smart_wakeup_zero()
916 if (sma->sems[i].semval == 0) { in do_smart_wakeup_zero()
918 semop_completed |= wake_const_ops(sma, i, wake_q); in do_smart_wakeup_zero()
927 semop_completed |= wake_const_ops(sma, -1, wake_q); in do_smart_wakeup_zero()
949 static int update_queue(struct sem_array *sma, int semnum, struct wake_q_head *wake_q) in update_queue() argument
956 pending_list = &sma->pending_alter; in update_queue()
958 pending_list = &sma->sems[semnum].pending_alter; in update_queue()
971 if (semnum != -1 && sma->sems[semnum].semval == 0) in update_queue()
974 error = perform_atomic_semop(sma, q); in update_queue()
980 unlink_queue(sma, q); in update_queue()
986 do_smart_wakeup_zero(sma, q->sops, q->nsops, wake_q); in update_queue()
987 restart = check_restart(sma, q); in update_queue()
1005 static void set_semotime(struct sem_array *sma, struct sembuf *sops) in set_semotime() argument
1008 sma->sems[0].sem_otime = ktime_get_real_seconds(); in set_semotime()
1010 sma->sems[sops[0].sem_num].sem_otime = in set_semotime()
1029 static void do_smart_update(struct sem_array *sma, struct sembuf *sops, int nsops, in do_smart_update() argument
1034 otime |= do_smart_wakeup_zero(sma, sops, nsops, wake_q); in do_smart_update()
1036 if (!list_empty(&sma->pending_alter)) { in do_smart_update()
1038 otime |= update_queue(sma, -1, wake_q); in do_smart_update()
1045 for (i = 0; i < sma->sem_nsems; i++) in do_smart_update()
1046 otime |= update_queue(sma, i, wake_q); in do_smart_update()
1059 otime |= update_queue(sma, in do_smart_update()
1066 set_semotime(sma, sops); in do_smart_update()
1072 static int check_qop(struct sem_array *sma, int semnum, struct sem_queue *q, in check_qop() argument
1106 static int count_semcnt(struct sem_array *sma, ushort semnum, in count_semcnt() argument
1116 l = &sma->sems[semnum].pending_const; in count_semcnt()
1118 l = &sma->sems[semnum].pending_alter; in count_semcnt()
1128 list_for_each_entry(q, &sma->pending_alter, list) { in count_semcnt()
1129 semcnt += check_qop(sma, semnum, q, count_zero); in count_semcnt()
1132 list_for_each_entry(q, &sma->pending_const, list) { in count_semcnt()
1133 semcnt += check_qop(sma, semnum, q, count_zero); in count_semcnt()
1147 struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm); in freeary() local
1152 ipc_assert_locked_object(&sma->sem_perm); in freeary()
1153 list_for_each_entry_safe(un, tu, &sma->list_id, list_id) { in freeary()
1163 list_for_each_entry_safe(q, tq, &sma->pending_const, list) { in freeary()
1164 unlink_queue(sma, q); in freeary()
1168 list_for_each_entry_safe(q, tq, &sma->pending_alter, list) { in freeary()
1169 unlink_queue(sma, q); in freeary()
1172 for (i = 0; i < sma->sem_nsems; i++) { in freeary()
1173 struct sem *sem = &sma->sems[i]; in freeary()
1175 unlink_queue(sma, q); in freeary()
1179 unlink_queue(sma, q); in freeary()
1186 sem_rmid(ns, sma); in freeary()
1187 sem_unlock(sma, -1); in freeary()
1191 ns->used_sems -= sma->sem_nsems; in freeary()
1192 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in freeary()
1219 static time64_t get_semotime(struct sem_array *sma) in get_semotime() argument
1224 res = sma->sems[0].sem_otime; in get_semotime()
1225 for (i = 1; i < sma->sem_nsems; i++) { in get_semotime()
1226 time64_t to = sma->sems[i].sem_otime; in get_semotime()
1237 struct sem_array *sma; in semctl_stat() local
1245 sma = sem_obtain_object(ns, semid); in semctl_stat()
1246 if (IS_ERR(sma)) { in semctl_stat()
1247 err = PTR_ERR(sma); in semctl_stat()
1251 sma = sem_obtain_object_check(ns, semid); in semctl_stat()
1252 if (IS_ERR(sma)) { in semctl_stat()
1253 err = PTR_ERR(sma); in semctl_stat()
1260 audit_ipc_obj(&sma->sem_perm); in semctl_stat()
1263 if (ipcperms(ns, &sma->sem_perm, S_IRUGO)) in semctl_stat()
1267 err = security_sem_semctl(&sma->sem_perm, cmd); in semctl_stat()
1271 ipc_lock_object(&sma->sem_perm); in semctl_stat()
1273 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_stat()
1274 ipc_unlock_object(&sma->sem_perm); in semctl_stat()
1279 kernel_to_ipc64_perm(&sma->sem_perm, &semid64->sem_perm); in semctl_stat()
1280 semotime = get_semotime(sma); in semctl_stat()
1282 semid64->sem_ctime = sma->sem_ctime; in semctl_stat()
1285 semid64->sem_ctime_high = sma->sem_ctime >> 32; in semctl_stat()
1287 semid64->sem_nsems = sma->sem_nsems; in semctl_stat()
1300 err = sma->sem_perm.id; in semctl_stat()
1302 ipc_unlock_object(&sma->sem_perm); in semctl_stat()
1347 struct sem_array *sma; in semctl_setval() local
1356 sma = sem_obtain_object_check(ns, semid); in semctl_setval()
1357 if (IS_ERR(sma)) { in semctl_setval()
1359 return PTR_ERR(sma); in semctl_setval()
1362 if (semnum < 0 || semnum >= sma->sem_nsems) { in semctl_setval()
1368 if (ipcperms(ns, &sma->sem_perm, S_IWUGO)) { in semctl_setval()
1373 err = security_sem_semctl(&sma->sem_perm, SETVAL); in semctl_setval()
1379 sem_lock(sma, NULL, -1); in semctl_setval()
1381 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_setval()
1382 sem_unlock(sma, -1); in semctl_setval()
1387 semnum = array_index_nospec(semnum, sma->sem_nsems); in semctl_setval()
1388 curr = &sma->sems[semnum]; in semctl_setval()
1390 ipc_assert_locked_object(&sma->sem_perm); in semctl_setval()
1391 list_for_each_entry(un, &sma->list_id, list_id) in semctl_setval()
1396 sma->sem_ctime = ktime_get_real_seconds(); in semctl_setval()
1398 do_smart_update(sma, NULL, 0, 0, &wake_q); in semctl_setval()
1399 sem_unlock(sma, -1); in semctl_setval()
1408 struct sem_array *sma; in semctl_main() local
1416 sma = sem_obtain_object_check(ns, semid); in semctl_main()
1417 if (IS_ERR(sma)) { in semctl_main()
1419 return PTR_ERR(sma); in semctl_main()
1422 nsems = sma->sem_nsems; in semctl_main()
1425 if (ipcperms(ns, &sma->sem_perm, cmd == SETALL ? S_IWUGO : S_IRUGO)) in semctl_main()
1428 err = security_sem_semctl(&sma->sem_perm, cmd); in semctl_main()
1438 sem_lock(sma, NULL, -1); in semctl_main()
1439 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1444 if (!ipc_rcu_getref(&sma->sem_perm)) { in semctl_main()
1448 sem_unlock(sma, -1); in semctl_main()
1453 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1458 sem_lock_and_putref(sma); in semctl_main()
1459 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1464 for (i = 0; i < sma->sem_nsems; i++) in semctl_main()
1465 sem_io[i] = sma->sems[i].semval; in semctl_main()
1466 sem_unlock(sma, -1); in semctl_main()
1478 if (!ipc_rcu_getref(&sma->sem_perm)) { in semctl_main()
1488 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1494 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1501 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1507 sem_lock_and_putref(sma); in semctl_main()
1508 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1514 sma->sems[i].semval = sem_io[i]; in semctl_main()
1515 ipc_update_pid(&sma->sems[i].sempid, task_tgid(current)); in semctl_main()
1518 ipc_assert_locked_object(&sma->sem_perm); in semctl_main()
1519 list_for_each_entry(un, &sma->list_id, list_id) { in semctl_main()
1523 sma->sem_ctime = ktime_get_real_seconds(); in semctl_main()
1525 do_smart_update(sma, NULL, 0, 0, &wake_q); in semctl_main()
1535 sem_lock(sma, NULL, -1); in semctl_main()
1536 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1542 curr = &sma->sems[semnum]; in semctl_main()
1552 err = count_semcnt(sma, semnum, 0); in semctl_main()
1555 err = count_semcnt(sma, semnum, 1); in semctl_main()
1560 sem_unlock(sma, -1); in semctl_main()
1604 struct sem_array *sma; in semctl_down() local
1618 sma = container_of(ipcp, struct sem_array, sem_perm); in semctl_down()
1620 err = security_sem_semctl(&sma->sem_perm, cmd); in semctl_down()
1626 sem_lock(sma, NULL, -1); in semctl_down()
1631 sem_lock(sma, NULL, -1); in semctl_down()
1635 sma->sem_ctime = ktime_get_real_seconds(); in semctl_down()
1643 sem_unlock(sma, -1); in semctl_down()
1908 struct sem_array *sma; in find_alloc_undo() local
1926 sma = sem_obtain_object_check(ns, semid); in find_alloc_undo()
1927 if (IS_ERR(sma)) { in find_alloc_undo()
1929 return ERR_CAST(sma); in find_alloc_undo()
1932 nsems = sma->sem_nsems; in find_alloc_undo()
1933 if (!ipc_rcu_getref(&sma->sem_perm)) { in find_alloc_undo()
1944 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in find_alloc_undo()
1950 sem_lock_and_putref(sma); in find_alloc_undo()
1951 if (!ipc_valid_object(&sma->sem_perm)) { in find_alloc_undo()
1952 sem_unlock(sma, -1); in find_alloc_undo()
1975 ipc_assert_locked_object(&sma->sem_perm); in find_alloc_undo()
1976 list_add(&new->list_id, &sma->list_id); in find_alloc_undo()
1980 sem_unlock(sma, -1); in find_alloc_undo()
1990 struct sem_array *sma; in __do_semtimedop() local
2049 sma = sem_obtain_object_check(ns, semid); in __do_semtimedop()
2050 if (IS_ERR(sma)) { in __do_semtimedop()
2052 error = PTR_ERR(sma); in __do_semtimedop()
2057 if (max >= sma->sem_nsems) { in __do_semtimedop()
2063 if (ipcperms(ns, &sma->sem_perm, alter ? S_IWUGO : S_IRUGO)) { in __do_semtimedop()
2068 error = security_sem_semop(&sma->sem_perm, sops, nsops, alter); in __do_semtimedop()
2075 locknum = sem_lock(sma, sops, nsops); in __do_semtimedop()
2084 if (!ipc_valid_object(&sma->sem_perm)) in __do_semtimedop()
2103 error = perform_atomic_semop(sma, &queue); in __do_semtimedop()
2112 do_smart_update(sma, sops, nsops, 1, &wake_q); in __do_semtimedop()
2114 set_semotime(sma, sops); in __do_semtimedop()
2116 sem_unlock(sma, locknum); in __do_semtimedop()
2131 int idx = array_index_nospec(sops->sem_num, sma->sem_nsems); in __do_semtimedop()
2132 curr = &sma->sems[idx]; in __do_semtimedop()
2135 if (sma->complex_count) { in __do_semtimedop()
2137 &sma->pending_alter); in __do_semtimedop()
2147 if (!sma->complex_count) in __do_semtimedop()
2148 merge_queues(sma); in __do_semtimedop()
2151 list_add_tail(&queue.list, &sma->pending_alter); in __do_semtimedop()
2153 list_add_tail(&queue.list, &sma->pending_const); in __do_semtimedop()
2155 sma->complex_count++; in __do_semtimedop()
2165 sem_unlock(sma, locknum); in __do_semtimedop()
2191 locknum = sem_lock(sma, sops, nsops); in __do_semtimedop()
2193 if (!ipc_valid_object(&sma->sem_perm)) in __do_semtimedop()
2215 unlink_queue(sma, &queue); in __do_semtimedop()
2218 sem_unlock(sma, locknum); in __do_semtimedop()
2350 struct sem_array *sma; in exit_sem() local
2382 sma = sem_obtain_object_check(tsk->nsproxy->ipc_ns, semid); in exit_sem()
2384 if (IS_ERR(sma)) { in exit_sem()
2389 sem_lock(sma, NULL, -1); in exit_sem()
2391 if (!ipc_valid_object(&sma->sem_perm)) { in exit_sem()
2392 sem_unlock(sma, -1); in exit_sem()
2401 sem_unlock(sma, -1); in exit_sem()
2407 ipc_assert_locked_object(&sma->sem_perm); in exit_sem()
2415 for (i = 0; i < sma->sem_nsems; i++) { in exit_sem()
2416 struct sem *semaphore = &sma->sems[i]; in exit_sem()
2440 do_smart_update(sma, NULL, 0, 1, &wake_q); in exit_sem()
2441 sem_unlock(sma, -1); in exit_sem()
2455 struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm); in sysvipc_sem_proc_show() local
2465 complexmode_enter(sma); in sysvipc_sem_proc_show()
2467 sem_otime = get_semotime(sma); in sysvipc_sem_proc_show()
2471 sma->sem_perm.key, in sysvipc_sem_proc_show()
2472 sma->sem_perm.id, in sysvipc_sem_proc_show()
2473 sma->sem_perm.mode, in sysvipc_sem_proc_show()
2474 sma->sem_nsems, in sysvipc_sem_proc_show()
2475 from_kuid_munged(user_ns, sma->sem_perm.uid), in sysvipc_sem_proc_show()
2476 from_kgid_munged(user_ns, sma->sem_perm.gid), in sysvipc_sem_proc_show()
2477 from_kuid_munged(user_ns, sma->sem_perm.cuid), in sysvipc_sem_proc_show()
2478 from_kgid_munged(user_ns, sma->sem_perm.cgid), in sysvipc_sem_proc_show()
2480 sma->sem_ctime); in sysvipc_sem_proc_show()
2482 complexmode_tryleave(sma); in sysvipc_sem_proc_show()