Lines Matching refs:sma
258 static void unmerge_queues(struct sem_array *sma) in unmerge_queues() argument
263 if (sma->complex_count) in unmerge_queues()
270 list_for_each_entry_safe(q, tq, &sma->pending_alter, list) { in unmerge_queues()
272 curr = &sma->sems[q->sops[0].sem_num]; in unmerge_queues()
276 INIT_LIST_HEAD(&sma->pending_alter); in unmerge_queues()
288 static void merge_queues(struct sem_array *sma) in merge_queues() argument
291 for (i = 0; i < sma->sem_nsems; i++) { in merge_queues()
292 struct sem *sem = &sma->sems[i]; in merge_queues()
294 list_splice_init(&sem->pending_alter, &sma->pending_alter); in merge_queues()
301 struct sem_array *sma = container_of(p, struct sem_array, sem_perm); in sem_rcu_free() local
303 security_sem_free(&sma->sem_perm); in sem_rcu_free()
304 kvfree(sma); in sem_rcu_free()
311 static void complexmode_enter(struct sem_array *sma) in complexmode_enter() argument
316 if (sma->use_global_lock > 0) { in complexmode_enter()
322 sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS; in complexmode_enter()
325 sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS; in complexmode_enter()
327 for (i = 0; i < sma->sem_nsems; i++) { in complexmode_enter()
328 sem = &sma->sems[i]; in complexmode_enter()
338 static void complexmode_tryleave(struct sem_array *sma) in complexmode_tryleave() argument
340 if (sma->complex_count) { in complexmode_tryleave()
346 if (sma->use_global_lock == 1) { in complexmode_tryleave()
353 smp_store_release(&sma->use_global_lock, 0); in complexmode_tryleave()
355 sma->use_global_lock--; in complexmode_tryleave()
367 static inline int sem_lock(struct sem_array *sma, struct sembuf *sops, in sem_lock() argument
375 ipc_lock_object(&sma->sem_perm); in sem_lock()
378 complexmode_enter(sma); in sem_lock()
389 idx = array_index_nospec(sops->sem_num, sma->sem_nsems); in sem_lock()
390 sem = &sma->sems[idx]; in sem_lock()
396 if (!sma->use_global_lock) { in sem_lock()
404 if (!smp_load_acquire(&sma->use_global_lock)) { in sem_lock()
412 ipc_lock_object(&sma->sem_perm); in sem_lock()
414 if (sma->use_global_lock == 0) { in sem_lock()
426 ipc_unlock_object(&sma->sem_perm); in sem_lock()
438 static inline void sem_unlock(struct sem_array *sma, int locknum) in sem_unlock() argument
441 unmerge_queues(sma); in sem_unlock()
442 complexmode_tryleave(sma); in sem_unlock()
443 ipc_unlock_object(&sma->sem_perm); in sem_unlock()
445 struct sem *sem = &sma->sems[locknum]; in sem_unlock()
477 static inline void sem_lock_and_putref(struct sem_array *sma) in sem_lock_and_putref() argument
479 sem_lock(sma, NULL, -1); in sem_lock_and_putref()
480 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in sem_lock_and_putref()
490 struct sem_array *sma; in sem_alloc() local
493 if (nsems > (INT_MAX - sizeof(*sma)) / sizeof(sma->sems[0])) in sem_alloc()
496 size = sizeof(*sma) + nsems * sizeof(sma->sems[0]); in sem_alloc()
497 sma = kvmalloc(size, GFP_KERNEL); in sem_alloc()
498 if (unlikely(!sma)) in sem_alloc()
501 memset(sma, 0, size); in sem_alloc()
503 return sma; in sem_alloc()
516 struct sem_array *sma; in newary() local
527 sma = sem_alloc(nsems); in newary()
528 if (!sma) in newary()
531 sma->sem_perm.mode = (semflg & S_IRWXUGO); in newary()
532 sma->sem_perm.key = key; in newary()
534 sma->sem_perm.security = NULL; in newary()
535 retval = security_sem_alloc(&sma->sem_perm); in newary()
537 kvfree(sma); in newary()
542 INIT_LIST_HEAD(&sma->sems[i].pending_alter); in newary()
543 INIT_LIST_HEAD(&sma->sems[i].pending_const); in newary()
544 spin_lock_init(&sma->sems[i].lock); in newary()
547 sma->complex_count = 0; in newary()
548 sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS; in newary()
549 INIT_LIST_HEAD(&sma->pending_alter); in newary()
550 INIT_LIST_HEAD(&sma->pending_const); in newary()
551 INIT_LIST_HEAD(&sma->list_id); in newary()
552 sma->sem_nsems = nsems; in newary()
553 sma->sem_ctime = ktime_get_real_seconds(); in newary()
556 retval = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni); in newary()
558 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in newary()
563 sem_unlock(sma, -1); in newary()
566 return sma->sem_perm.id; in newary()
576 struct sem_array *sma; in sem_more_checks() local
578 sma = container_of(ipcp, struct sem_array, sem_perm); in sem_more_checks()
579 if (params->u.nsems > sma->sem_nsems) in sem_more_checks()
629 static int perform_atomic_semop_slow(struct sem_array *sma, struct sem_queue *q) in perform_atomic_semop_slow() argument
643 int idx = array_index_nospec(sop->sem_num, sma->sem_nsems); in perform_atomic_semop_slow()
644 curr = &sma->sems[idx]; in perform_atomic_semop_slow()
671 ipc_update_pid(&sma->sems[sop->sem_num].sempid, pid); in perform_atomic_semop_slow()
693 sma->sems[sop->sem_num].semval -= sem_op; in perform_atomic_semop_slow()
702 static int perform_atomic_semop(struct sem_array *sma, struct sem_queue *q) in perform_atomic_semop() argument
715 return perform_atomic_semop_slow(sma, q); in perform_atomic_semop()
724 int idx = array_index_nospec(sop->sem_num, sma->sem_nsems); in perform_atomic_semop()
726 curr = &sma->sems[idx]; in perform_atomic_semop()
750 curr = &sma->sems[sop->sem_num]; in perform_atomic_semop()
784 static void unlink_queue(struct sem_array *sma, struct sem_queue *q) in unlink_queue() argument
788 sma->complex_count--; in unlink_queue()
801 static inline int check_restart(struct sem_array *sma, struct sem_queue *q) in check_restart() argument
804 if (!list_empty(&sma->pending_alter)) in check_restart()
839 static int wake_const_ops(struct sem_array *sma, int semnum, in wake_const_ops() argument
847 pending_list = &sma->pending_const; in wake_const_ops()
849 pending_list = &sma->sems[semnum].pending_const; in wake_const_ops()
852 int error = perform_atomic_semop(sma, q); in wake_const_ops()
857 unlink_queue(sma, q); in wake_const_ops()
878 static int do_smart_wakeup_zero(struct sem_array *sma, struct sembuf *sops, in do_smart_wakeup_zero() argument
890 if (sma->sems[num].semval == 0) { in do_smart_wakeup_zero()
892 semop_completed |= wake_const_ops(sma, num, wake_q); in do_smart_wakeup_zero()
900 for (i = 0; i < sma->sem_nsems; i++) { in do_smart_wakeup_zero()
901 if (sma->sems[i].semval == 0) { in do_smart_wakeup_zero()
903 semop_completed |= wake_const_ops(sma, i, wake_q); in do_smart_wakeup_zero()
912 semop_completed |= wake_const_ops(sma, -1, wake_q); in do_smart_wakeup_zero()
934 static int update_queue(struct sem_array *sma, int semnum, struct wake_q_head *wake_q) in update_queue() argument
941 pending_list = &sma->pending_alter; in update_queue()
943 pending_list = &sma->sems[semnum].pending_alter; in update_queue()
956 if (semnum != -1 && sma->sems[semnum].semval == 0) in update_queue()
959 error = perform_atomic_semop(sma, q); in update_queue()
965 unlink_queue(sma, q); in update_queue()
971 do_smart_wakeup_zero(sma, q->sops, q->nsops, wake_q); in update_queue()
972 restart = check_restart(sma, q); in update_queue()
990 static void set_semotime(struct sem_array *sma, struct sembuf *sops) in set_semotime() argument
993 sma->sems[0].sem_otime = ktime_get_real_seconds(); in set_semotime()
995 sma->sems[sops[0].sem_num].sem_otime = in set_semotime()
1014 static void do_smart_update(struct sem_array *sma, struct sembuf *sops, int nsops, in do_smart_update() argument
1019 otime |= do_smart_wakeup_zero(sma, sops, nsops, wake_q); in do_smart_update()
1021 if (!list_empty(&sma->pending_alter)) { in do_smart_update()
1023 otime |= update_queue(sma, -1, wake_q); in do_smart_update()
1030 for (i = 0; i < sma->sem_nsems; i++) in do_smart_update()
1031 otime |= update_queue(sma, i, wake_q); in do_smart_update()
1044 otime |= update_queue(sma, in do_smart_update()
1051 set_semotime(sma, sops); in do_smart_update()
1057 static int check_qop(struct sem_array *sma, int semnum, struct sem_queue *q, in check_qop() argument
1091 static int count_semcnt(struct sem_array *sma, ushort semnum, in count_semcnt() argument
1101 l = &sma->sems[semnum].pending_const; in count_semcnt()
1103 l = &sma->sems[semnum].pending_alter; in count_semcnt()
1113 list_for_each_entry(q, &sma->pending_alter, list) { in count_semcnt()
1114 semcnt += check_qop(sma, semnum, q, count_zero); in count_semcnt()
1117 list_for_each_entry(q, &sma->pending_const, list) { in count_semcnt()
1118 semcnt += check_qop(sma, semnum, q, count_zero); in count_semcnt()
1132 struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm); in freeary() local
1137 ipc_assert_locked_object(&sma->sem_perm); in freeary()
1138 list_for_each_entry_safe(un, tu, &sma->list_id, list_id) { in freeary()
1148 list_for_each_entry_safe(q, tq, &sma->pending_const, list) { in freeary()
1149 unlink_queue(sma, q); in freeary()
1153 list_for_each_entry_safe(q, tq, &sma->pending_alter, list) { in freeary()
1154 unlink_queue(sma, q); in freeary()
1157 for (i = 0; i < sma->sem_nsems; i++) { in freeary()
1158 struct sem *sem = &sma->sems[i]; in freeary()
1160 unlink_queue(sma, q); in freeary()
1164 unlink_queue(sma, q); in freeary()
1171 sem_rmid(ns, sma); in freeary()
1172 sem_unlock(sma, -1); in freeary()
1176 ns->used_sems -= sma->sem_nsems; in freeary()
1177 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in freeary()
1204 static time64_t get_semotime(struct sem_array *sma) in get_semotime() argument
1209 res = sma->sems[0].sem_otime; in get_semotime()
1210 for (i = 1; i < sma->sem_nsems; i++) { in get_semotime()
1211 time64_t to = sma->sems[i].sem_otime; in get_semotime()
1222 struct sem_array *sma; in semctl_stat() local
1230 sma = sem_obtain_object(ns, semid); in semctl_stat()
1231 if (IS_ERR(sma)) { in semctl_stat()
1232 err = PTR_ERR(sma); in semctl_stat()
1236 sma = sem_obtain_object_check(ns, semid); in semctl_stat()
1237 if (IS_ERR(sma)) { in semctl_stat()
1238 err = PTR_ERR(sma); in semctl_stat()
1245 audit_ipc_obj(&sma->sem_perm); in semctl_stat()
1248 if (ipcperms(ns, &sma->sem_perm, S_IRUGO)) in semctl_stat()
1252 err = security_sem_semctl(&sma->sem_perm, cmd); in semctl_stat()
1256 ipc_lock_object(&sma->sem_perm); in semctl_stat()
1258 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_stat()
1259 ipc_unlock_object(&sma->sem_perm); in semctl_stat()
1264 kernel_to_ipc64_perm(&sma->sem_perm, &semid64->sem_perm); in semctl_stat()
1265 semotime = get_semotime(sma); in semctl_stat()
1267 semid64->sem_ctime = sma->sem_ctime; in semctl_stat()
1270 semid64->sem_ctime_high = sma->sem_ctime >> 32; in semctl_stat()
1272 semid64->sem_nsems = sma->sem_nsems; in semctl_stat()
1285 err = sma->sem_perm.id; in semctl_stat()
1287 ipc_unlock_object(&sma->sem_perm); in semctl_stat()
1332 struct sem_array *sma; in semctl_setval() local
1341 sma = sem_obtain_object_check(ns, semid); in semctl_setval()
1342 if (IS_ERR(sma)) { in semctl_setval()
1344 return PTR_ERR(sma); in semctl_setval()
1347 if (semnum < 0 || semnum >= sma->sem_nsems) { in semctl_setval()
1353 if (ipcperms(ns, &sma->sem_perm, S_IWUGO)) { in semctl_setval()
1358 err = security_sem_semctl(&sma->sem_perm, SETVAL); in semctl_setval()
1364 sem_lock(sma, NULL, -1); in semctl_setval()
1366 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_setval()
1367 sem_unlock(sma, -1); in semctl_setval()
1372 semnum = array_index_nospec(semnum, sma->sem_nsems); in semctl_setval()
1373 curr = &sma->sems[semnum]; in semctl_setval()
1375 ipc_assert_locked_object(&sma->sem_perm); in semctl_setval()
1376 list_for_each_entry(un, &sma->list_id, list_id) in semctl_setval()
1381 sma->sem_ctime = ktime_get_real_seconds(); in semctl_setval()
1383 do_smart_update(sma, NULL, 0, 0, &wake_q); in semctl_setval()
1384 sem_unlock(sma, -1); in semctl_setval()
1393 struct sem_array *sma; in semctl_main() local
1401 sma = sem_obtain_object_check(ns, semid); in semctl_main()
1402 if (IS_ERR(sma)) { in semctl_main()
1404 return PTR_ERR(sma); in semctl_main()
1407 nsems = sma->sem_nsems; in semctl_main()
1410 if (ipcperms(ns, &sma->sem_perm, cmd == SETALL ? S_IWUGO : S_IRUGO)) in semctl_main()
1413 err = security_sem_semctl(&sma->sem_perm, cmd); in semctl_main()
1424 sem_lock(sma, NULL, -1); in semctl_main()
1425 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1430 if (!ipc_rcu_getref(&sma->sem_perm)) { in semctl_main()
1434 sem_unlock(sma, -1); in semctl_main()
1439 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1444 sem_lock_and_putref(sma); in semctl_main()
1445 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1450 for (i = 0; i < sma->sem_nsems; i++) in semctl_main()
1451 sem_io[i] = sma->sems[i].semval; in semctl_main()
1452 sem_unlock(sma, -1); in semctl_main()
1464 if (!ipc_rcu_getref(&sma->sem_perm)) { in semctl_main()
1474 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1480 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1487 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1493 sem_lock_and_putref(sma); in semctl_main()
1494 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1500 sma->sems[i].semval = sem_io[i]; in semctl_main()
1501 ipc_update_pid(&sma->sems[i].sempid, task_tgid(current)); in semctl_main()
1504 ipc_assert_locked_object(&sma->sem_perm); in semctl_main()
1505 list_for_each_entry(un, &sma->list_id, list_id) { in semctl_main()
1509 sma->sem_ctime = ktime_get_real_seconds(); in semctl_main()
1511 do_smart_update(sma, NULL, 0, 0, &wake_q); in semctl_main()
1521 sem_lock(sma, NULL, -1); in semctl_main()
1522 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1528 curr = &sma->sems[semnum]; in semctl_main()
1538 err = count_semcnt(sma, semnum, 0); in semctl_main()
1541 err = count_semcnt(sma, semnum, 1); in semctl_main()
1546 sem_unlock(sma, -1); in semctl_main()
1590 struct sem_array *sma; in semctl_down() local
1604 sma = container_of(ipcp, struct sem_array, sem_perm); in semctl_down()
1606 err = security_sem_semctl(&sma->sem_perm, cmd); in semctl_down()
1612 sem_lock(sma, NULL, -1); in semctl_down()
1617 sem_lock(sma, NULL, -1); in semctl_down()
1621 sma->sem_ctime = ktime_get_real_seconds(); in semctl_down()
1629 sem_unlock(sma, -1); in semctl_down()
1867 struct sem_array *sma; in find_alloc_undo() local
1885 sma = sem_obtain_object_check(ns, semid); in find_alloc_undo()
1886 if (IS_ERR(sma)) { in find_alloc_undo()
1888 return ERR_CAST(sma); in find_alloc_undo()
1891 nsems = sma->sem_nsems; in find_alloc_undo()
1892 if (!ipc_rcu_getref(&sma->sem_perm)) { in find_alloc_undo()
1902 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in find_alloc_undo()
1908 sem_lock_and_putref(sma); in find_alloc_undo()
1909 if (!ipc_valid_object(&sma->sem_perm)) { in find_alloc_undo()
1910 sem_unlock(sma, -1); in find_alloc_undo()
1932 ipc_assert_locked_object(&sma->sem_perm); in find_alloc_undo()
1933 list_add(&new->list_id, &sma->list_id); in find_alloc_undo()
1938 sem_unlock(sma, -1); in find_alloc_undo()
1947 struct sem_array *sma; in do_semtimedop() local
2018 sma = sem_obtain_object_check(ns, semid); in do_semtimedop()
2019 if (IS_ERR(sma)) { in do_semtimedop()
2021 error = PTR_ERR(sma); in do_semtimedop()
2026 if (max >= sma->sem_nsems) { in do_semtimedop()
2032 if (ipcperms(ns, &sma->sem_perm, alter ? S_IWUGO : S_IRUGO)) { in do_semtimedop()
2037 error = security_sem_semop(&sma->sem_perm, sops, nsops, alter); in do_semtimedop()
2044 locknum = sem_lock(sma, sops, nsops); in do_semtimedop()
2053 if (!ipc_valid_object(&sma->sem_perm)) in do_semtimedop()
2072 error = perform_atomic_semop(sma, &queue); in do_semtimedop()
2081 do_smart_update(sma, sops, nsops, 1, &wake_q); in do_semtimedop()
2083 set_semotime(sma, sops); in do_semtimedop()
2085 sem_unlock(sma, locknum); in do_semtimedop()
2100 int idx = array_index_nospec(sops->sem_num, sma->sem_nsems); in do_semtimedop()
2101 curr = &sma->sems[idx]; in do_semtimedop()
2104 if (sma->complex_count) { in do_semtimedop()
2106 &sma->pending_alter); in do_semtimedop()
2116 if (!sma->complex_count) in do_semtimedop()
2117 merge_queues(sma); in do_semtimedop()
2120 list_add_tail(&queue.list, &sma->pending_alter); in do_semtimedop()
2122 list_add_tail(&queue.list, &sma->pending_const); in do_semtimedop()
2124 sma->complex_count++; in do_semtimedop()
2132 sem_unlock(sma, locknum); in do_semtimedop()
2164 locknum = sem_lock(sma, sops, nsops); in do_semtimedop()
2166 if (!ipc_valid_object(&sma->sem_perm)) in do_semtimedop()
2185 unlink_queue(sma, &queue); in do_semtimedop()
2188 sem_unlock(sma, locknum); in do_semtimedop()
2288 struct sem_array *sma; in exit_sem() local
2320 sma = sem_obtain_object_check(tsk->nsproxy->ipc_ns, semid); in exit_sem()
2322 if (IS_ERR(sma)) { in exit_sem()
2327 sem_lock(sma, NULL, -1); in exit_sem()
2329 if (!ipc_valid_object(&sma->sem_perm)) { in exit_sem()
2330 sem_unlock(sma, -1); in exit_sem()
2339 sem_unlock(sma, -1); in exit_sem()
2345 ipc_assert_locked_object(&sma->sem_perm); in exit_sem()
2355 for (i = 0; i < sma->sem_nsems; i++) { in exit_sem()
2356 struct sem *semaphore = &sma->sems[i]; in exit_sem()
2380 do_smart_update(sma, NULL, 0, 1, &wake_q); in exit_sem()
2381 sem_unlock(sma, -1); in exit_sem()
2395 struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm); in sysvipc_sem_proc_show() local
2404 complexmode_enter(sma); in sysvipc_sem_proc_show()
2406 sem_otime = get_semotime(sma); in sysvipc_sem_proc_show()
2410 sma->sem_perm.key, in sysvipc_sem_proc_show()
2411 sma->sem_perm.id, in sysvipc_sem_proc_show()
2412 sma->sem_perm.mode, in sysvipc_sem_proc_show()
2413 sma->sem_nsems, in sysvipc_sem_proc_show()
2414 from_kuid_munged(user_ns, sma->sem_perm.uid), in sysvipc_sem_proc_show()
2415 from_kgid_munged(user_ns, sma->sem_perm.gid), in sysvipc_sem_proc_show()
2416 from_kuid_munged(user_ns, sma->sem_perm.cuid), in sysvipc_sem_proc_show()
2417 from_kgid_munged(user_ns, sma->sem_perm.cgid), in sysvipc_sem_proc_show()
2419 sma->sem_ctime); in sysvipc_sem_proc_show()
2421 complexmode_tryleave(sma); in sysvipc_sem_proc_show()