1 // SPDX-License-Identifier: GPL-2.0
9  * SMP-threaded, sysctl's added
30  * - FIFO ordering for semop() operations (just FIFO, not starvation
32  * - multiple semaphore operations that alter the same semaphore in
34  * - sem_ctime (time of last semctl()) is updated in the IPC_SET, SETVAL and
36  * - two Linux specific semctl() commands: SEM_STAT, SEM_INFO.
37  * - undo adjustments at process exit are limited to 0..SEMVMX.
38  * - namespace are supported.
39  * - SEMMSL, SEMMNS, SEMOPM and SEMMNI can be configured at runtime by writing
41  * - statistics about the usage are reported in /proc/sysvipc/sem.
44  * - scalability:
45  *   - all global variables are read-mostly.
46  *   - semop() calls and semctl(RMID) are synchronized by RCU.
47  *   - most operations do write operations (actually: spin_lock calls) to
48  *     the per-semaphore array structure.
52  * - semncnt and semzcnt are calculated on demand in count_semcnt()
53  * - the task that performs a successful semop() scans the list of all
57  * - To improve the scalability, the actual wake-up calls are performed after
59  * - All work is done by the waker, the woken up task does not have to do
60  *   anything - not even acquiring a lock or dropping a refcount.
61  * - A woken up task may not even touch the semaphore array anymore, it may
63  * - UNDO values are stored in an array (one per process and per
67  * - There are two lists of the pending operations: a per-array list
68  *   and per-semaphore list (stored in the array). This allows to achieve FIFO
70  *   The worst-case behavior is nevertheless O(N^2) for N wakeups.
100 	 *  - semop
101 	 *  - semctl, via SETVAL and SETALL.
102 	 *  - at task exit when performing undo adjustments (see exit_sem).
105 	spinlock_t	lock;	/* spinlock for fine-grained semtimedop */
106 	struct list_head pending_alter; /* pending single-sop operations */
108 	struct list_head pending_const; /* pending single-sop operations */
124 	unsigned int		use_global_lock;/* >0: global lock required */
147 	struct list_head	list_proc;	/* per-process list: *
151 	struct sem_undo_list	*ulp;		/* back ptr to sem_undo_list */  member
169 #define sem_ids(ns)	((ns)->ids[IPC_SEM_IDS])
200  *	* undo_list->lock for write
211  * Setting it from non-zero to 0 is a RELEASE, this is ensured by
212  * using smp_store_release(): Immediately after setting it to 0,
214  * Testing if it is non-zero is an ACQUIRE, this is ensured by using
216  * Setting it from 0 to non-zero must be ordered with regards to
218  * is inside a spin_lock() and after a write from 0 to non-zero a
220  * To prevent the compiler/cpu temporarily writing 0 to use_global_lock,
235  * 3) current->state:
236  * current->state is set to TASK_INTERRUPTIBLE while holding sem_lock().
244 #define sc_semmsl	sem_ctls[0]
251 	ns->sc_semmsl = SEMMSL;  in sem_init_ns()
252 	ns->sc_semmns = SEMMNS;  in sem_init_ns()
253 	ns->sc_semopm = SEMOPM;  in sem_init_ns()
254 	ns->sc_semmni = SEMMNI;  in sem_init_ns()
255 	ns->used_sems = 0;  in sem_init_ns()
256 	ipc_init_ids(&ns->ids[IPC_SEM_IDS]);  in sem_init_ns()
263 	idr_destroy(&ns->ids[IPC_SEM_IDS].ipcs_idr);  in sem_exit_ns()
264 	rhashtable_destroy(&ns->ids[IPC_SEM_IDS].key_ht);  in sem_exit_ns()
277  * unmerge_queues - unmerge queues, if possible.
280  * The function unmerges the wait queues if complex_count is 0.
288 	if (sma->complex_count)  in unmerge_queues()
292 	 * Move all pending operation back into the per-semaphore  in unmerge_queues()
295 	list_for_each_entry_safe(q, tq, &sma->pending_alter, list) {  in unmerge_queues()
297 		curr = &sma->sems[q->sops[0].sem_num];  in unmerge_queues()
299 		list_add_tail(&q->list, &curr->pending_alter);  in unmerge_queues()
301 	INIT_LIST_HEAD(&sma->pending_alter);  in unmerge_queues()
305  * merge_queues - merge single semop queues into global queue
308  * This function merges all per-semaphore queues into the global queue.
309  * It is necessary to achieve FIFO ordering for the pending single-sop
310  * operations when a multi-semop operation must sleep.
316 	for (i = 0; i < sma->sem_nsems; i++) {  in merge_queues()
317 		struct sem *sem = &sma->sems[i];  in merge_queues()
319 		list_splice_init(&sem->pending_alter, &sma->pending_alter);  in merge_queues()
328 	security_sem_free(&sma->sem_perm);  in sem_rcu_free()
333  * Enter the mode suitable for non-simple operations:
341 	if (sma->use_global_lock > 0)  {  in complexmode_enter()
347 		WRITE_ONCE(sma->use_global_lock, USE_GLOBAL_LOCK_HYSTERESIS);  in complexmode_enter()
350 	WRITE_ONCE(sma->use_global_lock, USE_GLOBAL_LOCK_HYSTERESIS);  in complexmode_enter()
352 	for (i = 0; i < sma->sem_nsems; i++) {  in complexmode_enter()
353 		sem = &sma->sems[i];  in complexmode_enter()
354 		spin_lock(&sem->lock);  in complexmode_enter()
355 		spin_unlock(&sem->lock);  in complexmode_enter()
365 	if (sma->complex_count)  {  in complexmode_tryleave()
371 	if (sma->use_global_lock == 1) {  in complexmode_tryleave()
374 		smp_store_release(&sma->use_global_lock, 0);  in complexmode_tryleave()
376 		WRITE_ONCE(sma->use_global_lock,  in complexmode_tryleave()
377 				sma->use_global_lock-1);  in complexmode_tryleave()
381 #define SEM_GLOBAL_LOCK	(-1)
396 		/* Complex operation - acquire a full lock */  in sem_lock()
397 		ipc_lock_object(&sma->sem_perm);  in sem_lock()
405 	 * Only one semaphore affected - try to optimize locking.  in sem_lock()
411 	idx = array_index_nospec(sops->sem_num, sma->sem_nsems);  in sem_lock()
412 	sem = &sma->sems[idx];  in sem_lock()
418 	if (!READ_ONCE(sma->use_global_lock)) {  in sem_lock()
421 		 * Acquire the per-semaphore lock.  in sem_lock()
423 		spin_lock(&sem->lock);  in sem_lock()
426 		if (!smp_load_acquire(&sma->use_global_lock)) {  in sem_lock()
428 			return sops->sem_num;  in sem_lock()
430 		spin_unlock(&sem->lock);  in sem_lock()
434 	ipc_lock_object(&sma->sem_perm);  in sem_lock()
436 	if (sma->use_global_lock == 0) {  in sem_lock()
439 		 * sma->sem_perm.lock. Thus we must switch to locking  in sem_lock()
440 		 * with sem->lock.  in sem_lock()
442 		 * sma->use_global_lock after we have acquired sem->lock:  in sem_lock()
443 		 * We own sma->sem_perm.lock, thus use_global_lock cannot  in sem_lock()
446 		spin_lock(&sem->lock);  in sem_lock()
448 		ipc_unlock_object(&sma->sem_perm);  in sem_lock()
449 		return sops->sem_num;  in sem_lock()
454 		 * the caller that has set use_global_mode to non-zero.  in sem_lock()
465 		ipc_unlock_object(&sma->sem_perm);  in sem_unlock()
467 		struct sem *sem = &sma->sems[locknum];  in sem_unlock()
468 		spin_unlock(&sem->lock);  in sem_unlock()
501 	sem_lock(sma, NULL, -1);  in sem_lock_and_putref()
502 	ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);  in sem_lock_and_putref()
507 	ipc_rmid(&sem_ids(ns), &s->sem_perm);  in sem_rmid()
514 	if (nsems > (INT_MAX - sizeof(*sma)) / sizeof(sma->sems[0]))  in sem_alloc()
525  * newary - Create a new semaphore set
535 	key_t key = params->key;  in newary()
536 	int nsems = params->u.nsems;  in newary()
537 	int semflg = params->flg;  in newary()
541 		return -EINVAL;  in newary()
542 	if (ns->used_sems + nsems > ns->sc_semmns)  in newary()
543 		return -ENOSPC;  in newary()
547 		return -ENOMEM;  in newary()
549 	sma->sem_perm.mode = (semflg & S_IRWXUGO);  in newary()
550 	sma->sem_perm.key = key;  in newary()
552 	sma->sem_perm.security = NULL;  in newary()
553 	retval = security_sem_alloc(&sma->sem_perm);  in newary()
559 	for (i = 0; i < nsems; i++) {  in newary()
560 		INIT_LIST_HEAD(&sma->sems[i].pending_alter);  in newary()
561 		INIT_LIST_HEAD(&sma->sems[i].pending_const);  in newary()
562 		spin_lock_init(&sma->sems[i].lock);  in newary()
565 	sma->complex_count = 0;  in newary()
566 	sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS;  in newary()
567 	INIT_LIST_HEAD(&sma->pending_alter);  in newary()
568 	INIT_LIST_HEAD(&sma->pending_const);  in newary()
569 	INIT_LIST_HEAD(&sma->list_id);  in newary()
570 	sma->sem_nsems = nsems;  in newary()
571 	sma->sem_ctime = ktime_get_real_seconds();  in newary()
574 	retval = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni);  in newary()
575 	if (retval < 0) {  in newary()
576 		ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);  in newary()
579 	ns->used_sems += nsems;  in newary()
581 	sem_unlock(sma, -1);  in newary()
584 	return sma->sem_perm.id;  in newary()
596 	if (params->u.nsems > sma->sem_nsems)  in sem_more_checks()
597 		return -EINVAL;  in sem_more_checks()
599 	return 0;  in sem_more_checks()
612 	ns = current->nsproxy->ipc_ns;  in ksys_semget()
614 	if (nsems < 0 || nsems > ns->sc_semmsl)  in ksys_semget()
615 		return -EINVAL;  in ksys_semget()
630  * perform_atomic_semop[_slow] - Attempt to perform semaphore
638  *  (1) >0 never blocks.
639  *  (2)  0 (wait-for-zero operation): semval is non-zero.
640  *  (3) <0 attempting to decrement semval to a value smaller than zero.
642  * Returns 0 if the operation was possible.
644  * Returns <0 for error codes.
655 	sops = q->sops;  in perform_atomic_semop_slow()
656 	nsops = q->nsops;  in perform_atomic_semop_slow()
657 	un = q->undo;  in perform_atomic_semop_slow()
660 		int idx = array_index_nospec(sop->sem_num, sma->sem_nsems);  in perform_atomic_semop_slow()
661 		curr = &sma->sems[idx];  in perform_atomic_semop_slow()
662 		sem_op = sop->sem_op;  in perform_atomic_semop_slow()
663 		result = curr->semval;  in perform_atomic_semop_slow()
669 		if (result < 0)  in perform_atomic_semop_slow()
674 		if (sop->sem_flg & SEM_UNDO) {  in perform_atomic_semop_slow()
675 			int undo = un->semadj[sop->sem_num] - sem_op;  in perform_atomic_semop_slow()
677 			if (undo < (-SEMAEM - 1) || undo > SEMAEM)  in perform_atomic_semop_slow()
679 			un->semadj[sop->sem_num] = undo;  in perform_atomic_semop_slow()
682 		curr->semval = result;  in perform_atomic_semop_slow()
685 	sop--;  in perform_atomic_semop_slow()
686 	pid = q->pid;  in perform_atomic_semop_slow()
688 		ipc_update_pid(&sma->sems[sop->sem_num].sempid, pid);  in perform_atomic_semop_slow()
689 		sop--;  in perform_atomic_semop_slow()
692 	return 0;  in perform_atomic_semop_slow()
695 	result = -ERANGE;  in perform_atomic_semop_slow()
699 	q->blocking = sop;  in perform_atomic_semop_slow()
701 	if (sop->sem_flg & IPC_NOWAIT)  in perform_atomic_semop_slow()
702 		result = -EAGAIN;  in perform_atomic_semop_slow()
707 	sop--;  in perform_atomic_semop_slow()
709 		sem_op = sop->sem_op;  in perform_atomic_semop_slow()
710 		sma->sems[sop->sem_num].semval -= sem_op;  in perform_atomic_semop_slow()
711 		if (sop->sem_flg & SEM_UNDO)  in perform_atomic_semop_slow()
712 			un->semadj[sop->sem_num] += sem_op;  in perform_atomic_semop_slow()
713 		sop--;  in perform_atomic_semop_slow()
727 	sops = q->sops;  in perform_atomic_semop()
728 	nsops = q->nsops;  in perform_atomic_semop()
729 	un = q->undo;  in perform_atomic_semop()
731 	if (unlikely(q->dupsop))  in perform_atomic_semop()
741 		int idx = array_index_nospec(sop->sem_num, sma->sem_nsems);  in perform_atomic_semop()
743 		curr = &sma->sems[idx];  in perform_atomic_semop()
744 		sem_op = sop->sem_op;  in perform_atomic_semop()
745 		result = curr->semval;  in perform_atomic_semop()
748 			goto would_block; /* wait-for-zero */  in perform_atomic_semop()
751 		if (result < 0)  in perform_atomic_semop()
755 			return -ERANGE;  in perform_atomic_semop()
757 		if (sop->sem_flg & SEM_UNDO) {  in perform_atomic_semop()
758 			int undo = un->semadj[sop->sem_num] - sem_op;  in perform_atomic_semop()
761 			if (undo < (-SEMAEM - 1) || undo > SEMAEM)  in perform_atomic_semop()
762 				return -ERANGE;  in perform_atomic_semop()
767 		curr = &sma->sems[sop->sem_num];  in perform_atomic_semop()
768 		sem_op = sop->sem_op;  in perform_atomic_semop()
769 		result = curr->semval;  in perform_atomic_semop()
771 		if (sop->sem_flg & SEM_UNDO) {  in perform_atomic_semop()
772 			int undo = un->semadj[sop->sem_num] - sem_op;  in perform_atomic_semop()
774 			un->semadj[sop->sem_num] = undo;  in perform_atomic_semop()
776 		curr->semval += sem_op;  in perform_atomic_semop()
777 		ipc_update_pid(&curr->sempid, q->pid);  in perform_atomic_semop()
780 	return 0;  in perform_atomic_semop()
783 	q->blocking = sop;  in perform_atomic_semop()
784 	return sop->sem_flg & IPC_NOWAIT ? -EAGAIN : 1;  in perform_atomic_semop()
792 	sleeper = get_task_struct(q->sleeper);  in wake_up_sem_queue_prepare()
795 	smp_store_release(&q->status, error);  in wake_up_sem_queue_prepare()
802 	list_del(&q->list);  in unlink_queue()
803 	if (q->nsops > 1)  in unlink_queue()
804 		sma->complex_count--;  in unlink_queue()
815  * Note that wait-for-zero operations are handled without restart.
820 	if (!list_empty(&sma->pending_alter))  in check_restart()
824 	if (q->nsops > 1)  in check_restart()
828 	 * - complex operations always restart.  in check_restart()
829 	 * - wait-for-zero are handled separately.  in check_restart()
830 	 * - q is a previously sleeping simple operation that  in check_restart()
833 	 * - If there are older (higher priority) decrements  in check_restart()
836 	 *   decremented to value - thus they won't proceed either.  in check_restart()
838 	return 0;  in check_restart()
842  * wake_const_ops - wake up non-alter tasks
845  * @wake_q: lockless wake-queue head.
848  * was set to 0. If complex const operations are pending, wake_const_ops must
849  * be called with semnum = -1, as well as with the number of each modified
852  * is stored in q->pid.
860 	int semop_completed = 0;  in wake_const_ops()
862 	if (semnum == -1)  in wake_const_ops()
863 		pending_list = &sma->pending_const;  in wake_const_ops()
865 		pending_list = &sma->sems[semnum].pending_const;  in wake_const_ops()
870 		if (error > 0)  in wake_const_ops()
876 		if (error == 0)  in wake_const_ops()
884  * do_smart_wakeup_zero - wakeup all wait for zero tasks
888  * @wake_q: lockless wake-queue head
890  * Checks all required queue for wait-for-zero operations, based
898 	int semop_completed = 0;  in do_smart_wakeup_zero()
899 	int got_zero = 0;  in do_smart_wakeup_zero()
901 	/* first: the per-semaphore queues, if known */  in do_smart_wakeup_zero()
903 		for (i = 0; i < nsops; i++) {  in do_smart_wakeup_zero()
906 			if (sma->sems[num].semval == 0) {  in do_smart_wakeup_zero()
916 		for (i = 0; i < sma->sem_nsems; i++) {  in do_smart_wakeup_zero()
917 			if (sma->sems[i].semval == 0) {  in do_smart_wakeup_zero()
924 	 * If one of the modified semaphores got 0,  in do_smart_wakeup_zero()
928 		semop_completed |= wake_const_ops(sma, -1, wake_q);  in do_smart_wakeup_zero()
935  * update_queue - look for tasks that can be completed.
938  * @wake_q: lockless wake-queue head.
942  * be called with semnum = -1, as well as with the number of each modified
945  * is stored in q->pid.
954 	int semop_completed = 0;  in update_queue()
956 	if (semnum == -1)  in update_queue()
957 		pending_list = &sma->pending_alter;  in update_queue()
959 		pending_list = &sma->sems[semnum].pending_alter;  in update_queue()
965 		/* If we are scanning the single sop, per-semaphore list of  in update_queue()
966 		 * one semaphore and that semaphore is 0, then it is not  in update_queue()
970 		 * cannot be successful if the value is already 0.  in update_queue()
972 		if (semnum != -1 && sma->sems[semnum].semval == 0)  in update_queue()
977 		/* Does q->sleeper still need to sleep? */  in update_queue()
978 		if (error > 0)  in update_queue()
984 			restart = 0;  in update_queue()
987 			do_smart_wakeup_zero(sma, q->sops, q->nsops, wake_q);  in update_queue()
999  * set_semotime - set sem_otime
1009 		sma->sems[0].sem_otime = ktime_get_real_seconds();  in set_semotime()
1011 		sma->sems[sops[0].sem_num].sem_otime =  in set_semotime()
1017  * do_smart_update - optimized update_queue
1022  * @wake_q: lockless wake-queue head
1026  * Note that the function does not do the actual wake-up: the caller is
1037 	if (!list_empty(&sma->pending_alter)) {  in do_smart_update()
1038 		/* semaphore array uses the global queue - just process it. */  in do_smart_update()
1039 		otime |= update_queue(sma, -1, wake_q);  in do_smart_update()
1046 			for (i = 0; i < sma->sem_nsems; i++)  in do_smart_update()
1051 			 * - No complex ops, thus all sleeping ops are  in do_smart_update()
1053 			 * - if we decreased the value, then any sleeping  in do_smart_update()
1058 			for (i = 0; i < nsops; i++) {  in do_smart_update()
1059 				if (sops[i].sem_op > 0) {  in do_smart_update()
1076 	struct sembuf *sop = q->blocking;  in check_qop()
1087 			current->comm, task_pid_nr(current));  in check_qop()
1089 	if (sop->sem_num != semnum)  in check_qop()
1090 		return 0;  in check_qop()
1092 	if (count_zero && sop->sem_op == 0)  in check_qop()
1094 	if (!count_zero && sop->sem_op < 0)  in check_qop()
1097 	return 0;  in check_qop()
1114 	semcnt = 0;  in count_semcnt()
1117 		l = &sma->sems[semnum].pending_const;  in count_semcnt()
1119 		l = &sma->sems[semnum].pending_alter;  in count_semcnt()
1122 		/* all task on a per-semaphore list sleep on exactly  in count_semcnt()
1129 	list_for_each_entry(q, &sma->pending_alter, list) {  in count_semcnt()
1133 		list_for_each_entry(q, &sma->pending_const, list) {  in count_semcnt()
1153 	ipc_assert_locked_object(&sma->sem_perm);  in freeary()
1154 	list_for_each_entry_safe(un, tu, &sma->list_id, list_id) {  in freeary()
1155 		list_del(&un->list_id);  in freeary()
1156 		spin_lock(&un->ulp->lock);  in freeary()
1157 		un->semid = -1;  in freeary()
1158 		list_del_rcu(&un->list_proc);  in freeary()
1159 		spin_unlock(&un->ulp->lock);  in freeary()
1164 	list_for_each_entry_safe(q, tq, &sma->pending_const, list) {  in freeary()
1166 		wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);  in freeary()
1169 	list_for_each_entry_safe(q, tq, &sma->pending_alter, list) {  in freeary()
1171 		wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);  in freeary()
1173 	for (i = 0; i < sma->sem_nsems; i++) {  in freeary()
1174 		struct sem *sem = &sma->sems[i];  in freeary()
1175 		list_for_each_entry_safe(q, tq, &sem->pending_const, list) {  in freeary()
1177 			wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);  in freeary()
1179 		list_for_each_entry_safe(q, tq, &sem->pending_alter, list) {  in freeary()
1181 			wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);  in freeary()
1183 		ipc_update_pid(&sem->sempid, NULL);  in freeary()
1188 	sem_unlock(sma, -1);  in freeary()
1192 	ns->used_sems -= sma->sem_nsems;  in freeary()
1193 	ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);  in freeary()
1205 		memset(&out, 0, sizeof(out));  in copy_semid_to_user()
1207 		ipc64_perm_to_ipc_perm(&in->sem_perm, &out.sem_perm);  in copy_semid_to_user()
1209 		out.sem_otime	= in->sem_otime;  in copy_semid_to_user()
1210 		out.sem_ctime	= in->sem_ctime;  in copy_semid_to_user()
1211 		out.sem_nsems	= in->sem_nsems;  in copy_semid_to_user()
1216 		return -EINVAL;  in copy_semid_to_user()
1225 	res = sma->sems[0].sem_otime;  in get_semotime()
1226 	for (i = 1; i < sma->sem_nsems; i++) {  in get_semotime()
1227 		time64_t to = sma->sems[i].sem_otime;  in get_semotime()
1242 	memset(semid64, 0, sizeof(*semid64));  in semctl_stat()
1261 		audit_ipc_obj(&sma->sem_perm);  in semctl_stat()
1263 		err = -EACCES;  in semctl_stat()
1264 		if (ipcperms(ns, &sma->sem_perm, S_IRUGO))  in semctl_stat()
1268 	err = security_sem_semctl(&sma->sem_perm, cmd);  in semctl_stat()
1272 	ipc_lock_object(&sma->sem_perm);  in semctl_stat()
1274 	if (!ipc_valid_object(&sma->sem_perm)) {  in semctl_stat()
1275 		ipc_unlock_object(&sma->sem_perm);  in semctl_stat()
1276 		err = -EIDRM;  in semctl_stat()
1280 	kernel_to_ipc64_perm(&sma->sem_perm, &semid64->sem_perm);  in semctl_stat()
1282 	semid64->sem_otime = semotime;  in semctl_stat()
1283 	semid64->sem_ctime = sma->sem_ctime;  in semctl_stat()
1285 	semid64->sem_otime_high = semotime >> 32;  in semctl_stat()
1286 	semid64->sem_ctime_high = sma->sem_ctime >> 32;  in semctl_stat()
1288 	semid64->sem_nsems = sma->sem_nsems;  in semctl_stat()
1293 		 * Return 0 on success  in semctl_stat()
1295 		err = 0;  in semctl_stat()
1301 		err = sma->sem_perm.id;  in semctl_stat()
1303 	ipc_unlock_object(&sma->sem_perm);  in semctl_stat()
1320 	memset(&seminfo, 0, sizeof(seminfo));  in semctl_info()
1321 	seminfo.semmni = ns->sc_semmni;  in semctl_info()
1322 	seminfo.semmns = ns->sc_semmns;  in semctl_info()
1323 	seminfo.semmsl = ns->sc_semmsl;  in semctl_info()
1324 	seminfo.semopm = ns->sc_semopm;  in semctl_info()
1332 		seminfo.semaem = ns->used_sems;  in semctl_info()
1340 		return -EFAULT;  in semctl_info()
1341 	return (max_idx < 0) ? 0 : max_idx;  in semctl_info()
1353 	if (val > SEMVMX || val < 0)  in semctl_setval()
1354 		return -ERANGE;  in semctl_setval()
1363 	if (semnum < 0 || semnum >= sma->sem_nsems) {  in semctl_setval()
1365 		return -EINVAL;  in semctl_setval()
1369 	if (ipcperms(ns, &sma->sem_perm, S_IWUGO)) {  in semctl_setval()
1371 		return -EACCES;  in semctl_setval()
1374 	err = security_sem_semctl(&sma->sem_perm, SETVAL);  in semctl_setval()
1377 		return -EACCES;  in semctl_setval()
1380 	sem_lock(sma, NULL, -1);  in semctl_setval()
1382 	if (!ipc_valid_object(&sma->sem_perm)) {  in semctl_setval()
1383 		sem_unlock(sma, -1);  in semctl_setval()
1385 		return -EIDRM;  in semctl_setval()
1388 	semnum = array_index_nospec(semnum, sma->sem_nsems);  in semctl_setval()
1389 	curr = &sma->sems[semnum];  in semctl_setval()
1391 	ipc_assert_locked_object(&sma->sem_perm);  in semctl_setval()
1392 	list_for_each_entry(un, &sma->list_id, list_id)  in semctl_setval()
1393 		un->semadj[semnum] = 0;  in semctl_setval()
1395 	curr->semval = val;  in semctl_setval()
1396 	ipc_update_pid(&curr->sempid, task_tgid(current));  in semctl_setval()
1397 	sma->sem_ctime = ktime_get_real_seconds();  in semctl_setval()
1398 	/* maybe some queued-up processes were waiting for this */  in semctl_setval()
1399 	do_smart_update(sma, NULL, 0, 0, &wake_q);  in semctl_setval()
1400 	sem_unlock(sma, -1);  in semctl_setval()
1403 	return 0;  in semctl_setval()
1423 	nsems = sma->sem_nsems;  in semctl_main()
1425 	err = -EACCES;  in semctl_main()
1426 	if (ipcperms(ns, &sma->sem_perm, cmd == SETALL ? S_IWUGO : S_IRUGO))  in semctl_main()
1429 	err = security_sem_semctl(&sma->sem_perm, cmd);  in semctl_main()
1433 	err = -EACCES;  in semctl_main()
1440 		sem_lock(sma, NULL, -1);  in semctl_main()
1441 		if (!ipc_valid_object(&sma->sem_perm)) {  in semctl_main()
1442 			err = -EIDRM;  in semctl_main()
1446 			if (!ipc_rcu_getref(&sma->sem_perm)) {  in semctl_main()
1447 				err = -EIDRM;  in semctl_main()
1450 			sem_unlock(sma, -1);  in semctl_main()
1455 				ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);  in semctl_main()
1456 				return -ENOMEM;  in semctl_main()
1461 			if (!ipc_valid_object(&sma->sem_perm)) {  in semctl_main()
1462 				err = -EIDRM;  in semctl_main()
1466 		for (i = 0; i < sma->sem_nsems; i++)  in semctl_main()
1467 			sem_io[i] = sma->sems[i].semval;  in semctl_main()
1468 		sem_unlock(sma, -1);  in semctl_main()
1470 		err = 0;  in semctl_main()
1472 			err = -EFAULT;  in semctl_main()
1480 		if (!ipc_rcu_getref(&sma->sem_perm)) {  in semctl_main()
1481 			err = -EIDRM;  in semctl_main()
1490 				ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);  in semctl_main()
1491 				return -ENOMEM;  in semctl_main()
1496 			ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);  in semctl_main()
1497 			err = -EFAULT;  in semctl_main()
1501 		for (i = 0; i < nsems; i++) {  in semctl_main()
1503 				ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);  in semctl_main()
1504 				err = -ERANGE;  in semctl_main()
1510 		if (!ipc_valid_object(&sma->sem_perm)) {  in semctl_main()
1511 			err = -EIDRM;  in semctl_main()
1515 		for (i = 0; i < nsems; i++) {  in semctl_main()
1516 			sma->sems[i].semval = sem_io[i];  in semctl_main()
1517 			ipc_update_pid(&sma->sems[i].sempid, task_tgid(current));  in semctl_main()
1520 		ipc_assert_locked_object(&sma->sem_perm);  in semctl_main()
1521 		list_for_each_entry(un, &sma->list_id, list_id) {  in semctl_main()
1522 			for (i = 0; i < nsems; i++)  in semctl_main()
1523 				un->semadj[i] = 0;  in semctl_main()
1525 		sma->sem_ctime = ktime_get_real_seconds();  in semctl_main()
1526 		/* maybe some queued-up processes were waiting for this */  in semctl_main()
1527 		do_smart_update(sma, NULL, 0, 0, &wake_q);  in semctl_main()
1528 		err = 0;  in semctl_main()
1531 	/* GETVAL, GETPID, GETNCTN, GETZCNT: fall-through */  in semctl_main()
1533 	err = -EINVAL;  in semctl_main()
1534 	if (semnum < 0 || semnum >= nsems)  in semctl_main()
1537 	sem_lock(sma, NULL, -1);  in semctl_main()
1538 	if (!ipc_valid_object(&sma->sem_perm)) {  in semctl_main()
1539 		err = -EIDRM;  in semctl_main()
1544 	curr = &sma->sems[semnum];  in semctl_main()
1548 		err = curr->semval;  in semctl_main()
1551 		err = pid_vnr(curr->sempid);  in semctl_main()
1554 		err = count_semcnt(sma, semnum, 0);  in semctl_main()
1562 	sem_unlock(sma, -1);  in semctl_main()
1578 			return -EFAULT;  in copy_semid_from_user()
1579 		return 0;  in copy_semid_from_user()
1585 			return -EFAULT;  in copy_semid_from_user()
1587 		out->sem_perm.uid	= tbuf_old.sem_perm.uid;  in copy_semid_from_user()
1588 		out->sem_perm.gid	= tbuf_old.sem_perm.gid;  in copy_semid_from_user()
1589 		out->sem_perm.mode	= tbuf_old.sem_perm.mode;  in copy_semid_from_user()
1591 		return 0;  in copy_semid_from_user()
1594 		return -EINVAL;  in copy_semid_from_user()
1614 				      &semid64->sem_perm, 0);  in semctl_down()
1622 	err = security_sem_semctl(&sma->sem_perm, cmd);  in semctl_down()
1628 		sem_lock(sma, NULL, -1);  in semctl_down()
1633 		sem_lock(sma, NULL, -1);  in semctl_down()
1634 		err = ipc_update_perm(&semid64->sem_perm, ipcp);  in semctl_down()
1637 		sma->sem_ctime = ktime_get_real_seconds();  in semctl_down()
1640 		err = -EINVAL;  in semctl_down()
1645 	sem_unlock(sma, -1);  in semctl_down()
1660 	if (semid < 0)  in ksys_semctl()
1661 		return -EINVAL;  in ksys_semctl()
1663 	ns = current->nsproxy->ipc_ns;  in ksys_semctl()
1673 		if (err < 0)  in ksys_semctl()
1676 			err = -EFAULT;  in ksys_semctl()
1688 		/* big-endian 64bit */  in ksys_semctl()
1691 		/* 32bit or little-endian 64bit */  in ksys_semctl()
1698 			return -EFAULT;  in ksys_semctl()
1703 		return -EINVAL;  in ksys_semctl()
1742 	memset(out, 0, sizeof(*out));  in copy_compat_semid_from_user()
1745 		return get_compat_ipc64_perm(&out->sem_perm, &p->sem_perm);  in copy_compat_semid_from_user()
1748 		return get_compat_ipc_perm(&out->sem_perm, &p->sem_perm);  in copy_compat_semid_from_user()
1757 		memset(&v, 0, sizeof(v));  in copy_compat_semid_to_user()
1758 		to_compat_ipc64_perm(&v.sem_perm, &in->sem_perm);  in copy_compat_semid_to_user()
1759 		v.sem_otime	 = lower_32_bits(in->sem_otime);  in copy_compat_semid_to_user()
1760 		v.sem_otime_high = upper_32_bits(in->sem_otime);  in copy_compat_semid_to_user()
1761 		v.sem_ctime	 = lower_32_bits(in->sem_ctime);  in copy_compat_semid_to_user()
1762 		v.sem_ctime_high = upper_32_bits(in->sem_ctime);  in copy_compat_semid_to_user()
1763 		v.sem_nsems = in->sem_nsems;  in copy_compat_semid_to_user()
1767 		memset(&v, 0, sizeof(v));  in copy_compat_semid_to_user()
1768 		to_compat_ipc_perm(&v.sem_perm, &in->sem_perm);  in copy_compat_semid_to_user()
1769 		v.sem_otime = in->sem_otime;  in copy_compat_semid_to_user()
1770 		v.sem_ctime = in->sem_ctime;  in copy_compat_semid_to_user()
1771 		v.sem_nsems = in->sem_nsems;  in copy_compat_semid_to_user()
1783 	ns = current->nsproxy->ipc_ns;  in compat_ksys_semctl()
1785 	if (semid < 0)  in compat_ksys_semctl()
1786 		return -EINVAL;  in compat_ksys_semctl()
1796 		if (err < 0)  in compat_ksys_semctl()
1799 			err = -EFAULT;  in compat_ksys_semctl()
1812 			return -EFAULT;  in compat_ksys_semctl()
1817 		return -EINVAL;  in compat_ksys_semctl()
1856 	undo_list = current->sysvsem.undo_list;  in get_undo_list()
1860 			return -ENOMEM;  in get_undo_list()
1861 		spin_lock_init(&undo_list->lock);  in get_undo_list()
1862 		refcount_set(&undo_list->refcnt, 1);  in get_undo_list()
1863 		INIT_LIST_HEAD(&undo_list->list_proc);  in get_undo_list()
1865 		current->sysvsem.undo_list = undo_list;  in get_undo_list()
1868 	return 0;  in get_undo_list()
1871 static struct sem_undo *__lookup_undo(struct sem_undo_list *ulp, int semid)  in __lookup_undo()  argument
1875 	list_for_each_entry_rcu(un, &ulp->list_proc, list_proc,  in __lookup_undo()
1876 				spin_is_locked(&ulp->lock)) {  in __lookup_undo()
1877 		if (un->semid == semid)  in __lookup_undo()
1883 static struct sem_undo *lookup_undo(struct sem_undo_list *ulp, int semid)  in lookup_undo()  argument
1887 	assert_spin_locked(&ulp->lock);  in lookup_undo()
1889 	un = __lookup_undo(ulp, semid);  in lookup_undo()
1891 		list_del_rcu(&un->list_proc);  in lookup_undo()
1892 		list_add_rcu(&un->list_proc, &ulp->list_proc);  in lookup_undo()
1898  * find_alloc_undo - lookup (and if not present create) undo array
1905  * Lifetime-rules: sem_undo is rcu-protected, on success, the function
1911 	struct sem_undo_list *ulp;  in find_alloc_undo()  local
1915 	error = get_undo_list(&ulp);  in find_alloc_undo()
1920 	spin_lock(&ulp->lock);  in find_alloc_undo()
1921 	un = lookup_undo(ulp, semid);  in find_alloc_undo()
1922 	spin_unlock(&ulp->lock);  in find_alloc_undo()
1926 	/* no undo structure around - allocate one. */  in find_alloc_undo()
1934 	nsems = sma->sem_nsems;  in find_alloc_undo()
1935 	if (!ipc_rcu_getref(&sma->sem_perm)) {  in find_alloc_undo()
1937 		un = ERR_PTR(-EIDRM);  in find_alloc_undo()
1946 		ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);  in find_alloc_undo()
1947 		return ERR_PTR(-ENOMEM);  in find_alloc_undo()
1953 	if (!ipc_valid_object(&sma->sem_perm)) {  in find_alloc_undo()
1954 		sem_unlock(sma, -1);  in find_alloc_undo()
1957 		un = ERR_PTR(-EIDRM);  in find_alloc_undo()
1960 	spin_lock(&ulp->lock);  in find_alloc_undo()
1965 	un = lookup_undo(ulp, semid);  in find_alloc_undo()
1971 	new->semadj = (short *) &new[1];  in find_alloc_undo()
1972 	new->ulp = ulp;  in find_alloc_undo()
1973 	new->semid = semid;  in find_alloc_undo()
1974 	assert_spin_locked(&ulp->lock);  in find_alloc_undo()
1975 	list_add_rcu(&new->list_proc, &ulp->list_proc);  in find_alloc_undo()
1976 	ipc_assert_locked_object(&sma->sem_perm);  in find_alloc_undo()
1977 	list_add(&new->list_id, &sma->list_id);  in find_alloc_undo()
1981 	spin_unlock(&ulp->lock);  in find_alloc_undo()
1982 	sem_unlock(sma, -1);  in find_alloc_undo()
1991 	int error = -EINVAL;  in __do_semtimedop()
1998 	unsigned long dup = 0, jiffies_left = 0;  in __do_semtimedop()
2000 	if (nsops < 1 || semid < 0)  in __do_semtimedop()
2001 		return -EINVAL;  in __do_semtimedop()
2002 	if (nsops > ns->sc_semopm)  in __do_semtimedop()
2003 		return -E2BIG;  in __do_semtimedop()
2006 		if (timeout->tv_sec < 0 || timeout->tv_nsec < 0 ||  in __do_semtimedop()
2007 			timeout->tv_nsec >= 1000000000L) {  in __do_semtimedop()
2008 			error = -EINVAL;  in __do_semtimedop()
2015 	max = 0;  in __do_semtimedop()
2017 		unsigned long mask = 1ULL << ((sop->sem_num) % BITS_PER_LONG);  in __do_semtimedop()
2019 		if (sop->sem_num >= max)  in __do_semtimedop()
2020 			max = sop->sem_num;  in __do_semtimedop()
2021 		if (sop->sem_flg & SEM_UNDO)  in __do_semtimedop()
2032 		if (sop->sem_op != 0) {  in __do_semtimedop()
2057 	error = -EFBIG;  in __do_semtimedop()
2058 	if (max >= sma->sem_nsems) {  in __do_semtimedop()
2063 	error = -EACCES;  in __do_semtimedop()
2064 	if (ipcperms(ns, &sma->sem_perm, alter ? S_IWUGO : S_IRUGO)) {  in __do_semtimedop()
2069 	error = security_sem_semop(&sma->sem_perm, sops, nsops, alter);  in __do_semtimedop()
2075 	error = -EIDRM;  in __do_semtimedop()
2081 	 * only a per-semaphore lock is held and it's OK to proceed with the  in __do_semtimedop()
2085 	if (!ipc_valid_object(&sma->sem_perm))  in __do_semtimedop()
2088 	 * semid identifiers are not unique - find_alloc_undo may have  in __do_semtimedop()
2091 	 * This case can be detected checking un->semid. The existence of  in __do_semtimedop()
2094 	if (un && un->semid == -1)  in __do_semtimedop()
2105 	if (error == 0) { /* non-blocking successful path */  in __do_semtimedop()
2123 	if (error < 0) /* non-blocking error path */  in __do_semtimedop()
2132 		int idx = array_index_nospec(sops->sem_num, sma->sem_nsems);  in __do_semtimedop()
2133 		curr = &sma->sems[idx];  in __do_semtimedop()
2136 			if (sma->complex_count) {  in __do_semtimedop()
2138 						&sma->pending_alter);  in __do_semtimedop()
2142 						&curr->pending_alter);  in __do_semtimedop()
2145 			list_add_tail(&queue.list, &curr->pending_const);  in __do_semtimedop()
2148 		if (!sma->complex_count)  in __do_semtimedop()
2152 			list_add_tail(&queue.list, &sma->pending_alter);  in __do_semtimedop()
2154 			list_add_tail(&queue.list, &sma->pending_const);  in __do_semtimedop()
2156 		sma->complex_count++;  in __do_semtimedop()
2161 		WRITE_ONCE(queue.status, -EINTR);  in __do_semtimedop()
2186 		if (error != -EINTR) {  in __do_semtimedop()
2195 		if (!ipc_valid_object(&sma->sem_perm))  in __do_semtimedop()
2204 		 * If queue.status != -EINTR we are woken up by another process.  in __do_semtimedop()
2207 		if (error != -EINTR)  in __do_semtimedop()
2213 		if (timeout && jiffies_left == 0)  in __do_semtimedop()
2214 			error = -EAGAIN;  in __do_semtimedop()
2215 	} while (error == -EINTR && !signal_pending(current)); /* spurious */  in __do_semtimedop()
2234 	ns = current->nsproxy->ipc_ns;  in do_semtimedop()
2235 	if (nsops > ns->sc_semopm)  in do_semtimedop()
2236 		return -E2BIG;  in do_semtimedop()
2238 		return -EINVAL;  in do_semtimedop()
2243 			return -ENOMEM;  in do_semtimedop()
2247 		ret =  -EFAULT;  in do_semtimedop()
2266 			return -EFAULT;  in ksys_semtimedop()
2286 			return -EFAULT;  in compat_ksys_semtimedop()
2319 		refcount_inc(&undo_list->refcnt);  in copy_semundo()
2320 		tsk->sysvsem.undo_list = undo_list;  in copy_semundo()
2322 		tsk->sysvsem.undo_list = NULL;  in copy_semundo()
2324 	return 0;  in copy_semundo()
2341 	struct sem_undo_list *ulp;  in exit_sem()  local
2343 	ulp = tsk->sysvsem.undo_list;  in exit_sem()
2344 	if (!ulp)  in exit_sem()
2346 	tsk->sysvsem.undo_list = NULL;  in exit_sem()
2348 	if (!refcount_dec_and_test(&ulp->refcnt))  in exit_sem()
2360 		un = list_entry_rcu(ulp->list_proc.next,  in exit_sem()
2362 		if (&un->list_proc == &ulp->list_proc) {  in exit_sem()
2364 			 * We must wait for freeary() before freeing this ulp,  in exit_sem()
2369 			spin_lock(&ulp->lock);  in exit_sem()
2370 			spin_unlock(&ulp->lock);  in exit_sem()
2374 		spin_lock(&ulp->lock);  in exit_sem()
2375 		semid = un->semid;  in exit_sem()
2376 		spin_unlock(&ulp->lock);  in exit_sem()
2379 		if (semid == -1) {  in exit_sem()
2384 		sma = sem_obtain_object_check(tsk->nsproxy->ipc_ns, semid);  in exit_sem()
2391 		sem_lock(sma, NULL, -1);  in exit_sem()
2393 		if (!ipc_valid_object(&sma->sem_perm)) {  in exit_sem()
2394 			sem_unlock(sma, -1);  in exit_sem()
2398 		un = __lookup_undo(ulp, semid);  in exit_sem()
2403 			sem_unlock(sma, -1);  in exit_sem()
2409 		ipc_assert_locked_object(&sma->sem_perm);  in exit_sem()
2410 		list_del(&un->list_id);  in exit_sem()
2412 		spin_lock(&ulp->lock);  in exit_sem()
2413 		list_del_rcu(&un->list_proc);  in exit_sem()
2414 		spin_unlock(&ulp->lock);  in exit_sem()
2417 		for (i = 0; i < sma->sem_nsems; i++) {  in exit_sem()
2418 			struct sem *semaphore = &sma->sems[i];  in exit_sem()
2419 			if (un->semadj[i]) {  in exit_sem()
2420 				semaphore->semval += un->semadj[i];  in exit_sem()
2424 				 * - Some unices ignore the undo entirely  in exit_sem()
2426 				 * - some cap the value (e.g. FreeBSD caps  in exit_sem()
2427 				 *   at 0, but doesn't enforce SEMVMX)  in exit_sem()
2429 				 * Linux caps the semaphore value, both at 0  in exit_sem()
2434 				if (semaphore->semval < 0)  in exit_sem()
2435 					semaphore->semval = 0;  in exit_sem()
2436 				if (semaphore->semval > SEMVMX)  in exit_sem()
2437 					semaphore->semval = SEMVMX;  in exit_sem()
2438 				ipc_update_pid(&semaphore->sempid, task_tgid(current));  in exit_sem()
2441 		/* maybe some queued-up processes were waiting for this */  in exit_sem()
2442 		do_smart_update(sma, NULL, 0, 1, &wake_q);  in exit_sem()
2443 		sem_unlock(sma, -1);  in exit_sem()
2449 	kfree(ulp);  in exit_sem()
2462 	 * ipc_lock_object(), i.e. spin_lock(&sma->sem_perm.lock).  in sysvipc_sem_proc_show()
2473 		   sma->sem_perm.key,  in sysvipc_sem_proc_show()
2474 		   sma->sem_perm.id,  in sysvipc_sem_proc_show()
2475 		   sma->sem_perm.mode,  in sysvipc_sem_proc_show()
2476 		   sma->sem_nsems,  in sysvipc_sem_proc_show()
2477 		   from_kuid_munged(user_ns, sma->sem_perm.uid),  in sysvipc_sem_proc_show()
2478 		   from_kgid_munged(user_ns, sma->sem_perm.gid),  in sysvipc_sem_proc_show()
2479 		   from_kuid_munged(user_ns, sma->sem_perm.cuid),  in sysvipc_sem_proc_show()
2480 		   from_kgid_munged(user_ns, sma->sem_perm.cgid),  in sysvipc_sem_proc_show()
2482 		   sma->sem_ctime);  in sysvipc_sem_proc_show()
2486 	return 0;  in sysvipc_sem_proc_show()