sem.c 40.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8
/*
 * linux/ipc/sem.c
 * Copyright (C) 1992 Krishna Balasubramanian
 * Copyright (C) 1995 Eric Schenk, Bruno Haible
 *
 * /proc/sysvipc/sem support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
 *
 * SMP-threaded, sysctl's added
9
 * (c) 1999 Manfred Spraul <manfred@colorfullife.com>
L
Linus Torvalds 已提交
10
 * Enforced range limit on SEM_UNDO
A
Alan Cox 已提交
11
 * (c) 2001 Red Hat Inc
L
Linus Torvalds 已提交
12 13
 * Lockless wakeup
 * (c) 2003 Manfred Spraul <manfred@colorfullife.com>
14 15
 * Further wakeup optimizations, documentation
 * (c) 2010 Manfred Spraul <manfred@colorfullife.com>
S
Steve Grubb 已提交
16 17 18
 *
 * support for audit of ipc object properties and permission changes
 * Dustin Kirkland <dustin.kirkland@us.ibm.com>
K
Kirill Korotaev 已提交
19 20 21 22
 *
 * namespaces support
 * OpenVZ, SWsoft Inc.
 * Pavel Emelianov <xemul@openvz.org>
23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73
 *
 * Implementation notes: (May 2010)
 * This file implements System V semaphores.
 *
 * User space visible behavior:
 * - FIFO ordering for semop() operations (just FIFO, not starvation
 *   protection)
 * - multiple semaphore operations that alter the same semaphore in
 *   one semop() are handled.
 * - sem_ctime (time of last semctl()) is updated in the IPC_SET, SETVAL and
 *   SETALL calls.
 * - two Linux specific semctl() commands: SEM_STAT, SEM_INFO.
 * - undo adjustments at process exit are limited to 0..SEMVMX.
 * - namespace are supported.
 * - SEMMSL, SEMMNS, SEMOPM and SEMMNI can be configured at runtine by writing
 *   to /proc/sys/kernel/sem.
 * - statistics about the usage are reported in /proc/sysvipc/sem.
 *
 * Internals:
 * - scalability:
 *   - all global variables are read-mostly.
 *   - semop() calls and semctl(RMID) are synchronized by RCU.
 *   - most operations do write operations (actually: spin_lock calls) to
 *     the per-semaphore array structure.
 *   Thus: Perfect SMP scaling between independent semaphore arrays.
 *         If multiple semaphores in one array are used, then cache line
 *         trashing on the semaphore array spinlock will limit the scaling.
 * - semncnt and semzcnt are calculated on demand in count_semncnt() and
 *   count_semzcnt()
 * - the task that performs a successful semop() scans the list of all
 *   sleeping tasks and completes any pending operations that can be fulfilled.
 *   Semaphores are actively given to waiting tasks (necessary for FIFO).
 *   (see update_queue())
 * - To improve the scalability, the actual wake-up calls are performed after
 *   dropping all locks. (see wake_up_sem_queue_prepare(),
 *   wake_up_sem_queue_do())
 * - All work is done by the waker, the woken up task does not have to do
 *   anything - not even acquiring a lock or dropping a refcount.
 * - A woken up task may not even touch the semaphore array anymore, it may
 *   have been destroyed already by a semctl(RMID).
 * - The synchronizations between wake-ups due to a timeout/signal and a
 *   wake-up due to a completed semaphore operation is achieved by using an
 *   intermediate state (IN_WAKEUP).
 * - UNDO values are stored in an array (one per process and per
 *   semaphore array, lazily allocated). For backwards compatibility, multiple
 *   modes for the UNDO variables are supported (per process, per thread)
 *   (see copy_semundo, CLONE_SYSVSEM)
 * - There are two lists of the pending operations: a per-array list
 *   and per-semaphore list (stored in the array). This allows to achieve FIFO
 *   ordering without always scanning all pending operations.
 *   The worst-case behavior is nevertheless O(N^2) for N wakeups.
L
Linus Torvalds 已提交
74 75 76 77 78 79 80 81 82 83
 */

#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/init.h>
#include <linux/proc_fs.h>
#include <linux/time.h>
#include <linux/security.h>
#include <linux/syscalls.h>
#include <linux/audit.h>
84
#include <linux/capability.h>
85
#include <linux/seq_file.h>
N
Nadia Derbey 已提交
86
#include <linux/rwsem.h>
K
Kirill Korotaev 已提交
87
#include <linux/nsproxy.h>
88
#include <linux/ipc_namespace.h>
I
Ingo Molnar 已提交
89

L
Linus Torvalds 已提交
90 91 92
#include <asm/uaccess.h>
#include "util.h"

93
#define sem_ids(ns)	((ns)->ids[IPC_SEM_IDS])
K
Kirill Korotaev 已提交
94 95

#define sem_unlock(sma)		ipc_unlock(&(sma)->sem_perm)
N
Nadia Derbey 已提交
96
#define sem_checkid(sma, semid)	ipc_checkid(&sma->sem_perm, semid)
L
Linus Torvalds 已提交
97

N
Nadia Derbey 已提交
98
static int newary(struct ipc_namespace *, struct ipc_params *);
99
static void freeary(struct ipc_namespace *, struct kern_ipc_perm *);
L
Linus Torvalds 已提交
100
#ifdef CONFIG_PROC_FS
101
static int sysvipc_sem_proc_show(struct seq_file *s, void *it);
L
Linus Torvalds 已提交
102 103 104 105 106 107 108 109 110 111 112 113 114 115
#endif

#define SEMMSL_FAST	256 /* 512 bytes on stack */
#define SEMOPM_FAST	64  /* ~ 372 bytes on stack */

/*
 * linked list protection:
 *	sem_undo.id_next,
 *	sem_array.sem_pending{,last},
 *	sem_array.sem_undo: sem_lock() for read/write
 *	sem_undo.proc_next: only "current" is allowed to read/write that field.
 *	
 */

K
Kirill Korotaev 已提交
116 117 118 119 120
#define sc_semmsl	sem_ctls[0]
#define sc_semmns	sem_ctls[1]
#define sc_semopm	sem_ctls[2]
#define sc_semmni	sem_ctls[3]

121
void sem_init_ns(struct ipc_namespace *ns)
K
Kirill Korotaev 已提交
122 123 124 125 126 127
{
	ns->sc_semmsl = SEMMSL;
	ns->sc_semmns = SEMMNS;
	ns->sc_semopm = SEMOPM;
	ns->sc_semmni = SEMMNI;
	ns->used_sems = 0;
128
	ipc_init_ids(&ns->ids[IPC_SEM_IDS]);
K
Kirill Korotaev 已提交
129 130
}

131
#ifdef CONFIG_IPC_NS
K
Kirill Korotaev 已提交
132 133
void sem_exit_ns(struct ipc_namespace *ns)
{
134
	free_ipcs(ns, &sem_ids(ns), freeary);
S
Serge E. Hallyn 已提交
135
	idr_destroy(&ns->ids[IPC_SEM_IDS].ipcs_idr);
K
Kirill Korotaev 已提交
136
}
137
#endif
L
Linus Torvalds 已提交
138 139 140

void __init sem_init (void)
{
141
	sem_init_ns(&init_ipc_ns);
142 143
	ipc_init_proc_interface("sysvipc/sem",
				"       key      semid perms      nsems   uid   gid  cuid  cgid      otime      ctime\n",
K
Kirill Korotaev 已提交
144
				IPC_SEM_IDS, sysvipc_sem_proc_show);
L
Linus Torvalds 已提交
145 146
}

N
Nadia Derbey 已提交
147 148 149 150
/*
 * sem_lock_(check_) routines are called in the paths where the rw_mutex
 * is not held.
 */
151 152
static inline struct sem_array *sem_lock(struct ipc_namespace *ns, int id)
{
N
Nadia Derbey 已提交
153 154
	struct kern_ipc_perm *ipcp = ipc_lock(&sem_ids(ns), id);

155 156 157
	if (IS_ERR(ipcp))
		return (struct sem_array *)ipcp;

N
Nadia Derbey 已提交
158
	return container_of(ipcp, struct sem_array, sem_perm);
159 160 161 162 163
}

static inline struct sem_array *sem_lock_check(struct ipc_namespace *ns,
						int id)
{
N
Nadia Derbey 已提交
164 165
	struct kern_ipc_perm *ipcp = ipc_lock_check(&sem_ids(ns), id);

166 167 168
	if (IS_ERR(ipcp))
		return (struct sem_array *)ipcp;

N
Nadia Derbey 已提交
169
	return container_of(ipcp, struct sem_array, sem_perm);
170 171
}

172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190
static inline void sem_lock_and_putref(struct sem_array *sma)
{
	ipc_lock_by_ptr(&sma->sem_perm);
	ipc_rcu_putref(sma);
}

static inline void sem_getref_and_unlock(struct sem_array *sma)
{
	ipc_rcu_getref(sma);
	ipc_unlock(&(sma)->sem_perm);
}

static inline void sem_putref(struct sem_array *sma)
{
	ipc_lock_by_ptr(&sma->sem_perm);
	ipc_rcu_putref(sma);
	ipc_unlock(&(sma)->sem_perm);
}

N
Nadia Derbey 已提交
191 192 193 194 195
static inline void sem_rmid(struct ipc_namespace *ns, struct sem_array *s)
{
	ipc_rmid(&sem_ids(ns), &s->sem_perm);
}

L
Linus Torvalds 已提交
196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
/*
 * Lockless wakeup algorithm:
 * Without the check/retry algorithm a lockless wakeup is possible:
 * - queue.status is initialized to -EINTR before blocking.
 * - wakeup is performed by
 *	* unlinking the queue entry from sma->sem_pending
 *	* setting queue.status to IN_WAKEUP
 *	  This is the notification for the blocked thread that a
 *	  result value is imminent.
 *	* call wake_up_process
 *	* set queue.status to the final value.
 * - the previously blocked thread checks queue.status:
 *   	* if it's IN_WAKEUP, then it must wait until the value changes
 *   	* if it's not -EINTR, then the operation was completed by
 *   	  update_queue. semtimedop can return queue.status without
I
Ingo Molnar 已提交
211
 *   	  performing any operation on the sem array.
L
Linus Torvalds 已提交
212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
 *   	* otherwise it must acquire the spinlock and check what's up.
 *
 * The two-stage algorithm is necessary to protect against the following
 * races:
 * - if queue.status is set after wake_up_process, then the woken up idle
 *   thread could race forward and try (and fail) to acquire sma->lock
 *   before update_queue had a chance to set queue.status
 * - if queue.status is written before wake_up_process and if the
 *   blocked process is woken up by a signal between writing
 *   queue.status and the wake_up_process, then the woken up
 *   process could return from semtimedop and die by calling
 *   sys_exit before wake_up_process is called. Then wake_up_process
 *   will oops, because the task structure is already invalid.
 *   (yes, this happened on s390 with sysv msg).
 *
 */
#define IN_WAKEUP	1

N
Nadia Derbey 已提交
230 231 232 233 234
/**
 * newary - Create a new semaphore set
 * @ns: namespace
 * @params: ptr to the structure that contains key, semflg and nsems
 *
N
Nadia Derbey 已提交
235
 * Called with sem_ids.rw_mutex held (as a writer)
N
Nadia Derbey 已提交
236 237
 */

N
Nadia Derbey 已提交
238
static int newary(struct ipc_namespace *ns, struct ipc_params *params)
L
Linus Torvalds 已提交
239 240 241 242 243
{
	int id;
	int retval;
	struct sem_array *sma;
	int size;
N
Nadia Derbey 已提交
244 245 246
	key_t key = params->key;
	int nsems = params->u.nsems;
	int semflg = params->flg;
247
	int i;
L
Linus Torvalds 已提交
248 249 250

	if (!nsems)
		return -EINVAL;
K
Kirill Korotaev 已提交
251
	if (ns->used_sems + nsems > ns->sc_semmns)
L
Linus Torvalds 已提交
252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
		return -ENOSPC;

	size = sizeof (*sma) + nsems * sizeof (struct sem);
	sma = ipc_rcu_alloc(size);
	if (!sma) {
		return -ENOMEM;
	}
	memset (sma, 0, size);

	sma->sem_perm.mode = (semflg & S_IRWXUGO);
	sma->sem_perm.key = key;

	sma->sem_perm.security = NULL;
	retval = security_sem_alloc(sma);
	if (retval) {
		ipc_rcu_putref(sma);
		return retval;
	}

K
Kirill Korotaev 已提交
271
	id = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni);
272
	if (id < 0) {
L
Linus Torvalds 已提交
273 274
		security_sem_free(sma);
		ipc_rcu_putref(sma);
275
		return id;
L
Linus Torvalds 已提交
276
	}
K
Kirill Korotaev 已提交
277
	ns->used_sems += nsems;
L
Linus Torvalds 已提交
278 279

	sma->sem_base = (struct sem *) &sma[1];
280 281 282 283 284

	for (i = 0; i < nsems; i++)
		INIT_LIST_HEAD(&sma->sem_base[i].sem_pending);

	sma->complex_count = 0;
285
	INIT_LIST_HEAD(&sma->sem_pending);
286
	INIT_LIST_HEAD(&sma->list_id);
L
Linus Torvalds 已提交
287 288 289 290
	sma->sem_nsems = nsems;
	sma->sem_ctime = get_seconds();
	sem_unlock(sma);

N
Nadia Derbey 已提交
291
	return sma->sem_perm.id;
L
Linus Torvalds 已提交
292 293
}

N
Nadia Derbey 已提交
294

N
Nadia Derbey 已提交
295
/*
N
Nadia Derbey 已提交
296
 * Called with sem_ids.rw_mutex and ipcp locked.
N
Nadia Derbey 已提交
297
 */
N
Nadia Derbey 已提交
298
static inline int sem_security(struct kern_ipc_perm *ipcp, int semflg)
N
Nadia Derbey 已提交
299
{
N
Nadia Derbey 已提交
300 301 302 303
	struct sem_array *sma;

	sma = container_of(ipcp, struct sem_array, sem_perm);
	return security_sem_associate(sma, semflg);
N
Nadia Derbey 已提交
304 305
}

N
Nadia Derbey 已提交
306
/*
N
Nadia Derbey 已提交
307
 * Called with sem_ids.rw_mutex and ipcp locked.
N
Nadia Derbey 已提交
308
 */
N
Nadia Derbey 已提交
309 310
static inline int sem_more_checks(struct kern_ipc_perm *ipcp,
				struct ipc_params *params)
N
Nadia Derbey 已提交
311
{
N
Nadia Derbey 已提交
312 313 314 315
	struct sem_array *sma;

	sma = container_of(ipcp, struct sem_array, sem_perm);
	if (params->u.nsems > sma->sem_nsems)
N
Nadia Derbey 已提交
316 317 318 319 320
		return -EINVAL;

	return 0;
}

321
SYSCALL_DEFINE3(semget, key_t, key, int, nsems, int, semflg)
L
Linus Torvalds 已提交
322
{
K
Kirill Korotaev 已提交
323
	struct ipc_namespace *ns;
N
Nadia Derbey 已提交
324 325
	struct ipc_ops sem_ops;
	struct ipc_params sem_params;
K
Kirill Korotaev 已提交
326 327

	ns = current->nsproxy->ipc_ns;
L
Linus Torvalds 已提交
328

K
Kirill Korotaev 已提交
329
	if (nsems < 0 || nsems > ns->sc_semmsl)
L
Linus Torvalds 已提交
330
		return -EINVAL;
N
Nadia Derbey 已提交
331

N
Nadia Derbey 已提交
332 333 334 335 336 337 338
	sem_ops.getnew = newary;
	sem_ops.associate = sem_security;
	sem_ops.more_checks = sem_more_checks;

	sem_params.key = key;
	sem_params.flg = semflg;
	sem_params.u.nsems = nsems;
L
Linus Torvalds 已提交
339

N
Nadia Derbey 已提交
340
	return ipcget(ns, &sem_ids(ns), &sem_ops, &sem_params);
L
Linus Torvalds 已提交
341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
}

/*
 * Determine whether a sequence of semaphore operations would succeed
 * all at once. Return 0 if yes, 1 if need to sleep, else return error code.
 */

static int try_atomic_semop (struct sem_array * sma, struct sembuf * sops,
			     int nsops, struct sem_undo *un, int pid)
{
	int result, sem_op;
	struct sembuf *sop;
	struct sem * curr;

	for (sop = sops; sop < sops + nsops; sop++) {
		curr = sma->sem_base + sop->sem_num;
		sem_op = sop->sem_op;
		result = curr->semval;
  
		if (!sem_op && result)
			goto would_block;

		result += sem_op;
		if (result < 0)
			goto would_block;
		if (result > SEMVMX)
			goto out_of_range;
		if (sop->sem_flg & SEM_UNDO) {
			int undo = un->semadj[sop->sem_num] - sem_op;
			/*
	 		 *	Exceeding the undo range is an error.
			 */
			if (undo < (-SEMAEM - 1) || undo > SEMAEM)
				goto out_of_range;
		}
		curr->semval = result;
	}

	sop--;
	while (sop >= sops) {
		sma->sem_base[sop->sem_num].sempid = pid;
		if (sop->sem_flg & SEM_UNDO)
			un->semadj[sop->sem_num] -= sop->sem_op;
		sop--;
	}
	
	return 0;

out_of_range:
	result = -ERANGE;
	goto undo;

would_block:
	if (sop->sem_flg & IPC_NOWAIT)
		result = -EAGAIN;
	else
		result = 1;

undo:
	sop--;
	while (sop >= sops) {
		sma->sem_base[sop->sem_num].semval -= sop->sem_op;
		sop--;
	}

	return result;
}

409 410 411 412 413
/** wake_up_sem_queue_prepare(q, error): Prepare wake-up
 * @q: queue entry that must be signaled
 * @error: Error value for the signal
 *
 * Prepare the wake-up of the queue entry q.
N
Nick Piggin 已提交
414
 */
415 416
static void wake_up_sem_queue_prepare(struct list_head *pt,
				struct sem_queue *q, int error)
N
Nick Piggin 已提交
417
{
418 419 420 421 422 423 424
	if (list_empty(pt)) {
		/*
		 * Hold preempt off so that we don't get preempted and have the
		 * wakee busy-wait until we're scheduled back on.
		 */
		preempt_disable();
	}
N
Nick Piggin 已提交
425
	q->status = IN_WAKEUP;
426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453
	q->pid = error;

	list_add_tail(&q->simple_list, pt);
}

/**
 * wake_up_sem_queue_do(pt) - do the actual wake-up
 * @pt: list of tasks to be woken up
 *
 * Do the actual wake-up.
 * The function is called without any locks held, thus the semaphore array
 * could be destroyed already and the tasks can disappear as soon as the
 * status is set to the actual return code.
 */
static void wake_up_sem_queue_do(struct list_head *pt)
{
	struct sem_queue *q, *t;
	int did_something;

	did_something = !list_empty(pt);
	list_for_each_entry_safe(q, t, pt, simple_list) {
		wake_up_process(q->sleeper);
		/* q can disappear immediately after writing q->status. */
		smp_wmb();
		q->status = q->pid;
	}
	if (did_something)
		preempt_enable();
N
Nick Piggin 已提交
454 455
}

456 457 458 459 460 461 462 463 464
static void unlink_queue(struct sem_array *sma, struct sem_queue *q)
{
	list_del(&q->list);
	if (q->nsops == 1)
		list_del(&q->simple_list);
	else
		sma->complex_count--;
}

465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527
/** check_restart(sma, q)
 * @sma: semaphore array
 * @q: the operation that just completed
 *
 * update_queue is O(N^2) when it restarts scanning the whole queue of
 * waiting operations. Therefore this function checks if the restart is
 * really necessary. It is called after a previously waiting operation
 * was completed.
 */
static int check_restart(struct sem_array *sma, struct sem_queue *q)
{
	struct sem *curr;
	struct sem_queue *h;

	/* if the operation didn't modify the array, then no restart */
	if (q->alter == 0)
		return 0;

	/* pending complex operations are too difficult to analyse */
	if (sma->complex_count)
		return 1;

	/* we were a sleeping complex operation. Too difficult */
	if (q->nsops > 1)
		return 1;

	curr = sma->sem_base + q->sops[0].sem_num;

	/* No-one waits on this queue */
	if (list_empty(&curr->sem_pending))
		return 0;

	/* the new semaphore value */
	if (curr->semval) {
		/* It is impossible that someone waits for the new value:
		 * - q is a previously sleeping simple operation that
		 *   altered the array. It must be a decrement, because
		 *   simple increments never sleep.
		 * - The value is not 0, thus wait-for-zero won't proceed.
		 * - If there are older (higher priority) decrements
		 *   in the queue, then they have observed the original
		 *   semval value and couldn't proceed. The operation
		 *   decremented to value - thus they won't proceed either.
		 */
		BUG_ON(q->sops[0].sem_op >= 0);
		return 0;
	}
	/*
	 * semval is 0. Check if there are wait-for-zero semops.
	 * They must be the first entries in the per-semaphore simple queue
	 */
	h = list_first_entry(&curr->sem_pending, struct sem_queue, simple_list);
	BUG_ON(h->nsops != 1);
	BUG_ON(h->sops[0].sem_num != q->sops[0].sem_num);

	/* Yes, there is a wait-for-zero semop. Restart */
	if (h->sops[0].sem_op == 0)
		return 1;

	/* Again - no-one is waiting for the new value. */
	return 0;
}

528 529 530 531 532

/**
 * update_queue(sma, semnum): Look for tasks that can be completed.
 * @sma: semaphore array.
 * @semnum: semaphore that was modified.
533
 * @pt: list head for the tasks that must be woken up.
534 535 536 537
 *
 * update_queue must be called after a semaphore in a semaphore array
 * was modified. If multiple semaphore were modified, then @semnum
 * must be set to -1.
538 539 540
 * The tasks that must be woken up are added to @pt. The return code
 * is stored in q->pid.
 * The function return 1 if at least one semop was completed successfully.
L
Linus Torvalds 已提交
541
 */
542
static int update_queue(struct sem_array *sma, int semnum, struct list_head *pt)
L
Linus Torvalds 已提交
543
{
544 545 546 547
	struct sem_queue *q;
	struct list_head *walk;
	struct list_head *pending_list;
	int offset;
548
	int semop_completed = 0;
549 550 551 552 553 554 555 556 557 558 559 560 561 562 563

	/* if there are complex operations around, then knowing the semaphore
	 * that was modified doesn't help us. Assume that multiple semaphores
	 * were modified.
	 */
	if (sma->complex_count)
		semnum = -1;

	if (semnum == -1) {
		pending_list = &sma->sem_pending;
		offset = offsetof(struct sem_queue, list);
	} else {
		pending_list = &sma->sem_base[semnum].sem_pending;
		offset = offsetof(struct sem_queue, simple_list);
	}
N
Nick Piggin 已提交
564 565

again:
566 567
	walk = pending_list->next;
	while (walk != pending_list) {
568
		int error, restart;
569 570 571

		q = (struct sem_queue *)((char *)walk - offset);
		walk = walk->next;
L
Linus Torvalds 已提交
572

573 574 575 576 577 578 579 580 581 582 583
		/* If we are scanning the single sop, per-semaphore list of
		 * one semaphore and that semaphore is 0, then it is not
		 * necessary to scan the "alter" entries: simple increments
		 * that affect only one entry succeed immediately and cannot
		 * be in the  per semaphore pending queue, and decrements
		 * cannot be successful if the value is already 0.
		 */
		if (semnum != -1 && sma->sem_base[semnum].semval == 0 &&
				q->alter)
			break;

L
Linus Torvalds 已提交
584 585 586 587
		error = try_atomic_semop(sma, q->sops, q->nsops,
					 q->undo, q->pid);

		/* Does q->sleeper still need to sleep? */
N
Nick Piggin 已提交
588 589 590
		if (error > 0)
			continue;

591
		unlink_queue(sma, q);
N
Nick Piggin 已提交
592

593
		if (error) {
594
			restart = 0;
595 596
		} else {
			semop_completed = 1;
597
			restart = check_restart(sma, q);
598
		}
599

600
		wake_up_sem_queue_prepare(pt, q, error);
601
		if (restart)
N
Nick Piggin 已提交
602
			goto again;
L
Linus Torvalds 已提交
603
	}
604
	return semop_completed;
L
Linus Torvalds 已提交
605 606
}

607 608
/**
 * do_smart_update(sma, sops, nsops, otime, pt) - optimized update_queue
609 610 611
 * @sma: semaphore array
 * @sops: operations that were performed
 * @nsops: number of operations
612 613
 * @otime: force setting otime
 * @pt: list head of the tasks that must be woken up.
614 615 616
 *
 * do_smart_update() does the required called to update_queue, based on the
 * actual changes that were performed on the semaphore array.
617 618 619
 * Note that the function does not do the actual wake-up: the caller is
 * responsible for calling wake_up_sem_queue_do(@pt).
 * It is safe to perform this call after dropping all locks.
620
 */
621 622
static void do_smart_update(struct sem_array *sma, struct sembuf *sops, int nsops,
			int otime, struct list_head *pt)
623 624 625 626
{
	int i;

	if (sma->complex_count || sops == NULL) {
627 628 629
		if (update_queue(sma, -1, pt))
			otime = 1;
		goto done;
630 631 632 633 634 635
	}

	for (i = 0; i < nsops; i++) {
		if (sops[i].sem_op > 0 ||
			(sops[i].sem_op < 0 &&
				sma->sem_base[sops[i].sem_num].semval == 0))
636 637
			if (update_queue(sma, sops[i].sem_num, pt))
				otime = 1;
638
	}
639 640 641
done:
	if (otime)
		sma->sem_otime = get_seconds();
642 643 644
}


L
Linus Torvalds 已提交
645 646 647 648 649 650 651 652 653 654 655 656 657 658 659
/* The following counts are associated to each semaphore:
 *   semncnt        number of tasks waiting on semval being nonzero
 *   semzcnt        number of tasks waiting on semval being zero
 * This model assumes that a task waits on exactly one semaphore.
 * Since semaphore operations are to be performed atomically, tasks actually
 * wait on a whole sequence of semaphores simultaneously.
 * The counts we return here are a rough approximation, but still
 * warrant that semncnt+semzcnt>0 if the task is on the pending queue.
 */
static int count_semncnt (struct sem_array * sma, ushort semnum)
{
	int semncnt;
	struct sem_queue * q;

	semncnt = 0;
660
	list_for_each_entry(q, &sma->sem_pending, list) {
L
Linus Torvalds 已提交
661 662 663 664 665 666 667 668 669 670 671
		struct sembuf * sops = q->sops;
		int nsops = q->nsops;
		int i;
		for (i = 0; i < nsops; i++)
			if (sops[i].sem_num == semnum
			    && (sops[i].sem_op < 0)
			    && !(sops[i].sem_flg & IPC_NOWAIT))
				semncnt++;
	}
	return semncnt;
}
672

L
Linus Torvalds 已提交
673 674 675 676 677 678
static int count_semzcnt (struct sem_array * sma, ushort semnum)
{
	int semzcnt;
	struct sem_queue * q;

	semzcnt = 0;
679
	list_for_each_entry(q, &sma->sem_pending, list) {
L
Linus Torvalds 已提交
680 681 682 683 684 685 686 687 688 689 690 691
		struct sembuf * sops = q->sops;
		int nsops = q->nsops;
		int i;
		for (i = 0; i < nsops; i++)
			if (sops[i].sem_num == semnum
			    && (sops[i].sem_op == 0)
			    && !(sops[i].sem_flg & IPC_NOWAIT))
				semzcnt++;
	}
	return semzcnt;
}

A
Adrian Bunk 已提交
692
static void free_un(struct rcu_head *head)
693 694 695 696 697
{
	struct sem_undo *un = container_of(head, struct sem_undo, rcu);
	kfree(un);
}

N
Nadia Derbey 已提交
698 699 700
/* Free a semaphore set. freeary() is called with sem_ids.rw_mutex locked
 * as a writer and the spinlock for this semaphore set hold. sem_ids.rw_mutex
 * remains locked on exit.
L
Linus Torvalds 已提交
701
 */
702
static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
L
Linus Torvalds 已提交
703
{
704 705
	struct sem_undo *un, *tu;
	struct sem_queue *q, *tq;
706
	struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm);
707
	struct list_head tasks;
L
Linus Torvalds 已提交
708

709
	/* Free the existing undo structures for this semaphore set.  */
710
	assert_spin_locked(&sma->sem_perm.lock);
711 712 713
	list_for_each_entry_safe(un, tu, &sma->list_id, list_id) {
		list_del(&un->list_id);
		spin_lock(&un->ulp->lock);
L
Linus Torvalds 已提交
714
		un->semid = -1;
715 716 717 718
		list_del_rcu(&un->list_proc);
		spin_unlock(&un->ulp->lock);
		call_rcu(&un->rcu, free_un);
	}
L
Linus Torvalds 已提交
719 720

	/* Wake up all pending processes and let them fail with EIDRM. */
721
	INIT_LIST_HEAD(&tasks);
722
	list_for_each_entry_safe(q, tq, &sma->sem_pending, list) {
723
		unlink_queue(sma, q);
724
		wake_up_sem_queue_prepare(&tasks, q, -EIDRM);
L
Linus Torvalds 已提交
725 726
	}

N
Nadia Derbey 已提交
727 728
	/* Remove the semaphore set from the IDR */
	sem_rmid(ns, sma);
L
Linus Torvalds 已提交
729 730
	sem_unlock(sma);

731
	wake_up_sem_queue_do(&tasks);
K
Kirill Korotaev 已提交
732
	ns->used_sems -= sma->sem_nsems;
L
Linus Torvalds 已提交
733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758
	security_sem_free(sma);
	ipc_rcu_putref(sma);
}

static unsigned long copy_semid_to_user(void __user *buf, struct semid64_ds *in, int version)
{
	switch(version) {
	case IPC_64:
		return copy_to_user(buf, in, sizeof(*in));
	case IPC_OLD:
	    {
		struct semid_ds out;

		ipc64_perm_to_ipc_perm(&in->sem_perm, &out.sem_perm);

		out.sem_otime	= in->sem_otime;
		out.sem_ctime	= in->sem_ctime;
		out.sem_nsems	= in->sem_nsems;

		return copy_to_user(buf, &out, sizeof(out));
	    }
	default:
		return -EINVAL;
	}
}

759 760
static int semctl_nolock(struct ipc_namespace *ns, int semid,
			 int cmd, int version, union semun arg)
L
Linus Torvalds 已提交
761
{
762
	int err;
L
Linus Torvalds 已提交
763 764 765 766 767 768 769 770 771 772 773 774 775 776
	struct sem_array *sma;

	switch(cmd) {
	case IPC_INFO:
	case SEM_INFO:
	{
		struct seminfo seminfo;
		int max_id;

		err = security_sem_semctl(NULL, cmd);
		if (err)
			return err;
		
		memset(&seminfo,0,sizeof(seminfo));
K
Kirill Korotaev 已提交
777 778 779 780
		seminfo.semmni = ns->sc_semmni;
		seminfo.semmns = ns->sc_semmns;
		seminfo.semmsl = ns->sc_semmsl;
		seminfo.semopm = ns->sc_semopm;
L
Linus Torvalds 已提交
781 782 783 784
		seminfo.semvmx = SEMVMX;
		seminfo.semmnu = SEMMNU;
		seminfo.semmap = SEMMAP;
		seminfo.semume = SEMUME;
N
Nadia Derbey 已提交
785
		down_read(&sem_ids(ns).rw_mutex);
L
Linus Torvalds 已提交
786
		if (cmd == SEM_INFO) {
K
Kirill Korotaev 已提交
787 788
			seminfo.semusz = sem_ids(ns).in_use;
			seminfo.semaem = ns->used_sems;
L
Linus Torvalds 已提交
789 790 791 792
		} else {
			seminfo.semusz = SEMUSZ;
			seminfo.semaem = SEMAEM;
		}
N
Nadia Derbey 已提交
793
		max_id = ipc_get_maxid(&sem_ids(ns));
N
Nadia Derbey 已提交
794
		up_read(&sem_ids(ns).rw_mutex);
L
Linus Torvalds 已提交
795 796 797 798
		if (copy_to_user (arg.__buf, &seminfo, sizeof(struct seminfo))) 
			return -EFAULT;
		return (max_id < 0) ? 0: max_id;
	}
799
	case IPC_STAT:
L
Linus Torvalds 已提交
800 801 802 803 804
	case SEM_STAT:
	{
		struct semid64_ds tbuf;
		int id;

805 806 807 808 809 810 811 812 813 814 815
		if (cmd == SEM_STAT) {
			sma = sem_lock(ns, semid);
			if (IS_ERR(sma))
				return PTR_ERR(sma);
			id = sma->sem_perm.id;
		} else {
			sma = sem_lock_check(ns, semid);
			if (IS_ERR(sma))
				return PTR_ERR(sma);
			id = 0;
		}
L
Linus Torvalds 已提交
816 817 818 819 820 821 822 823 824

		err = -EACCES;
		if (ipcperms (&sma->sem_perm, S_IRUGO))
			goto out_unlock;

		err = security_sem_semctl(sma, cmd);
		if (err)
			goto out_unlock;

825 826
		memset(&tbuf, 0, sizeof(tbuf));

L
Linus Torvalds 已提交
827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843
		kernel_to_ipc64_perm(&sma->sem_perm, &tbuf.sem_perm);
		tbuf.sem_otime  = sma->sem_otime;
		tbuf.sem_ctime  = sma->sem_ctime;
		tbuf.sem_nsems  = sma->sem_nsems;
		sem_unlock(sma);
		if (copy_semid_to_user (arg.buf, &tbuf, version))
			return -EFAULT;
		return id;
	}
	default:
		return -EINVAL;
	}
out_unlock:
	sem_unlock(sma);
	return err;
}

K
Kirill Korotaev 已提交
844 845
static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
		int cmd, int version, union semun arg)
L
Linus Torvalds 已提交
846 847 848 849 850 851 852
{
	struct sem_array *sma;
	struct sem* curr;
	int err;
	ushort fast_sem_io[SEMMSL_FAST];
	ushort* sem_io = fast_sem_io;
	int nsems;
853
	struct list_head tasks;
L
Linus Torvalds 已提交
854

855 856 857
	sma = sem_lock_check(ns, semid);
	if (IS_ERR(sma))
		return PTR_ERR(sma);
L
Linus Torvalds 已提交
858

859
	INIT_LIST_HEAD(&tasks);
L
Linus Torvalds 已提交
860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877
	nsems = sma->sem_nsems;

	err = -EACCES;
	if (ipcperms (&sma->sem_perm, (cmd==SETVAL||cmd==SETALL)?S_IWUGO:S_IRUGO))
		goto out_unlock;

	err = security_sem_semctl(sma, cmd);
	if (err)
		goto out_unlock;

	err = -EACCES;
	switch (cmd) {
	case GETALL:
	{
		ushort __user *array = arg.array;
		int i;

		if(nsems > SEMMSL_FAST) {
878
			sem_getref_and_unlock(sma);
L
Linus Torvalds 已提交
879 880 881

			sem_io = ipc_alloc(sizeof(ushort)*nsems);
			if(sem_io == NULL) {
882
				sem_putref(sma);
L
Linus Torvalds 已提交
883 884 885
				return -ENOMEM;
			}

886
			sem_lock_and_putref(sma);
L
Linus Torvalds 已提交
887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906
			if (sma->sem_perm.deleted) {
				sem_unlock(sma);
				err = -EIDRM;
				goto out_free;
			}
		}

		for (i = 0; i < sma->sem_nsems; i++)
			sem_io[i] = sma->sem_base[i].semval;
		sem_unlock(sma);
		err = 0;
		if(copy_to_user(array, sem_io, nsems*sizeof(ushort)))
			err = -EFAULT;
		goto out_free;
	}
	case SETALL:
	{
		int i;
		struct sem_undo *un;

907
		sem_getref_and_unlock(sma);
L
Linus Torvalds 已提交
908 909 910 911

		if(nsems > SEMMSL_FAST) {
			sem_io = ipc_alloc(sizeof(ushort)*nsems);
			if(sem_io == NULL) {
912
				sem_putref(sma);
L
Linus Torvalds 已提交
913 914 915 916 917
				return -ENOMEM;
			}
		}

		if (copy_from_user (sem_io, arg.array, nsems*sizeof(ushort))) {
918
			sem_putref(sma);
L
Linus Torvalds 已提交
919 920 921 922 923 924
			err = -EFAULT;
			goto out_free;
		}

		for (i = 0; i < nsems; i++) {
			if (sem_io[i] > SEMVMX) {
925
				sem_putref(sma);
L
Linus Torvalds 已提交
926 927 928 929
				err = -ERANGE;
				goto out_free;
			}
		}
930
		sem_lock_and_putref(sma);
L
Linus Torvalds 已提交
931 932 933 934 935 936 937 938
		if (sma->sem_perm.deleted) {
			sem_unlock(sma);
			err = -EIDRM;
			goto out_free;
		}

		for (i = 0; i < nsems; i++)
			sma->sem_base[i].semval = sem_io[i];
939 940 941

		assert_spin_locked(&sma->sem_perm.lock);
		list_for_each_entry(un, &sma->list_id, list_id) {
L
Linus Torvalds 已提交
942 943
			for (i = 0; i < nsems; i++)
				un->semadj[i] = 0;
944
		}
L
Linus Torvalds 已提交
945 946
		sma->sem_ctime = get_seconds();
		/* maybe some queued-up processes were waiting for this */
947
		do_smart_update(sma, NULL, 0, 0, &tasks);
L
Linus Torvalds 已提交
948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975
		err = 0;
		goto out_unlock;
	}
	/* GETVAL, GETPID, GETNCTN, GETZCNT, SETVAL: fall-through */
	}
	err = -EINVAL;
	if(semnum < 0 || semnum >= nsems)
		goto out_unlock;

	curr = &sma->sem_base[semnum];

	switch (cmd) {
	case GETVAL:
		err = curr->semval;
		goto out_unlock;
	case GETPID:
		err = curr->sempid;
		goto out_unlock;
	case GETNCNT:
		err = count_semncnt(sma,semnum);
		goto out_unlock;
	case GETZCNT:
		err = count_semzcnt(sma,semnum);
		goto out_unlock;
	case SETVAL:
	{
		int val = arg.val;
		struct sem_undo *un;
976

L
Linus Torvalds 已提交
977 978 979 980
		err = -ERANGE;
		if (val > SEMVMX || val < 0)
			goto out_unlock;

981 982
		assert_spin_locked(&sma->sem_perm.lock);
		list_for_each_entry(un, &sma->list_id, list_id)
L
Linus Torvalds 已提交
983
			un->semadj[semnum] = 0;
984

L
Linus Torvalds 已提交
985
		curr->semval = val;
986
		curr->sempid = task_tgid_vnr(current);
L
Linus Torvalds 已提交
987 988
		sma->sem_ctime = get_seconds();
		/* maybe some queued-up processes were waiting for this */
989
		do_smart_update(sma, NULL, 0, 0, &tasks);
L
Linus Torvalds 已提交
990 991 992 993 994 995
		err = 0;
		goto out_unlock;
	}
	}
out_unlock:
	sem_unlock(sma);
996 997
	wake_up_sem_queue_do(&tasks);

L
Linus Torvalds 已提交
998 999 1000 1001 1002 1003
out_free:
	if(sem_io != fast_sem_io)
		ipc_free(sem_io, sizeof(ushort)*nsems);
	return err;
}

1004 1005
static inline unsigned long
copy_semid_from_user(struct semid64_ds *out, void __user *buf, int version)
L
Linus Torvalds 已提交
1006 1007 1008
{
	switch(version) {
	case IPC_64:
1009
		if (copy_from_user(out, buf, sizeof(*out)))
L
Linus Torvalds 已提交
1010 1011 1012 1013 1014 1015 1016 1017 1018
			return -EFAULT;
		return 0;
	case IPC_OLD:
	    {
		struct semid_ds tbuf_old;

		if(copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
			return -EFAULT;

1019 1020 1021
		out->sem_perm.uid	= tbuf_old.sem_perm.uid;
		out->sem_perm.gid	= tbuf_old.sem_perm.gid;
		out->sem_perm.mode	= tbuf_old.sem_perm.mode;
L
Linus Torvalds 已提交
1022 1023 1024 1025 1026 1027 1028 1029

		return 0;
	    }
	default:
		return -EINVAL;
	}
}

1030 1031 1032 1033 1034
/*
 * This function handles some semctl commands which require the rw_mutex
 * to be held in write mode.
 * NOTE: no locks must be held, the rw_mutex is taken inside this function.
 */
1035 1036
static int semctl_down(struct ipc_namespace *ns, int semid,
		       int cmd, int version, union semun arg)
L
Linus Torvalds 已提交
1037 1038 1039
{
	struct sem_array *sma;
	int err;
1040
	struct semid64_ds semid64;
L
Linus Torvalds 已提交
1041 1042 1043
	struct kern_ipc_perm *ipcp;

	if(cmd == IPC_SET) {
1044
		if (copy_semid_from_user(&semid64, arg.buf, version))
L
Linus Torvalds 已提交
1045 1046
			return -EFAULT;
	}
S
Steve Grubb 已提交
1047

1048 1049 1050
	ipcp = ipcctl_pre_down(&sem_ids(ns), semid, cmd, &semid64.sem_perm, 0);
	if (IS_ERR(ipcp))
		return PTR_ERR(ipcp);
S
Steve Grubb 已提交
1051

1052
	sma = container_of(ipcp, struct sem_array, sem_perm);
L
Linus Torvalds 已提交
1053 1054 1055 1056 1057 1058 1059

	err = security_sem_semctl(sma, cmd);
	if (err)
		goto out_unlock;

	switch(cmd){
	case IPC_RMID:
1060
		freeary(ns, ipcp);
1061
		goto out_up;
L
Linus Torvalds 已提交
1062
	case IPC_SET:
1063
		ipc_update_perm(&semid64.sem_perm, ipcp);
L
Linus Torvalds 已提交
1064 1065 1066 1067 1068 1069 1070 1071
		sma->sem_ctime = get_seconds();
		break;
	default:
		err = -EINVAL;
	}

out_unlock:
	sem_unlock(sma);
1072 1073
out_up:
	up_write(&sem_ids(ns).rw_mutex);
L
Linus Torvalds 已提交
1074 1075 1076
	return err;
}

1077
SYSCALL_DEFINE(semctl)(int semid, int semnum, int cmd, union semun arg)
L
Linus Torvalds 已提交
1078 1079 1080
{
	int err = -EINVAL;
	int version;
K
Kirill Korotaev 已提交
1081
	struct ipc_namespace *ns;
L
Linus Torvalds 已提交
1082 1083 1084 1085 1086

	if (semid < 0)
		return -EINVAL;

	version = ipc_parse_version(&cmd);
K
Kirill Korotaev 已提交
1087
	ns = current->nsproxy->ipc_ns;
L
Linus Torvalds 已提交
1088 1089 1090 1091

	switch(cmd) {
	case IPC_INFO:
	case SEM_INFO:
1092
	case IPC_STAT:
L
Linus Torvalds 已提交
1093
	case SEM_STAT:
1094
		err = semctl_nolock(ns, semid, cmd, version, arg);
L
Linus Torvalds 已提交
1095 1096 1097 1098 1099 1100 1101 1102
		return err;
	case GETALL:
	case GETVAL:
	case GETPID:
	case GETNCNT:
	case GETZCNT:
	case SETVAL:
	case SETALL:
K
Kirill Korotaev 已提交
1103
		err = semctl_main(ns,semid,semnum,cmd,version,arg);
L
Linus Torvalds 已提交
1104 1105 1106
		return err;
	case IPC_RMID:
	case IPC_SET:
1107
		err = semctl_down(ns, semid, cmd, version, arg);
L
Linus Torvalds 已提交
1108 1109 1110 1111 1112
		return err;
	default:
		return -EINVAL;
	}
}
1113 1114 1115 1116 1117 1118 1119
#ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
asmlinkage long SyS_semctl(int semid, int semnum, int cmd, union semun arg)
{
	return SYSC_semctl((int) semid, (int) semnum, (int) cmd, arg);
}
SYSCALL_ALIAS(sys_semctl, SyS_semctl);
#endif
L
Linus Torvalds 已提交
1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137

/* If the task doesn't already have a undo_list, then allocate one
 * here.  We guarantee there is only one thread using this undo list,
 * and current is THE ONE
 *
 * If this allocation and assignment succeeds, but later
 * portions of this code fail, there is no need to free the sem_undo_list.
 * Just let it stay associated with the task, and it'll be freed later
 * at exit time.
 *
 * This can block, so callers must hold no locks.
 */
static inline int get_undo_list(struct sem_undo_list **undo_listp)
{
	struct sem_undo_list *undo_list;

	undo_list = current->sysvsem.undo_list;
	if (!undo_list) {
1138
		undo_list = kzalloc(sizeof(*undo_list), GFP_KERNEL);
L
Linus Torvalds 已提交
1139 1140
		if (undo_list == NULL)
			return -ENOMEM;
I
Ingo Molnar 已提交
1141
		spin_lock_init(&undo_list->lock);
L
Linus Torvalds 已提交
1142
		atomic_set(&undo_list->refcnt, 1);
1143 1144
		INIT_LIST_HEAD(&undo_list->list_proc);

L
Linus Torvalds 已提交
1145 1146 1147 1148 1149 1150
		current->sysvsem.undo_list = undo_list;
	}
	*undo_listp = undo_list;
	return 0;
}

1151
static struct sem_undo *__lookup_undo(struct sem_undo_list *ulp, int semid)
L
Linus Torvalds 已提交
1152
{
1153
	struct sem_undo *un;
1154

1155 1156 1157
	list_for_each_entry_rcu(un, &ulp->list_proc, list_proc) {
		if (un->semid == semid)
			return un;
L
Linus Torvalds 已提交
1158
	}
1159
	return NULL;
L
Linus Torvalds 已提交
1160 1161
}

1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175
static struct sem_undo *lookup_undo(struct sem_undo_list *ulp, int semid)
{
	struct sem_undo *un;

  	assert_spin_locked(&ulp->lock);

	un = __lookup_undo(ulp, semid);
	if (un) {
		list_del_rcu(&un->list_proc);
		list_add_rcu(&un->list_proc, &ulp->list_proc);
	}
	return un;
}

1176 1177 1178 1179 1180 1181 1182 1183
/**
 * find_alloc_undo - Lookup (and if not present create) undo array
 * @ns: namespace
 * @semid: semaphore array id
 *
 * The function looks up (and if not present creates) the undo structure.
 * The size of the undo structure depends on the size of the semaphore
 * array, thus the alloc path is not that straightforward.
1184 1185
 * Lifetime-rules: sem_undo is rcu-protected, on success, the function
 * performs a rcu_read_lock().
1186 1187
 */
static struct sem_undo *find_alloc_undo(struct ipc_namespace *ns, int semid)
L
Linus Torvalds 已提交
1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198
{
	struct sem_array *sma;
	struct sem_undo_list *ulp;
	struct sem_undo *un, *new;
	int nsems;
	int error;

	error = get_undo_list(&ulp);
	if (error)
		return ERR_PTR(error);

1199
	rcu_read_lock();
1200
	spin_lock(&ulp->lock);
L
Linus Torvalds 已提交
1201
	un = lookup_undo(ulp, semid);
1202
	spin_unlock(&ulp->lock);
L
Linus Torvalds 已提交
1203 1204
	if (likely(un!=NULL))
		goto out;
1205
	rcu_read_unlock();
L
Linus Torvalds 已提交
1206 1207

	/* no undo structure around - allocate one. */
1208
	/* step 1: figure out the size of the semaphore array */
1209 1210
	sma = sem_lock_check(ns, semid);
	if (IS_ERR(sma))
J
Julia Lawall 已提交
1211
		return ERR_CAST(sma);
1212

L
Linus Torvalds 已提交
1213
	nsems = sma->sem_nsems;
1214
	sem_getref_and_unlock(sma);
L
Linus Torvalds 已提交
1215

1216
	/* step 2: allocate new undo structure */
1217
	new = kzalloc(sizeof(struct sem_undo) + sizeof(short)*nsems, GFP_KERNEL);
L
Linus Torvalds 已提交
1218
	if (!new) {
1219
		sem_putref(sma);
L
Linus Torvalds 已提交
1220 1221 1222
		return ERR_PTR(-ENOMEM);
	}

1223
	/* step 3: Acquire the lock on semaphore array */
1224
	sem_lock_and_putref(sma);
L
Linus Torvalds 已提交
1225 1226 1227 1228 1229 1230
	if (sma->sem_perm.deleted) {
		sem_unlock(sma);
		kfree(new);
		un = ERR_PTR(-EIDRM);
		goto out;
	}
1231 1232 1233 1234 1235 1236 1237 1238 1239 1240
	spin_lock(&ulp->lock);

	/*
	 * step 4: check for races: did someone else allocate the undo struct?
	 */
	un = lookup_undo(ulp, semid);
	if (un) {
		kfree(new);
		goto success;
	}
1241 1242
	/* step 5: initialize & link new undo structure */
	new->semadj = (short *) &new[1];
1243
	new->ulp = ulp;
1244 1245
	new->semid = semid;
	assert_spin_locked(&ulp->lock);
1246
	list_add_rcu(&new->list_proc, &ulp->list_proc);
1247 1248
	assert_spin_locked(&sma->sem_perm.lock);
	list_add(&new->list_id, &sma->list_id);
1249
	un = new;
1250

1251
success:
1252
	spin_unlock(&ulp->lock);
1253 1254
	rcu_read_lock();
	sem_unlock(sma);
L
Linus Torvalds 已提交
1255 1256 1257 1258
out:
	return un;
}

1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285

/**
 * get_queue_result - Retrieve the result code from sem_queue
 * @q: Pointer to queue structure
 *
 * Retrieve the return code from the pending queue. If IN_WAKEUP is found in
 * q->status, then we must loop until the value is replaced with the final
 * value: This may happen if a task is woken up by an unrelated event (e.g.
 * signal) and in parallel the task is woken up by another task because it got
 * the requested semaphores.
 *
 * The function can be called with or without holding the semaphore spinlock.
 */
static int get_queue_result(struct sem_queue *q)
{
	int error;

	error = q->status;
	while (unlikely(error == IN_WAKEUP)) {
		cpu_relax();
		error = q->status;
	}

	return error;
}


1286 1287
SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
		unsigned, nsops, const struct timespec __user *, timeout)
L
Linus Torvalds 已提交
1288 1289 1290 1291 1292 1293
{
	int error = -EINVAL;
	struct sem_array *sma;
	struct sembuf fast_sops[SEMOPM_FAST];
	struct sembuf* sops = fast_sops, *sop;
	struct sem_undo *un;
1294
	int undos = 0, alter = 0, max;
L
Linus Torvalds 已提交
1295 1296
	struct sem_queue queue;
	unsigned long jiffies_left = 0;
K
Kirill Korotaev 已提交
1297
	struct ipc_namespace *ns;
1298
	struct list_head tasks;
K
Kirill Korotaev 已提交
1299 1300

	ns = current->nsproxy->ipc_ns;
L
Linus Torvalds 已提交
1301 1302 1303

	if (nsops < 1 || semid < 0)
		return -EINVAL;
K
Kirill Korotaev 已提交
1304
	if (nsops > ns->sc_semopm)
L
Linus Torvalds 已提交
1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332
		return -E2BIG;
	if(nsops > SEMOPM_FAST) {
		sops = kmalloc(sizeof(*sops)*nsops,GFP_KERNEL);
		if(sops==NULL)
			return -ENOMEM;
	}
	if (copy_from_user (sops, tsops, nsops * sizeof(*tsops))) {
		error=-EFAULT;
		goto out_free;
	}
	if (timeout) {
		struct timespec _timeout;
		if (copy_from_user(&_timeout, timeout, sizeof(*timeout))) {
			error = -EFAULT;
			goto out_free;
		}
		if (_timeout.tv_sec < 0 || _timeout.tv_nsec < 0 ||
			_timeout.tv_nsec >= 1000000000L) {
			error = -EINVAL;
			goto out_free;
		}
		jiffies_left = timespec_to_jiffies(&_timeout);
	}
	max = 0;
	for (sop = sops; sop < sops + nsops; sop++) {
		if (sop->sem_num >= max)
			max = sop->sem_num;
		if (sop->sem_flg & SEM_UNDO)
1333 1334
			undos = 1;
		if (sop->sem_op != 0)
L
Linus Torvalds 已提交
1335 1336 1337 1338
			alter = 1;
	}

	if (undos) {
1339
		un = find_alloc_undo(ns, semid);
L
Linus Torvalds 已提交
1340 1341 1342 1343 1344 1345 1346
		if (IS_ERR(un)) {
			error = PTR_ERR(un);
			goto out_free;
		}
	} else
		un = NULL;

1347 1348
	INIT_LIST_HEAD(&tasks);

1349 1350
	sma = sem_lock_check(ns, semid);
	if (IS_ERR(sma)) {
1351 1352
		if (un)
			rcu_read_unlock();
1353
		error = PTR_ERR(sma);
L
Linus Torvalds 已提交
1354
		goto out_free;
1355 1356
	}

L
Linus Torvalds 已提交
1357
	/*
1358
	 * semid identifiers are not unique - find_alloc_undo may have
L
Linus Torvalds 已提交
1359
	 * allocated an undo structure, it was invalidated by an RMID
1360
	 * and now a new array with received the same id. Check and fail.
1361 1362
	 * This case can be detected checking un->semid. The existance of
	 * "un" itself is guaranteed by rcu.
L
Linus Torvalds 已提交
1363
	 */
1364
	error = -EIDRM;
1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380
	if (un) {
		if (un->semid == -1) {
			rcu_read_unlock();
			goto out_unlock_free;
		} else {
			/*
			 * rcu lock can be released, "un" cannot disappear:
			 * - sem_lock is acquired, thus IPC_RMID is
			 *   impossible.
			 * - exit_sem is impossible, it always operates on
			 *   current (or a dead task).
			 */

			rcu_read_unlock();
		}
	}
1381

L
Linus Torvalds 已提交
1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393
	error = -EFBIG;
	if (max >= sma->sem_nsems)
		goto out_unlock_free;

	error = -EACCES;
	if (ipcperms(&sma->sem_perm, alter ? S_IWUGO : S_IRUGO))
		goto out_unlock_free;

	error = security_sem_semop(sma, sops, nsops, alter);
	if (error)
		goto out_unlock_free;

1394
	error = try_atomic_semop (sma, sops, nsops, un, task_tgid_vnr(current));
L
Linus Torvalds 已提交
1395 1396
	if (error <= 0) {
		if (alter && error == 0)
1397
			do_smart_update(sma, sops, nsops, 1, &tasks);
1398

L
Linus Torvalds 已提交
1399 1400 1401 1402 1403 1404 1405 1406 1407 1408
		goto out_unlock_free;
	}

	/* We need to sleep on this operation, so we put the current
	 * task into the pending queue and go to sleep.
	 */
		
	queue.sops = sops;
	queue.nsops = nsops;
	queue.undo = un;
1409
	queue.pid = task_tgid_vnr(current);
L
Linus Torvalds 已提交
1410 1411
	queue.alter = alter;
	if (alter)
1412
		list_add_tail(&queue.list, &sma->sem_pending);
L
Linus Torvalds 已提交
1413
	else
1414
		list_add(&queue.list, &sma->sem_pending);
L
Linus Torvalds 已提交
1415

1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428
	if (nsops == 1) {
		struct sem *curr;
		curr = &sma->sem_base[sops->sem_num];

		if (alter)
			list_add_tail(&queue.simple_list, &curr->sem_pending);
		else
			list_add(&queue.simple_list, &curr->sem_pending);
	} else {
		INIT_LIST_HEAD(&queue.simple_list);
		sma->complex_count++;
	}

L
Linus Torvalds 已提交
1429 1430 1431 1432 1433 1434 1435 1436 1437 1438
	queue.status = -EINTR;
	queue.sleeper = current;
	current->state = TASK_INTERRUPTIBLE;
	sem_unlock(sma);

	if (timeout)
		jiffies_left = schedule_timeout(jiffies_left);
	else
		schedule();

1439
	error = get_queue_result(&queue);
L
Linus Torvalds 已提交
1440 1441 1442

	if (error != -EINTR) {
		/* fast path: update_queue already obtained all requested
1443 1444 1445 1446 1447 1448 1449 1450
		 * resources.
		 * Perform a smp_mb(): User space could assume that semop()
		 * is a memory barrier: Without the mb(), the cpu could
		 * speculatively read in user space stale data that was
		 * overwritten by the previous owner of the semaphore.
		 */
		smp_mb();

L
Linus Torvalds 已提交
1451 1452 1453
		goto out_free;
	}

K
Kirill Korotaev 已提交
1454
	sma = sem_lock(ns, semid);
1455
	if (IS_ERR(sma)) {
L
Linus Torvalds 已提交
1456 1457 1458 1459
		error = -EIDRM;
		goto out_free;
	}

1460 1461
	error = get_queue_result(&queue);

L
Linus Torvalds 已提交
1462 1463 1464
	/*
	 * If queue.status != -EINTR we are woken up by another process
	 */
1465

L
Linus Torvalds 已提交
1466 1467 1468 1469 1470 1471 1472 1473 1474
	if (error != -EINTR) {
		goto out_unlock_free;
	}

	/*
	 * If an interrupt occurred we have to clean up the queue
	 */
	if (timeout && jiffies_left == 0)
		error = -EAGAIN;
1475
	unlink_queue(sma, &queue);
L
Linus Torvalds 已提交
1476 1477 1478

out_unlock_free:
	sem_unlock(sma);
1479 1480

	wake_up_sem_queue_do(&tasks);
L
Linus Torvalds 已提交
1481 1482 1483 1484 1485 1486
out_free:
	if(sops != fast_sops)
		kfree(sops);
	return error;
}

1487 1488
SYSCALL_DEFINE3(semop, int, semid, struct sembuf __user *, tsops,
		unsigned, nsops)
L
Linus Torvalds 已提交
1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527
{
	return sys_semtimedop(semid, tsops, nsops, NULL);
}

/* If CLONE_SYSVSEM is set, establish sharing of SEM_UNDO state between
 * parent and child tasks.
 */

int copy_semundo(unsigned long clone_flags, struct task_struct *tsk)
{
	struct sem_undo_list *undo_list;
	int error;

	if (clone_flags & CLONE_SYSVSEM) {
		error = get_undo_list(&undo_list);
		if (error)
			return error;
		atomic_inc(&undo_list->refcnt);
		tsk->sysvsem.undo_list = undo_list;
	} else 
		tsk->sysvsem.undo_list = NULL;

	return 0;
}

/*
 * add semadj values to semaphores, free undo structures.
 * undo structures are not freed when semaphore arrays are destroyed
 * so some of them may be out of date.
 * IMPLEMENTATION NOTE: There is some confusion over whether the
 * set of adjustments that needs to be done should be done in an atomic
 * manner or not. That is, if we are attempting to decrement the semval
 * should we queue up and wait until we can do so legally?
 * The original implementation attempted to do this (queue and wait).
 * The current implementation does not do so. The POSIX standard
 * and SVID should be consulted to determine what behavior is mandated.
 */
void exit_sem(struct task_struct *tsk)
{
1528
	struct sem_undo_list *ulp;
L
Linus Torvalds 已提交
1529

1530 1531
	ulp = tsk->sysvsem.undo_list;
	if (!ulp)
L
Linus Torvalds 已提交
1532
		return;
1533
	tsk->sysvsem.undo_list = NULL;
L
Linus Torvalds 已提交
1534

1535
	if (!atomic_dec_and_test(&ulp->refcnt))
L
Linus Torvalds 已提交
1536 1537
		return;

1538
	for (;;) {
L
Linus Torvalds 已提交
1539
		struct sem_array *sma;
1540
		struct sem_undo *un;
1541
		struct list_head tasks;
1542
		int semid;
1543 1544
		int i;

1545
		rcu_read_lock();
1546 1547
		un = list_entry_rcu(ulp->list_proc.next,
				    struct sem_undo, list_proc);
1548 1549 1550 1551 1552
		if (&un->list_proc == &ulp->list_proc)
			semid = -1;
		 else
			semid = un->semid;
		rcu_read_unlock();
1553

1554 1555
		if (semid == -1)
			break;
L
Linus Torvalds 已提交
1556

1557
		sma = sem_lock_check(tsk->nsproxy->ipc_ns, un->semid);
L
Linus Torvalds 已提交
1558

1559 1560 1561
		/* exit_sem raced with IPC_RMID, nothing to do */
		if (IS_ERR(sma))
			continue;
L
Linus Torvalds 已提交
1562

1563
		un = __lookup_undo(ulp, semid);
1564 1565 1566 1567 1568 1569 1570 1571 1572
		if (un == NULL) {
			/* exit_sem raced with IPC_RMID+semget() that created
			 * exactly the same semid. Nothing to do.
			 */
			sem_unlock(sma);
			continue;
		}

		/* remove un from the linked lists */
1573 1574 1575
		assert_spin_locked(&sma->sem_perm.lock);
		list_del(&un->list_id);

1576 1577 1578 1579
		spin_lock(&ulp->lock);
		list_del_rcu(&un->list_proc);
		spin_unlock(&ulp->lock);

1580 1581
		/* perform adjustments registered in un */
		for (i = 0; i < sma->sem_nsems; i++) {
I
Ingo Molnar 已提交
1582
			struct sem * semaphore = &sma->sem_base[i];
1583 1584
			if (un->semadj[i]) {
				semaphore->semval += un->semadj[i];
L
Linus Torvalds 已提交
1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597
				/*
				 * Range checks of the new semaphore value,
				 * not defined by sus:
				 * - Some unices ignore the undo entirely
				 *   (e.g. HP UX 11i 11.22, Tru64 V5.1)
				 * - some cap the value (e.g. FreeBSD caps
				 *   at 0, but doesn't enforce SEMVMX)
				 *
				 * Linux caps the semaphore value, both at 0
				 * and at SEMVMX.
				 *
				 * 	Manfred <manfred@colorfullife.com>
				 */
I
Ingo Molnar 已提交
1598 1599 1600 1601
				if (semaphore->semval < 0)
					semaphore->semval = 0;
				if (semaphore->semval > SEMVMX)
					semaphore->semval = SEMVMX;
1602
				semaphore->sempid = task_tgid_vnr(current);
L
Linus Torvalds 已提交
1603 1604 1605
			}
		}
		/* maybe some queued-up processes were waiting for this */
1606 1607
		INIT_LIST_HEAD(&tasks);
		do_smart_update(sma, NULL, 0, 1, &tasks);
L
Linus Torvalds 已提交
1608
		sem_unlock(sma);
1609
		wake_up_sem_queue_do(&tasks);
1610 1611

		call_rcu(&un->rcu, free_un);
L
Linus Torvalds 已提交
1612
	}
1613
	kfree(ulp);
L
Linus Torvalds 已提交
1614 1615 1616
}

#ifdef CONFIG_PROC_FS
1617
static int sysvipc_sem_proc_show(struct seq_file *s, void *it)
L
Linus Torvalds 已提交
1618
{
1619 1620 1621
	struct sem_array *sma = it;

	return seq_printf(s,
1622
			  "%10d %10d  %4o %10u %5u %5u %5u %5u %10lu %10lu\n",
1623
			  sma->sem_perm.key,
N
Nadia Derbey 已提交
1624
			  sma->sem_perm.id,
1625 1626 1627 1628 1629 1630 1631 1632
			  sma->sem_perm.mode,
			  sma->sem_nsems,
			  sma->sem_perm.uid,
			  sma->sem_perm.gid,
			  sma->sem_perm.cuid,
			  sma->sem_perm.cgid,
			  sma->sem_otime,
			  sma->sem_ctime);
L
Linus Torvalds 已提交
1633 1634
}
#endif