work around broken kernel struct ipc_perm on some big endian archs

the mode member of struct ipc_perm is specified by POSIX to have type
mode_t, which is uniformly defined as unsigned int. however, Linux
defines it with type __kernel_mode_t, and defines __kernel_mode_t as
unsigned short on some archs. since there is a subsequent padding
field, treating it as a 32-bit unsigned int works on little endian
archs, but the order is backwards on big endian archs with the
erroneous definition.

since multiple archs are affected, remedy the situation with fixup
code in the affected functions (shmctl, semctl, and msgctl) rather
than repeating the same shims in syscall_arch.h for every affected
arch.

arch/arm/syscall_arch.h

@@ -103,3 +103,5 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo
 #define VDSO_CGT_VER "LINUX_2.6"
 
 #define SYSCALL_FADVISE_6_ARG
+
+#define SYSCALL_IPC_BROKEN_MODE

arch/m68k/syscall_arch.h

@@ -88,3 +88,4 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo
 }
 
 #define SYSCALL_USE_SOCKETCALL
+#define SYSCALL_IPC_BROKEN_MODE

arch/microblaze/syscall_arch.h

@@ -102,3 +102,5 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo
 #define SYSCALL_NO_INLINE
 
 #endif
+
+#define SYSCALL_IPC_BROKEN_MODE

arch/sh/syscall_arch.h

@@ -86,3 +86,5 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo
 	register long r1 __asm__("r1") = f;
 	__asm_syscall(22, "r"(r3), "r"(r4), "r"(r5), "r"(r6), "r"(r7), "0"(r0), "r"(r1));
 }
+
+#define SYSCALL_IPC_BROKEN_MODE

src/ipc/msgctl.c

 #include <sys/msg.h>
+#include <endian.h>
 #include "syscall.h"
 #include "ipc.h"
 
+#if __BYTE_ORDER != __BIG_ENDIAN
+#undef SYSCALL_IPC_BROKEN_MODE
+#endif
+
 int msgctl(int q, int cmd, struct msqid_ds *buf)
 {
+#ifdef SYSCALL_IPC_BROKEN_MODE
+	struct msqid_ds tmp;
+	if (cmd == IPC_SET) {
+		tmp = *buf;
+		tmp.msg_perm.mode *= 0x10000U;
+		buf = &tmp;
+	}
+#endif
 #ifdef SYS_msgctl
-	return syscall(SYS_msgctl, q, cmd | IPC_64, buf);
+	int r = __syscall(SYS_msgctl, q, cmd | IPC_64, buf);
 #else
-	return syscall(SYS_ipc, IPCOP_msgctl, q, cmd | IPC_64, 0, buf, 0);
+	int r = __syscall(SYS_ipc, IPCOP_msgctl, q, cmd | IPC_64, 0, buf, 0);
+#endif
+#ifdef SYSCALL_IPC_BROKEN_MODE
+	if (r >= 0) switch (cmd) {
+	case IPC_STAT:
+	case MSG_STAT:
+	case MSG_STAT_ANY:
+		buf->msg_perm.mode >>= 16;
+	}
 #endif
+	return __syscall_ret(r);
 }

src/ipc/semctl.c

 #include <sys/sem.h>
 #include <stdarg.h>
+#include <endian.h>
 #include "syscall.h"
 #include "ipc.h"
 
+#if __BYTE_ORDER != __BIG_ENDIAN
+#undef SYSCALL_IPC_BROKEN_MODE
+#endif
+
 union semun {
 	int val;
 	struct semid_ds *buf;
@@ -20,9 +25,26 @@ int semctl(int id, int num, int cmd, ...)
 		arg = va_arg(ap, union semun);
 		va_end(ap);
 	}
+#ifdef SYSCALL_IPC_BROKEN_MODE
+	struct semid_ds tmp;
+	if (cmd == IPC_SET) {
+		tmp = *arg.buf;
+		tmp.sem_perm.mode *= 0x10000U;
+		arg.buf = &tmp;
+	}
+#endif
 #ifdef SYS_semctl
-	return syscall(SYS_semctl, id, num, cmd | IPC_64, arg.buf);
+	int r = __syscall(SYS_semctl, id, num, cmd | IPC_64, arg.buf);
 #else
-	return syscall(SYS_ipc, IPCOP_semctl, id, num, cmd | IPC_64, &arg.buf);
+	int r = __syscall(SYS_ipc, IPCOP_semctl, id, num, cmd | IPC_64, &arg.buf);
+#endif
+#ifdef SYSCALL_IPC_BROKEN_MODE
+	if (r >= 0) switch (cmd) {
+	case IPC_STAT:
+	case SEM_STAT:
+	case SEM_STAT_ANY:
+		arg.buf->sem_perm.mode >>= 16;
+	}
 #endif
+	return __syscall_ret(r);
 }

src/ipc/shmctl.c

 #include <sys/shm.h>
+#include <endian.h>
 #include "syscall.h"
 #include "ipc.h"
 
+#if __BYTE_ORDER != __BIG_ENDIAN
+#undef SYSCALL_IPC_BROKEN_MODE
+#endif
+
 int shmctl(int id, int cmd, struct shmid_ds *buf)
 {
+#ifdef SYSCALL_IPC_BROKEN_MODE
+	struct shmid_ds tmp;
+	if (cmd == IPC_SET) {
+		tmp = *buf;
+		tmp.shm_perm.mode *= 0x10000U;
+		buf = &tmp;
+	}
+#endif
 #ifdef SYS_shmctl
-	return syscall(SYS_shmctl, id, cmd | IPC_64, buf);
+	int r = __syscall(SYS_shmctl, id, cmd | IPC_64, buf);
 #else
-	return syscall(SYS_ipc, IPCOP_shmctl, id, cmd | IPC_64, 0, buf, 0);
+	int r = __syscall(SYS_ipc, IPCOP_shmctl, id, cmd | IPC_64, 0, buf, 0);
+#endif
+#ifdef SYSCALL_IPC_BROKEN_MODE
+	if (r >= 0) switch (cmd) {
+	case IPC_STAT:
+	case SHM_STAT:
+	case SHM_STAT_ANY:
+		buf->shm_perm.mode >>= 16;
+	}
 #endif
+	return __syscall_ret(r);
 }