fix for broken kernel side RLIM_INFINITY on mips

On 32 bit mips the kernel uses -1UL/2 to mark RLIM_INFINITY (and
this is the definition in the userspace api), but since it is in
the middle of the valid range of limits and limits are often
compared with relational operators, various kernel side logic is
broken if larger than -1UL/2 limits are used. So we truncate the
limits to -1UL/2 in get/setrlimit and prlimit.

Even if the kernel side logic consistently treated -1UL/2 as greater
than any other limit value, there wouldn't be any clean workaround
that allowed using large limits:
* using -1UL/2 as RLIM_INFINITY in userspace would mean different
infinity value for get/setrlimt and prlimit (where infinity is always
-1ULL) and userspace logic could break easily (just like the kernel
is broken now) and more special case code would be needed for mips.
* translating -1UL/2 kernel side value to -1ULL in userspace would
mean that -1UL/2 limit cannot be set (eg. -1UL/2+1 had to be passed
to the kernel instead).

arch/mips/syscall_arch.h

@@ -5,6 +5,8 @@
 
 long (__syscall)(long, ...);
 
+#define SYSCALL_RLIM_INFINITY (-1UL/2)
+
 #ifndef __clang__
 
 #define __asm_syscall(...) do { \

src/internal/syscall.h

@@ -4,6 +4,10 @@
 #include <sys/syscall.h>
 #include "syscall_arch.h"
 
+#ifndef SYSCALL_RLIM_INFINITY
+#define SYSCALL_RLIM_INFINITY (~0ULL)
+#endif
+
 #ifndef __scc
 #define __scc(X) ((long) (X))
 typedef long syscall_arg_t;

src/linux/prlimit.c

@@ -3,9 +3,24 @@
 #include "syscall.h"
 #include "libc.h"
 
+#define FIX(x) do{ if ((x)>=SYSCALL_RLIM_INFINITY) (x)=RLIM_INFINITY; }while(0)
+
 int prlimit(pid_t pid, int resource, const struct rlimit *new_limit, struct rlimit *old_limit)
 {
-	return syscall(SYS_prlimit64, pid, resource, new_limit, old_limit);
+	struct rlimit tmp;
+	int r;
+	if (new_limit && SYSCALL_RLIM_INFINITY != RLIM_INFINITY) {
+		tmp = *new_limit;
+		FIX(tmp.rlim_cur);
+		FIX(tmp.rlim_max);
+		new_limit = &tmp;
+	}
+	r = syscall(SYS_prlimit64, pid, resource, new_limit, old_limit);
+	if (!r && old_limit && SYSCALL_RLIM_INFINITY != RLIM_INFINITY) {
+		FIX(old_limit->rlim_cur);
+		FIX(old_limit->rlim_max);
+	}
+	return r;
 }
 
 #undef prlimit64

src/misc/getrlimit.c

@@ -3,16 +3,24 @@
 #include "syscall.h"
 #include "libc.h"
 
+#define FIX(x) do{ if ((x)>=SYSCALL_RLIM_INFINITY) (x)=RLIM_INFINITY; }while(0)
+
 int getrlimit(int resource, struct rlimit *rlim)
 {
 	unsigned long k_rlim[2];
 	int ret = syscall(SYS_prlimit64, 0, resource, 0, rlim);
+	if (!ret) {
+		FIX(rlim->rlim_cur);
+		FIX(rlim->rlim_max);
+	}
 	if (!ret || errno != ENOSYS)
 		return ret;
 	if (syscall(SYS_getrlimit, resource, k_rlim) < 0)
 		return -1;
 	rlim->rlim_cur = k_rlim[0] == -1UL ? RLIM_INFINITY : k_rlim[0];
 	rlim->rlim_max = k_rlim[1] == -1UL ? RLIM_INFINITY : k_rlim[1];
+	FIX(rlim->rlim_cur);
+	FIX(rlim->rlim_max);
 	return 0;
 }
 

src/misc/setrlimit.c

@@ -4,14 +4,22 @@
 #include "libc.h"
 
 #define MIN(a, b) ((a)<(b) ? (a) : (b))
+#define FIX(x) do{ if ((x)>=SYSCALL_RLIM_INFINITY) (x)=RLIM_INFINITY; }while(0)
 
 int __setrlimit(int resource, const struct rlimit *rlim)
 {
 	unsigned long k_rlim[2];
+	struct rlimit tmp;
+	if (SYSCALL_RLIM_INFINITY != RLIM_INFINITY) {
+		tmp = *rlim;
+		FIX(tmp.rlim_cur);
+		FIX(tmp.rlim_max);
+		rlim = &tmp;
+	}
 	int ret = __syscall(SYS_prlimit64, 0, resource, rlim, 0);
 	if (ret != -ENOSYS) return ret;
-	k_rlim[0] = MIN(rlim->rlim_cur, -1UL);
-	k_rlim[1] = MIN(rlim->rlim_max, -1UL);
+	k_rlim[0] = MIN(rlim->rlim_cur, MIN(-1UL, SYSCALL_RLIM_INFINITY));
+	k_rlim[1] = MIN(rlim->rlim_max, MIN(-1UL, SYSCALL_RLIM_INFINITY));
 	return __syscall(SYS_setrlimit, resource, k_rlim);
 }