combine arch ABI's DTP_OFFSET into DTV pointers

as explained in commit 6ba5517a, some
archs use an offset (typicaly -0x8000) with their DTPOFF relocations,
which __tls_get_addr needs to invert. on affected archs, which lack
direct support for large immediates, this can cost multiple extra
instructions in the hot path. instead, incorporate the DTP_OFFSET into
the DTV entries. this means they are no longer valid pointers, so
store them as an array of uintptr_t rather than void *; this also
makes it easier to access slot 0 as a valid slot count.

commit e75b16cf left behind cruft in
two places, __reset_tls and __tls_get_new, from back when it was
possible to have uninitialized gap slots indicated by a null pointer
in the DTV. since the concept of null pointer is no longer meaningful
with an offset applied, remove this cruft.

presently there are no archs with both TLSDESC and nonzero DTP_OFFSET,
but the dynamic TLSDESC relocation code is also updated to apply an
inverted offset to its offset field, so that the offset DTV would not
impose a runtime cost in TLSDESC resolver functions.

ldso/dynlink.c

@@ -72,7 +72,7 @@ struct dso {
 	struct tls_module tls;
 	size_t tls_id;
 	size_t relro_start, relro_end;
-	void **new_dtv;
+	uintptr_t *new_dtv;
 	unsigned char *new_tls;
 	volatile int new_dtv_idx, new_tls_idx;
 	struct td_index *td_index;
@@ -445,7 +445,7 @@ static void do_relocs(struct dso *dso, size_t *rel, size_t rel_size, size_t stri
 				new->next = dso->td_index;
 				dso->td_index = new;
 				new->args[0] = def.dso->tls_id;
-				new->args[1] = tls_val + addend;
+				new->args[1] = tls_val + addend - DTP_OFFSET;
 				reloc_addr[0] = (size_t)__tlsdesc_dynamic;
 				reloc_addr[1] = (size_t)new;
 			} else {
@@ -1345,9 +1345,9 @@ hidden void *__tls_get_new(tls_mod_off_t *v)
 	/* Block signals to make accessing new TLS async-signal-safe */
 	sigset_t set;
 	__block_all_sigs(&set);
-	if (v[0]<=(size_t)self->dtv[0]) {
+	if (v[0] <= self->dtv[0]) {
 		__restore_sigs(&set);
-		return (char *)self->dtv[v[0]]+v[1]+DTP_OFFSET;
+		return (void *)(self->dtv[v[0]] + v[1]);
 	}
 
 	/* This is safe without any locks held because, if the caller
@@ -1357,15 +1357,12 @@ hidden void *__tls_get_new(tls_mod_off_t *v)
 	struct dso *p;
 	for (p=head; p->tls_id != v[0]; p=p->next);
 
-	/* Get new DTV space from new DSO if needed */
-	if (v[0] > (size_t)self->dtv[0]) {
-		void **newdtv = p->new_dtv +
-			(v[0]+1)*a_fetch_add(&p->new_dtv_idx,1);
-		memcpy(newdtv, self->dtv,
-			((size_t)self->dtv[0]+1) * sizeof(void *));
-		newdtv[0] = (void *)v[0];
-		self->dtv = self->dtv_copy = newdtv;
-	}
+	/* Get new DTV space from new DSO */
+	uintptr_t *newdtv = p->new_dtv +
+		(v[0]+1)*a_fetch_add(&p->new_dtv_idx,1);
+	memcpy(newdtv, self->dtv, (self->dtv[0]+1) * sizeof(uintptr_t));
+	newdtv[0] = v[0];
+	self->dtv = self->dtv_copy = newdtv;
 
 	/* Get new TLS memory from all new DSOs up to the requested one */
 	unsigned char *mem;
@@ -1375,7 +1372,7 @@ hidden void *__tls_get_new(tls_mod_off_t *v)
 			* a_fetch_add(&p->new_tls_idx,1);
 		mem += ((uintptr_t)p->tls.image - (uintptr_t)mem)
 			& (p->tls.align-1);
-		self->dtv[p->tls_id] = mem;
+		self->dtv[p->tls_id] = (uintptr_t)mem + DTP_OFFSET;
 		memcpy(mem, p->tls.image, p->tls.len);
 		if (p->tls_id == v[0]) break;
 	}

src/env/__init_tls.c

@@ -36,32 +36,32 @@ void *__copy_tls(unsigned char *mem)
 	pthread_t td;
 	struct tls_module *p;
 	size_t i;
-	void **dtv;
+	uintptr_t *dtv;
 
 #ifdef TLS_ABOVE_TP
-	dtv = (void **)(mem + libc.tls_size) - (libc.tls_cnt + 1);
+	dtv = (uintptr_t*)(mem + libc.tls_size) - (libc.tls_cnt + 1);
 
 	mem += -((uintptr_t)mem + sizeof(struct pthread)) & (libc.tls_align-1);
 	td = (pthread_t)mem;
 	mem += sizeof(struct pthread);
 
 	for (i=1, p=libc.tls_head; p; i++, p=p->next) {
-		dtv[i] = mem + p->offset;
-		memcpy(dtv[i], p->image, p->len);
+		dtv[i] = (uintptr_t)(mem + p->offset) + DTP_OFFSET;
+		memcpy(mem + p->offset, p->image, p->len);
 	}
 #else
-	dtv = (void **)mem;
+	dtv = (uintptr_t *)mem;
 
 	mem += libc.tls_size - sizeof(struct pthread);
 	mem -= (uintptr_t)mem & (libc.tls_align-1);
 	td = (pthread_t)mem;
 
 	for (i=1, p=libc.tls_head; p; i++, p=p->next) {
-		dtv[i] = mem - p->offset;
-		memcpy(dtv[i], p->image, p->len);
+		dtv[i] = (uintptr_t)(mem - p->offset) + DTP_OFFSET;
+		memcpy(mem - p->offset, p->image, p->len);
 	}
 #endif
-	dtv[0] = (void *)libc.tls_cnt;
+	dtv[0] = libc.tls_cnt;
 	td->dtv = td->dtv_copy = dtv;
 	return td;
 }

src/env/__reset_tls.c

@@ -6,11 +6,10 @@ void __reset_tls()
 {
 	pthread_t self = __pthread_self();
 	struct tls_module *p;
-	size_t i, n = (size_t)self->dtv[0];
+	size_t i, n = self->dtv[0];
 	if (n) for (p=libc.tls_head, i=1; i<=n; i++, p=p->next) {
-		if (!self->dtv[i]) continue;
-		memcpy(self->dtv[i], p->image, p->len);
-		memset((char *)self->dtv[i]+p->len, 0,
-			p->size - p->len);
+		char *mem = (char *)(self->dtv[i] - DTP_OFFSET);
+		memcpy(mem, p->image, p->len);
+		memset(mem+p->len, 0, p->size - p->len);
 	}
 }

src/internal/pthread_impl.h

@@ -17,7 +17,8 @@ struct pthread {
 	/* Part 1 -- these fields may be external or
 	 * internal (accessed via asm) ABI. Do not change. */
 	struct pthread *self;
-	void **dtv, *unused1, *unused2;
+	uintptr_t *dtv;
+	void *unused1, *unused2;
 	uintptr_t sysinfo;
 	uintptr_t canary, canary2;
 
@@ -54,7 +55,7 @@ struct pthread {
 	/* Part 3 -- the positions of these fields relative to
 	 * the end of the structure is external and internal ABI. */
 	uintptr_t canary_at_end;
-	void **dtv_copy;
+	uintptr_t *dtv_copy;
 };
 
 struct start_sched_args {

src/thread/__tls_get_addr.c

@@ -4,8 +4,8 @@
 void *__tls_get_addr(tls_mod_off_t *v)
 {
 	pthread_t self = __pthread_self();
-	if (v[0]<=(size_t)self->dtv[0])
-		return (char *)self->dtv[v[0]]+v[1]+DTP_OFFSET;
+	if (v[0] <= self->dtv[0])
+		return (void *)(self->dtv[v[0]] + v[1]);
 	return __tls_get_new(v);
 }