Fix crash that invalid func ptr when dso has dlclosed

Use atexit to register some callback from dso, but there is no
unregister interface. After this dso closed, this callback in
atexit callback chain is invalidate. This workround will save
the dso that the callback come from, then when dso closed, the
relative callbacks would cleared.

Issue： #I6UEZQ
Test：register atexit cb, then dlclose this dso
Signed-off-by: Nhhj <huanghuijin@huawei.com>
Change-Id: I0dfc15366dd51bbe677b9700131fbaf57622499c

libc-test/src/functional/BUILD.gn

@@ -30,11 +30,14 @@ test_sharedlib("tls_init_dso") {
 }
 test_sharedlib("tls_align_dso") {
 }
+test_sharedlib("atexit_dlclose_dso") {
+}
 
 group("dso_shared") {
   testonly = true
 
   deps = [
+    ":atexit_dlclose_dso",
     ":dlclose_reset_dso",
     ":dlopen_dso",
     ":dlopen_ns_dso",

libc-test/src/functional/atexit_dlclose.c

+/*
+ * Copyright (c) 2023 Huawei Device Co., Ltd.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <dlfcn.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <sys/wait.h>
+#include <string.h>
+#include <errno.h>
+#include "test.h"
+
+typedef void (* ATEXIT_CB)();
+
+#define TEST_DSO "libatexit_dlclose_dso.so"
+#define ATEXIT_CB_NAME "atexit_cb"
+#define ATEXIT_WATCHPOINT_NAME "g_watchpoint"
+
+int fork_main(char *exe)
+{
+	char buf[512];
+	void *handler = NULL;
+	ATEXIT_CB cb = NULL;
+	unsigned int *wp_ptr = NULL;
+	unsigned int wp = 0;
+
+	int err = 0;
+
+	if(!t_pathrel(buf, sizeof(buf), exe, TEST_DSO)) {
+		t_error("failed to obtain relative path to " TEST_DSO "\n");
+		return 1;
+	}
+
+	handler = dlopen(buf, RTLD_LAZY|RTLD_LOCAL);
+	if(!handler) {
+		t_error("dlopen %s failed: %s\n", buf, dlerror());
+		return 2;
+	}
+
+	cb = (ATEXIT_CB)dlsym(handler, ATEXIT_CB_NAME);
+	if (!cb) {
+		t_error("dlsym %s failed: %s\n", ATEXIT_CB_NAME, dlerror());
+		return 3;
+	}
+
+	wp_ptr = (unsigned int *)dlsym(handler, ATEXIT_WATCHPOINT_NAME);
+	if (!wp_ptr) {
+		t_error("dlsym %s failed: %s\n", ATEXIT_WATCHPOINT_NAME, dlerror());
+		return 3;
+	}
+
+	wp_ptr = &wp;
+
+	err = atexit(cb);
+
+	if(dlclose( handler)) {
+		t_error("dlclose failed: %s\n", dlerror());
+		return 4;
+	}
+
+	if (wp == 0xFFFF) {
+		t_error("error, atexit callback called");
+		return 5;
+	}
+
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	pid_t pid, w;
+	int err;
+	int wstatus;
+
+	pid = fork();
+	if (pid == 0) { // child process
+		return fork_main(argv[0]);
+	} else if (pid > 0) { // parent process
+		w = waitpid(pid, &wstatus, 0);
+		if (w == -1) {
+			t_error("wait for child process failed");
+			return 1;
+		}
+
+		if (WIFEXITED(wstatus)) {
+			err = WEXITSTATUS(wstatus);
+			t_error("exited with status=%d\n", err);
+			return err;
+		} else if (WIFSIGNALED(wstatus)) {
+			t_error("killed by signal %d\n", WTERMSIG(wstatus));
+			return 9;
+		} else if (WIFSTOPPED(wstatus)) {
+			t_error("stopped by signal %d\n", WSTOPSIG(wstatus));
+			return 9;
+		} else {
+			t_error("stopped by signal %d\n", WSTOPSIG(wstatus));
+			return 9;
+		}
+	} else {
+		t_error("fork failed: %s\n", strerror(errno));
+		return 1;
+	}
+}
+

libc-test/src/functional/atexit_dlclose_dso.c

+/*
+ * Copyright (C) 2023 Huawei Device Co., Ltd.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdio.h>
+
+unsigned int *g_watchpoint = NULL;
+
+void atexit_cb()
+{
+    if (g_watchpoint) {
+        *g_watchpoint = 0xFFFF;
+    }
+}
+
+__attribute__((constructor)) void atexit_dso_ctor()
+{
+	   printf("before main\n");
+}
+
+__attribute__((destructor)) void atexit_dso_dtor()
+{
+	   printf("after main\n");
+}

libc-test/src/functional/dlclose_reset.c

@@ -3,10 +3,10 @@
 #include "test.h"
 
 
-typedef int (* TEST_S_VAR)();  
+typedef int (* TEST_S_VAR)();
 typedef int (* TEST_G_VAR)();
-typedef int (* INSP_S_VAR)();  
-typedef int (* INSP_G_VAR)();  
+typedef int (* INSP_S_VAR)();
+typedef int (* INSP_G_VAR)();
 
 
 int main(int argc, char *argv[])
@@ -16,9 +16,9 @@ int main(int argc, char *argv[])
 	void *g = NULL;
 	TEST_S_VAR test_s_var = NULL;
 	TEST_G_VAR test_g_var = NULL;
-	INSP_S_VAR insp_s_var = NULL;  
-	INSP_G_VAR insp_g_var = NULL;  
-	
+	INSP_S_VAR insp_s_var = NULL;
+	INSP_G_VAR insp_g_var = NULL;
+
 	int s_var = 0;
 	int g_var = 0;
 
@@ -48,7 +48,7 @@ int main(int argc, char *argv[])
 	g = dlopen(buf, RTLD_LAZY|RTLD_LOCAL);
 	if(!g)
         t_error("dlopen %s failed: %s\n", buf, dlerror());
-	
+
 	insp_s_var = (INSP_S_VAR)dlsym( g, "ret_static");
 	if (!insp_s_var)
 		t_error("dlsym ret_static failed: %s\n", dlerror());
@@ -56,7 +56,7 @@ int main(int argc, char *argv[])
 	insp_g_var = (INSP_G_VAR)dlsym( g, "ret_global");
 	if (!insp_g_var)
 		t_error("dlsyn global_var failed: %s\n", dlerror());
-	
+
 	s_var = insp_s_var();
 	g_var = insp_g_var();
 

libc-test/src/functional/test_src_functional.gni

@@ -79,6 +79,7 @@ functional_list = [
   "wcsstr",
   "wcstol",
   "dlclose_reset",
+  "atexit_dlclose",
 ]
 
 malloc_stats_list = [

porting/linux/user/ldso/dynlink.c

@@ -3540,6 +3540,8 @@ static void *do_dlsym(struct dso *p, const char *s, const char *v, void *ra)
 	return laddr(def.dso, def.sym->st_value);
 }
 
+extern int invalidate_exit_funcs(struct dso *p);
+
 static int dlclose_impl(struct dso *p)
 {
 	size_t n;
@@ -3574,6 +3576,9 @@ static int dlclose_impl(struct dso *p)
 		p->constructed = 0;
 	}
 
+	/* after destruct, invalidate atexit funcs which belong to this dso */
+	invalidate_exit_funcs(p);
+
 	/* remove dso symbols from global list */
 	if (p->syms_next) {
 		for (d = head; d->syms_next != p; d = d->syms_next)

porting/linux/user/src/exit/atexit.c

-#include <stdlib.h>
-#include <stdint.h>
-#include "libc.h"
-#include "lock.h"
-#include "fork_impl.h"
-
-#define malloc __libc_malloc
-#define calloc __libc_calloc
-#define realloc undef
-#define free undef
-
-/* Ensure that at least 32 atexit handlers can be registered without malloc */
-#define COUNT 32
-
-static struct fl
-{
-	struct fl *next;
-	void (*f[COUNT])(void *);
-	void *a[COUNT];
-	void *dso[COUNT];
-} builtin, *head;
-
-static int slot;
-static volatile int lock[1];
-volatile int *const __atexit_lockptr = lock;
-
-void __funcs_on_exit()
-{
-	void (*func)(void *), *arg;
-	LOCK(lock);
-	for (; head; head=head->next, slot=COUNT) while(slot-->0) {
-		if (head->f[slot] != NULL) {
-			func = head->f[slot];
-			arg = head->a[slot];
-			UNLOCK(lock);
-			func(arg);
-			LOCK(lock);
-		}
-	}
-	UNLOCK(lock);
-}
-
-void __cxa_finalize(void *dso)
-{
-	void (*func)(void *), *arg;
-	struct fl *head_tmp = head;
-	int slot_tmp = slot;
-
-	LOCK(lock);
-	for (; head_tmp; head_tmp=head_tmp->next, slot_tmp=COUNT) while(slot_tmp-->0) {
-		if (dso == head_tmp->dso[slot_tmp]) {
-			func = head_tmp->f[slot_tmp];
-			arg = head_tmp->a[slot_tmp];
-			UNLOCK(lock);
-			func(arg);
-			LOCK(lock);
-
-			head_tmp->dso[slot_tmp] = NULL;
-			head_tmp->f[slot_tmp] = NULL;
-		}
-	}
-	UNLOCK(lock);
-}
-
-int __cxa_atexit(void (*func)(void *), void *arg, void *dso)
-{
-	LOCK(lock);
-
-	/* Defer initialization of head so it can be in BSS */
-	if (!head) head = &builtin;
-
-	/* If the current function list is full, add a new one */
-	if (slot==COUNT) {
-		struct fl *new_fl = calloc(sizeof(struct fl), 1);
-		if (!new_fl) {
-			UNLOCK(lock);
-			return -1;
-		}
-		new_fl->next = head;
-		head = new_fl;
-		slot = 0;
-	}
-
-	/* Append function to the list. */
-	head->f[slot] = func;
-	head->a[slot] = arg;
-	head->dso[slot] = dso;
-
-	slot++;
-
-	UNLOCK(lock);
-	return 0;
-}
-
-static void call(void *p)
-{
-	((void (*)(void))(uintptr_t)p)();
-}
-
-int atexit(void (*func)(void))
-{
-	return __cxa_atexit(call, (void *)(uintptr_t)func, 0);
-}
\ No newline at end of file
+#include <stdlib.h>
+#include <stdint.h>
+#include "libc.h"
+#include "lock.h"
+#include "fork_impl.h"
+#include "dynlink.h"
+#include "musl_log.h"
+
+#define malloc __libc_malloc
+#define calloc __libc_calloc
+#define realloc undef
+#define free undef
+
+/* Ensure that at least 32 atexit handlers can be registered without malloc */
+#define COUNT 32
+
+static struct fl
+{
+	struct fl *next;
+	void (*f[COUNT])(void *);
+	void *a[COUNT];
+	void *dso[COUNT];
+	struct dso *internal_dso[COUNT]; // the internal dso weekptr, used for dlclose
+} builtin, *head;
+
+static int slot;
+static volatile int lock[1];
+volatile int *const __atexit_lockptr = lock;
+
+void __funcs_on_exit()
+{
+	void (*func)(void *), *arg;
+	LOCK(lock);
+	for (; head; head=head->next, slot=COUNT) while(slot-->0) {
+		if (head->f[slot] != NULL) {
+			func = head->f[slot];
+			arg = head->a[slot];
+			UNLOCK(lock);
+			func(arg);
+			LOCK(lock);
+		}
+	}
+	UNLOCK(lock);
+}
+
+void __cxa_finalize(void *dso)
+{
+	void (*func)(void *), *arg;
+	struct fl *head_tmp = head;
+	int slot_tmp = slot;
+
+	LOCK(lock);
+	for (; head_tmp; head_tmp=head_tmp->next, slot_tmp=COUNT) while(slot_tmp-->0) {
+		if (dso == head_tmp->dso[slot_tmp]) {
+			func = head_tmp->f[slot_tmp];
+			arg = head_tmp->a[slot_tmp];
+			UNLOCK(lock);
+			func(arg);
+			LOCK(lock);
+
+			head_tmp->dso[slot_tmp] = NULL;
+			head_tmp->f[slot_tmp] = NULL;
+		}
+	}
+	UNLOCK(lock);
+}
+
+static void call(void *p);
+
+int __cxa_atexit(void (*func)(void *), void *arg, void *dso)
+{
+	struct dso *p = NULL;
+	LOCK(lock);
+
+	/* Defer initialization of head so it can be in BSS */
+	if (!head) head = &builtin;
+
+	// if called from atexit, check callback ptr mem range.
+	if ((func == (void *)call) && (dso == NULL)) {
+		p = addr2dso((size_t)arg);
+		if (p == NULL) {
+			UNLOCK(lock);
+			MUSL_LOGE("call atexit with invalid callback ptr=%{public}p", arg);
+			return -1;
+		}
+	}
+
+	/* If the current function list is full, add a new one */
+	if (slot==COUNT) {
+		struct fl *new_fl = calloc(sizeof(struct fl), 1);
+		if (!new_fl) {
+			UNLOCK(lock);
+			return -1;
+		}
+		new_fl->next = head;
+		head = new_fl;
+		slot = 0;
+	}
+
+	/* Append function to the list. */
+	head->f[slot] = func;
+	head->a[slot] = arg;
+	head->dso[slot] = dso;
+	head->internal_dso[slot] = p;
+
+	slot++;
+
+	UNLOCK(lock);
+	return 0;
+}
+
+static void call(void *p)
+{
+	if (p != NULL)
+		((void (*)(void))(uintptr_t)p)();
+}
+
+int atexit(void (*func)(void))
+{
+	return __cxa_atexit(call, (void *)(uintptr_t)func, 0);
+}
+
+int invalidate_exit_funcs(struct dso *p)
+{
+	struct fl *head_tmp = head;
+	int slot_tmp = slot;
+
+	LOCK(lock);
+	for (; head_tmp; head_tmp=head_tmp->next, slot_tmp=COUNT) {
+		while(slot_tmp-->0) {
+			// if found exit callback relative to this dso, and
+			if (p == head_tmp->internal_dso[slot_tmp]) {
+				if ((head_tmp->dso[slot_tmp] == NULL) && head_tmp->f[slot_tmp] == (void *)call) {
+					MUSL_LOGD("invalidate callback ptr=%{public}p when uninstall %{public}%s", head_tmp->a[slot_tmp], p->name);
+					head_tmp->a[slot_tmp] = NULL;
+				}
+			}
+		}
+	}
+	UNLOCK(lock);
+
+	return 0;
+}