Fix crash that invalid func ptr when dso has dlclosed

Use atexit to register some callback from dso, but there is no
unregister interface. After this dso closed, this callback in
atexit callback chain is invalidate. This workround will save
the dso that the callback come from, then when dso closed, the
relative callbacks would cleared.

Issue： #I6UEZQ
Test：register atexit cb, then dlclose this dso
Signed-off-by: Nhhj <huanghuijin@huawei.com>
Change-Id: I0dfc15366dd51bbe677b9700131fbaf57622499c

libc-test/src/functional/BUILD.gn

@@ -30,11 +30,14 @@ test_sharedlib("tls_init_dso") {
 }
 test_sharedlib("tls_align_dso") {
 }
+test_sharedlib("atexit_dlclose_dso") {
+}
 
 group("dso_shared") {
   testonly = true
 
   deps = [
+    ":atexit_dlclose_dso",
     ":dlclose_reset_dso",
     ":dlopen_dso",
     ":dlopen_ns_dso",

libc-test/src/functional/atexit_dlclose.c

+/*
+ * Copyright (c) 2023 Huawei Device Co., Ltd.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <dlfcn.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <sys/wait.h>
+#include <string.h>
+#include <errno.h>
+#include "test.h"
+
+typedef void (* ATEXIT_CB)();
+
+#define TEST_DSO "libatexit_dlclose_dso.so"
+#define ATEXIT_CB_NAME "atexit_cb"
+#define ATEXIT_WATCHPOINT_NAME "g_watchpoint"
+
+int fork_main(char *exe)
+{
+	char buf[512];
+	void *handler = NULL;
+	ATEXIT_CB cb = NULL;
+	unsigned int *wp_ptr = NULL;
+	unsigned int wp = 0;
+
+	int err = 0;
+
+	if(!t_pathrel(buf, sizeof(buf), exe, TEST_DSO)) {
+		t_error("failed to obtain relative path to " TEST_DSO "\n");
+		return 1;
+	}
+
+	handler = dlopen(buf, RTLD_LAZY|RTLD_LOCAL);
+	if(!handler) {
+		t_error("dlopen %s failed: %s\n", buf, dlerror());
+		return 2;
+	}
+
+	cb = (ATEXIT_CB)dlsym(handler, ATEXIT_CB_NAME);
+	if (!cb) {
+		t_error("dlsym %s failed: %s\n", ATEXIT_CB_NAME, dlerror());
+		return 3;
+	}
+
+	wp_ptr = (unsigned int *)dlsym(handler, ATEXIT_WATCHPOINT_NAME);
+	if (!wp_ptr) {
+		t_error("dlsym %s failed: %s\n", ATEXIT_WATCHPOINT_NAME, dlerror());
+		return 3;
+	}
+
+	wp_ptr = &wp;
+
+	err = atexit(cb);
+
+	if(dlclose( handler)) {
+		t_error("dlclose failed: %s\n", dlerror());
+		return 4;
+	}
+
+	if (wp == 0xFFFF) {
+		t_error("error, atexit callback called");
+		return 5;
+	}
+
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	pid_t pid, w;
+	int err;
+	int wstatus;
+
+	pid = fork();
+	if (pid == 0) { // child process
+		return fork_main(argv[0]);
+	} else if (pid > 0) { // parent process
+		w = waitpid(pid, &wstatus, 0);
+		if (w == -1) {
+			t_error("wait for child process failed");
+			return 1;
+		}
+
+		if (WIFEXITED(wstatus)) {
+			err = WEXITSTATUS(wstatus);
+			t_error("exited with status=%d\n", err);
+			return err;
+		} else if (WIFSIGNALED(wstatus)) {
+			t_error("killed by signal %d\n", WTERMSIG(wstatus));
+			return 9;
+		} else if (WIFSTOPPED(wstatus)) {
+			t_error("stopped by signal %d\n", WSTOPSIG(wstatus));
+			return 9;
+		} else {
+			t_error("stopped by signal %d\n", WSTOPSIG(wstatus));
+			return 9;
+		}
+	} else {
+		t_error("fork failed: %s\n", strerror(errno));
+		return 1;
+	}
+}
+

libc-test/src/functional/atexit_dlclose_dso.c

+/*
+ * Copyright (C) 2023 Huawei Device Co., Ltd.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdio.h>
+
+unsigned int *g_watchpoint = NULL;
+
+void atexit_cb()
+{
+    if (g_watchpoint) {
+        *g_watchpoint = 0xFFFF;
+    }
+}
+
+__attribute__((constructor)) void atexit_dso_ctor()
+{
+	   printf("before main\n");
+}
+
+__attribute__((destructor)) void atexit_dso_dtor()
+{
+	   printf("after main\n");
+}

libc-test/src/functional/test_src_functional.gni

@@ -79,6 +79,7 @@ functional_list = [
   "wcsstr",
   "wcstol",
   "dlclose_reset",
+  "atexit_dlclose",
 ]
 
 malloc_stats_list = [

porting/linux/user/ldso/dynlink.c

@@ -3540,6 +3540,8 @@ static void *do_dlsym(struct dso *p, const char *s, const char *v, void *ra)
 	return laddr(def.dso, def.sym->st_value);
 }
 
+extern int invalidate_exit_funcs(struct dso *p);
+
 static int dlclose_impl(struct dso *p)
 {
 	size_t n;
@@ -3574,6 +3576,9 @@ static int dlclose_impl(struct dso *p)
 		p->constructed = 0;
 	}
 
+	/* after destruct, invalidate atexit funcs which belong to this dso */
+	invalidate_exit_funcs(p);
+
 	/* remove dso symbols from global list */
 	if (p->syms_next) {
 		for (d = head; d->syms_next != p; d = d->syms_next)

porting/linux/user/src/exit/atexit.c

@@ -3,6 +3,8 @@
 #include "libc.h"
 #include "lock.h"
 #include "fork_impl.h"
+#include "dynlink.h"
+#include "musl_log.h"
 
 #define malloc __libc_malloc
 #define calloc __libc_calloc
@@ -18,6 +20,7 @@ static struct fl
 	void (*f[COUNT])(void *);
 	void *a[COUNT];
 	void *dso[COUNT];
+	struct dso *internal_dso[COUNT]; // the internal dso weekptr, used for dlclose
 } builtin, *head;
 
 static int slot;
@@ -62,13 +65,26 @@ void __cxa_finalize(void *dso)
 	UNLOCK(lock);
 }
 
+static void call(void *p);
+
 int __cxa_atexit(void (*func)(void *), void *arg, void *dso)
 {
+	struct dso *p = NULL;
 	LOCK(lock);
 
 	/* Defer initialization of head so it can be in BSS */
 	if (!head) head = &builtin;
 
+	// if called from atexit, check callback ptr mem range.
+	if ((func == (void *)call) && (dso == NULL)) {
+		p = addr2dso((size_t)arg);
+		if (p == NULL) {
+			UNLOCK(lock);
+			MUSL_LOGE("call atexit with invalid callback ptr=%{public}p", arg);
+			return -1;
+		}
+	}
+
 	/* If the current function list is full, add a new one */
 	if (slot==COUNT) {
 		struct fl *new_fl = calloc(sizeof(struct fl), 1);
@@ -85,6 +101,7 @@ int __cxa_atexit(void (*func)(void *), void *arg, void *dso)
 	head->f[slot] = func;
 	head->a[slot] = arg;
 	head->dso[slot] = dso;
+	head->internal_dso[slot] = p;
 
 	slot++;
 
@@ -94,6 +111,7 @@ int __cxa_atexit(void (*func)(void *), void *arg, void *dso)
 
 static void call(void *p)
 {
+	if (p != NULL)
 		((void (*)(void))(uintptr_t)p)();
 }
 
@@ -101,3 +119,25 @@ int atexit(void (*func)(void))
 {
 	return __cxa_atexit(call, (void *)(uintptr_t)func, 0);
 }
+
+int invalidate_exit_funcs(struct dso *p)
+{
+	struct fl *head_tmp = head;
+	int slot_tmp = slot;
+
+	LOCK(lock);
+	for (; head_tmp; head_tmp=head_tmp->next, slot_tmp=COUNT) {
+		while(slot_tmp-->0) {
+			// if found exit callback relative to this dso, and
+			if (p == head_tmp->internal_dso[slot_tmp]) {
+				if ((head_tmp->dso[slot_tmp] == NULL) && head_tmp->f[slot_tmp] == (void *)call) {
+					MUSL_LOGD("invalidate callback ptr=%{public}p when uninstall %{public}%s", head_tmp->a[slot_tmp], p->name);
+					head_tmp->a[slot_tmp] = NULL;
+				}
+			}
+		}
+	}
+	UNLOCK(lock);
+
+	return 0;
+}