simplify logic, slightly optimize contended case for non-default mutex types

src/thread/pthread_mutex_trylock.c

@@ -9,15 +9,13 @@ int pthread_mutex_trylock(pthread_mutex_t *m)
 
 	tid = pthread_self()->tid;
 
-	if (m->_m_owner == tid) {
-		if (m->_m_type != PTHREAD_MUTEX_RECURSIVE)
-			return EBUSY;
+	if (m->_m_owner == tid && m->_m_type == PTHREAD_MUTEX_RECURSIVE) {
 		if ((unsigned)m->_m_count >= INT_MAX) return EAGAIN;
 		m->_m_count++;
 		return 0;
 	}
 
-	if (a_xchg(&m->_m_lock, 1)) return EBUSY;
+	if (m->_m_owner || a_xchg(&m->_m_lock, 1)) return EBUSY;
 	m->_m_owner = tid;
 	m->_m_count = 1;
 	return 0;