apps_ui.c: Improve error handling and return value of setup_ui_method()

Reviewed-by: NTomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/12493)

apps/lib/apps_ui.c

@@ -106,11 +106,13 @@ int setup_ui_method(void)
     ui_fallback_method = UI_OpenSSL();
 #endif
     ui_method = UI_create_method("OpenSSL application user interface");
-    UI_method_set_opener(ui_method, ui_open);
-    UI_method_set_reader(ui_method, ui_read);
-    UI_method_set_writer(ui_method, ui_write);
-    UI_method_set_closer(ui_method, ui_close);
-    return 0;
+    return ui_method != NULL
+        && 0 == UI_method_set_opener(ui_method, ui_open)
+        && 0 == UI_method_set_reader(ui_method, ui_read)
+        && 0 == UI_method_set_writer(ui_method, ui_write)
+        && 0 == UI_method_set_closer(ui_method, ui_close)
+        && 0 == UI_method_set_prompt_constructor(ui_method,
+                                                 ui_prompt_construct);
 }
 
 void destroy_ui_method(void)

apps/openssl.c

@@ -68,7 +68,7 @@ static int apps_startup(void)
                           | OPENSSL_INIT_LOAD_CONFIG, NULL))
         return 0;
 
-    setup_ui_method();
+    (void)setup_ui_method();
 
     /*
      * NOTE: This is an undocumented feature required for testing only.

test/uitest.c

@@ -78,7 +78,7 @@ static int test_new_ui(void)
     char pass[16];
     int ok = 0;
 
-    setup_ui_method();
+    (void)setup_ui_method();
     if (TEST_int_gt(password_callback(pass, sizeof(pass), 0, &cb_data), 0)
             && TEST_str_eq(pass, cb_data.password))
         ok = 1;