Check directory is able to create files for various -out option

This is to address issue #3404, only works in Unix-like platforms
Reviewed-by: NMatt Caswell <matt@openssl.org>
Reviewed-by: NRichard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3709)

apps/apps.c

@@ -2340,6 +2340,73 @@ int app_isdir(const char *name)
 }
 #endif
 
+/* app_dirname section */
+
+/*
+ * This exactly follows what POSIX's
+ * dirname does, but is implemented
+ * in a more platform independent way.
+ *
+ * path        dirname
+ * /usr/lib    /usr
+ * /usr/       /
+ * usr         .
+ * /           /
+ * .           .
+ * ..          .
+ * ""          .
+ *
+ * Note: this function also keeps the
+ * possibility of modifying the 'path'
+ * string same as POSIX dirname.
+ */
+static char *posix_dirname(char *path)
+{
+    size_t l;
+    char *ret = ".";
+
+    l = strlen(path);
+    if (l == 0)
+        goto out;
+    if (strcmp(path, ".") == 0)
+        goto out;
+    if (strcmp(path, "..") == 0)
+        goto out;
+    if (strcmp(path, "/") == 0) {
+        ret = "/";
+        goto out;
+    }
+    if (path[l - 1] == '/') {
+        /* /usr/ */
+        path[l - 1] = '\0';
+    }
+    if ((ret = strrchr(path, '/')) == NULL) {
+        /* usr */
+        ret = ".";
+    } else if (ret == path) {
+        /* /usr */
+        *++ret = '\0';
+        ret = path;
+    } else {
+        /* /usr/lib */
+        *ret = '\0';
+        ret = path;
+    }
+ out:
+    return ret;
+}
+
+/*
+ * TODO: implement app_dirname for Windows
+ * and VMS.
+ */
+#if !defined(_WIN32) && !defined(__VMS)
+char *app_dirname(char *path)
+{
+    return posix_dirname(path);
+}
+#endif
+
 /* raw_read|write section */
 #if defined(__VMS)
 # include "vms_term_sock.h"

apps/apps.h

@@ -594,6 +594,7 @@ void store_setup_crl_download(X509_STORE *st);
 
 int app_isdir(const char *);
 int app_access(const char *, int flag);
+char *app_dirname(char *path);
 int fileno_stdin(void);
 int fileno_stdout(void);
 int raw_read_stdin(void *, int);

apps/opt.c

@@ -613,13 +613,17 @@ int opt_verify(int opt, X509_VERIFY_PARAM *vpm)
  */
 int opt_next(void)
 {
-    char *p;
+    char *p, *estr;
     const OPTIONS *o;
     int ival;
     long lval;
     unsigned long ulval;
     ossl_intmax_t imval;
     ossl_uintmax_t umval;
+#if !defined(_WIN32) && !defined(__VMS)
+    char *c;
+    int oerrno;
+#endif
 
     /* Look at current arg; at end of the list? */
     arg = NULL;
@@ -676,13 +680,13 @@ int opt_next(void)
             /* Just a string. */
             break;
         case '/':
-            if (app_isdir(arg) >= 0)
+            if (app_isdir(arg) > 0)
                 break;
             BIO_printf(bio_err, "%s: Not a directory: %s\n", prog, arg);
             return -1;
         case '<':
             /* Input file. */
-            if (strcmp(arg, "-") == 0 || app_access(arg, R_OK) >= 0)
+            if (strcmp(arg, "-") == 0 || app_access(arg, R_OK) == 0)
                 break;
             BIO_printf(bio_err,
                        "%s: Cannot open input file %s, %s\n",
@@ -690,11 +694,38 @@ int opt_next(void)
             return -1;
         case '>':
             /* Output file. */
-            if (strcmp(arg, "-") == 0 || app_access(arg, W_OK) >= 0 || errno == ENOENT)
+#if !defined(_WIN32) && !defined(__VMS)
+            c = OPENSSL_strdup(arg);
+            if (c == NULL) {
+                BIO_printf(bio_err,
+                           "%s: Memory allocation failure\n", prog);
+                return -1;
+            }
+            oerrno = errno;
+            errno = 0;
+            if (strcmp(arg, "-") == 0
+                || (app_access(app_dirname(c), W_OK) == 0
+                    && app_isdir(arg) <= 0
+                    && (app_access(arg, W_OK) == 0 || errno == ENOENT))) {
+                OPENSSL_free(c);
                 break;
+            }
+            OPENSSL_free(c);
+            if (errno == 0)
+                /* only possible if 'arg' is a directory */
+                estr = "is a directory";
+            else
+                estr = strerror(errno);
+            errno = oerrno;
+#else
+            if (strcmp(arg, "-") == 0 || app_access(arg, W_OK) == 0
+                || errno == ENOENT)
+                break;
+            estr = strerror(errno);
+#endif
             BIO_printf(bio_err,
                        "%s: Cannot open output file %s, %s\n",
-                       prog, arg, strerror(errno));
+                       prog, arg, estr);
             return -1;
         case 'p':
         case 'n':