Merge pull request #2827 from bdesplanq/fix-engine-support

Fix engine keyfile support.

apps/mosquitto_ctrl/options.c

@@ -593,6 +593,11 @@ int client_opts_set(struct mosquitto *mosq, struct mosq_config *cfg)
 		return 1;
 	}
 #ifdef WITH_TLS
+	if(cfg->keyform && mosquitto_string_option(mosq, MOSQ_OPT_TLS_KEYFORM, cfg->keyform)){
+		fprintf(stderr, "Error: Problem setting key form, it must be one of 'pem' or 'engine'.\n");
+		mosquitto_lib_cleanup();
+		return 1;
+	}
 	if(cfg->cafile || cfg->capath){
 		rc = mosquitto_tls_set(mosq, cfg->cafile, cfg->capath, cfg->certfile, cfg->keyfile, NULL);
 		if(rc){
@@ -615,11 +620,6 @@ int client_opts_set(struct mosquitto *mosq, struct mosq_config *cfg)
 		mosquitto_lib_cleanup();
 		return 1;
 	}
-	if(cfg->keyform && mosquitto_string_option(mosq, MOSQ_OPT_TLS_KEYFORM, cfg->keyform)){
-		fprintf(stderr, "Error: Problem setting key form, it must be one of 'pem' or 'engine'.\n");
-		mosquitto_lib_cleanup();
-		return 1;
-	}
 	if(cfg->tls_engine_kpass_sha1 && mosquitto_string_option(mosq, MOSQ_OPT_TLS_ENGINE_KPASS_SHA1, cfg->tls_engine_kpass_sha1)){
 		fprintf(stderr, "Error: Problem setting TLS engine key pass sha, is it a 40 character hex string?\n");
 		mosquitto_lib_cleanup();

client/client_shared.c

@@ -1253,6 +1253,11 @@ int client_opts_set(struct mosquitto *mosq, struct mosq_config *cfg)
 		return 1;
 	}
 #ifdef WITH_TLS
+	if(cfg->keyform && mosquitto_string_option(mosq, MOSQ_OPT_TLS_KEYFORM, cfg->keyform)){
+		err_printf(cfg, "Error: Problem setting key form, it must be one of 'pem' or 'engine'.\n");
+		mosquitto_lib_cleanup();
+		return 1;
+	}
 	if(cfg->cafile || cfg->capath){
 		rc = mosquitto_tls_set(mosq, cfg->cafile, cfg->capath, cfg->certfile, cfg->keyfile, NULL);
 		if(rc){
@@ -1289,11 +1294,6 @@ int client_opts_set(struct mosquitto *mosq, struct mosq_config *cfg)
 		mosquitto_lib_cleanup();
 		return 1;
 	}
-	if(cfg->keyform && mosquitto_string_option(mosq, MOSQ_OPT_TLS_KEYFORM, cfg->keyform)){
-		err_printf(cfg, "Error: Problem setting key form, it must be one of 'pem' or 'engine'.\n");
-		mosquitto_lib_cleanup();
-		return 1;
-	}
 	if(cfg->tls_engine_kpass_sha1 && mosquitto_string_option(mosq, MOSQ_OPT_TLS_ENGINE_KPASS_SHA1, cfg->tls_engine_kpass_sha1)){
 		err_printf(cfg, "Error: Problem setting TLS engine key pass sha, is it a 40 character hex string?\n");
 		mosquitto_lib_cleanup();

lib/options.c

@@ -179,19 +179,21 @@ int mosquitto_tls_set(struct mosquitto *mosq, const char *cafile, const char *ca
 	mosquitto__free(mosq->tls_keyfile);
 	mosq->tls_keyfile = NULL;
 	if(keyfile){
-		fptr = mosquitto__fopen(keyfile, "rt", false);
-		if(fptr){
-			fclose(fptr);
-		}else{
-			mosquitto__free(mosq->tls_cafile);
-			mosq->tls_cafile = NULL;
+		if(mosq->tls_keyform == mosq_k_pem){
+			fptr = mosquitto__fopen(keyfile, "rt", false);
+			if(fptr){
+				fclose(fptr);
+			}else{
+				mosquitto__free(mosq->tls_cafile);
+				mosq->tls_cafile = NULL;
 
-			mosquitto__free(mosq->tls_capath);
-			mosq->tls_capath = NULL;
+				mosquitto__free(mosq->tls_capath);
+				mosq->tls_capath = NULL;
 
-			mosquitto__free(mosq->tls_certfile);
-			mosq->tls_certfile = NULL;
-			return MOSQ_ERR_INVAL;
+				mosquitto__free(mosq->tls_certfile);
+				mosq->tls_certfile = NULL;
+				return MOSQ_ERR_INVAL;
+			}
 		}
 		mosq->tls_keyfile = mosquitto__strdup(keyfile);
 		if(!mosq->tls_keyfile){
@@ -290,6 +292,11 @@ int mosquitto_string_option(struct mosquitto *mosq, enum mosq_opt_t option, cons
 #if defined(WITH_TLS) && !defined(OPENSSL_NO_ENGINE)
 			mosquitto__free(mosq->tls_engine);
 			if(value){
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+				/* The "Dynamic" OpenSSL engine is not initialized by default but
+				   is required by ENGINE_by_id() to find dynamically loadable engines */
+				OPENSSL_init_crypto(OPENSSL_INIT_ENGINE_DYNAMIC, NULL);
+#endif
 				eng = ENGINE_by_id(value);
 				if(!eng){
 					return MOSQ_ERR_INVAL;

man/mosquitto.conf.5.xml

@@ -1391,9 +1391,12 @@ openssl dhparam -out dhparam.pem 2048</programlisting>
 					<term><option>keyfile</option> <replaceable>file path</replaceable></term>
 					<listitem>
 						<para>
-							Path to the PEM encoded server key. This
-							option and <option>certfile</option> must be present
-							to enable certificate based TLS encryption.
+							If <option>tls_keyform</option> equals "pem" this is the
+							path to the PEM encoded server key. This option
+							and <option>certfile</option> must be present
+							to enable certificate based TLS encryption. If
+							<option>tls_keyform</option> is "engine" this represents
+							the engine handle of the private key.
 						</para>
 						<para>
 							The private key pointed to by this option will be