Add TLS tests for RSA-PSS Restricted certificates

Reviewed-by: NPaul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/9553)

(cherry picked from commit 20946b94658416d2fed0b9d9c7adfbe4b7d70515)

test/ssl-tests/20-cert-select.conf.in

@@ -36,6 +36,12 @@ my $server_pss_only = {
     "PrivateKey" => test_pem("server-pss-key.pem"),
 };
 
+my $server_pss_restrict_only = {
+    "Certificate" => test_pem("server-pss-restrict-cert.pem"),
+    "PrivateKey" => test_pem("server-pss-restrict-key.pem"),
+};
+
+
 my $server_rsa_all = {
     "PSS.Certificate" => test_pem("server-pss-cert.pem"),
     "PSS.PrivateKey" => test_pem("server-pss-key.pem"),
@@ -379,6 +385,19 @@ our @tests = (
             "ExpectedResult" => "Success"
         },
     },
+    {
+        name => "Only RSA-PSS Certificate Valid Signature Algorithms",
+        server => $server_pss_only,
+        client => {
+            "SignatureAlgorithms" => "rsa_pss_pss_sha512",
+        },
+        test   => {
+            "ExpectedServerCertType" => "RSA-PSS",
+            "ExpectedServerSignHash" => "SHA512",
+            "ExpectedServerSignType" => "RSA-PSS",
+            "ExpectedResult" => "Success"
+        },
+    },
     {
         name => "RSA-PSS Certificate, no PSS signature algorithms",
         server => $server_pss_only,
@@ -389,6 +408,53 @@ our @tests = (
             "ExpectedResult" => "ServerFail"
         },
     },
+    {
+        name => "Only RSA-PSS Restricted Certificate",
+        server => $server_pss_restrict_only,
+        client => {},
+        test   => {
+            "ExpectedServerCertType" => "RSA-PSS",
+            "ExpectedServerSignHash" => "SHA256",
+            "ExpectedServerSignType" => "RSA-PSS",
+            "ExpectedResult" => "Success"
+        },
+    },
+    {
+        name => "RSA-PSS Restricted Certificate Valid Signature Algorithms",
+        server => $server_pss_restrict_only,
+        client => {
+            "SignatureAlgorithms" => "rsa_pss_pss_sha256:rsa_pss_pss_sha512",
+        },
+        test   => {
+            "ExpectedServerCertType" => "RSA-PSS",
+            "ExpectedServerSignHash" => "SHA256",
+            "ExpectedServerSignType" => "RSA-PSS",
+            "ExpectedResult" => "Success"
+        },
+    },
+    {
+        name => "RSA-PSS Restricted Cert client prefers invalid Signature Algorithm",
+        server => $server_pss_restrict_only,
+        client => {
+            "SignatureAlgorithms" => "rsa_pss_pss_sha512:rsa_pss_pss_sha256",
+        },
+        test   => {
+            "ExpectedServerCertType" => "RSA-PSS",
+            "ExpectedServerSignHash" => "SHA256",
+            "ExpectedServerSignType" => "RSA-PSS",
+            "ExpectedResult" => "Success"
+        },
+    },
+    {
+        name => "RSA-PSS Restricted Certificate Invalid Signature Algorithms",
+        server => $server_pss_restrict_only,
+        client => {
+            "SignatureAlgorithms" => "rsa_pss_pss_sha512",
+        },
+        test   => {
+            "ExpectedResult" => "ServerFail"
+        },
+    },
     {
         name => "RSA key exchange with all RSA certificate types",
         server => $server_rsa_all,