add PSS key tests

Reviewed-by: NRich Salz <rsalz@openssl.org>
Reviewed-by: NMatt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2177)

test/evptests.txt

@@ -2968,6 +2968,92 @@ Input="0123456789ABCDEF0123456789ABCDEF"
 Output=4DE433D5844043EF08D354DA03CB29068780D52706D7D1E4D50EFB7D58C9D547D83A747DDD0635A96B28F854E50145518482CB49E963054621B53C60C498D07C16E9C2789C893CF38D4D86900DE71BDE463BD2761D1271E358C7480A1AC0BAB930DDF39602AD1BC165B5D7436B516B7A7858E8EB7AB1C420EEB482F4D207F0E462B1724959320A084E13848D11D10FB593E66BF680BF6D3F345FC3E9C3DE60ABBAC37E1C6EC80A268C8D9FC49626C679097AA690BC1AA662B95EB8DB70390861AA0898229F9349B4B5FDD030D4928C47084708A933144BE23BD3C6E661B85B2C0EF9ED36D498D5B7320E8194D363D4AD478C059BAE804181965E0B81B663158A
 Result = VERIFY_ERROR
 
+# RSA PSS key tests
+
+# PSS only key, no parameter restrictions
+PrivateKey = RSA-PSS
+-----BEGIN PRIVATE KEY-----
+MIIEugIBADALBgkqhkiG9w0BAQoEggSmMIIEogIBAAKCAQEAzQCB6nsq4eoG1Z98
+c9n/uUoJYVwuS6fGNs7wjdNTPsMYVSWwFcdpuZp31nJb+cNTKptuX2Yn1fuFFgdo
+092py9NZdFEXF9w9MJ0vxH7kH5fjKtt/ndhkocR2emZuzXG8Gqz151F/SzhZT+qb
+BeQtWtqZEgCAE+RTFqTZu47QhriNKHWLrK+SLUaoaLSF0jnJuusOK2RZJxD0Ky0e
+oKS0gCwL7Ksyj4posAc721Rv7qmAnShJkSs5DBUyvH4px2WPgXX65G80My/4e8qz
+5AZJuYV3hp2g6nGDU/ByJ1SIaRNkh2DRIr5nbg/Eg90g/8Mb2pajGWbJqi51rQPe
+R+HETwIDAQABAoIBAGApeseZGxZ6BtayR1i4y+IIvrmy2eyXOL2A+QouNQBd184p
+LZ4puohb0xb+8fIJE7wKyQ1rCAiyQU2CEERB2GJKM84CM8j3gKSLN1r/AtdnEiKK
+cCSE2z+evszPu+4XCduhgoANlJ6eQhbgv/NVg4j4vZDaNzodgnQ+w/vdFCf9FoJa
+ZXoxaRLoaVNlEXyi+EXJCUBfysVfiV/BXSA4bCbueMnpkHUCmheKbB5M8MIA6KnP
+sn6dFW+G5sKtwisahKHNXKWyeQh115QHyEs1I5XLgcw/7VuwQ7ae3gwHIEVQAlzu
+jF9EAXC2Egu0jg90e82PUiEQhQ3wQ8Qo39GHBTECgYEA9vlhtHy8A1067evH3oUK
+lWtl7Nuc9gdkBj8VqkhVPFjZcv5mdQVuNd39w3vzufL2Iu4nEzclaEnJvvIXb+j3
+w/i7kbo3TdU7rz3sgU0r3sEMH9yIzdFodvJrHt+j8JQZft9NQv8fspcRA7iYyoWc
+QnKHCGqEKrQQu2nPLTWva+MCgYEA1H5ySn/0EEiycMJSSkEBh4tzFZu3PT28GHsi
+DmNbNTT5biQ6GE2T+GC2v7trccHtmh4fRYWDAjwwHpamksGgi1PQ7JypEBANgEUe
+O33GoBusSuzvjfeYhGvCNaCMu6LPTAaATMESGelWCMcU4/FDDUkfrbujKldRoE+X
+dFg0yaUCgYAh8kUrubld/QKMkUv3mfHKd+ialdUNPBbThPhFX4vXr56z36PVkdmE
+Le8jX3YwqOSMCI/2ZC4QF5RTWpM+HpdvqFCfxyiy2gxKGgjX/PN6uq4f8wAayh3B
+u7Bdnf+6oaCff7Hu84I32evMxyK5M4Q23ecRkRJ5jCaAnBqN7EMgYQKBgB91EKpi
+wtjeSjxTKCeB9B4C0Oi0Aq54Qy5EnEgRAWGhFAPwLQGICo3Mk4FS15chpHEaYHrE
+Rx6/lkgQ+VvkekXmBJnin0yXc8g3c0BPYGY3cowtA1G7A8MmyLtzpyHn+lRA6iFy
+u6FGX8ww3LDZ+Jkw6BWqH3+XKahX4A4DON1ZAoGATR8NdW/nfgEJmmUvUKiLe2hd
+xb8AmB1dI3b9DG/inNW2OHNEeTBac608FZnTnq47rgNfvW/tB8KN5wWTOHmgbkjm
+pgNobtjiVgpfavHywk+vSqlg44IYbxXu3OmiSRrnMGgN1M93i3D6qGgmqzIjR3zJ
+E3exmm1aLq6iGXYL7tU=
+-----END PRIVATE KEY-----
+
+# PSS public key default parameters
+PublicKey = RSA-PSS-DEFAULT
+-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQowAAOCAQ8AMIIBCgKCAQEAzQCB6nsq4eoG1Z98c9n/
+uUoJYVwuS6fGNs7wjdNTPsMYVSWwFcdpuZp31nJb+cNTKptuX2Yn1fuFFgdo092p
+y9NZdFEXF9w9MJ0vxH7kH5fjKtt/ndhkocR2emZuzXG8Gqz151F/SzhZT+qbBeQt
+WtqZEgCAE+RTFqTZu47QhriNKHWLrK+SLUaoaLSF0jnJuusOK2RZJxD0Ky0eoKS0
+gCwL7Ksyj4posAc721Rv7qmAnShJkSs5DBUyvH4px2WPgXX65G80My/4e8qz5AZJ
+uYV3hp2g6nGDU/ByJ1SIaRNkh2DRIr5nbg/Eg90g/8Mb2pajGWbJqi51rQPeR+HE
+TwIDAQAB
+-----END PUBLIC KEY-----
+
+# Verify using default parameters
+Verify = RSA-PSS-DEFAULT
+Input="0123456789ABCDEF0123"
+Output = 3EFE09D88509027D837BFA5F8471CF7B69E6DF395DD999BB9CA42021F15722D9AC76670507C6BCFB73F64FB2211B611B8F140E76EBDB064BD762FDBA89D019E304A0D6B274E1C2FE1DF50005598A0306AF805416094E2A5BA60BC72BDE38CE061E853ED40F14967A8B9CA4DC739B462F89558F12FDF2D8D19FBEF16AD66FE2DDDA8BEE983ECBD873064244849D8D94B5B33F45E076871A47ED653E73257A2BE2DB3C0878094B0D2B6B682C8007DFD989425FB39A1FEEC9EED5876414601A49176EC344F5E3EDEE81CA2DDD29B7364F4638112CB3A547E2BC170E28CB66BDABE863754BE8AD5BA230567B575266F4B6B4CF81F28310ABF05351CC9E2DB85D00BF
+
+# Verify using default parameters, explicitly setting parameters
+Verify = RSA-PSS-DEFAULT
+Ctrl = rsa_padding_mode:pss
+Ctrl = rsa_pss_saltlen:20
+Ctrl = digest:sha1
+Input="0123456789ABCDEF0123"
+Output = 3EFE09D88509027D837BFA5F8471CF7B69E6DF395DD999BB9CA42021F15722D9AC76670507C6BCFB73F64FB2211B611B8F140E76EBDB064BD762FDBA89D019E304A0D6B274E1C2FE1DF50005598A0306AF805416094E2A5BA60BC72BDE38CE061E853ED40F14967A8B9CA4DC739B462F89558F12FDF2D8D19FBEF16AD66FE2DDDA8BEE983ECBD873064244849D8D94B5B33F45E076871A47ED653E73257A2BE2DB3C0878094B0D2B6B682C8007DFD989425FB39A1FEEC9EED5876414601A49176EC344F5E3EDEE81CA2DDD29B7364F4638112CB3A547E2BC170E28CB66BDABE863754BE8AD5BA230567B575266F4B6B4CF81F28310ABF05351CC9E2DB85D00BF
+
+# Verify using salt length larger than minimum
+Verify = RSA-PSS-DEFAULT
+Ctrl = rsa_pss_saltlen:30
+Input="0123456789ABCDEF0123"
+Output = 6BF7EDC63A0BA184EEEC7F3020FEC8F5EBF38C2B76481881F48BCCE5796E7AB294548BA9AE810457C7723CABD1BDE94CF59CF7C0FC7461B22760C8ED703DD98E97BFDD61FA8D1181C411F6DEE5FF159F4850746D78EDEE385A363DC28E2CB373D5CAD7953F3BD5E639BE345732C03A1BDEA268814DA036EB1891C82D4012F3B903D86636055F87B96FC98806AD1B217685A4D754046A5DE0B0D7870664BE07902153EC85BA457BE7D7F89D7FE0F626D02A9CBBB2BB479DDA1A5CAE75247FB7BF6BFB15C1D3FD9E6B1573CCDBC72011C3B97716058BB11C7EA2E4E56ADAFE1F5DE6A7FD405AC5890100F9C3408EFFB5C73BF73F48177FF743B4B819D0699D507B
+
+# Attempt to change salt length below minimum
+Verify = RSA-PSS-DEFAULT
+Ctrl = rsa_pss_saltlen:0
+Result = PKEY_CTRL_ERROR
+
+# Attempt to change padding mode
+Verify = RSA-PSS-DEFAULT
+Ctrl = rsa_padding_mode:pkcs1
+Result = PKEY_CTRL_INVALID
+
+# Attempt to change digest
+Verify = RSA-PSS-DEFAULT
+Ctrl = digest:sha256
+Result = PKEY_CTRL_ERROR
+
+# Illegal decrypt
+
+Decrypt = RSA-PSS
+Result = KEYOP_INIT_ERROR
+Function = EVP_PKEY_decrypt_init
+Reason = operation not supported for this keytype
+
 # scrypt tests from draft-josefsson-scrypt-kdf-03
 PBE = scrypt
 Password = ""