[tls] Add support for RSA-PSS signature scheme

The RSA-PSS signature scheme is crowbarred somewhat awkwardly into TLS
version 1.2.  Certificates with the standard rsaEncryption OID in the
public key may be used with either PKCS#1 or RSA-PSS, which breaks the
straightforward mapping between the OID and the signature algorithm.

Extend the definition of a TLS signature hash algorithm to include a
required OID-identified algorithm in the certificate's public key.
This allows us to define signature schemes such as rsa_pss_rsae_sha256
where the signature scheme uses an algorithm that differs from the
algorithm identified in the certificate's public key.

Signed-off-by: Michael Brown <mcb30@ipxe.org>

src/crypto/mishmash/ecdsa_sha224.c

@@ -45,6 +45,7 @@ struct asn1_algorithm ecdsa_with_sha224_algorithm __asn1_algorithm = {
 struct tls_signature_hash_algorithm
 tls_ecdsa_sha224 __tls_sig_hash_algorithm = {
 	.code = htons ( TLS_ECDSA_SHA224_ALGORITHM ),
+	.algorithm = &ecpubkey_algorithm,
 	.pubkey = &ecdsa_algorithm,
 	.digest = &sha224_algorithm,
 };

src/crypto/mishmash/ecdsa_sha256.c

@@ -45,6 +45,7 @@ struct asn1_algorithm ecdsa_with_sha256_algorithm __asn1_algorithm = {
 struct tls_signature_hash_algorithm
 tls_ecdsa_sha256 __tls_sig_hash_algorithm = {
 	.code = htons ( TLS_ECDSA_SHA256_ALGORITHM ),
+	.algorithm = &ecpubkey_algorithm,
 	.pubkey = &ecdsa_algorithm,
 	.digest = &sha256_algorithm,
 };

src/crypto/mishmash/ecdsa_sha384.c

@@ -45,6 +45,7 @@ struct asn1_algorithm ecdsa_with_sha384_algorithm __asn1_algorithm = {
 struct tls_signature_hash_algorithm
 tls_ecdsa_sha384 __tls_sig_hash_algorithm = {
 	.code = htons ( TLS_ECDSA_SHA384_ALGORITHM ),
+	.algorithm = &ecpubkey_algorithm,
 	.pubkey = &ecdsa_algorithm,
 	.digest = &sha384_algorithm,
 };

src/crypto/mishmash/ecdsa_sha512.c

@@ -45,6 +45,7 @@ struct asn1_algorithm ecdsa_with_sha512_algorithm __asn1_algorithm = {
 struct tls_signature_hash_algorithm
 tls_ecdsa_sha512 __tls_sig_hash_algorithm = {
 	.code = htons ( TLS_ECDSA_SHA512_ALGORITHM ),
+	.algorithm = &ecpubkey_algorithm,
 	.pubkey = &ecdsa_algorithm,
 	.digest = &sha512_algorithm,
 };

src/crypto/mishmash/oid_rsa_pss.c

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2026 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+FILE_SECBOOT ( PERMITTED );
+
+#include <ipxe/rsa.h>
+#include <ipxe/asn1.h>
+
+/** "rsassa-pss" object identifier */
+static uint8_t oid_rsassa_pss[] = { ASN1_OID_RSASSA_PSS };
+
+/** "rsassa-pss" OID-identified algorithm */
+struct asn1_algorithm rsassa_pss_algorithm __asn1_algorithm = {
+	.name = "rsassa-pss",
+	.pubkey = &rsa_pss_algorithm,
+	.digest = NULL,
+	.oid = ASN1_CURSOR ( oid_rsassa_pss ),
+};

src/crypto/mishmash/rsa_sha1.c

@@ -56,6 +56,7 @@ struct rsa_digestinfo_prefix rsa_sha1_prefix __rsa_digestinfo_prefix = {
 /** RSA with SHA-1 signature hash algorithm */
 struct tls_signature_hash_algorithm tls_rsa_sha1 __tls_sig_hash_algorithm = {
 	.code = htons ( TLS_RSA_SHA1_ALGORITHM ),
+	.algorithm = &rsa_encryption_algorithm,
 	.pubkey = &rsa_algorithm,
 	.digest = &sha1_algorithm,
 };

src/crypto/mishmash/rsa_sha224.c

@@ -56,6 +56,7 @@ struct rsa_digestinfo_prefix rsa_sha224_prefix __rsa_digestinfo_prefix = {
 /** RSA with SHA-224 signature hash algorithm */
 struct tls_signature_hash_algorithm tls_rsa_sha224 __tls_sig_hash_algorithm = {
 	.code = htons ( TLS_RSA_SHA224_ALGORITHM ),
+	.algorithm = &rsa_encryption_algorithm,
 	.pubkey = &rsa_algorithm,
 	.digest = &sha224_algorithm,
 };

src/crypto/mishmash/rsa_sha256.c

@@ -56,6 +56,25 @@ struct rsa_digestinfo_prefix rsa_sha256_prefix __rsa_digestinfo_prefix = {
 /** RSA with SHA-256 signature hash algorithm */
 struct tls_signature_hash_algorithm tls_rsa_sha256 __tls_sig_hash_algorithm = {
 	.code = htons ( TLS_RSA_SHA256_ALGORITHM ),
+	.algorithm = &rsa_encryption_algorithm,
 	.pubkey = &rsa_algorithm,
 	.digest = &sha256_algorithm,
 };
+
+/** RSA-PSS with rsaEncryption OID and SHA-256 signature hash algorithm */
+struct tls_signature_hash_algorithm
+tls_rsa_pss_rsae_sha256 __tls_sig_hash_algorithm = {
+	.code = htons ( TLS_RSA_PSS_RSAE_SHA256_ALGORITHM ),
+	.algorithm = &rsa_encryption_algorithm,
+	.pubkey = &rsa_pss_algorithm,
+	.digest = &sha256_algorithm,
+};
+
+/** RSA-PSS with RSASSA-PSS OID and SHA-256 signature hash algorithm */
+struct tls_signature_hash_algorithm
+tls_rsa_pss_pss_sha256 __tls_sig_hash_algorithm = {
+	.code = htons ( TLS_RSA_PSS_PSS_SHA256_ALGORITHM ),
+	.algorithm = &rsassa_pss_algorithm,
+	.pubkey = &rsa_pss_algorithm,
+	.digest = &sha256_algorithm,
+};

src/crypto/mishmash/rsa_sha384.c

@@ -56,6 +56,25 @@ struct rsa_digestinfo_prefix rsa_sha384_prefix __rsa_digestinfo_prefix = {
 /** RSA with SHA-384 signature hash algorithm */
 struct tls_signature_hash_algorithm tls_rsa_sha384 __tls_sig_hash_algorithm = {
 	.code = htons ( TLS_RSA_SHA384_ALGORITHM ),
+	.algorithm = &rsa_encryption_algorithm,
 	.pubkey = &rsa_algorithm,
 	.digest = &sha384_algorithm,
 };
+
+/** RSA-PSS with rsaEncryption OID and SHA-384 signature hash algorithm */
+struct tls_signature_hash_algorithm
+tls_rsa_pss_rsae_sha384 __tls_sig_hash_algorithm = {
+	.code = htons ( TLS_RSA_PSS_RSAE_SHA384_ALGORITHM ),
+	.algorithm = &rsa_encryption_algorithm,
+	.pubkey = &rsa_pss_algorithm,
+	.digest = &sha384_algorithm,
+};
+
+/** RSA-PSS with RSASSA-PSS OID and SHA-384 signature hash algorithm */
+struct tls_signature_hash_algorithm
+tls_rsa_pss_pss_sha384 __tls_sig_hash_algorithm = {
+	.code = htons ( TLS_RSA_PSS_PSS_SHA384_ALGORITHM ),
+	.algorithm = &rsassa_pss_algorithm,
+	.pubkey = &rsa_pss_algorithm,
+	.digest = &sha384_algorithm,
+};

src/crypto/mishmash/rsa_sha512.c

@@ -56,6 +56,25 @@ struct rsa_digestinfo_prefix rsa_sha512_prefix __rsa_digestinfo_prefix = {
 /** RSA with SHA-512 signature hash algorithm */
 struct tls_signature_hash_algorithm tls_rsa_sha512 __tls_sig_hash_algorithm = {
 	.code = htons ( TLS_RSA_SHA512_ALGORITHM ),
+	.algorithm = &rsa_encryption_algorithm,
 	.pubkey = &rsa_algorithm,
 	.digest = &sha512_algorithm,
 };
+
+/** RSA-PSS with rsaEncryption OID and SHA-512 signature hash algorithm */
+struct tls_signature_hash_algorithm
+tls_rsa_pss_rsae_sha512 __tls_sig_hash_algorithm = {
+	.code = htons ( TLS_RSA_PSS_RSAE_SHA512_ALGORITHM ),
+	.algorithm = &rsa_encryption_algorithm,
+	.pubkey = &rsa_pss_algorithm,
+	.digest = &sha512_algorithm,
+};
+
+/** RSA-PSS with RSASSA-PSS OID and SHA-512 signature hash algorithm */
+struct tls_signature_hash_algorithm
+tls_rsa_pss_pss_sha512 __tls_sig_hash_algorithm = {
+	.code = htons ( TLS_RSA_PSS_PSS_SHA512_ALGORITHM ),
+	.algorithm = &rsassa_pss_algorithm,
+	.pubkey = &rsa_pss_algorithm,
+	.digest = &sha512_algorithm,
+};

src/include/ipxe/asn1.h

@@ -182,6 +182,12 @@ struct asn1_builder_header {
 	ASN1_OID_TRIPLE ( 113549 ), ASN1_OID_SINGLE ( 1 ),	\
 	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 5 )
 
+/** ASN.1 OID for rsassa-pss (1.2.840.113549.1.1.10) */
+#define ASN1_OID_RSASSA_PSS					\
+	ASN1_OID_INITIAL ( 1, 2 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_TRIPLE ( 113549 ), ASN1_OID_SINGLE ( 1 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 10 )
+
 /** ASN.1 OID for sha256WithRSAEncryption (1.2.840.113549.1.1.11) */
 #define ASN1_OID_SHA256WITHRSAENCRYPTION			\
 	ASN1_OID_INITIAL ( 1, 2 ), ASN1_OID_DOUBLE ( 840 ),	\
@@ -437,6 +443,7 @@ struct asn1_algorithm {
 
 /* ASN.1 OID-identified algorithms */
 extern struct asn1_algorithm rsa_encryption_algorithm __asn1_algorithm;
+extern struct asn1_algorithm rsassa_pss_algorithm __asn1_algorithm;
 extern struct asn1_algorithm md5_with_rsa_encryption_algorithm __asn1_algorithm;
 extern struct asn1_algorithm
 sha1_with_rsa_encryption_algorithm __asn1_algorithm;

src/include/ipxe/tls.h

@@ -123,6 +123,12 @@ struct tls_header {
 #define TLS_ECDSA_SHA384_ALGORITHM 0x0503
 #define TLS_RSA_SHA512_ALGORITHM 0x0601
 #define TLS_ECDSA_SHA512_ALGORITHM 0x0603
+#define TLS_RSA_PSS_RSAE_SHA256_ALGORITHM 0x0804
+#define TLS_RSA_PSS_RSAE_SHA384_ALGORITHM 0x0805
+#define TLS_RSA_PSS_RSAE_SHA512_ALGORITHM 0x0806
+#define TLS_RSA_PSS_PSS_SHA256_ALGORITHM 0x0809
+#define TLS_RSA_PSS_PSS_SHA384_ALGORITHM 0x080a
+#define TLS_RSA_PSS_PSS_SHA512_ALGORITHM 0x080b
 
 /* TLS server name extension */
 #define TLS_SERVER_NAME 0
@@ -284,6 +290,8 @@ struct tls_signature_hash_algorithm {
 	struct digest_algorithm *digest;
 	/** Public-key algorithm */
 	struct pubkey_algorithm *pubkey;
+	/** Required certificate OID-identified algorithm */
+	struct asn1_algorithm *algorithm;
 	/** Numeric code (in network-endian order) */
 	uint16_t code;
 };
@@ -392,6 +400,8 @@ struct tls_server {
 	struct x509_root *root;
 	/** Certificate chain */
 	struct x509_chain *chain;
+	/** Public key algorithm (within server certificate) */
+	struct asn1_algorithm *algorithm;
 	/** Public key (within server certificate) */
 	struct asn1_cursor key;
 	/** Certificate validator */

src/net/tls.c

@@ -1463,10 +1463,9 @@ static int tls_verify_dh_params ( struct tls_connection *tls,
 		digest = sig_hash->digest;
 		DBGC ( tls, "TLS %p using signature hash %s-%s\n",
 		       tls, pubkey->name, digest->name );
-		if ( pubkey != cipherspec->suite->pubkey ) {
-			DBGC ( tls, "TLS %p ServerKeyExchange incorrect "
-			       "signature algorithm %s (expected %s)\n", tls,
-			       pubkey->name, cipherspec->suite->pubkey->name );
+		if ( sig_hash->algorithm != tls->server.algorithm ) {
+			DBGC ( tls, "TLS %p cannot use %s public key\n",
+			       tls, tls->server.algorithm->name );
 			return -EPERM_KEY_EXCHANGE;
 		}
 	} else {
@@ -2331,6 +2330,7 @@ static int tls_parse_chain ( struct tls_connection *tls,
 	memset ( &tls->server.key, 0, sizeof ( tls->server.key ) );
 	x509_chain_put ( tls->server.chain );
 	tls->server.chain = NULL;
+	tls->server.algorithm = NULL;
 
 	/* Create certificate chain */
 	tls->server.chain = x509_alloc_chain();
@@ -2391,6 +2391,7 @@ static int tls_parse_chain ( struct tls_connection *tls,
 	memset ( &tls->server.key, 0, sizeof ( tls->server.key ) );
 	x509_chain_put ( tls->server.chain );
 	tls->server.chain = NULL;
+	tls->server.algorithm = NULL;
  err_alloc_chain:
 	return rc;
 }
@@ -3685,6 +3686,7 @@ static void tls_validator_done ( struct tls_connection *tls, int rc ) {
 	}
 
 	/* Extract the now trusted server public key */
+	tls->server.algorithm = cert->subject.public_key.algorithm;
 	memcpy ( &tls->server.key, &cert->subject.public_key.raw,
 		 sizeof ( tls->server.key ) );