zero pad DHE public key in ServerKeyExchange message for interop

Some versions of the Microsoft TLS stack have problems when the DHE public key
is encoded with fewer bytes than the DHE prime. (Backported from master)

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/1350)
This commit is contained in:
russor 2016-07-25 13:11:28 -07:00 committed by Rich Salz
parent 70705b291c
commit 5bbedd3cc1

View File

@ -1601,6 +1601,9 @@ int ssl3_send_server_key_exchange(SSL *s)
unsigned int u;
#endif
#ifndef OPENSSL_NO_DH
# ifdef OPENSSL_NO_RSA
int j;
# endif
DH *dh = NULL, *dhp;
#endif
#ifndef OPENSSL_NO_ECDH
@ -1861,6 +1864,16 @@ int ssl3_send_server_key_exchange(SSL *s)
if ((i == 2) && (type & SSL_kSRP))
n += 1 + nr[i];
else
#endif
#ifndef OPENSSL_NO_DH
/*
* for interoperability with some versions of the Microsoft TLS
* stack, we need to zero pad the DHE pub key to the same length
* as the prime, so use the length of the prime here
*/
if ((i == 2) && (type & (SSL_kEDH)))
n += 2 + nr[0];
else
#endif
n += 2 + nr[i];
}
@ -1895,6 +1908,20 @@ int ssl3_send_server_key_exchange(SSL *s)
*p = nr[i];
p++;
} else
#endif
#ifndef OPENSSL_NO_DH
/*
* for interoperability with some versions of the Microsoft TLS
* stack, we need to zero pad the DHE pub key to the same length
* as the prime
*/
if ((i == 2) && (type & (SSL_kEDH))) {
s2n(nr[0], p);
for (j = 0; j < (nr[0] - nr[2]); ++j) {
*p = 0;
++p;
}
} else
#endif
s2n(nr[i], p);
BN_bn2bin(r[i], p);