X509_NAME_cmp: restrict normal return values to {-1,0,1} to avoid confusion with -2 for error

Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/12769)
This commit is contained in:
Dr. David von Oheimb 2020-09-02 13:12:22 +02:00
parent 2aa91df406
commit bc64c5a69b
2 changed files with 21 additions and 22 deletions

View File

@ -30,8 +30,8 @@ int X509_issuer_and_serial_cmp(const X509 *a, const X509 *b)
ai = &a->cert_info; ai = &a->cert_info;
bi = &b->cert_info; bi = &b->cert_info;
i = ASN1_INTEGER_cmp(&ai->serialNumber, &bi->serialNumber); i = ASN1_INTEGER_cmp(&ai->serialNumber, &bi->serialNumber);
if (i) if (i != 0)
return i; return i < 0 ? -1 : 1;
return X509_NAME_cmp(ai->issuer, bi->issuer); return X509_NAME_cmp(ai->issuer, bi->issuer);
} }
@ -83,7 +83,9 @@ int X509_CRL_cmp(const X509_CRL *a, const X509_CRL *b)
int X509_CRL_match(const X509_CRL *a, const X509_CRL *b) int X509_CRL_match(const X509_CRL *a, const X509_CRL *b)
{ {
return memcmp(a->sha1_hash, b->sha1_hash, 20); int rv = memcmp(a->sha1_hash, b->sha1_hash, 20);
return rv < 0 ? -1 : rv > 0;
} }
X509_NAME *X509_get_issuer_name(const X509 *a) X509_NAME *X509_get_issuer_name(const X509 *a)
@ -149,18 +151,18 @@ int X509_cmp(const X509 *a, const X509 *b)
return -2; return -2;
rv = memcmp(a->sha1_hash, b->sha1_hash, SHA_DIGEST_LENGTH); rv = memcmp(a->sha1_hash, b->sha1_hash, SHA_DIGEST_LENGTH);
if (rv) if (rv != 0)
return rv; return rv < 0 ? -1 : 1;
/* Check for match against stored encoding too */ /* Check for match against stored encoding too */
if (!a->cert_info.enc.modified && !b->cert_info.enc.modified) { if (!a->cert_info.enc.modified && !b->cert_info.enc.modified) {
if (a->cert_info.enc.len < b->cert_info.enc.len) if (a->cert_info.enc.len < b->cert_info.enc.len)
return -1; return -1;
if (a->cert_info.enc.len > b->cert_info.enc.len) if (a->cert_info.enc.len > b->cert_info.enc.len)
return 1; return 1;
return memcmp(a->cert_info.enc.enc, b->cert_info.enc.enc, rv = memcmp(a->cert_info.enc.enc,
a->cert_info.enc.len); b->cert_info.enc.enc, a->cert_info.enc.len);
} }
return rv; return rv < 0 ? -1 : rv > 0;
} }
int X509_add_cert_new(STACK_OF(X509) **sk, X509 *cert, int flags) int X509_add_cert_new(STACK_OF(X509) **sk, X509 *cert, int flags)
@ -242,12 +244,10 @@ int X509_NAME_cmp(const X509_NAME *a, const X509_NAME *b)
} }
ret = a->canon_enclen - b->canon_enclen; ret = a->canon_enclen - b->canon_enclen;
if (ret == 0 && a->canon_enclen != 0)
ret = memcmp(a->canon_enc, b->canon_enc, a->canon_enclen);
if (ret != 0 || a->canon_enclen == 0) return ret < 0 ? -1 : ret > 0;
return ret;
return memcmp(a->canon_enc, b->canon_enc, a->canon_enclen);
} }
unsigned long X509_NAME_hash(const X509_NAME *x) unsigned long X509_NAME_hash(const X509_NAME *x)
@ -410,9 +410,9 @@ static int check_suite_b(EVP_PKEY *pkey, int sign_nid, unsigned long *pflags)
return X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM; return X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM;
if (!(*pflags & X509_V_FLAG_SUITEB_128_LOS_ONLY)) if (!(*pflags & X509_V_FLAG_SUITEB_128_LOS_ONLY))
return X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED; return X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED;
} else } else {
return X509_V_ERR_SUITE_B_INVALID_CURVE; return X509_V_ERR_SUITE_B_INVALID_CURVE;
}
return X509_V_OK; return X509_V_OK;
} }
@ -430,9 +430,9 @@ int X509_chain_check_suiteb(int *perror_depth, X509 *x, STACK_OF(X509) *chain,
if (x == NULL) { if (x == NULL) {
x = sk_X509_value(chain, 0); x = sk_X509_value(chain, 0);
i = 1; i = 1;
} else } else {
i = 0; i = 0;
}
pk = X509_get0_pubkey(x); pk = X509_get0_pubkey(x);
/* /*

View File

@ -47,9 +47,8 @@ of just the issuer name.
=head1 RETURN VALUES =head1 RETURN VALUES
Like common memory comparison functions, the B<X509> comparison functions return The B<X509> comparison functions return B<-1>, B<0>, or B<1> if object B<a> is
an integer less than, equal to, or greater than zero if object B<a> is found to found to be less than, to match, or be greater than object B<b>, respectively.
be less than, to match, or be greater than object B<b>, respectively.
X509_NAME_cmp(), X509_issuer_and_serial_cmp(), X509_issuer_name_cmp(), X509_NAME_cmp(), X509_issuer_and_serial_cmp(), X509_issuer_name_cmp(),
X509_subject_name_cmp() and X509_CRL_cmp() may return B<-2> to indicate an error. X509_subject_name_cmp() and X509_CRL_cmp() may return B<-2> to indicate an error.