ENCODER / DECODER: Add functions to encode/decode to/from a buffer

This adds OSSL_ENCODER_to_data() and OSSL_DECODER_from_data().  These
functions allow fairly simple rewrites of type-specific i2d and d2i
calls.

Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/13094)

crypto/encode_decode/decoder_lib.c

@@ -77,6 +77,27 @@ int OSSL_DECODER_from_fp(OSSL_DECODER_CTX *ctx, FILE *fp)
 }
 #endif
 
+int OSSL_DECODER_from_data(OSSL_DECODER_CTX *ctx, const unsigned char **pdata,
+                           size_t *pdata_len)
+{
+    BIO *membio;
+    int ret = 0;
+
+    if (pdata == NULL || *pdata == NULL || pdata_len == NULL) {
+        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_PASSED_NULL_PARAMETER);
+        return 0;
+    }
+
+    membio = BIO_new_mem_buf(*pdata, (int)*pdata_len);
+    if (OSSL_DECODER_from_bio(ctx, membio)) {
+        *pdata_len = (size_t)BIO_get_mem_data(membio, pdata);
+        ret = 1;
+    }
+    BIO_free(membio);
+
+    return ret;
+}
+
 int OSSL_DECODER_CTX_set_input_type(OSSL_DECODER_CTX *ctx,
                                     const char *input_type)
 {

crypto/encode_decode/encoder_lib.c

@@ -49,6 +49,54 @@ int OSSL_ENCODER_to_fp(OSSL_ENCODER_CTX *ctx, FILE *fp)
 }
 #endif
 
+int OSSL_ENCODER_to_data(OSSL_ENCODER_CTX *ctx, unsigned char **pdata,
+                         size_t *pdata_len)
+{
+    BIO *out = BIO_new(BIO_s_mem());
+    BUF_MEM *buf = NULL;
+    int ret = 0;
+
+    if (pdata_len == NULL) {
+        ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_PASSED_NULL_PARAMETER);
+        return 0;
+    }
+
+    if (OSSL_ENCODER_to_bio(ctx, out)
+        && BIO_get_mem_ptr(out, &buf) > 0) {
+        ret = 1; /* Hope for the best. A too small buffer will clear this */
+
+        if (pdata != NULL && *pdata != NULL) {
+            if (*pdata_len < buf->length)
+                /*
+                 * It's tempting to do |*pdata_len = (size_t)buf->length|
+                 * However, it's believed to be confusing more than helpful,
+                 * so we don't.
+                 */
+                ret = 0;
+            else
+                *pdata_len -= buf->length;
+        } else {
+            /* The buffer with the right size is already allocated for us */
+            *pdata_len = (size_t)buf->length;
+        }
+
+        if (ret) {
+            if (pdata != NULL) {
+                if (*pdata != NULL) {
+                    memcpy(*pdata, buf->data, buf->length);
+                    *pdata += buf->length;
+                } else {
+                    /* In this case, we steal the data from BIO_s_mem() */
+                    *pdata = (unsigned char *)buf->data;
+                    buf->data = NULL;
+                }
+            }
+        }
+    }
+    BIO_free(out);
+    return ret;
+}
+
 int OSSL_ENCODER_CTX_set_output_type(OSSL_ENCODER_CTX *ctx,
                                      const char *output_type)
 {

doc/man3/OSSL_DECODER_from_bio.pod

@@ -2,6 +2,7 @@
 
 =head1 NAME
 
+OSSL_DECODER_from_data,
 OSSL_DECODER_from_bio,
 OSSL_DECODER_from_fp
 - Routines to perform a decoding
@@ -12,6 +13,8 @@ OSSL_DECODER_from_fp
 
  int OSSL_DECODER_from_bio(OSSL_DECODER_CTX *ctx, BIO *in);
  int OSSL_DECODER_from_fp(OSSL_DECODER_CTX *ctx, FILE *fp);
+ int OSSL_DECODER_from_data(OSSL_DECODER_CTX *ctx, const unsigned char **pdata,
+                            size_t *pdata_len);
 
 Feature availability macros:
 
@@ -24,6 +27,12 @@ is undefined.
 
 =head1 DESCRIPTION
 
+OSSL_DECODER_from_data() runs the decoding process for the context I<ctx>,
+with input coming from I<*pdata>, I<*pdata_len> bytes long.  Both I<*pdata>
+and I<*pdata_len> must be non-NULL.  When OSSL_DECODER_from_data() returns,
+I<*pdata> is updated to point at the location after what has been decoded,
+and I<*pdata_len> to have the number of remaining bytes.
+
 OSSL_DECODER_from_bio() runs the decoding process for the context I<ctx>,
 with the input coming from the B<BIO> I<in>.  Should it make a difference,
 it's recommended to have the BIO set in binary mode rather than text mode.

doc/man3/OSSL_ENCODER_to_bio.pod

@@ -2,6 +2,7 @@
 
 =head1 NAME
 
+OSSL_ENCODER_to_data,
 OSSL_ENCODER_to_bio,
 OSSL_ENCODER_to_fp
 - Routines to perform an encoding
@@ -10,6 +11,8 @@ OSSL_ENCODER_to_fp
 
  #include <openssl/encoder.h>
 
+ int OSSL_ENCODER_to_data(OSSL_ENCODER_CTX *ctx, unsigned char **pdata,
+                          size_t *pdata_len);
  int OSSL_ENCODER_to_bio(OSSL_ENCODER_CTX *ctx, BIO *out);
  int OSSL_ENCODER_to_fp(OSSL_ENCODER_CTX *ctx, FILE *fp);
 
@@ -24,20 +27,33 @@ is undefined.
 
 =head1 DESCRIPTION
 
+OSSL_ENCODER_to_data() runs the encoding process for the context I<ctx>,
+with the output going to the I<*pdata> and I<*pdata_len>.
+If I<*pdata> is NULL when OSSL_ENCODER_to_data() is called, a buffer will be
+allocated using L<OPENSSL_zalloc(3)>, and I<*pdata> will be set to point at
+the start of that buffer, and I<*pdata_len> will be assigned its length when
+OSSL_ENCODER_to_data() returns.
+If I<*pdata> is non-NULL when OSSL_ENCODER_to_data() is called, I<*pdata_len>
+is assumed to have its size.  In this case, I<*pdata> will be set to point
+after the encoded bytes, and I<*pdata_len> will be assigned the number of
+remaining bytes.
+
 OSSL_ENCODER_to_bio() runs the encoding process for the context I<ctx>, with
-the output going to the B<BIO> I<out>.  The application is required to set
-up the B<BIO> properly, for example to have it in text or binary mode if
-that's appropriate.
+the output going to the B<BIO> I<out>.
+
+OSSL_ENCODER_to_fp() does the same thing as OSSL_ENCODER_to_bio(), except
+that the output is going to the B<FILE> I<fp>.
 
 =for comment Know your encoder!
 
-OSSL_ENCODER_to_fp() does the same thing as OSSL_ENCODER_to_bio(),
-except that the output is going to the B<FILE> I<fp>.
+For OSSL_ENCODER_to_bio() and OSSL_ENCODER_to_fp(), the application is
+required to set up the B<BIO> or B<FILE> properly, for example to have
+it in text or binary mode as is appropriate for the encoder output type.
 
 =head1 RETURN VALUES
 
-OSSL_ENCODER_to_bio() and OSSL_ENCODER_to_fp() return 1 on success, or 0 on
-failure.
+OSSL_ENCODER_to_bio(), OSSL_ENCODER_to_fp() and OSSL_ENCODER_to_data()
+return 1 on success, or 0 on failure.
 
 =begin comment TODO(3.0) Add examples!
 

include/openssl/decoder.h

@@ -106,6 +106,8 @@ int OSSL_DECODER_from_bio(OSSL_DECODER_CTX *ctx, BIO *in);
 #ifndef OPENSSL_NO_STDIO
 int OSSL_DECODER_from_fp(OSSL_DECODER_CTX *ctx, FILE *in);
 #endif
+int OSSL_DECODER_from_data(OSSL_DECODER_CTX *ctx, const unsigned char **pdata,
+                           size_t *pdata_len);
 
 /*
  * Create the OSSL_DECODER_CTX with an associated type.  This will perform

include/openssl/encoder.h

@@ -101,6 +101,8 @@ int OSSL_ENCODER_to_bio(OSSL_ENCODER_CTX *ctx, BIO *out);
 #ifndef OPENSSL_NO_STDIO
 int OSSL_ENCODER_to_fp(OSSL_ENCODER_CTX *ctx, FILE *fp);
 #endif
+int OSSL_ENCODER_to_data(OSSL_ENCODER_CTX *ctx, unsigned char **pdata,
+                         size_t *pdata_len);
 
 /*
  * Create the OSSL_ENCODER_CTX with an associated type.  This will perform

util/libcrypto.num

@@ -5286,3 +5286,5 @@ EVP_KEM_gettable_ctx_params             ?	3_0_0	EXIST::FUNCTION:
 EVP_KEM_settable_ctx_params             ?	3_0_0	EXIST::FUNCTION:
 PKCS7_type_is_other                     ?	3_0_0	EXIST::FUNCTION:
 PKCS7_get_octet_string                  ?	3_0_0	EXIST::FUNCTION:
+OSSL_DECODER_from_data                  ?	3_0_0	EXIST::FUNCTION:
+OSSL_ENCODER_to_data                    ?	3_0_0	EXIST::FUNCTION: