From f2d5df8a302dcd940689310ef8623d9b48bbc68a Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Tue, 24 Nov 2020 15:07:53 +0100
Subject: [PATCH] homed: move helper calls for RSA encryption to shared code

---
 src/home/homectl-pkcs11.c | 42 ++-------------------------------------
 src/shared/meson.build    |  1 +
 src/shared/openssl-util.c | 41 ++++++++++++++++++++++++++++++++++++++
 src/shared/openssl-util.h |  4 ++++
 4 files changed, 48 insertions(+), 40 deletions(-)
 create mode 100644 src/shared/openssl-util.c

diff --git a/src/home/homectl-pkcs11.c b/src/home/homectl-pkcs11.c
index 4b7f8336aa..f4cfb94d2c 100644
--- a/src/home/homectl-pkcs11.c
+++ b/src/home/homectl-pkcs11.c
@@ -93,43 +93,6 @@ static int acquire_pkcs11_certificate(
 #endif
 }
 
-static int encrypt_bytes(
-                EVP_PKEY *pkey,
-                const void *decrypted_key,
-                size_t decrypted_key_size,
-                void **ret_encrypt_key,
-                size_t *ret_encrypt_key_size) {
-
-        _cleanup_(EVP_PKEY_CTX_freep) EVP_PKEY_CTX *ctx = NULL;
-        _cleanup_free_ void *b = NULL;
-        size_t l;
-
-        ctx = EVP_PKEY_CTX_new(pkey, NULL);
-        if (!ctx)
-                return log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to allocate public key context");
-
-        if (EVP_PKEY_encrypt_init(ctx) <= 0)
-                return log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to initialize public key context");
-
-        if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING) <= 0)
-                return log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to configure PKCS#1 padding");
-
-        if (EVP_PKEY_encrypt(ctx, NULL, &l, decrypted_key, decrypted_key_size) <= 0)
-                return log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to determine encrypted key size");
-
-        b = malloc(l);
-        if (!b)
-                return log_oom();
-
-        if (EVP_PKEY_encrypt(ctx, b, &l, decrypted_key, decrypted_key_size) <= 0)
-                return log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to determine encrypted key size");
-
-        *ret_encrypt_key = TAKE_PTR(b);
-        *ret_encrypt_key_size = l;
-
-        return 0;
-}
-
 static int add_pkcs11_encrypted_key(
                 JsonVariant **v,
                 const char *uri,
@@ -267,9 +230,8 @@ int identity_add_pkcs11_key_data(JsonVariant **v, const char *uri) {
         size_t decrypted_key_size, encrypted_key_size;
         _cleanup_(X509_freep) X509 *cert = NULL;
         EVP_PKEY *pkey;
+        int bits, r;
         RSA *rsa;
-        int bits;
-        int r;
 
         assert(v);
 
@@ -308,7 +270,7 @@ int identity_add_pkcs11_key_data(JsonVariant **v, const char *uri) {
         if (r < 0)
                 return log_error_errno(r, "Failed to generate random key: %m");
 
-        r = encrypt_bytes(pkey, decrypted_key, decrypted_key_size, &encrypted_key, &encrypted_key_size);
+        r = rsa_encrypt_bytes(pkey, decrypted_key, decrypted_key_size, &encrypted_key, &encrypted_key_size);
         if (r < 0)
                 return log_error_errno(r, "Failed to encrypt key: %m");
 
diff --git a/src/shared/meson.build b/src/shared/meson.build
index cc844eb09e..ebe98df24a 100644
--- a/src/shared/meson.build
+++ b/src/shared/meson.build
@@ -183,6 +183,7 @@ shared_sources = files('''
         nsflags.h
         numa-util.c
         numa-util.h
+        openssl-util.c
         openssl-util.h
         os-util.c
         os-util.h
diff --git a/src/shared/openssl-util.c b/src/shared/openssl-util.c
new file mode 100644
index 0000000000..1e2aaa2130
--- /dev/null
+++ b/src/shared/openssl-util.c
@@ -0,0 +1,41 @@
+#include "openssl-util.h"
+#include "alloc-util.h"
+
+#if HAVE_OPENSSL
+int rsa_encrypt_bytes(
+                EVP_PKEY *pkey,
+                const void *decrypted_key,
+                size_t decrypted_key_size,
+                void **ret_encrypt_key,
+                size_t *ret_encrypt_key_size) {
+
+        _cleanup_(EVP_PKEY_CTX_freep) EVP_PKEY_CTX *ctx = NULL;
+        _cleanup_free_ void *b = NULL;
+        size_t l;
+
+        ctx = EVP_PKEY_CTX_new(pkey, NULL);
+        if (!ctx)
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to allocate public key context");
+
+        if (EVP_PKEY_encrypt_init(ctx) <= 0)
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to initialize public key context");
+
+        if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING) <= 0)
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to configure PKCS#1 padding");
+
+        if (EVP_PKEY_encrypt(ctx, NULL, &l, decrypted_key, decrypted_key_size) <= 0)
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to determine encrypted key size");
+
+        b = malloc(l);
+        if (!b)
+                return -ENOMEM;
+
+        if (EVP_PKEY_encrypt(ctx, b, &l, decrypted_key, decrypted_key_size) <= 0)
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to determine encrypted key size");
+
+        *ret_encrypt_key = TAKE_PTR(b);
+        *ret_encrypt_key_size = l;
+
+        return 0;
+}
+#endif
diff --git a/src/shared/openssl-util.h b/src/shared/openssl-util.h
index 1b49834dd8..a669b0926f 100644
--- a/src/shared/openssl-util.h
+++ b/src/shared/openssl-util.h
@@ -1,6 +1,8 @@
 /* SPDX-License-Identifier: LGPL-2.1-or-later */
 #pragma once
 
+#include "macro.h"
+
 #if HAVE_OPENSSL
 #  include <openssl/pem.h>
 
@@ -9,4 +11,6 @@ DEFINE_TRIVIAL_CLEANUP_FUNC(X509_NAME*, X509_NAME_free);
 DEFINE_TRIVIAL_CLEANUP_FUNC(EVP_PKEY_CTX*, EVP_PKEY_CTX_free);
 DEFINE_TRIVIAL_CLEANUP_FUNC(EVP_CIPHER_CTX*, EVP_CIPHER_CTX_free);
 
+int rsa_encrypt_bytes(EVP_PKEY *pkey, const void *decrypted_key, size_t decrypted_key_size, void **ret_encrypt_key, size_t *ret_encrypt_key_size);
+
 #endif