homed: split out code that determines suitable LUKS passphrase size from RSA key

We can use this in cryptenroll later on, hence let's make this generic.

src/home/homectl-pkcs11.c

@@ -148,8 +148,7 @@ int identity_add_pkcs11_key_data(JsonVariant **v, const char *uri) {
         size_t decrypted_key_size, encrypted_key_size;
         _cleanup_(X509_freep) X509 *cert = NULL;
         EVP_PKEY *pkey;
-        int bits, r;
-        RSA *rsa;
+        int r;
 
         assert(v);
 
@@ -161,22 +160,9 @@ int identity_add_pkcs11_key_data(JsonVariant **v, const char *uri) {
         if (!pkey)
                 return log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to extract public key from X.509 certificate.");
 
-        if (EVP_PKEY_base_id(pkey) != EVP_PKEY_RSA)
-                return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), "X.509 certificate does not refer to RSA key.");
-
-        rsa = EVP_PKEY_get0_RSA(pkey);
-        if (!rsa)
-                return log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to acquire RSA public key from X.509 certificate.");
-
-        bits = RSA_bits(rsa);
-        log_debug("Bits in RSA key: %i", bits);
-
-        /* We use PKCS#1 padding for the RSA cleartext, hence let's leave some extra space for it, hence only
-         * generate a random key half the size of the RSA length */
-        decrypted_key_size = bits / 8 / 2;
-
-        if (decrypted_key_size < 1)
-                return log_error_errno(SYNTHETIC_ERRNO(EIO), "Uh, RSA key size too short?");
+        r = rsa_pkey_to_suitable_key_size(pkey, &decrypted_key_size);
+        if (r < 0)
+                return log_error_errno(r, "Failed to extract RSA key size from X509 certificate.");
 
         log_debug("Generating %zu bytes random key.", decrypted_key_size);
 

src/shared/openssl-util.c

@@ -38,4 +38,39 @@ int rsa_encrypt_bytes(
 
         return 0;
 }
+
+int rsa_pkey_to_suitable_key_size(
+                EVP_PKEY *pkey,
+                size_t *ret_suitable_key_size) {
+
+        size_t suitable_key_size;
+        RSA *rsa;
+        int bits;
+
+        assert_se(pkey);
+        assert_se(ret_suitable_key_size);
+
+        /* Analyzes the specified public key and that it is RSA. If so, will return a suitable size for a
+         * disk encryption key to encrypt with RSA for use in PKCS#11 security token schemes. */
+
+        if (EVP_PKEY_base_id(pkey) != EVP_PKEY_RSA)
+                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "X.509 certificate does not refer to RSA key.");
+
+        rsa = EVP_PKEY_get0_RSA(pkey);
+        if (!rsa)
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to acquire RSA public key from X.509 certificate.");
+
+        bits = RSA_bits(rsa);
+        log_debug("Bits in RSA key: %i", bits);
+
+        /* We use PKCS#1 padding for the RSA cleartext, hence let's leave some extra space for it, hence only
+         * generate a random key half the size of the RSA length */
+        suitable_key_size = bits / 8 / 2;
+
+        if (suitable_key_size < 1)
+                return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Uh, RSA key size too short?");
+
+        *ret_suitable_key_size = suitable_key_size;
+        return 0;
+}
 #endif

src/shared/openssl-util.h

@@ -13,4 +13,6 @@ DEFINE_TRIVIAL_CLEANUP_FUNC(EVP_CIPHER_CTX*, EVP_CIPHER_CTX_free);
 
 int rsa_encrypt_bytes(EVP_PKEY *pkey, const void *decrypted_key, size_t decrypted_key_size, void **ret_encrypt_key, size_t *ret_encrypt_key_size);
 
+int rsa_pkey_to_suitable_key_size(EVP_PKEY *pkey, size_t *ret_suitable_key_size);
+
 #endif