Skip to content

Fix misleading error in private_encrypt with default OAEP padding #177

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
timlegge merged 3 commits into from
Apr 23, 2026
Merged

Fix misleading error in private_encrypt with default OAEP padding #177

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0791e70
Add tests for private_encrypt error message ordering
toddr-bot Apr 23, 2026
fc75575
Check padding compatibility before message length in private_encrypt()
toddr-bot Apr 23, 2026
0c36f0b
Add OAEP/PSS padding checks to public_decrypt()
toddr-bot Apr 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions RSA.xs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1196,6 +1196,14 @@ private_encrypt(p_rsa, p_plaintext)
{
croak("Public keys cannot private_encrypt");
}
if (p_rsa->padding == RSA_PKCS1_OAEP_PADDING) {
croak("OAEP padding is not supported for private_encrypt/public_decrypt. "
"Call use_no_padding() or use_pkcs1_padding() first.");
}
if (p_rsa->padding == RSA_PKCS1_PSS_PADDING) {
croak("PSS padding with private_encrypt/public_decrypt is not supported. "
"Use sign()/verify() for PSS signatures.");
}
check_max_message_length(p_rsa, sv_len(p_plaintext));
#if OPENSSL_VERSION_NUMBER >= 0x30000000L
RETVAL = rsa_crypt(p_rsa, p_plaintext, EVP_PKEY_sign, EVP_PKEY_sign_init, 0 /* is_encrypt */);
Expand All @@ -1210,6 +1218,14 @@ public_decrypt(p_rsa, p_ciphertext)
rsaData* p_rsa;
SV* p_ciphertext;
CODE:
if (p_rsa->padding == RSA_PKCS1_OAEP_PADDING) {
croak("OAEP padding is not supported for private_encrypt/public_decrypt. "
"Call use_no_padding() or use_pkcs1_padding() first.");
}
if (p_rsa->padding == RSA_PKCS1_PSS_PADDING) {
croak("PSS padding with private_encrypt/public_decrypt is not supported. "
"Use sign()/verify() for PSS signatures.");
}
#if OPENSSL_VERSION_NUMBER >= 0x30000000L
RETVAL = rsa_crypt(p_rsa, p_ciphertext, EVP_PKEY_verify_recover, EVP_PKEY_verify_recover_init, 0 /* is_encrypt */);
#else
Expand Down
60 changes: 32 additions & 28 deletions t/private_encrypt.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,7 @@ use Test::More;

use Crypt::OpenSSL::Random;
use Crypt::OpenSSL::RSA;
use Crypt::OpenSSL::Guess qw(openssl_version);

my ($major) = openssl_version();

plan tests => 10;
plan tests => 14;

Crypt::OpenSSL::Random::random_seed("OpenSSL needs at least 32 bytes.");
Crypt::OpenSSL::RSA->import_random_seed();
Expand All @@ -29,43 +25,51 @@ is($dec, $data, "public_decrypt(private_encrypt(data)) round-trips with no_paddi

$rsa->use_pkcs1_oaep_padding();
eval { $rsa->private_encrypt("test") };
if ($major ge '3') {
like($@, qr/OAEP padding is not supported for private_encrypt/,
"private_encrypt with OAEP croaks with clear message on OpenSSL 3.x");
} else {
# Pre-3.x uses RSA_private_encrypt which handles padding differently
ok(1, "private_encrypt with OAEP behavior on pre-3.x (skipped)");
}
like($@, qr/OAEP padding is not supported for private_encrypt/,
"private_encrypt with OAEP croaks with clear message");

# --- OAEP: should croak for public_decrypt ---

eval { $rsa->public_decrypt("test" x 64) };
if ($major ge '3') {
like($@, qr/OAEP padding is not supported for private_encrypt\/public_decrypt/,
"public_decrypt with OAEP croaks with clear message on OpenSSL 3.x");
} else {
ok(1, "public_decrypt with OAEP behavior on pre-3.x (skipped)");
}
like($@, qr/OAEP padding is not supported for private_encrypt\/public_decrypt/,
"public_decrypt with OAEP croaks with clear message");

# --- PSS: should croak for private_encrypt ---

$rsa->use_pkcs1_pss_padding();
eval { $rsa->private_encrypt("test") };
if ($major ge '3') {
like($@, qr/PSS padding with private_encrypt\/public_decrypt is not supported/,
"private_encrypt with PSS croaks with clear message on OpenSSL 3.x");
} else {
ok(1, "private_encrypt with PSS behavior on pre-3.x (skipped)");
}
like($@, qr/PSS padding with private_encrypt\/public_decrypt is not supported/,
"private_encrypt with PSS croaks with clear message");

# --- PSS: should croak for public_decrypt ---

eval { $rsa->public_decrypt("test" x 64) };
if ($major ge '3') {
like($@, qr/PSS padding with private_encrypt\/public_decrypt is not supported/,
"public_decrypt with PSS croaks with clear message");

# --- Error ordering: padding error must come before length error ---

{
my $rsa_fresh = Crypt::OpenSSL::RSA->generate_key(2048);
my $large_data = "x" x 250; # exceeds OAEP limit (256 - 42 = 214)
eval { $rsa_fresh->private_encrypt($large_data) };
like($@, qr/OAEP padding is not supported for private_encrypt/,
"private_encrypt with default OAEP + oversized data gives padding error, not length error");

$rsa_fresh->use_pkcs1_pss_padding();
eval { $rsa_fresh->private_encrypt($large_data) };
like($@, qr/PSS padding with private_encrypt\/public_decrypt is not supported/,
"public_decrypt with PSS croaks with clear message on OpenSSL 3.x");
} else {
ok(1, "public_decrypt with PSS behavior on pre-3.x (skipped)");
"private_encrypt with PSS + oversized data gives padding error, not length error");

$rsa_fresh->use_pkcs1_oaep_padding();
my $exact_data = "y" x 215; # just over OAEP limit
eval { $rsa_fresh->private_encrypt($exact_data) };
like($@, qr/OAEP padding is not supported for private_encrypt/,
"private_encrypt with OAEP + barely-over-limit data gives padding error, not length error");

$rsa_fresh->use_pkcs1_padding();
eval { $rsa_fresh->private_encrypt("test") };
ok(!$@, "private_encrypt with PKCS#1 v1.5 padding works");
}

# --- Encryption operations still work correctly ---
Expand Down
Loading