Generating RSA signature w/manual padding

[jyri]

I'm experimenting with some code to generate an RSA signature using
RSA_NO_PADDING. Instead, I use RSA_padding_add_PKCS1_type_1() to 
manually add padding to the hash prior to calling EVP_PKEY_sign().

Attempting to then verify the generated signature conventionally
(setting padding to RSA_PKCS1_PADDING and letting EVP_PKEY_verify()
handle it) fails to verify the signature.

Any thoughts why this doesn't match? Attached is minimal sample test
case to illustrate the scenario. All error checking is stripped out
for brevity, but in the real code all return codes are being checked
and there are no errors, the signature just doesn't verify.

Thanks,
-- 
Jyri J. Virkki - Santa Cruz, CA






-- 

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <openssl/evp.h>
#include <openssl/rsa.h>
#include <openssl/sha.h>

#define DATALEN 512

void main()
{
  EVP_PKEY_CTX * ctx = NULL;
  EVP_PKEY * key = NULL;
  int i;

  // Some data to sign
  unsigned char data[DATALEN];
  for (i = 0; i < DATALEN; i++) { data[i]=i; }


  printf("Generating a key...\n");
  ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL);
  EVP_PKEY_keygen_init(ctx);
  EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, 2048);
  EVP_PKEY_keygen(ctx, &key);
  EVP_PKEY_CTX_free(ctx);


  printf("Hashing data...\n");
  unsigned char hash[SHA256_DIGEST_LENGTH];
  SHA256_CTX sha256;
  SHA256_Init(&sha256);
  SHA256_Update(&sha256, data, DATALEN);
  SHA256_Final(hash, &sha256);


  printf("Adding padding...\n");
  size_t buflen = 2048/8;
  unsigned char buffer[buflen];
  RSA_padding_add_PKCS1_type_1(buffer, buflen, hash, SHA256_DIGEST_LENGTH);


  printf("Signing pre-padded buffer...\n");
  ctx = EVP_PKEY_CTX_new(key, NULL);
  EVP_PKEY_sign_init(ctx);
  EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_NO_PADDING);
  //  EVP_PKEY_CTX_set_signature_md(ctx, EVP_sha256()); // fails
  size_t siglen;
  EVP_PKEY_sign(ctx, NULL, &siglen, buffer, buflen);
  printf("siglen = %zu\n", siglen);
  unsigned char signature[siglen];
  EVP_PKEY_sign(ctx, signature, &siglen, buffer, buflen);
  EVP_PKEY_CTX_free(ctx);


  printf("Now trying to verify signature directly...\n");
  ctx = EVP_PKEY_CTX_new(key, NULL);
  EVP_PKEY_verify_init(ctx);
  EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING);
  EVP_PKEY_CTX_set_signature_md(ctx, EVP_sha256());

  if (EVP_PKEY_verify(ctx, signature, siglen, hash, SHA256_DIGEST_LENGTH) <= 0) {
    printf("Signature did not verify\n");
  } else {
    printf("Signature OK!\n");
  }

  EVP_PKEY_CTX_free(ctx);
  exit(0);
}