RSA PKCS1 OAEP padding with SHA512 algorithm

RSA_padding_add_PKCS1_OAEP_SHA512.c

static int MGF1_SHA512(unsigned char *mask, long len, const unsigned char *seed, long seedlen)
{
    return PKCS1_MGF1(mask, len, seed, seedlen, EVP_sha512());
}

static int RSA_padding_add_PKCS1_OAEP_SHA512(unsigned char *to, int tlen,
                                             const unsigned char *from, int flen,
                                             const unsigned char *param, int plen)
{
    int i, emlen = tlen - 1;
    unsigned char *db, *seed;
    unsigned char *dbmask, seedmask[SHA512_DIGEST_LENGTH];

    if (flen > emlen - 2 * SHA512_DIGEST_LENGTH - 1)
    {
        RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP,
            RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
        return 0;
    }

    if (emlen < 2 * SHA512_DIGEST_LENGTH + 1)
    {
        RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, RSA_R_KEY_SIZE_TOO_SMALL);
        return 0;
    }

    to[0] = 0;
    seed = to + 1;
    db = to + SHA512_DIGEST_LENGTH + 1;

    if (!EVP_Digest((void *)param, plen, db, NULL, EVP_sha512(), NULL))
        return 0;
    memset(db + SHA512_DIGEST_LENGTH, 0,
        emlen - flen - 2 * SHA512_DIGEST_LENGTH - 1);
    db[emlen - flen - SHA512_DIGEST_LENGTH - 1] = 0x01;
    memcpy(db + emlen - flen - SHA512_DIGEST_LENGTH, from, (unsigned int) flen);
    if (RAND_bytes(seed, SHA512_DIGEST_LENGTH) <= 0)
        return 0;

    dbmask = (unsigned char*)OPENSSL_malloc(emlen - SHA512_DIGEST_LENGTH);
    if (dbmask == NULL)
    {
        RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);
        return 0;
    }

    if (MGF1_SHA512(dbmask, emlen - SHA512_DIGEST_LENGTH, seed, SHA512_DIGEST_LENGTH) < 0)
        return 0;
    for (i = 0; i < emlen - SHA512_DIGEST_LENGTH; i++)
        db[i] ^= dbmask[i];

    if (MGF1_SHA512(seedmask, SHA512_DIGEST_LENGTH, db, emlen - SHA512_DIGEST_LENGTH) < 0)
        return 0;
    for (i = 0; i < SHA512_DIGEST_LENGTH; i++)
        seed[i] ^= seedmask[i];

    OPENSSL_free(dbmask);
    return 1;
}