ECDSA Signature Verification fails using openssl

Question

I am not able to verify the NIST signature for ECDSA algorithm and below I am presenting the source code for reference.

I have tried few vectors available from the ACVP server for testing

sample vector for verification

Msg =  F5DFA7CFB5DB85155217FAA1279AEFCC5AC204B3D8B372BACF009E70B97C0063AB6EC0C96E4EDA7E8AEE2FD10824B2A2F56B06DB92D79BB6A8151E905A0F0482E903C6DA3DE538619E978B73691BD0EE90F97FBEFFE9677148FDAEA6AF7915156299EDD913439B752A05C5FA49E3B457BCCE096F24671CD6DCD91DD95A626763
Qx = CDCEA8541A3A37BB2622F8EA2A291C3C8B299C3365130575
Qy = D7F60186A5BE701CE677FBE4BDF2A6FF5B05C62128FA6384
R = 1C5C6EC56A58407BC87CF867A3028166136A18C8D619DF2E
S = 723B2868060482E93CBAA70E6E1118EE5FA807ECFA7D8A55
actual Result = PASS
output from the ecdsa_do_verify_fn() = Signature verification FAIL which should evaluate PASS

Msg = EA454ED1B4F2A734F849CBEC31C94478DFB1DEBB8B576D5C6DA51C6E1DFFEC88A39CD2E90BC0D075B2553A6E0CE867A31F2241221FEAF0BC2F5A918468598932B8D8D4996C649A20D3FEA67B149DB6FE2A85C8A732A46078363E4EA6E0A47458D04E0BC3E69128E8C973C6806326FB7D017228DCB15A81C39CBB2B944C63949A
Qx = B2F4E1C905A40E7576EB365AA84A087B9D8C5E65DEB0761B
Qy = 48EC67C72F1B0AEB14898D4FCC0D6C29125D78A99804641E
R = 119BE9DD9CC83A6596FCA21CC3FE8903C0906B2A19FBFF2B
S = 0DC49D486DF1708E0E3A4BEEFA9D9EF7C8F455BD4C83646E
actual RESULT = FAIL
output from the ecdsa_do_verify_fn() = Signature verification is also FAIL

I have also generated my own test vectors for ECDSA and which all of them are verified correctly without any problem with the same code and I wonder whats wrong with code not able to verify NIST test vectors and request anybody to help me to point out or give suggestion what can be done to verify the NIST test vectors

Here I am reading the above sample vectors Values Msg, Qx, Qy, R and S values to corresponding character array and passing these values to the function ecdsa_do_verify_fn() along with the curve name P-224

#include <openssl/err.h>
#include <openssl/obj_mac.h>
#include <openssl/bn.h>
#include <openssl/ec.h>
#include <openssl/evp.h>
#include <openssl/crypto.h>
#include <openssl/rand.h>
#include <openssl/sha.h>
#include <openssl/ecdsa.h>

#include <stdlib.h>
#include <stdio.h>
#include <string.h>


int main()
{
unsigned char *dgst;
int dgst_len;
char c_name[10] = "P-192";
char in_msg[270] = "F5DFA7CFB5DB85155217FAA1279AEFCC5AC204B3D8B372BACF009E70B97C0063AB6EC0C96E4EDA7E8AEE2FD10824B2A2F56B06DB92D79BB6A8151E905A0F0482E903C6DA3DE538619E978B73691BD0EE90F97FBEFFE9677148FDAEA6AF7915156299EDD913439B752A05C5FA49E3B457BCCE096F24671CD6DCD91DD95A626763";
int msg_len = 256;

char in_Qx[100] = "CDCEA8541A3A37BB2622F8EA2A291C3C8B299C3365130575";
char in_Qy[100] = "D7F60186A5BE701CE677FBE4BDF2A6FF5B05C62128FA6384";
char rr[100] = "1C5C6EC56A58407BC87CF867A3028166136A18C8D619DF2E";
char ss[100] = "723B2868060482E93CBAA70E6E1118EE5FA807ECFA7D8A55";

dgst = get_dgst_224_R(in_msg,msg_len);
dgst_len = SHA224_DIGEST_LENGTH;

int ret = ecdsa_do_verify_fn(c_name,dgst,dgst_len,in_Qx,in_Qy,rr,ss);

if(ret == 0)
 printf("Signature verification pass\n");
else
printf("Signature verification FAIL\n");

return 0;
}

int fips_curve_name(char *curve_name)
{
    char temp_curve[20];

    strcpy(temp_curve,curve_name);
    
    int nid;



        if(!strncmp(temp_curve,"[P-192",5))
        {
            return(EC_curve_nist2nid("P-192"));
        }
        if(!strncmp(temp_curve,"[P-224",5))
        {
            return(EC_curve_nist2nid("P-224"));
        }
        if(!strncmp(temp_curve,"[P-256",5))
        {
            return(EC_curve_nist2nid("P-256"));
        }
        if(!strncmp(temp_curve,"[P-384",5))
        {
            return(EC_curve_nist2nid("P-384"));
        }
        if(!strncmp(temp_curve,"[P-521",5))
        {
            return(EC_curve_nist2nid("P-521"));
        }
        if(!strncmp(temp_curve,"[K-163",5))
        {
            return(EC_curve_nist2nid("K-163"));
        }
        if(!strncmp(temp_curve,"[K-233",5))
        {
            return(EC_curve_nist2nid("K-233"));
        }
        if(!strncmp(temp_curve,"[K-283",5))
        {
            return(EC_curve_nist2nid("K-283"));
        }
        if(!strncmp(temp_curve,"[K-409",5))
        {
            return(EC_curve_nist2nid("K-409"));
        }
        if(!strncmp(temp_curve,"[K-571",5))
        {
            return(EC_curve_nist2nid("K-571"));
        }
        if(!strncmp(temp_curve,"[B-163",5))
        {
            return(EC_curve_nist2nid("B-163"));
        }
        if(!strncmp(temp_curve,"[B-233",5))
        {
            return(EC_curve_nist2nid("B-233"));
        }
        if(!strncmp(temp_curve,"[B-283",5))
        {
            return(EC_curve_nist2nid("B-283"));
        }
        if(!strncmp(temp_curve,"[B-409",5))
        {
            return(EC_curve_nist2nid("B-409"));
        }
        if(!strncmp(temp_curve,"[B-571",5))
        {
            return(EC_curve_nist2nid("B-571"));
        }

        else return 0;


}
    
unsigned char* get_dgst_224_R(const char msg[],int msg_len)
{
    SHA256_CTX ctx;

    int i;

    static unsigned char digest[SHA224_DIGEST_LENGTH];
    
    SHA224_Init(&ctx);
    SHA224_Update(&ctx, msg,msg_len);
    SHA224_Final(digest, &ctx);


    
    printf("MD244 = ");
    for(i = 0; i < SHA224_DIGEST_LENGTH ; i++)
    {
        printf("%02x",digest[i]);
    }printf("\n");

    return digest; 

}               

int ecdsa_do_verify_fn(char *c_name,unsigned char *dgst, int dgst_len,char *in_Qx,char *in_Qy,char *rr, char *ss)
{
int ret = -1, i;
BN_CTX   *ctx;
BIGNUM   *order, *u1, *u2, *m, *X;
EC_POINT *point = NULL;
const EC_GROUP *ec_group;
const EC_POINT *pub_key;
int nid_num;


    if((c_name == NULL) || (dgst == NULL) || (dgst_len == 0) || (in_Qx == NULL) || (in_Qy == NULL )|| (rr == NULL) || (ss == NULL))
    {
        ret = 1;
        goto err1;
    }
    else
    {
        EC_KEY *ec_key = EC_KEY_new();


        nid_num = EC_curve_nist2nid(c_name);
        //  printf("nid_num = %d\n",nid_num);

        ec_key = EC_KEY_new_by_curve_name(nid_num);//(NID_sect233k1);//EC_KEY_new();
        if(ec_key == NULL)
        {
            printf("Error in memory allocation for EC key\n");
            return -1;
        }
        ec_group = EC_KEY_get0_group(ec_key);//EC_GROUP_new_by_curve_name(nid_num);
        //EC_KEY_set_group(ec_key, ec_group);//added

        //ctx=NULL;
        if ((ctx = BN_CTX_new()) == NULL)
        {
            printf("Error in context memory allocation \n");
        }



        //setting pub_key
        BIGNUM *pub_X = BN_new();
        BIGNUM *pub_Y = BN_new();
        if(pub_X == NULL || pub_Y == NULL)
        {
            printf("Error in memory allocation for X and Y coordinates\n");
            goto err;
        }
        BN_hex2bn(&pub_X, in_Qx);
        BN_hex2bn(&pub_Y, in_Qy);
        
        pub_key      = EC_POINT_new(ec_group);
        EC_POINT_set_affine_coordinates(ec_group, pub_key, pub_X, pub_Y, NULL);
    
        EC_KEY_set_public_key(ec_key, pub_key);

        const BIGNUM *R = BN_new();
        if(R == NULL )
        {
            printf("Error in memory allocation for 'R' coordinates\n");
            goto err;
        }
        BN_hex2bn(&R, rr);      
                
        const BIGNUM *S = BN_new();
        if(S == NULL )
        {
            printf("Error in memory allocation for 'S' coordinates\n");
            goto err;
        }
        BN_hex2bn(&S, ss);
        
        BIGNUM *x = BN_new();
        BIGNUM *y = BN_new();
        if(x == NULL || y == NULL)
        {
            printf("Error in memory allocation for X and Y coordinates\n");
            return -1;
        }
            
        BN_CTX_start(ctx);
        order = BN_CTX_get(ctx);
        u1    = BN_CTX_get(ctx);
        u2    = BN_CTX_get(ctx);
        m     = BN_CTX_get(ctx);
        X     = BN_CTX_get(ctx);
        
        if (!X)
        {
            //ECDSAerr(ECDSA_F_ECDSA_DO_VERIFY, ERR_R_BN_LIB);
            printf("ECDSA_F_ECDSA_DO_VERIFY, ERR_R_BN_LIB\n");
            goto err;
        }

        if (!EC_GROUP_get_order(ec_group, order, ctx))
        {
            //ECDSAerr(ECDSA_F_ECDSA_DO_VERIFY, ERR_R_EC_LIB);
            printf("ECDSA_F_ECDSA_DO_VERIFY, ERR_R_EC_LIB\n");
            goto err;
        }

        if (BN_is_zero(R)          || BN_is_negative(R) ||
                BN_ucmp(R, order) >= 0 || BN_is_zero(S)  ||
                BN_is_negative(S)      || BN_ucmp(S, order) >= 0)
        {
            //ECDSAerr(ECDSA_F_ECDSA_DO_VERIFY, ECDSA_R_BAD_SIGNATURE);
            printf("ECDSA_F_ECDSA_DO_VERIFY, ECDSA_R_BAD_SIGNATURE\n");
            ret = 1;    /* signature is invalid */
            goto err;
        }
        /* calculate tmp1 = inv(S) mod order */
        //if (!BN_mod_inverse(u2, S, order, ctx))
        if (!BN_mod_inverse(u2, S, order, ctx))
        {
            //ECDSAerr(ECDSA_F_ECDSA_DO_VERIFY, ERR_R_BN_LIB);
            printf("ECDSA_F_ECDSA_DO_VERIFY, ERR_R_BN_LIB\n");
            goto err;
        }

        
        /* digest -> m */
        i = BN_num_bits(order);
        /* Need to truncate digest if it is too long: first truncate whole
         * bytes.
         */
        if (8 * dgst_len > i)
            dgst_len = (i + 7)/8;
        if (!BN_bin2bn(dgst, dgst_len, m))
        {
            //ECDSAerr(ECDSA_F_ECDSA_DO_VERIFY, ERR_R_BN_LIB);
            printf("ECDSA_F_ECDSA_DO_VERIFY, ERR_R_BN_LIB\n");
            goto err;
        }
        /* If still too long truncate remaining bits with a shift */
        if ((8 * dgst_len > i) && !BN_rshift(m, m, 8 - (i & 0x7)))
        {
            //ECDSAerr(ECDSA_F_ECDSA_DO_VERIFY, ERR_R_BN_LIB);
            printf("ECDSA_F_ECDSA_DO_VERIFY, ERR_R_BN_LIB\n");
            goto err;
        }
        /* u1 = m * tmp mod order */
        if (!BN_mod_mul(u1, m, u2, order, ctx))
        {
            //ECDSAerr(ECDSA_F_ECDSA_DO_VERIFY, ERR_R_BN_LIB);
            printf("ECDSA_F_ECDSA_DO_VERIFY, ERR_R_BN_LIB\n");
            goto err;
        }
        /* u2 = r * w mod q */
        if (!BN_mod_mul(u2, R, u2, order, ctx))
        {
            //ECDSAerr(ECDSA_F_ECDSA_DO_VERIFY, ERR_R_BN_LIB);
            printf("ECDSA_F_ECDSA_DO_VERIFY, ERR_R_BN_LIB\n");
            goto err;
        }

        if ((point =EC_GROUP_get0_generator(ec_group)) == NULL)
        {
            //ECDSAerr(ECDSA_F_ECDSA_DO_VERIFY, ERR_R_MALLOC_FAILURE);
            printf("ECDSA_F_ECDSA_DO_VERIFY, ERR_R_MALLOC_FAILURE\n");
            goto err;
        }

        EC_POINT_get_affine_coordinates(ec_group, point, x, y, NULL);
        

        if (!EC_POINT_mul(ec_group, point, u1, pub_key, u2, ctx))
        {
            //ECDSAerr(ECDSA_F_ECDSA_DO_VERIFY, ERR_R_EC_LIB);
            printf("ECDSA_F_ECDSA_DO_VERIFY, ERR_R_EC_LIB\n");
            goto err;
        }
        if (EC_METHOD_get_field_type(EC_GROUP_method_of(ec_group)) == NID_X9_62_prime_field)
        {
            if (!EC_POINT_get_affine_coordinates_GFp(ec_group,point, X, NULL, ctx))
            {
                //ECDSAerr(ECDSA_F_ECDSA_DO_VERIFY, ERR_R_EC_LIB);
                printf("ECDSA_F_ECDSA_DO_VERIFY, ERR_R_EC_LIB\n");
                goto err;
            }
        }
        else /* NID_X9_62_characteristic_two_field */
        {
            if (!EC_POINT_get_affine_coordinates_GF2m(ec_group,
                    point, X, NULL, ctx))
            {
                //ECDSAerr(ECDSA_F_ECDSA_DO_VERIFY, ERR_R_EC_LIB);
                printf("ECDSA_F_ECDSA_DO_VERIFY, ERR_R_EC_LIB\n");
                goto err;
            }
        }

        
        if (!BN_nnmod(u1, X, order, ctx))
        {
            //ECDSAerr(ECDSA_F_ECDSA_DO_VERIFY, ERR_R_BN_LIB);
            printf("ECDSA_F_ECDSA_DO_VERIFY, ERR_R_BN_LIB\n");
            goto err;
        }
        /*  if the signature is correct u1 is equal to sig->r */


        printf("u1 = ");
        BN_print_fp(stdout, u1);
        printf("\n");
        printf("R = ");
        BN_print_fp(stdout, R);
        //printf("ret = %d\n");
        printf("\n\n\n");

        ret =  BN_ucmp(u1, R);//(BN_ucmp(u1, r)== 0);
        /*printf("ret = %d\n",ret);
    if(ret == 0)
        fprintf(rsp,"Signature is valid !!\n\n");
    else
        fprintf(rsp,"signature is not valid !!\n\n");*/

        err:
        BN_CTX_end(ctx);
        BN_free(R);
        BN_free(S);
        BN_CTX_free(ctx);
        if (point)
            EC_POINT_free(point);
        EC_POINT_free(ec_key);
    //  ECDSA_SIG_free(signature);
        BN_free(pub_X);
        BN_free(pub_Y);
        BN_free(x);
        BN_free(y);
        //BN_free(BN_m);
    }
    err1: return ret;
}

Answer 1

Not a complete answer and probably not very near but too big for comments; I will delete if needed.

Semantic error(s): You use curve P-224, but your X,Y and R,S values correspond to a 192-bit curve (possibly P-192 which since 2013 is allowed only for verification, see SP800-131A and SP800-57). I don't use ACV(P,TS) but CAVP vectors have the Msg in hex, and if that's true here you need to decode it before hashing it; also msg in your code is severely truncated from the presumably correct value you posted above.

Programming errors: In get_digest_224_R you put the digest in a locally declared (i.e. auto) array digest and return that; in C this returns a pointer to storage which is then deallocated, and subsequently derefencing this pointer (as you do) is Undefined Behavior which in theory can destroy the universe but in practice mostly just gives you wrong data values (fortunately for the rest of us who also inhabit the universe). You also malloc hash for a certain size but then store to the byte after that size, which is also Undefined Behavior, and this time is likely to destroy other data; if your first for loop were uncommented, it would store in bytes much farther beyond the limit. (If you allocate (or declare an array of) SHA224_DIGEST_LENGTH bytes, the only valid byte subscripts are 0 .. SHA224_DIGEST_LENGTH -1; your for loop would use SHA224_DIGEST_LENGTH *2.) Finally you compute sizeof(msg) where msg is a parameter declared as an array; such a parameter declaration actually produces (is 'rewritten' to) a pointer, and this gives the size of the pointer not of the array it points to.

Answer 2

I could resolve the issue on my own by modifying the message format to hex values.