i am working on a program to authenticate an ENC signature file by using OpenSSL for windows, and specifically methods DSA_do_verify(...) and SHA1(...) hash algorithm, but is having problems as the result from DSA_do_verify is always 0 (invalid).

I am using the signature file of test set 4B from the IHO S-63 Data Protection Scheme, and also the SA public key (downloadable from IHO) for verification.

Below is my program, can anyone help to see where i have gone wrong as i have tried many ways but failed to get the verification to be valid, thanks..

The signature file from test set 4B

// Signature part R:
3F14 52CD AEC5 05B6 241A 02C7 614A D149 E7D6 C408.
// Signature part S:
44BB A3DB 8C46 8D11 B6DB 23BE 1A79 55E6 B083 7429.
// Signature part R:
93F5 EF86 1FF6 BA6F 1C2B B9BB 7F36 0C80 2F9B 2414.
// Signature part S:
4877 8130 12B4 50D8 3688 B52C 7A84 8E26 D442 8B6E.
// BIG p
C16C BAD3 4D47 5EC5 3966 95D6 94BC 8BC4 7E59 8E23 B5A9 D7C5 CEC8 2D65 B682 7D44 E953 7848 4730 C0BF F1F4 CB56 F47C 6E51 054B E892 00F3 0D43 DC4F EF96 24D4 665B.
// BIG q
B7B8 10B5 8C09 34F6 4287 8F36 0B96 D7CC 26B5 3E4D.
// BIG g
4C53 C726 BDBF BBA6 549D 7E73 1939 C6C9 3A86 9A27 C5DB 17BA 3CAC 589D 7B3E 003F A735 F290 CFD0 7A3E F10F 3515 5F1A 2EF7 0335 AF7B 6A52 11A1 1035 18FB A44E 9718.
// BIG y
15F8 A502 11C2 34BB DF19 B3CD 25D1 4413 F03D CF38 6FFC 7357 BCEE 59E4 EBFD B641 6726 5E5F 0682 47D4 B50B 3B86 7A85 FB4D 6E01 8329 A993 C36C FD9A BFB6 ED6D 29E0.

dataServer_pkeyfile.txt (extracted from above)

// BIG p
C16C BAD3 4D47 5EC5 3966 95D6 94BC 8BC4 7E59 8E23 B5A9 D7C5 CEC8 2D65 B682 7D44 E953 7848 4730 C0BF F1F4 CB56 F47C 6E51 054B E892 00F3 0D43 DC4F EF96 24D4 665B.
// BIG q
B7B8 10B5 8C09 34F6 4287 8F36 0B96 D7CC 26B5 3E4D.
// BIG g
4C53 C726 BDBF BBA6 549D 7E73 1939 C6C9 3A86 9A27 C5DB 17BA 3CAC 589D 7B3E 003F A735 F290 CFD0 7A3E F10F 3515 5F1A 2EF7 0335 AF7B 6A52 11A1 1035 18FB A44E 9718.
// BIG y
15F8 A502 11C2 34BB DF19 B3CD 25D1 4413 F03D CF38 6FFC 7357 BCEE 59E4 EBFD B641 6726 5E5F 0682 47D4 B50B 3B86 7A85 FB4D 6E01 8329 A993 C36C FD9A BFB6 ED6D 29E0.

Program abstract:

QbyteArray pk_data;
QFile pk_file("./dataServer_pkeyfile.txt");

if (pk_file.open(QIODevice::Text | QIODevice::ReadOnly))
{
   pk_data.append(pk_file.readAll());
}
pk_file.close();

unsigned char ptr_sha_hashed[20];
unsigned char *ptr_pk_data = (unsigned char *)pk_data.data();

// openssl SHA1 hashing algorithm
SHA1(ptr_pk_data, pk_data.length(), ptr_sha_hashed);

DSA_SIG *dsasig = DSA_SIG_new();

char ptr_r[] = "93F5EF861FF6BA6F1C2BB9BB7F360C802F9B2414"; //from tset 4B
char ptr_s[] = "4877813012B450D83688B52C7A848E26D4428B6E"; //from tset 4B

if (BN_hex2bn(&dsasig->r, ptr_r) == 0) return 0;
if (BN_hex2bn(&dsasig->s, ptr_s) == 0) return 0;

DSA *dsakeys = DSA_new();

//the following values are from the SA public key
char ptr_p[] = "FCA682CE8E12CABA26EFCCF7110E526DB078B05EDECBCD1EB4A208F3AE1617AE01F35B91A47E6DF63413C5E12ED0899BCD132ACD50D99151BDC43EE737592E17";
char ptr_q[] = "962EDDCC369CBA8EBB260EE6B6A126D9346E38C5";
char ptr_g[] = "678471B27A9CF44EE91A49C5147DB1A9AAF244F05A434D6486931D2D14271B9E35030B71FD73DA179069B32E2935630E1C2062354D0DA20A6C416E50BE794CA4";
char ptr_y[] = "963F14E32BA5372928F24F15B0730C49D31B28E5C7641002564DB95995B15CF8800ED54E354867B82BB9597B158269E079F0C4F4926B17761CC89EB77C9B7EF8";

if (BN_hex2bn(&dsakeys->p, ptr_p) == 0) return 0;
if (BN_hex2bn(&dsakeys->q, ptr_q) == 0) return 0;
if (BN_hex2bn(&dsakeys->g, ptr_g) == 0) return 0;
if (BN_hex2bn(&dsakeys->pub_key, ptr_y) == 0) return 0;

int result;  //valid = 1, invalid = 0, error = -1

result = DSA_do_verify(ptr_sha_hashed, 20, dsasig, dsakeys);

//result is 0 (invalid)