Add hash OID to signature verification (esp8266#6201)

Author: qistoph

cores/esp8266/Updater.h

@@ -38,6 +38,7 @@ class UpdaterHashClass {
     virtual void end() = 0;
     virtual int len() = 0;
     virtual const void *hash() = 0;
+    virtual const unsigned char *oid() = 0;
 };
 
 // Abstract class to implement a signature verifier

libraries/ESP8266WiFi/src/BearSSLHelpers.cpp

@@ -848,6 +848,10 @@ const void *HashSHA256::hash() {
   return (const void*) _sha256;
 }
 
+const unsigned char *HashSHA256::oid() {
+    return BR_HASH_OID_SHA256;
+}
+
 // SHA256 verifier
 uint32_t SigningVerifier::length()
 {
@@ -869,7 +873,7 @@ bool SigningVerifier::verify(UpdaterHashClass *hash, const void *signature, uint
     bool ret;
     unsigned char vrf[hash->len()];
     br_rsa_pkcs1_vrfy vrfy = br_rsa_pkcs1_vrfy_get_default();
-    ret = vrfy((const unsigned char *)signature, signatureLen, NULL, sizeof(vrf), _pubKey->getRSA(), vrf);
+    ret = vrfy((const unsigned char *)signature, signatureLen, hash->oid(), sizeof(vrf), _pubKey->getRSA(), vrf);
     if (!ret || memcmp(vrf, hash->hash(), sizeof(vrf)) ) {
       return false;
     } else {
@@ -896,4 +900,4 @@ make_stack_thunk(br_ssl_engine_sendrec_buf);
 
 #endif
 
-};
+};

libraries/ESP8266WiFi/src/BearSSLHelpers.h

@@ -146,6 +146,7 @@ class HashSHA256 : public UpdaterHashClass {
     virtual void end() override;
     virtual int len() override;
     virtual const void *hash() override;
+    virtual const unsigned char *oid() override;
   private:
     br_sha256_context _cc;
     unsigned char _sha256[32];

tools/signing.py

@@ -51,10 +51,9 @@ def main():
         try:
             with open(args.bin, "rb") as b:
                 bin = b.read()
-                sha256 = hashlib.sha256(bin)
-                signcmd = [ 'openssl', 'rsautl', '-sign', '-inkey', args.privatekey ]
+                signcmd = [ 'openssl', 'dgst', '-sha256', '-sign', args.privatekey ]
                 proc = subprocess.Popen(signcmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
-                signout, signerr = proc.communicate(input=sha256.digest())
+                signout, signerr = proc.communicate(input=bin)
                 if proc.returncode:
                     sys.stderr.write("OpenSSL returned an error signing the binary: " + str(proc.returncode) + "\nSTDERR: " + str(signerr))
                 else: