fix shift overflow in irep_hash_container

The return value of `inline size_t operator()(const packedt &p) const` is
`size_t`, the contents of `packedt` is `unsigned`. The result is computed as the
XOR of all contents of the `packedt` vector, shifted to the left `n` times,
where `n` is the index in the `packedt`. Shifting for more than the bits of the
type in `packedt` (here unsigned) is undefined behavior.

This patch removes the UB by i) using size_t type instead of unsigned int and
ii) by using irep_hash/hash_combine to compute the hash of the container, seeded
with the input container size.

The erroneous behaviour was observed in `ansi-c/extern1` regression test.

Author: Matthias Güdemann

src/util/irep_hash_container.cpp

@@ -21,7 +21,7 @@ Function: irep_hash_container_baset::number
 
 \*******************************************************************/
 
-unsigned irep_hash_container_baset::number(const irept &irep)
+size_t irep_hash_container_baset::number(const irept &irep)
 {
   // the ptr-hash provides a speedup of up to 3x
 
@@ -32,7 +32,7 @@ unsigned irep_hash_container_baset::number(const irept &irep)
 
   packedt packed;
   pack(irep, packed);
-  unsigned id=numbering.number(packed);
+  size_t id=numbering.number(packed);
 
   ptr_hash[&irep.read()]=id;
 

src/util/irep_hash_container.h

@@ -12,16 +12,17 @@ Author: Daniel Kroening, [email protected]
 #include <cstdlib>  // for size_t
 #include <vector>
 
+#include "irep_hash.h"
 #include "numbering.h"
 
 class irept;
 
 class irep_hash_container_baset
 {
 public:
-  unsigned number(const irept &irep);
+  size_t number(const irept &irep);
 
-  irep_hash_container_baset(bool _full):full(_full)
+  explicit irep_hash_container_baset(bool _full):full(_full)
   {
   }
 
@@ -44,20 +45,21 @@ class irep_hash_container_baset
     }
   };
 
-  typedef std::unordered_map<const void *, unsigned, pointer_hash> ptr_hasht;
+  typedef std::unordered_map<const void *, size_t, pointer_hash>
+    ptr_hasht;
   ptr_hasht ptr_hash;
 
   // this is the second level: content
 
-  typedef std::vector<unsigned> packedt;
+  typedef std::vector<size_t> packedt;
 
   struct vector_hash
   {
     inline size_t operator()(const packedt &p) const
     {
-      size_t result=p.size();
-      for(unsigned i=0; i<p.size(); i++)
-        result^=p[i]<<i;
+      size_t result=p.size(); // seed
+      for(auto elem : p)
+        result=hash_combine(result, elem);
       return result;
     }
   };