Changed HashMap's internal layout.

pczarn · pczarn · commit 8a3343b204f5 · 2015-02-27T16:41:51.000+01:00
diff --git a/src/libstd/collections/hash/table.rs b/src/libstd/collections/hash/table.rs
@@ -67,17 +67,14 @@ use core::nonzero::NonZero;
 pub struct RawTable<K, V> {
     capacity: usize,
     size:     usize,
-    hashes:   Unique<Option<SafeHash>>,
-    // Because K/V do not appear directly in any of the types in the struct,
-    // inform rustc that in fact instances of K and V are reachable from here.
-    marker:   marker::PhantomData<(K,V)>,
+    // NB. The table will probably need manual impls of Send and Sync if this
+    // field ever changes.
+    middle: Unique<(K, V)>,
 }
 
 struct RawBucket<K, V> {
     hash: *mut Option<SafeHash>,
-    key: *mut K,
-    val: *mut V,
-    _marker: marker::PhantomData<(K,V)>,
+    kval: *mut (K, V),
 }
 
 impl<K, V> Copy for RawBucket<K, V> {}
@@ -172,9 +169,7 @@ impl<K, V> RawBucket<K, V> {
     unsafe fn offset(self, count: isize) -> RawBucket<K, V> {
         RawBucket {
             hash: self.hash.offset(count),
-            key:  self.key.offset(count),
-            val:  self.val.offset(count),
-            _marker: marker::PhantomData,
+            kval: self.kval.offset(count),
         }
     }
 }
@@ -202,6 +197,7 @@ impl<K, V> Put for RawTable<K, V> {}
 impl<'t, K, V> Put for &'t mut RawTable<K, V> {}
 impl<K, V, M: Put> Put for Bucket<K, V, M> {}
 impl<K, V, M: Put> Put for FullBucket<K, V, M> {}
+
 // Buckets hold references to the table.
 impl<K, V, M, S> Bucket<K, V, M, S> {
     /// Borrow a reference to the table.
@@ -245,7 +241,6 @@ impl<K, V, M> Bucket<K, V, M> where M: Borrow<RawTable<K, V>> {
             capacity: capacity,
             table: table,
         };
-
         if capacity == 0 {
             Err(bucket.unsafe_cast())
         } else {
@@ -350,8 +345,7 @@ impl<K, V, M> EmptyBucket<K, V, M> where M: Borrow<RawTable<K, V>>, M: Put {
                -> FullBucket<K, V, M> {
         unsafe {
             *self.raw.hash = Some(hash);
-            ptr::write(self.0.raw.key, key);
-            ptr::write(self.0.raw.val, value);
+            ptr::write(self.raw.kval, (key, value));
         }
 
         self.table.size += 1;
@@ -375,56 +369,49 @@ impl<'t, K, V, M: 't> FullBucket<K, V, M> where M: Borrow<RawTable<K, V>> {
 
     /// Gets references to the key and value at a given index.
     pub fn read(&self) -> (&SafeHash, &K, &V) {
-        unsafe {
-            (&*(self.0.raw.hash as *mut SafeHash),
-             &*self.0.raw.key,
-             &*self.0.raw.val)
-        }
-    }
-}
-
-impl<K, V, M> FullBucket<K, V, M> where M: Borrow<RawTable<K, V>> {
-    /// Removes this bucket's key and value from the hashtable.
-    ///
-    /// This works similarly to `put`, building an `EmptyBucket` out of the
-    /// taken bucket.
-    pub fn take(mut self) -> (EmptyBucket<K, V, M>, K, V) {
-        self.0.table.size -= 1;
-
-        unsafe {
-            *self.0.raw.hash = None;
-            let k = ptr::read(self.0.raw.key);
-            let v = ptr::read(self.0.raw.val);
-            (Bucket(self.0), k, v)
-        }
-    }
-
-    /// Gets mutable references to the key and value at a given index.
-    pub fn read_mut(&mut self) -> (&mut SafeHash, &mut K, &mut V) {
-        unsafe {
-            (&mut *(self.0.raw.hash as *mut SafeHash),
-             &mut *self.0.raw.key,
-             &mut *self.0.raw.val)
-        }
+        let (&ref h, &(ref k, ref v)) = unsafe {
+            (&*(self.raw.hash as *mut SafeHash), &*self.raw.kval)
+        };
+        (h, k, v)
     }
-}
 
-impl<'t, K, V, M: 't> FullBucket<K, V, M> where M: Borrow<RawTable<K, V>> {
     /// Exchange a bucket state for immutable references into the table.
     /// Because the underlying reference to the table is also consumed,
     /// no further changes to the structure of the table are possible;
     /// in exchange for this, the returned references have a longer lifetime
     /// than the references returned by `read()`.
     pub fn into_refs(self) -> (&'t K, &'t V) {
-        self.0.raw.into_refs()
+        unsafe { (&(*self.raw.kval).0, &(*self.raw.kval).1) }
     }
 }
 
-impl<'t, K, V, M: 't> FullBucket<K, V, M> where M: Borrow<RawTable<K, V>> {
+impl<'t, K, V, M: 't> FullBucket<K, V, M> where M: BorrowMut<RawTable<K, V>> {
+    /// Gets mutable references to the key and value at a given index.
+    pub fn read_mut(&mut self) -> (&mut SafeHash, &mut K, &mut V) {
+        let (&mut ref mut h, &mut (ref mut k, ref mut v)) = unsafe {
+            (&mut *(self.raw.hash as *mut SafeHash), &mut *self.raw.kval)
+        };
+        (h, k, v)
+    }
+
     /// This works similarly to `into_refs`, exchanging a bucket state
     /// for mutable references into the table.
     pub fn into_mut_refs(self) -> (&'t mut K, &'t mut V) {
-        self.0.raw.into_mut_refs()
+        unsafe { (&mut (*self.raw.kval).0, &mut (*self.raw.kval).1) }
+    }
+
+    /// Removes this bucket's key and value from the hashtable.
+    ///
+    /// This works similarly to `put`, building an `EmptyBucket` out of the
+    /// taken bucket.
+    pub fn take(mut self) -> (EmptyBucket<K, V, M>, K, V) {
+        self.table.size -= 1;
+
+        unsafe {
+            *self.raw.hash = None;
+            let (k, v) = ptr::read(self.raw.kval);
+            (self.unsafe_cast(), k, v)
+        }
     }
 }
 
@@ -437,8 +424,7 @@ impl<K, V, M> GapThenFull<K, V, M> where M: Borrow<RawTable<K, V>> {
     pub fn shift(mut self) -> Option<GapThenFull<K, V, M>> {
         unsafe {
             *self.gap.raw.hash = mem::replace(&mut *self.full.raw.hash, None);
-            ptr::copy_nonoverlapping(self.gap.0.raw.key, self.full.0.raw.key, 1);
-            ptr::copy_nonoverlapping(self.gap.0.raw.val, self.full.0.raw.val, 1);
+            ptr::copy_nonoverlapping(self.gap.raw.kval, self.full.raw.kval, 1);
         }
 
         let Bucket { raw: prev_raw, idx: prev_idx, .. } = self.full;
@@ -457,117 +443,34 @@ impl<K, V, M> GapThenFull<K, V, M> where M: Borrow<RawTable<K, V>> {
     }
 }
 
-/// Rounds up to a multiple of a power of two. Returns the closest multiple
-/// of `target_alignment` that is higher or equal to `unrounded`.
-///
-/// # Panics
-///
-/// Panics if `target_alignment` is not a power of two.
-fn round_up_to_next(unrounded: usize, target_alignment: usize) -> usize {
-    assert!(target_alignment.is_power_of_two());
-    (unrounded + target_alignment - 1) & !(target_alignment - 1)
-}
-
-#[test]
-fn test_rounding() {
-    assert_eq!(round_up_to_next(0, 4), 0);
-    assert_eq!(round_up_to_next(1, 4), 4);
-    assert_eq!(round_up_to_next(2, 4), 4);
-    assert_eq!(round_up_to_next(3, 4), 4);
-    assert_eq!(round_up_to_next(4, 4), 4);
-    assert_eq!(round_up_to_next(5, 4), 8);
-}
-
-// Returns a tuple of (key_offset, val_offset),
-// from the start of a mallocated array.
-fn calculate_offsets(hashes_size: usize,
-                     keys_size: usize, keys_align: usize,
-                     vals_align: usize)
-                     -> (usize, usize) {
-    let keys_offset = round_up_to_next(hashes_size, keys_align);
-    let end_of_keys = keys_offset + keys_size;
-
-    let vals_offset = round_up_to_next(end_of_keys, vals_align);
-
-    (keys_offset, vals_offset)
-}
-
-// Returns a tuple of (minimum required malloc alignment, hash_offset,
-// array_size), from the start of a mallocated array.
-fn calculate_allocation(hash_size: usize, hash_align: usize,
-                        keys_size: usize, keys_align: usize,
-                        vals_size: usize, vals_align: usize)
-                        -> (usize, usize, usize) {
-    let hash_offset = 0;
-    let (_, vals_offset) = calculate_offsets(hash_size,
-                                             keys_size, keys_align,
-                                                        vals_align);
-    let end_of_vals = vals_offset + vals_size;
-
-    let min_align = cmp::max(hash_align, cmp::max(keys_align, vals_align));
-
-    (min_align, hash_offset, end_of_vals)
-}
-
-#[test]
-fn test_offset_calculation() {
-    assert_eq!(calculate_allocation(128, 8, 15, 1, 4,  4), (8, 0, 148));
-    assert_eq!(calculate_allocation(3,   1, 2,  1, 1,  1), (1, 0, 6));
-    assert_eq!(calculate_allocation(6,   2, 12, 4, 24, 8), (8, 0, 48));
-    assert_eq!(calculate_offsets(128, 15, 1, 4), (128, 144));
-    assert_eq!(calculate_offsets(3,   2,  1, 1), (3,   5));
-    assert_eq!(calculate_offsets(6,   12, 4, 8), (8,   24));
-}
-
 impl<K, V> RawTable<K, V> {
     /// Does not initialize the buckets.
-    unsafe fn new_uninitialized(capacity: uint) -> RawTable<K, V> {
-        if capacity == 0 {
-            return RawTable {
-                size: 0,
-                capacity: 0,
-                hashes: Unique::new(EMPTY as *mut u64),
-                marker: marker::PhantomData,
+    pub fn new_uninitialized(capacity: uint) -> PartialRawTable<K, V> {
+        unsafe {
+            let table = if capacity == 0 {
+                RawTable {
+                    size: 0,
+                    capacity: 0,
+                    middle: Unique::new(EMPTY as *mut _),
+                }
+            } else {
+                let hashes = allocate(checked_size_generic::<K, V>(capacity), align::<K, V>());
+                if hashes.is_null() { ::alloc::oom() }
+
+                RawTable {
+                    capacity: capacity,
+                    size:     0,
+                    middle:   Unique((hashes as *mut (K, V)).offset(capacity as isize)),
+                }
             };
-        }
 
-        // No need for `checked_mul` before a more restrictive check performed
-        // later in this method.
-        let hashes_size = capacity * size_of::<Option<SafeHash>>();
-        let keys_size   = capacity * size_of::<K>();
-        let vals_size   = capacity * size_of::<V>();
-
-        // Allocating hashmaps is a little tricky. We need to allocate three
-        // arrays, but since we know their sizes and alignments up front,
-        // we just allocate a single array, and then have the subarrays
-        // point into it.
-        //
-        // This is great in theory, but in practice getting the alignment
-        // right is a little subtle. Therefore, calculating offsets has been
-        // factored out into a different function.
-        let (malloc_alignment, hash_offset, size) =
-            calculate_allocation(
-                hashes_size, min_align_of::<Option<SafeHash>>(),
-                keys_size,   min_align_of::< K >(),
-                vals_size,   min_align_of::< V >());
-
-        // One check for overflow that covers calculation and rounding of size.
-        let size_of_bucket = size_of::<Option<SafeHash>>().checked_add(size_of::<K>()).unwrap()
-                                                          .checked_add(size_of::<V>()).unwrap();
-        assert!(size >= capacity.checked_mul(size_of_bucket)
-                                .expect("capacity overflow"),
-                "capacity overflow");
-
-        let buffer = allocate(size, malloc_alignment);
-        if buffer.is_null() { ::alloc::oom() }
-
-        let hashes = buffer.offset(hash_offset as isize) as *mut Option<SafeHash>;
-
-        RawTable {
-            capacity: capacity,
-            size:     0,
-            hashes:   Unique::new(hashes),
-            marker:   marker::PhantomData,
+            PartialRawTable {
+                front: table.first_bucket_raw(),
+                back: table.first_bucket_raw(),
+                front_num: 0,
+                back_num: capacity,
+                table: table,
+            }
         }
     }
 
@@ -582,23 +485,12 @@ impl<K, V> RawTable<K, V> {
         if self.capacity() == 0 {
             RawBucket {
                 hash: ptr::null_mut(),
-                key: ptr::null_mut(),
-                val: ptr::null_mut(),
+                kval: ptr::null_mut(),
             }
         } else {
-            let hashes_size = self.capacity * size_of::<Option<SafeHash>>();
-            let keys_size = self.capacity * size_of::<K>();
-
-            let buffer = *self.hashes as *mut u8;
-            let (keys_offset, vals_offset) = calculate_offsets(hashes_size,
-                                                               keys_size, min_align_of::<K>(),
-                                                               min_align_of::<V>());
-
             RawBucket {
-                hash: *self.middle as *mut Option<SafeHash>,
-                key:  buffer.offset(keys_offset as isize) as *mut K,
-                val:  buffer.offset(vals_offset as isize) as *mut V,
-                _marker: marker::PhantomData,
+                hash: self.middle.ptr as *mut Option<SafeHash>,
+                kval: self.middle.ptr.offset(-(self.capacity as isize)),
             }
         }
     }
@@ -613,14 +505,61 @@ impl<K, V> RawTable<K, V> {
     pub fn size(&self) -> usize {
         self.size
     }
+}
 
+/// Rounds up to a multiple of a power of two. Returns the closest multiple
+/// of `target_alignment` that is higher or equal to `unrounded`.
+///
+/// # Panics
+///
+/// Panics if `target_alignment` is not a power of two.
+fn round_up_to_next(unrounded: uint, target_alignment: uint) -> uint {
+    assert!(target_alignment.is_power_of_two());
+    (unrounded + target_alignment - 1) & !(target_alignment - 1)
 }
 
+#[test]
+fn test_rounding() {
+    assert_eq!(round_up_to_next(0, 4), 0);
+    assert_eq!(round_up_to_next(1, 4), 4);
+    assert_eq!(round_up_to_next(2, 4), 4);
+    assert_eq!(round_up_to_next(3, 4), 4);
+    assert_eq!(round_up_to_next(4, 4), 4);
+    assert_eq!(round_up_to_next(5, 4), 8);
+    assert_eq!(round_up_to_next(5, 8), 8);
+}
 
+#[inline]
+fn size_generic<K, V>(capacity: usize) -> usize {
+    let hash_align = min_align_of::<Option<SafeHash>>();
+    round_up_to_next(size_of::<(K, V)>() * capacity, hash_align) + size_of::<SafeHash>() * capacity
+}
 
+fn checked_size_generic<K, V>(capacity: usize) -> usize {
+    let size = size_generic::<K, V>(capacity);
+    let elem_size = size_of::<(K, V)>() + size_of::<SafeHash>();
+    assert!(size >= capacity.checked_mul(elem_size).expect("capacity overflow"),
+            "capacity overflow");
+    size
+}
+
+#[inline]
+fn align<K, V>() -> usize {
+    cmp::max(mem::min_align_of::<(K, V)>(), mem::min_align_of::<u64>())
+}
+
+/// A newtyped RawBucket. Not copyable.
+pub struct RawFullBucket<K, V, M>(RawBucket<K, V>);
+
+impl<'t, K, V, M: 't> RawFullBucket<K, V, M> where RawTable<K, V>: BorrowFrom<M> {
+    pub fn into_refs(self) -> (&'t K, &'t V) {
+        unsafe { (&(*self.0.kval).0, &(*self.0.kval).1) }
     }
 }
 
+impl<'t, K, V, M: 't> RawFullBucket<K, V, M> where RawTable<K, V>: BorrowFromMut<M> {
+    pub fn into_mut_refs(self) -> (&'t mut K, &'t mut V) {
+        unsafe { (&mut (*self.0.kval).0, &mut (*self.0.kval).1) }
     }
 }
 
