From 014c46bb4e91f5bb98ecea52757e9259e260f8d7 Mon Sep 17 00:00:00 2001 From: The8472 Date: Fri, 11 Oct 2019 20:42:32 +0200 Subject: [PATCH 01/54] unrelated typo fix --- src/liballoc/vec.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 26a7812f58e01..3418018a2aade 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2865,8 +2865,8 @@ where old_len: usize, /// The filter test predicate. pred: F, - /// A flag that indicates a panic has occurred in the filter test prodicate. - /// This is used as a hint in the drop implmentation to prevent consumption + /// A flag that indicates a panic has occurred in the filter test predicate. + /// This is used as a hint in the drop implementation to prevent consumption /// of the remainder of the `DrainFilter`. Any unprocessed items will be /// backshifted in the `vec`, but no further items will be dropped or /// tested by the filter predicate. From fa02f7292815bd060a02be335c5c1ec37f41ad6c Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 19 Oct 2019 00:00:45 +0200 Subject: [PATCH 02/54] bench --- src/liballoc/benches/vec.rs | 41 ++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/liballoc/benches/vec.rs b/src/liballoc/benches/vec.rs index a3da9e80cd0fc..258e9787b298c 100644 --- a/src/liballoc/benches/vec.rs +++ b/src/liballoc/benches/vec.rs @@ -1,5 +1,5 @@ use std::iter::{repeat, FromIterator}; -use test::Bencher; +use test::{black_box, Bencher}; #[bench] fn bench_new(b: &mut Bencher) { @@ -480,3 +480,42 @@ fn bench_clone_from_10_0100_0010(b: &mut Bencher) { fn bench_clone_from_10_1000_0100(b: &mut Bencher) { do_bench_clone_from(b, 10, 1000, 100) } + +macro_rules! bench_in_place { + ( + $($fname:ident, $type:ty , $count:expr, $init: expr);* + ) => { + $( + #[bench] + fn $fname(b: &mut Bencher) { + b.iter(|| { + let src: Vec<$type> = vec![$init; $count]; + black_box(src.into_iter() + .enumerate() + .map(|(idx, e)| { (idx as $type) ^ e }).collect::>()) + }); + } + )+ + }; +} + +bench_in_place![ + bench_in_place_xxu8_i0_0010, u8, 10, 0; + bench_in_place_xxu8_i0_0100, u8, 100, 0; + bench_in_place_xxu8_i0_1000, u8, 1000, 0; + bench_in_place_xxu8_i1_0010, u8, 10, 1; + bench_in_place_xxu8_i1_0100, u8, 100, 1; + bench_in_place_xxu8_i1_1000, u8, 1000, 1; + bench_in_place_xu32_i0_0010, u32, 10, 0; + bench_in_place_xu32_i0_0100, u32, 100, 0; + bench_in_place_xu32_i0_1000, u32, 1000, 0; + bench_in_place_xu32_i1_0010, u32, 10, 1; + bench_in_place_xu32_i1_0100, u32, 100, 1; + bench_in_place_xu32_i1_1000, u32, 1000, 1; + bench_in_place_u128_i0_0010, u128, 10, 0; + bench_in_place_u128_i0_0100, u128, 100, 0; + bench_in_place_u128_i0_1000, u128, 1000, 0; + bench_in_place_u128_i1_0010, u128, 10, 1; + bench_in_place_u128_i1_0100, u128, 100, 1; + bench_in_place_u128_i1_1000, u128, 1000, 1 +]; From 91413730f1b13e6b7e0f43ddd474a6442b02b033 Mon Sep 17 00:00:00 2001 From: The8472 Date: Fri, 11 Oct 2019 20:43:25 +0200 Subject: [PATCH 03/54] in-place collect for Vec. Box<[]> and BinaryHeap IntoIter and some adapters --- src/liballoc/collections/binary_heap.rs | 15 ++- src/liballoc/lib.rs | 1 + src/liballoc/tests/binary_heap.rs | 12 ++ src/liballoc/tests/lib.rs | 1 + src/liballoc/tests/slice.rs | 9 ++ src/liballoc/tests/vec.rs | 22 ++++ src/liballoc/vec.rs | 140 +++++++++++++++++------- src/libcore/iter/adapters/mod.rs | 122 ++++++++++++++++++++- src/libcore/iter/mod.rs | 5 + src/libcore/iter/traits/marker.rs | 12 ++ src/libcore/iter/traits/mod.rs | 2 + 11 files changed, 297 insertions(+), 44 deletions(-) diff --git a/src/liballoc/collections/binary_heap.rs b/src/liballoc/collections/binary_heap.rs index c527b378f7465..8c0d6d1f694a4 100644 --- a/src/liballoc/collections/binary_heap.rs +++ b/src/liballoc/collections/binary_heap.rs @@ -146,7 +146,7 @@ #![stable(feature = "rust1", since = "1.0.0")] use core::fmt; -use core::iter::{FromIterator, FusedIterator, TrustedLen}; +use core::iter::{FromIterator, FusedIterator, InPlaceIterable, SourceIter, TrustedLen}; use core::mem::{size_of, swap, ManuallyDrop}; use core::ops::{Deref, DerefMut}; use core::ptr; @@ -1145,6 +1145,19 @@ impl ExactSizeIterator for IntoIter { #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for IntoIter {} +#[unstable(issue = "0", feature = "inplace_iteration")] +impl SourceIter for IntoIter { + type Source = crate::vec::IntoIter; + + #[inline] + fn as_inner(&mut self) -> &mut Self::Source { + &mut self.iter + } +} + +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for IntoIter {} + #[unstable(feature = "binary_heap_into_iter_sorted", issue = "59278")] #[derive(Clone, Debug)] pub struct IntoIterSorted { diff --git a/src/liballoc/lib.rs b/src/liballoc/lib.rs index ffa4176cc7969..c37e474cbb9a9 100644 --- a/src/liballoc/lib.rs +++ b/src/liballoc/lib.rs @@ -121,6 +121,7 @@ #![feature(alloc_layout_extra)] #![feature(try_trait)] #![feature(associated_type_bounds)] +#![feature(inplace_iteration)] // Allow testing this library diff --git a/src/liballoc/tests/binary_heap.rs b/src/liballoc/tests/binary_heap.rs index f49ca7139212f..93eaff0054144 100644 --- a/src/liballoc/tests/binary_heap.rs +++ b/src/liballoc/tests/binary_heap.rs @@ -228,6 +228,18 @@ fn test_to_vec() { check_to_vec(vec![5, 4, 3, 2, 1, 5, 4, 3, 2, 1, 5, 4, 3, 2, 1]); } +#[test] +fn test_in_place_iterator_specialization() { + let src: Vec = vec![1, 2, 3]; + let src_ptr = src.as_ptr(); + let heap: BinaryHeap<_> = src.into_iter().map(std::convert::identity).collect(); + let heap_ptr = heap.iter().next().unwrap() as *const usize; + assert_eq!(src_ptr, heap_ptr); + let sink: Vec<_> = heap.into_iter().map(std::convert::identity).collect(); + let sink_ptr = sink.as_ptr(); + assert_eq!(heap_ptr, sink_ptr); +} + #[test] fn test_empty_pop() { let mut heap = BinaryHeap::::new(); diff --git a/src/liballoc/tests/lib.rs b/src/liballoc/tests/lib.rs index c1ae67a1a339f..966377543d580 100644 --- a/src/liballoc/tests/lib.rs +++ b/src/liballoc/tests/lib.rs @@ -12,6 +12,7 @@ #![feature(binary_heap_into_iter_sorted)] #![feature(binary_heap_drain_sorted)] #![feature(vec_remove_item)] +#![feature(inplace_iteration)] use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; diff --git a/src/liballoc/tests/slice.rs b/src/liballoc/tests/slice.rs index 51ddb5e7a4ec6..444d87d7172b4 100644 --- a/src/liballoc/tests/slice.rs +++ b/src/liballoc/tests/slice.rs @@ -1385,6 +1385,15 @@ fn test_to_vec() { assert_eq!(ys, [1, 2, 3]); } +#[test] +fn test_in_place_iterator_specialization() { + let src: Box<[usize]> = box [1, 2, 3]; + let src_ptr = src.as_ptr(); + let sink: Box<_> = src.into_vec().into_iter().map(std::convert::identity).collect(); + let sink_ptr = sink.as_ptr(); + assert_eq!(src_ptr, sink_ptr); +} + #[test] fn test_box_slice_clone() { let data = vec![vec![0, 1], vec![0], vec![1]]; diff --git a/src/liballoc/tests/vec.rs b/src/liballoc/tests/vec.rs index 2a9bfefc713e7..0f119a8a8e66f 100644 --- a/src/liballoc/tests/vec.rs +++ b/src/liballoc/tests/vec.rs @@ -1,5 +1,6 @@ use std::borrow::Cow; use std::collections::TryReserveError::*; +use std::iter::InPlaceIterable; use std::mem::size_of; use std::vec::{Drain, IntoIter}; use std::{isize, usize}; @@ -726,6 +727,27 @@ fn test_into_iter_clone() { assert_eq!(it.next(), None); } +#[test] +fn test_from_iter_specialization() { + let src: Vec = vec![0usize; 1]; + let srcptr = src.as_ptr(); + let sink = src.into_iter().collect::>(); + let sinkptr = sink.as_ptr(); + assert_eq!(srcptr, sinkptr); +} + +#[test] +fn test_from_iter_specialization_with_iterator_adapters() { + fn assert_in_place_trait(_: &T) {}; + let src: Vec = vec![0usize; 65535]; + let srcptr = src.as_ptr(); + let iter = src.into_iter().enumerate().map(|i| i.0 + i.1).peekable().skip(1); + assert_in_place_trait(&iter); + let sink = iter.collect::>(); + let sinkptr = sink.as_ptr(); + assert_eq!(srcptr, sinkptr); +} + #[test] fn test_cow_from() { let borrowed: &[_] = &["borrowed", "(slice)"]; diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 3418018a2aade..dc023160143fd 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -61,7 +61,7 @@ use core::cmp::{self, Ordering}; use core::fmt; use core::hash::{self, Hash}; use core::intrinsics::{arith_offset, assume}; -use core::iter::{FromIterator, FusedIterator, TrustedLen}; +use core::iter::{FromIterator, FusedIterator, InPlaceIterable, SourceIter, TrustedLen}; use core::marker::PhantomData; use core::mem; use core::ops::Bound::{Excluded, Included, Unbounded}; @@ -1916,7 +1916,7 @@ impl ops::DerefMut for Vec { impl FromIterator for Vec { #[inline] fn from_iter>(iter: I) -> Vec { - >::from_iter(iter.into_iter()) + >::from_iter(iter.into_iter()) } } @@ -1988,13 +1988,12 @@ impl Extend for Vec { } } -// Specialization trait used for Vec::from_iter and Vec::extend -trait SpecExtend { +// Specialization trait used for Vec::from_iter +trait SpecFrom { fn from_iter(iter: I) -> Self; - fn spec_extend(&mut self, iter: I); } -impl SpecExtend for Vec +impl SpecFrom for Vec where I: Iterator, { @@ -2019,7 +2018,87 @@ where as SpecExtend>::spec_extend(&mut vector, iterator); vector } +} + +fn from_into_iter_source(mut iterator: I) -> Vec +where + I: Iterator + InPlaceIterable + SourceIter>, +{ + let mut insert_pos = 0; + + // FIXME: how to drop values written into source when iteration panics? + // tail already gets cleaned by IntoIter::drop + while let Some(item) = iterator.next() { + let source_iter = iterator.as_inner(); + let src_buf = source_iter.buf.as_ptr(); + let src_idx = source_iter.ptr; + unsafe { + let dst = src_buf.offset(insert_pos as isize); + debug_assert!( + dst as *const _ < src_idx, + "InPlaceIterable implementation produced more\ + items than it consumed from the source" + ); + ptr::write(dst, item) + } + insert_pos += 1; + } + + let src = iterator.as_inner(); + let vec = unsafe { Vec::from_raw_parts(src.buf.as_ptr(), insert_pos, src.cap) }; + mem::forget(iterator); + vec +} + +impl SpecFrom> for Vec { + fn from_iter(iterator: IntoIter) -> Self { + // A common case is passing a vector into a function which immediately + // re-collects into a vector. We can short circuit this if the IntoIter + // has not been advanced at all. + if iterator.buf.as_ptr() as *const _ == iterator.ptr { + unsafe { + let vec = Vec::from_raw_parts(iterator.buf.as_ptr(), iterator.len(), iterator.cap); + mem::forget(iterator); + return vec; + } + } + + from_into_iter_source(iterator) + } +} + +// Further specialization potential once lattice specialization exists +// and https://github.com/rust-lang/rust/issues/62645 has been solved: +// This can be broadened to only require size and alignment equality between +// input and output Item types. +impl SpecFrom for Vec +where + I: Iterator + InPlaceIterable + SourceIter>, +{ + default fn from_iter(iterator: I) -> Self { + from_into_iter_source(iterator) + } +} + +impl<'a, T: 'a, I> SpecFrom<&'a T, I> for Vec +where + I: Iterator, + T: Clone, +{ + default fn from_iter(iterator: I) -> Self { + SpecFrom::from_iter(iterator.cloned()) + } +} +// Specialization trait used for Vec::extend +trait SpecExtend { + fn spec_extend(&mut self, iter: I); +} + +impl SpecExtend for Vec +where + I: Iterator, +{ default fn spec_extend(&mut self, iter: I) { self.extend_desugared(iter) } @@ -2029,12 +2108,6 @@ impl SpecExtend for Vec where I: TrustedLen, { - default fn from_iter(iterator: I) -> Self { - let mut vector = Vec::new(); - vector.spec_extend(iterator); - vector - } - default fn spec_extend(&mut self, iterator: I) { // This is the case for a TrustedLen iterator. let (low, high) = iterator.size_hint(); @@ -2064,41 +2137,11 @@ where } } -impl SpecExtend> for Vec { - fn from_iter(iterator: IntoIter) -> Self { - // A common case is passing a vector into a function which immediately - // re-collects into a vector. We can short circuit this if the IntoIter - // has not been advanced at all. - if iterator.buf.as_ptr() as *const _ == iterator.ptr { - unsafe { - let vec = Vec::from_raw_parts(iterator.buf.as_ptr(), iterator.len(), iterator.cap); - mem::forget(iterator); - vec - } - } else { - let mut vector = Vec::new(); - vector.spec_extend(iterator); - vector - } - } - - fn spec_extend(&mut self, mut iterator: IntoIter) { - unsafe { - self.append_elements(iterator.as_slice() as _); - } - iterator.ptr = iterator.end; - } -} - impl<'a, T: 'a, I> SpecExtend<&'a T, I> for Vec where I: Iterator, T: Clone, { - default fn from_iter(iterator: I) -> Self { - SpecExtend::from_iter(iterator.cloned()) - } - default fn spec_extend(&mut self, iterator: I) { self.spec_extend(iterator.cloned()) } @@ -2627,6 +2670,19 @@ unsafe impl<#[may_dangle] T> Drop for IntoIter { } } +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for IntoIter {} + +#[unstable(issue = "0", feature = "inplace_iteration")] +impl SourceIter for IntoIter { + type Source = IntoIter; + + #[inline] + fn as_inner(&mut self) -> &mut Self::Source { + self + } +} + /// A draining iterator for `Vec`. /// /// This `struct` is created by the [`drain`] method on [`Vec`]. diff --git a/src/libcore/iter/adapters/mod.rs b/src/libcore/iter/adapters/mod.rs index 7d10ef3d28219..31157dbbd707a 100644 --- a/src/libcore/iter/adapters/mod.rs +++ b/src/libcore/iter/adapters/mod.rs @@ -5,7 +5,9 @@ use crate::ops::{Add, AddAssign, Try}; use crate::usize; use super::{from_fn, LoopState}; -use super::{DoubleEndedIterator, ExactSizeIterator, FusedIterator, Iterator, TrustedLen}; +use super::{ + DoubleEndedIterator, ExactSizeIterator, FusedIterator, InPlaceIterable, Iterator, TrustedLen, +}; mod chain; mod flatten; @@ -17,6 +19,40 @@ pub use self::flatten::{FlatMap, Flatten}; pub(crate) use self::zip::TrustedRandomAccess; pub use self::zip::Zip; +/// This trait provides access to to the backing source of an interator-adapter pipeline +/// under the conditions that +/// * the iterator source `S` itself implements `SourceIter` +/// * there is a delegating implementation of this trait for each adapter in the pipeline +/// +/// This is useful for specializing [`FromIterator`] implementations or to retrieve +/// the remaining values from a source of a partially consumed iterator. +/// +/// # Examples +/// +/// Retrieving a partially consumed source and wrapping it into a different pipeline: +/// +/// ``` +/// # #![feature(inplace_iteration)] +/// # use std::iter::SourceIter; +/// +/// let mut iter = vec![9, 9, 9].into_iter().map(|i| i * i); +/// let first = iter.next().unwrap(); +/// let mut remainder = std::mem::replace(iter.as_inner(), Vec::new().into_iter()); +/// let second = remainder.map(|i| i + 1).next().unwrap(); +/// assert_eq!(first, 81); +/// assert_eq!(second, 10); +/// ``` +/// +/// [`FromIterator`]: trait.FromIterator.html +#[unstable(issue = "0", feature = "inplace_iteration")] +pub trait SourceIter { + /// The source iterator of the adapter. + type Source: Iterator; + + /// Recursively extract the source of an iterator pipeline. + fn as_inner(&mut self) -> &mut Self::Source; +} + /// A double-ended iterator with the direction inverted. /// /// This `struct` is created by the [`rev`] method on [`Iterator`]. See its @@ -880,6 +916,23 @@ where } } +#[unstable(issue = "0", feature = "inplace_iteration")] +impl SourceIter for Map +where + F: FnMut(I::Item) -> B, + I: SourceIter, +{ + type Source = S; + + #[inline] + fn as_inner(&mut self) -> &mut S { + SourceIter::as_inner(&mut self.iter) + } +} + +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Map where F: FnMut(I::Item) -> B {} + /// An iterator that filters the elements of `iter` with `predicate`. /// /// This `struct` is created by the [`filter`] method on [`Iterator`]. See its @@ -1348,6 +1401,22 @@ impl FusedIterator for Enumerate where I: FusedIterator {} #[unstable(feature = "trusted_len", issue = "37572")] unsafe impl TrustedLen for Enumerate where I: TrustedLen {} +#[unstable(issue = "0", feature = "inplace_iteration")] +impl SourceIter for Enumerate +where + I: SourceIter, +{ + type Source = S; + + #[inline] + fn as_inner(&mut self) -> &mut S { + SourceIter::as_inner(&mut self.iter) + } +} + +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Enumerate {} + /// An iterator with a `peek()` that returns an optional reference to the next /// element. /// @@ -1559,6 +1628,25 @@ impl Peekable { } } +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for Peekable where I: TrustedLen {} + +#[unstable(issue = "0", feature = "inplace_iteration")] +impl SourceIter for Peekable +where + I: SourceIter, +{ + type Source = S; + + #[inline] + fn as_inner(&mut self) -> &mut S { + SourceIter::as_inner(&mut self.iter) + } +} + +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Peekable {} + /// An iterator that rejects elements while `predicate` returns `true`. /// /// This `struct` is created by the [`skip_while`] method on [`Iterator`]. See its @@ -2030,6 +2118,22 @@ where #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for Skip where I: FusedIterator {} +#[unstable(issue = "0", feature = "inplace_iteration")] +impl SourceIter for Skip +where + I: SourceIter, +{ + type Source = S; + + #[inline] + fn as_inner(&mut self) -> &mut S { + SourceIter::as_inner(&mut self.iter) + } +} + +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Skip {} + /// An iterator that only iterates over the first `n` iterations of `iter`. /// /// This `struct` is created by the [`take`] method on [`Iterator`]. See its @@ -2513,6 +2617,22 @@ where } } +#[unstable(issue = "0", feature = "inplace_iteration")] +impl SourceIter for Fuse +where + I: SourceIter, +{ + type Source = S; + + #[inline] + fn as_inner(&mut self) -> &mut S { + SourceIter::as_inner(&mut self.iter) + } +} + +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Fuse {} + /// An iterator that calls a function with a reference to each element before /// yielding it. /// diff --git a/src/libcore/iter/mod.rs b/src/libcore/iter/mod.rs index d8a56cb3ae515..674be45f76d95 100644 --- a/src/libcore/iter/mod.rs +++ b/src/libcore/iter/mod.rs @@ -345,6 +345,9 @@ pub use self::traits::{DoubleEndedIterator, Extend, FromIterator, IntoIterator}; #[stable(feature = "rust1", since = "1.0.0")] pub use self::traits::{ExactSizeIterator, Product, Sum}; +#[unstable(issue = "0", feature = "inplace_iteration")] +pub use self::traits::InPlaceIterable; + #[stable(feature = "iter_cloned", since = "1.1.0")] pub use self::adapters::Cloned; #[stable(feature = "iter_copied", since = "1.36.0")] @@ -353,6 +356,8 @@ pub use self::adapters::Copied; pub use self::adapters::Flatten; #[unstable(feature = "iter_map_while", reason = "recently added", issue = "68537")] pub use self::adapters::MapWhile; +#[unstable(issue = "0", feature = "inplace_iteration")] +pub use self::adapters::SourceIter; #[stable(feature = "iterator_step_by", since = "1.28.0")] pub use self::adapters::StepBy; #[stable(feature = "rust1", since = "1.0.0")] diff --git a/src/libcore/iter/traits/marker.rs b/src/libcore/iter/traits/marker.rs index 404cc84495c96..b27cfc0d2d7c6 100644 --- a/src/libcore/iter/traits/marker.rs +++ b/src/libcore/iter/traits/marker.rs @@ -42,3 +42,15 @@ pub unsafe trait TrustedLen: Iterator {} #[unstable(feature = "trusted_len", issue = "37572")] unsafe impl TrustedLen for &mut I {} + +/// An iterator that when yielding an item will have taken at least one element +/// from its underlying [`SourceIter`]. +/// +/// Calling next() guarantees that at least one value of the iterator's underlying source +/// has been moved out and the result of the iterator chain could be inserted in its place, +/// assuming structural constraints of the source allow such an insertion. +/// In other words this trait indicates that an iterator pipeline can be collected in place. +/// +/// [`SourceIter`]: ../../std/iter/trait.SourceIter.html +#[unstable(issue = "0", feature = "inplace_iteration")] +pub unsafe trait InPlaceIterable: Iterator {} diff --git a/src/libcore/iter/traits/mod.rs b/src/libcore/iter/traits/mod.rs index efd1580a54807..9ed2de7313df1 100644 --- a/src/libcore/iter/traits/mod.rs +++ b/src/libcore/iter/traits/mod.rs @@ -11,5 +11,7 @@ pub use self::double_ended::DoubleEndedIterator; pub use self::exact_size::ExactSizeIterator; #[stable(feature = "rust1", since = "1.0.0")] pub use self::iterator::Iterator; +#[unstable(issue = "0", feature = "inplace_iteration")] +pub use self::marker::InPlaceIterable; #[stable(feature = "rust1", since = "1.0.0")] pub use self::marker::{FusedIterator, TrustedLen}; From 5d4f0558a1f3939acf5a0ceada4cfa7cc822e6e5 Mon Sep 17 00:00:00 2001 From: The8472 Date: Fri, 15 Nov 2019 01:18:30 +0100 Subject: [PATCH 04/54] mark SourceIter as unsafe, document invariants --- src/liballoc/collections/binary_heap.rs | 2 +- src/liballoc/vec.rs | 2 +- src/libcore/iter/adapters/mod.rs | 68 ++++++++++++++++++------- 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/src/liballoc/collections/binary_heap.rs b/src/liballoc/collections/binary_heap.rs index 8c0d6d1f694a4..6cc588bf8754f 100644 --- a/src/liballoc/collections/binary_heap.rs +++ b/src/liballoc/collections/binary_heap.rs @@ -1146,7 +1146,7 @@ impl ExactSizeIterator for IntoIter { impl FusedIterator for IntoIter {} #[unstable(issue = "0", feature = "inplace_iteration")] -impl SourceIter for IntoIter { +unsafe impl SourceIter for IntoIter { type Source = crate::vec::IntoIter; #[inline] diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index dc023160143fd..e8f375494f8cc 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2674,7 +2674,7 @@ unsafe impl<#[may_dangle] T> Drop for IntoIter { unsafe impl InPlaceIterable for IntoIter {} #[unstable(issue = "0", feature = "inplace_iteration")] -impl SourceIter for IntoIter { +unsafe impl SourceIter for IntoIter { type Source = IntoIter; #[inline] diff --git a/src/libcore/iter/adapters/mod.rs b/src/libcore/iter/adapters/mod.rs index 31157dbbd707a..15bca8933229c 100644 --- a/src/libcore/iter/adapters/mod.rs +++ b/src/libcore/iter/adapters/mod.rs @@ -19,37 +19,71 @@ pub use self::flatten::{FlatMap, Flatten}; pub(crate) use self::zip::TrustedRandomAccess; pub use self::zip::Zip; -/// This trait provides access to to the backing source of an interator-adapter pipeline +/// This trait provides transitive access to source-stages in an interator-adapter pipeline /// under the conditions that /// * the iterator source `S` itself implements `SourceIter` -/// * there is a delegating implementation of this trait for each adapter in the pipeline +/// * there is a delegating implementation of this trait for each adapter in the pipeline between +/// the source and the pipeline consumer. /// -/// This is useful for specializing [`FromIterator`] implementations or to retrieve -/// the remaining values from a source of a partially consumed iterator. +/// When the source is an owning iterator struct (commonly called `IntoIter`) then +/// this can be useful for specializing [`FromIterator`] implementations or recovering the +/// remaining elements after an iterator has been partially exhausted. +/// +/// Note that implementations do not necessarily have to provide access to the inner-most +/// source of a pipeline. A stateful intermediate adapter might eagerly evaluate a part +/// of the pipeline and expose its internal storage as source. +/// +/// The trait is unsafe because implementers must uphold additional safety properties. +/// See [`as_inner`] for details. /// /// # Examples /// -/// Retrieving a partially consumed source and wrapping it into a different pipeline: +/// Retrieving a partially consumed source: /// /// ``` /// # #![feature(inplace_iteration)] /// # use std::iter::SourceIter; /// /// let mut iter = vec![9, 9, 9].into_iter().map(|i| i * i); -/// let first = iter.next().unwrap(); +/// let _ = iter.next(); /// let mut remainder = std::mem::replace(iter.as_inner(), Vec::new().into_iter()); -/// let second = remainder.map(|i| i + 1).next().unwrap(); -/// assert_eq!(first, 81); -/// assert_eq!(second, 10); +/// println!("n = {} elements remaining", remainder.len()); /// ``` /// /// [`FromIterator`]: trait.FromIterator.html +/// [`as_inner`]: #method.as_inner #[unstable(issue = "0", feature = "inplace_iteration")] -pub trait SourceIter { - /// The source iterator of the adapter. +pub unsafe trait SourceIter { + /// A source stage in an iterator pipeline. type Source: Iterator; - /// Recursively extract the source of an iterator pipeline. + /// Extract the source of an iterator pipeline. + /// + /// Callers may assume that calls to [`next()`] or any method taking `&self` + /// does no replace the referenced value. + /// But callers may replace the referenced values as long they in turn do not + /// expose it through a delegating implementation of this trait. + /// Which means that while adapters may not modify the reference they cannot + /// rely on it not being modified. + /// + /// Adapters must not rely on exclusive ownership or immutability of the source. + /// For example a peeking adapter could either exploit [`TrustedRandomAccess`] to look ahead + /// or implement this trait, but it cannot do both because a caller could call `next()` or any + /// other mutating method on the source between iteration steps and thus invalidate the peeked + /// values. + /// The lack of exclusive ownership also requires that adapters must uphold the source's + /// public API even when they have crate- or module-internal access. + /// + /// Callers in turn must expect the source to be in any state that is consistent with + /// its public API since adapters sitting between it and the source have the same + /// access. In particular an adapter may have consumed more elements than strictly necessary. + /// + /// The overall goal of these requirements is to grant the consumer of a pipeline + /// access to the underlying storage of an iterator while restricting any statefulness + /// and side-effects of the pipeline stages from affecting or relying on that storage. + /// + /// [`TrustedRandomAccess`]: trait.TrustedRandomAccess.html + /// [`next`]: trait.Iterator.html#method.next fn as_inner(&mut self) -> &mut Self::Source; } @@ -917,7 +951,7 @@ where } #[unstable(issue = "0", feature = "inplace_iteration")] -impl SourceIter for Map +unsafe impl SourceIter for Map where F: FnMut(I::Item) -> B, I: SourceIter, @@ -1402,7 +1436,7 @@ impl FusedIterator for Enumerate where I: FusedIterator {} unsafe impl TrustedLen for Enumerate where I: TrustedLen {} #[unstable(issue = "0", feature = "inplace_iteration")] -impl SourceIter for Enumerate +unsafe impl SourceIter for Enumerate where I: SourceIter, { @@ -1632,7 +1666,7 @@ impl Peekable { unsafe impl TrustedLen for Peekable where I: TrustedLen {} #[unstable(issue = "0", feature = "inplace_iteration")] -impl SourceIter for Peekable +unsafe impl SourceIter for Peekable where I: SourceIter, { @@ -2119,7 +2153,7 @@ where impl FusedIterator for Skip where I: FusedIterator {} #[unstable(issue = "0", feature = "inplace_iteration")] -impl SourceIter for Skip +unsafe impl SourceIter for Skip where I: SourceIter, { @@ -2618,7 +2652,7 @@ where } #[unstable(issue = "0", feature = "inplace_iteration")] -impl SourceIter for Fuse +unsafe impl SourceIter for Fuse where I: SourceIter, { From f91bee50ac016047fd9a93d14045cf6db43b33b1 Mon Sep 17 00:00:00 2001 From: The8472 Date: Fri, 15 Nov 2019 01:27:41 +0100 Subject: [PATCH 05/54] assert that SourceIter requirements have not been violated by the pipeline --- src/liballoc/vec.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index e8f375494f8cc..0de06438f7552 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2025,12 +2025,14 @@ where I: Iterator + InPlaceIterable + SourceIter>, { let mut insert_pos = 0; + let original_ptr = iterator.as_inner().buf.as_ptr(); // FIXME: how to drop values written into source when iteration panics? // tail already gets cleaned by IntoIter::drop while let Some(item) = iterator.next() { let source_iter = iterator.as_inner(); let src_buf = source_iter.buf.as_ptr(); + debug_assert_eq!(original_ptr, src_buf); let src_idx = source_iter.ptr; unsafe { let dst = src_buf.offset(insert_pos as isize); From dd26d62b7ba97091df8bf777ac9fae955a644eb3 Mon Sep 17 00:00:00 2001 From: The8472 Date: Fri, 15 Nov 2019 02:37:55 +0100 Subject: [PATCH 06/54] implement drop handling --- src/liballoc/vec.rs | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 0de06438f7552..2dc1012bdc1e5 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2020,22 +2020,37 @@ where } } +struct InPlaceIterFront { + inner: *mut T, + count: usize, + did_panic: bool, +} + +impl Drop for InPlaceIterFront { + #[inline] + fn drop(&mut self) { + unsafe { + if mem::needs_drop::() && self.did_panic { + ptr::drop_in_place(slice::from_raw_parts_mut(self.inner, self.count) as *mut _); + } + } + } +} + fn from_into_iter_source(mut iterator: I) -> Vec where I: Iterator + InPlaceIterable + SourceIter>, { - let mut insert_pos = 0; let original_ptr = iterator.as_inner().buf.as_ptr(); + let mut front_buffer = InPlaceIterFront { inner: original_ptr, count: 0, did_panic: true }; - // FIXME: how to drop values written into source when iteration panics? - // tail already gets cleaned by IntoIter::drop while let Some(item) = iterator.next() { let source_iter = iterator.as_inner(); let src_buf = source_iter.buf.as_ptr(); debug_assert_eq!(original_ptr, src_buf); let src_idx = source_iter.ptr; unsafe { - let dst = src_buf.offset(insert_pos as isize); + let dst = src_buf.offset(front_buffer.count as isize); debug_assert!( dst as *const _ < src_idx, "InPlaceIterable implementation produced more\ @@ -2043,12 +2058,16 @@ where ); ptr::write(dst, item) } - insert_pos += 1; + front_buffer.count += 1; } let src = iterator.as_inner(); - let vec = unsafe { Vec::from_raw_parts(src.buf.as_ptr(), insert_pos, src.cap) }; - mem::forget(iterator); + front_buffer.did_panic = false; + let vec = unsafe { Vec::from_raw_parts(src.buf.as_ptr(), front_buffer.count, src.cap) }; + src.cap = 0; + src.buf = unsafe { NonNull::new_unchecked(RawVec::NEW.ptr()) }; + src.ptr = src.buf.as_ptr(); + src.end = src.buf.as_ptr(); vec } From af4ebcbb7abf8ec047b4954df953d729acd30622 Mon Sep 17 00:00:00 2001 From: The8472 Date: Fri, 15 Nov 2019 03:54:44 +0100 Subject: [PATCH 07/54] use add instead of offset --- src/liballoc/vec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 2dc1012bdc1e5..5dd2e254cfeb2 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2050,7 +2050,7 @@ where debug_assert_eq!(original_ptr, src_buf); let src_idx = source_iter.ptr; unsafe { - let dst = src_buf.offset(front_buffer.count as isize); + let dst = src_buf.add(front_buffer.count); debug_assert!( dst as *const _ < src_idx, "InPlaceIterable implementation produced more\ From 25dbd19655acc3bda87693ba4d6ec6c5652eb171 Mon Sep 17 00:00:00 2001 From: The8472 Date: Fri, 15 Nov 2019 04:13:11 +0100 Subject: [PATCH 08/54] do as tidy demands --- src/liballoc/vec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 5dd2e254cfeb2..3c2f968fe3eec 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -53,7 +53,7 @@ //! [`Index`]: ../../std/ops/trait.Index.html //! [`IndexMut`]: ../../std/ops/trait.IndexMut.html //! [`vec!`]: ../../std/macro.vec.html - +// ignore-tidy-filelength #![stable(feature = "rust1", since = "1.0.0")] use core::array::LengthAtMost32; From 331501e362836042f5de78fe5eab32a2dc497c67 Mon Sep 17 00:00:00 2001 From: The8472 Date: Fri, 15 Nov 2019 21:01:13 +0100 Subject: [PATCH 09/54] fix doc link --- src/libcore/iter/adapters/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcore/iter/adapters/mod.rs b/src/libcore/iter/adapters/mod.rs index 15bca8933229c..f2d66dda51eeb 100644 --- a/src/libcore/iter/adapters/mod.rs +++ b/src/libcore/iter/adapters/mod.rs @@ -83,7 +83,7 @@ pub unsafe trait SourceIter { /// and side-effects of the pipeline stages from affecting or relying on that storage. /// /// [`TrustedRandomAccess`]: trait.TrustedRandomAccess.html - /// [`next`]: trait.Iterator.html#method.next + /// [`next()`]: trait.Iterator.html#method.next fn as_inner(&mut self) -> &mut Self::Source; } From b446bd1be86c4ea09d80d31f6eaf2c14ae60b06d Mon Sep 17 00:00:00 2001 From: The8472 Date: Fri, 15 Nov 2019 21:02:54 +0100 Subject: [PATCH 10/54] simplify pointer arithmetic --- src/liballoc/vec.rs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 3c2f968fe3eec..d76c3b4939c9f 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2022,16 +2022,22 @@ where struct InPlaceIterFront { inner: *mut T, - count: usize, + dst: *mut T, did_panic: bool, } +impl InPlaceIterFront { + unsafe fn len(&self) -> usize { + self.dst.offset_from(self.inner) as usize + } +} + impl Drop for InPlaceIterFront { #[inline] fn drop(&mut self) { unsafe { if mem::needs_drop::() && self.did_panic { - ptr::drop_in_place(slice::from_raw_parts_mut(self.inner, self.count) as *mut _); + ptr::drop_in_place(slice::from_raw_parts_mut(self.inner, self.len()) as *mut _); } } } @@ -2042,28 +2048,26 @@ where I: Iterator + InPlaceIterable + SourceIter>, { let original_ptr = iterator.as_inner().buf.as_ptr(); - let mut front_buffer = InPlaceIterFront { inner: original_ptr, count: 0, did_panic: true }; + let mut front_buffer = + InPlaceIterFront { inner: original_ptr, dst: original_ptr, did_panic: true }; while let Some(item) = iterator.next() { let source_iter = iterator.as_inner(); - let src_buf = source_iter.buf.as_ptr(); - debug_assert_eq!(original_ptr, src_buf); - let src_idx = source_iter.ptr; + debug_assert_eq!(original_ptr, source_iter.buf.as_ptr()); unsafe { - let dst = src_buf.add(front_buffer.count); debug_assert!( - dst as *const _ < src_idx, + front_buffer.dst as *const _ < source_iter.ptr, "InPlaceIterable implementation produced more\ items than it consumed from the source" ); - ptr::write(dst, item) + ptr::write(front_buffer.dst, item); + front_buffer.dst = front_buffer.dst.add(1); } - front_buffer.count += 1; } let src = iterator.as_inner(); front_buffer.did_panic = false; - let vec = unsafe { Vec::from_raw_parts(src.buf.as_ptr(), front_buffer.count, src.cap) }; + let vec = unsafe { Vec::from_raw_parts(src.buf.as_ptr(), front_buffer.len(), src.cap) }; src.cap = 0; src.buf = unsafe { NonNull::new_unchecked(RawVec::NEW.ptr()) }; src.ptr = src.buf.as_ptr(); From 39ea563f1d913d5d9592759e62e982d739898a37 Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 16 Nov 2019 17:27:39 +0100 Subject: [PATCH 11/54] update benches --- src/liballoc/benches/vec.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/liballoc/benches/vec.rs b/src/liballoc/benches/vec.rs index 258e9787b298c..9efb95c63e68f 100644 --- a/src/liballoc/benches/vec.rs +++ b/src/liballoc/benches/vec.rs @@ -489,10 +489,11 @@ macro_rules! bench_in_place { #[bench] fn $fname(b: &mut Bencher) { b.iter(|| { - let src: Vec<$type> = vec![$init; $count]; - black_box(src.into_iter() + let src: Vec<$type> = black_box(vec![$init; $count]); + let mut sink = src.into_iter() .enumerate() - .map(|(idx, e)| { (idx as $type) ^ e }).collect::>()) + .map(|(idx, e)| { (idx as $type) ^ e }).collect::>(); + black_box(sink.as_mut_ptr()) }); } )+ From 2572f5884d7b38a7c8fbe984e58cc5e1389bb3b8 Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 16 Nov 2019 22:31:23 +0100 Subject: [PATCH 12/54] recover vectorization switch to try_fold and segregate the drop handling to keep collect::>() and similar optimizer-friendly It comes at the cost of less accurate debug_asserts and code complexity --- src/liballoc/lib.rs | 1 + src/liballoc/vec.rs | 76 +++++++++++++++++++++++++++++++-------------- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/src/liballoc/lib.rs b/src/liballoc/lib.rs index c37e474cbb9a9..489ced72ba184 100644 --- a/src/liballoc/lib.rs +++ b/src/liballoc/lib.rs @@ -122,6 +122,7 @@ #![feature(try_trait)] #![feature(associated_type_bounds)] #![feature(inplace_iteration)] +#![feature(never_type)] // Allow testing this library diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index d76c3b4939c9f..e7bc60cc98645 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2020,23 +2020,23 @@ where } } -struct InPlaceIterFront { +struct InPlaceDrop { inner: *mut T, dst: *mut T, did_panic: bool, } -impl InPlaceIterFront { +impl InPlaceDrop { unsafe fn len(&self) -> usize { self.dst.offset_from(self.inner) as usize } } -impl Drop for InPlaceIterFront { +impl Drop for InPlaceDrop { #[inline] fn drop(&mut self) { unsafe { - if mem::needs_drop::() && self.did_panic { + if self.did_panic { ptr::drop_in_place(slice::from_raw_parts_mut(self.inner, self.len()) as *mut _); } } @@ -2047,31 +2047,61 @@ fn from_into_iter_source(mut iterator: I) -> Vec where I: Iterator + InPlaceIterable + SourceIter>, { - let original_ptr = iterator.as_inner().buf.as_ptr(); - let mut front_buffer = - InPlaceIterFront { inner: original_ptr, dst: original_ptr, did_panic: true }; - - while let Some(item) = iterator.next() { - let source_iter = iterator.as_inner(); - debug_assert_eq!(original_ptr, source_iter.buf.as_ptr()); - unsafe { - debug_assert!( - front_buffer.dst as *const _ < source_iter.ptr, - "InPlaceIterable implementation produced more\ - items than it consumed from the source" - ); - ptr::write(front_buffer.dst, item); - front_buffer.dst = front_buffer.dst.add(1); - } - } + let src_buf = iterator.as_inner().buf.as_ptr(); + let src_end = iterator.as_inner().end; + let dst = src_buf; + + let dst = if mem::needs_drop::() { + // special-case drop handling since it prevents vectorization + let mut sink = InPlaceDrop { inner: src_buf, dst, did_panic: true }; + let _ = iterator.try_for_each::<_, Result<_, !>>(|item| { + unsafe { + debug_assert!( + sink.dst as *const _ <= src_end, + "InPlaceIterable contract violation" + ); + ptr::write(sink.dst, item); + sink.dst = sink.dst.add(1); + } + Ok(()) + }); + sink.did_panic = false; + sink.dst + } else { + // use try-fold since it vectorizes better, does not take ownership and lets us thread the + // write pointer through its innards + iterator + .try_fold::<_, _, Result<_, !>>(dst, move |mut dst, item| { + unsafe { + // the InPlaceIterable contract cannot be verified precisely here since + // try_fold has an exclusive reference to the source pointer + // all we can do is check if it's still in range + debug_assert!(dst as *const _ <= src_end, "InPlaceIterable contract violation"); + ptr::write(dst, item); + dst = dst.add(1); + } + Ok(dst) + }) + .unwrap() + }; let src = iterator.as_inner(); - front_buffer.did_panic = false; - let vec = unsafe { Vec::from_raw_parts(src.buf.as_ptr(), front_buffer.len(), src.cap) }; + // check if SourceIter and InPlaceIterable contracts were upheld. + // but if they weren't we may not even make it to this point + debug_assert_eq!(src_buf, src.buf.as_ptr()); + debug_assert!(dst as *const _ <= src.ptr, "InPlaceIterable contract violation"); + + let vec = unsafe { + let len = dst.offset_from(src_buf) as usize; + Vec::from_raw_parts(src.buf.as_ptr(), len, src.cap) + }; + // prevent drop of the underlying storage by turning the IntoIter into + // the equivalent of Vec::new().into_iter() src.cap = 0; src.buf = unsafe { NonNull::new_unchecked(RawVec::NEW.ptr()) }; src.ptr = src.buf.as_ptr(); src.end = src.buf.as_ptr(); + vec } From 1acb718335cd4e17507c360fe5605cd787a20347 Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 16 Nov 2019 22:32:07 +0100 Subject: [PATCH 13/54] replace mem::forget with mem::ManuallyDrop --- src/liballoc/vec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index e7bc60cc98645..1016fa0424ecc 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2112,8 +2112,8 @@ impl SpecFrom> for Vec { // has not been advanced at all. if iterator.buf.as_ptr() as *const _ == iterator.ptr { unsafe { + let iterator = mem::ManuallyDrop::new(iterator); let vec = Vec::from_raw_parts(iterator.buf.as_ptr(), iterator.len(), iterator.cap); - mem::forget(iterator); return vec; } } From b958beab5753d833960ca66b747087aebbdb130a Mon Sep 17 00:00:00 2001 From: The8472 Date: Sun, 17 Nov 2019 01:32:36 +0100 Subject: [PATCH 14/54] remove example that relied on non-public trait --- src/libcore/iter/adapters/mod.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/libcore/iter/adapters/mod.rs b/src/libcore/iter/adapters/mod.rs index f2d66dda51eeb..633601629ff8e 100644 --- a/src/libcore/iter/adapters/mod.rs +++ b/src/libcore/iter/adapters/mod.rs @@ -67,10 +67,6 @@ pub unsafe trait SourceIter { /// rely on it not being modified. /// /// Adapters must not rely on exclusive ownership or immutability of the source. - /// For example a peeking adapter could either exploit [`TrustedRandomAccess`] to look ahead - /// or implement this trait, but it cannot do both because a caller could call `next()` or any - /// other mutating method on the source between iteration steps and thus invalidate the peeked - /// values. /// The lack of exclusive ownership also requires that adapters must uphold the source's /// public API even when they have crate- or module-internal access. /// @@ -82,7 +78,6 @@ pub unsafe trait SourceIter { /// access to the underlying storage of an iterator while restricting any statefulness /// and side-effects of the pipeline stages from affecting or relying on that storage. /// - /// [`TrustedRandomAccess`]: trait.TrustedRandomAccess.html /// [`next()`]: trait.Iterator.html#method.next fn as_inner(&mut self) -> &mut Self::Source; } From fc807c4e720c7abba754e373d71311ef64551b45 Mon Sep 17 00:00:00 2001 From: The8472 Date: Sun, 17 Nov 2019 12:04:12 +0100 Subject: [PATCH 15/54] hide binary_heap::IntoIter internals behind impl Trait --- src/liballoc/collections/binary_heap.rs | 2 +- src/liballoc/lib.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/liballoc/collections/binary_heap.rs b/src/liballoc/collections/binary_heap.rs index 6cc588bf8754f..6e380bfca9bd7 100644 --- a/src/liballoc/collections/binary_heap.rs +++ b/src/liballoc/collections/binary_heap.rs @@ -1147,7 +1147,7 @@ impl FusedIterator for IntoIter {} #[unstable(issue = "0", feature = "inplace_iteration")] unsafe impl SourceIter for IntoIter { - type Source = crate::vec::IntoIter; + type Source = impl Iterator; #[inline] fn as_inner(&mut self) -> &mut Self::Source { diff --git a/src/liballoc/lib.rs b/src/liballoc/lib.rs index 489ced72ba184..8959eebff8e17 100644 --- a/src/liballoc/lib.rs +++ b/src/liballoc/lib.rs @@ -123,6 +123,7 @@ #![feature(associated_type_bounds)] #![feature(inplace_iteration)] #![feature(never_type)] +#![feature(type_alias_impl_trait)] // Allow testing this library From 603854203ab386aed5c9e327b937106510bec00c Mon Sep 17 00:00:00 2001 From: The8472 Date: Sun, 17 Nov 2019 12:49:18 +0100 Subject: [PATCH 16/54] restore Vec::extend specialization for vec::IntoIter sources that was lost during refactoring --- src/liballoc/vec.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 1016fa0424ecc..80c69c1e5a6ec 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2192,6 +2192,15 @@ where } } +impl SpecExtend> for Vec { + fn spec_extend(&mut self, mut iterator: IntoIter) { + unsafe { + self.append_elements(iterator.as_slice() as _); + } + iterator.ptr = iterator.end; + } +} + impl<'a, T: 'a, I> SpecExtend<&'a T, I> for Vec where I: Iterator, From 997e7952f03532af9438dc1ac029933a317c318a Mon Sep 17 00:00:00 2001 From: The8472 Date: Sun, 17 Nov 2019 14:50:48 +0100 Subject: [PATCH 17/54] use memmove instead of generic in-place iteration for IntoIter source this is the original SpecExtend<_, IntoIter> logic except generalizing the fast-path to include a memmove --- src/liballoc/vec.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 80c69c1e5a6ec..bc79c22430e15 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2110,15 +2110,23 @@ impl SpecFrom> for Vec { // A common case is passing a vector into a function which immediately // re-collects into a vector. We can short circuit this if the IntoIter // has not been advanced at all. - if iterator.buf.as_ptr() as *const _ == iterator.ptr { + // We can also reuse the memory and move the data to the front if + // allocating a new vector and moving to it would result in the same capacity + let non_zero_offset = iterator.buf.as_ptr() as *const _ != iterator.ptr; + if !non_zero_offset || iterator.len() >= iterator.cap / 2 { unsafe { let iterator = mem::ManuallyDrop::new(iterator); + if non_zero_offset { + ptr::copy(iterator.ptr, iterator.buf.as_ptr(), iterator.len()); + } let vec = Vec::from_raw_parts(iterator.buf.as_ptr(), iterator.len(), iterator.cap); return vec; } } - from_into_iter_source(iterator) + let mut vec = Vec::new(); + vec.extend(iterator); + vec } } From a6b9be86adea206c3098f86dffac165977d617c5 Mon Sep 17 00:00:00 2001 From: The8472 Date: Tue, 19 Nov 2019 23:10:43 +0100 Subject: [PATCH 18/54] add benches from bluss' gists --- src/liballoc/benches/vec.rs | 104 ++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/src/liballoc/benches/vec.rs b/src/liballoc/benches/vec.rs index 9efb95c63e68f..567e6e2a5bcf6 100644 --- a/src/liballoc/benches/vec.rs +++ b/src/liballoc/benches/vec.rs @@ -520,3 +520,107 @@ bench_in_place![ bench_in_place_u128_i1_0100, u128, 100, 1; bench_in_place_u128_i1_1000, u128, 1000, 1 ]; + +#[bench] +fn bench_chain_collect(b: &mut test::Bencher) { + let data = black_box([0; LEN]); + b.iter(|| data.iter().cloned().chain([1].iter().cloned()).collect::>()); +} + +#[bench] +fn bench_chain_chain_collect(b: &mut test::Bencher) { + let data = black_box([0; LEN]); + b.iter(|| { + data.iter() + .cloned() + .chain([1].iter().cloned()) + .chain([2].iter().cloned()) + .collect::>() + }); +} + +#[bench] +fn bench_nest_chain_chain_collect(b: &mut test::Bencher) { + let data = black_box([0; LEN]); + b.iter(|| { + data.iter().cloned().chain([1].iter().chain([2].iter()).cloned()).collect::>() + }); +} + +pub fn example_plain_slow(l: &[u32]) -> Vec { + let mut result = Vec::with_capacity(l.len()); + result.extend(l.iter().rev()); + result +} + +pub fn map_fast(l: &[(u32, u32)]) -> Vec { + let mut result = Vec::with_capacity(l.len()); + for i in 0..l.len() { + unsafe { + *result.get_unchecked_mut(i) = l[i].0; + result.set_len(i); + } + } + result +} + +const LEN: usize = 16384; + +#[bench] +fn bench_range_map_collect(b: &mut test::Bencher) { + b.iter(|| (0..LEN).map(|_| u32::default()).collect::>()); +} + +#[bench] +fn bench_chain_extend_ref(b: &mut test::Bencher) { + let data = black_box([0; LEN]); + b.iter(|| { + let mut v = Vec::::with_capacity(data.len() + 1); + v.extend(data.iter().chain([1].iter())); + v + }); +} + +#[bench] +fn bench_chain_extend_value(b: &mut test::Bencher) { + let data = black_box([0; LEN]); + b.iter(|| { + let mut v = Vec::::with_capacity(data.len() + 1); + v.extend(data.iter().cloned().chain(Some(1))); + v + }); +} + +#[bench] +fn bench_rev_1(b: &mut test::Bencher) { + let data = black_box([0; LEN]); + b.iter(|| { + let mut v = Vec::::new(); + v.extend(data.iter().rev()); + v + }); +} + +#[bench] +fn bench_rev_2(b: &mut test::Bencher) { + let data = black_box([0; LEN]); + b.iter(|| { + example_plain_slow(&data); + }); +} + +#[bench] +fn bench_map_regular(b: &mut test::Bencher) { + let data = black_box([(0, 0); LEN]); + b.iter(|| { + let mut v = Vec::::new(); + v.extend(data.iter().map(|t| t.1)); + v + }); +} + +#[bench] +fn bench_map_fast(b: &mut test::Bencher) { + let data = black_box([(0, 0); LEN]); + b.iter(|| map_fast(&data)); +} From 6696ae4754415787e621a15b1cb805719c11e7f3 Mon Sep 17 00:00:00 2001 From: The8472 Date: Wed, 20 Nov 2019 03:09:30 +0100 Subject: [PATCH 19/54] return the things under test so they get black_box()'ed --- src/liballoc/benches/vec.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/liballoc/benches/vec.rs b/src/liballoc/benches/vec.rs index 567e6e2a5bcf6..97df15016cb28 100644 --- a/src/liballoc/benches/vec.rs +++ b/src/liballoc/benches/vec.rs @@ -7,6 +7,7 @@ fn bench_new(b: &mut Bencher) { let v: Vec = Vec::new(); assert_eq!(v.len(), 0); assert_eq!(v.capacity(), 0); + v }) } @@ -17,6 +18,7 @@ fn do_bench_with_capacity(b: &mut Bencher, src_len: usize) { let v: Vec = Vec::with_capacity(src_len); assert_eq!(v.len(), 0); assert_eq!(v.capacity(), src_len); + v }) } @@ -47,6 +49,7 @@ fn do_bench_from_fn(b: &mut Bencher, src_len: usize) { let dst = (0..src_len).collect::>(); assert_eq!(dst.len(), src_len); assert!(dst.iter().enumerate().all(|(i, x)| i == *x)); + dst }) } @@ -77,6 +80,7 @@ fn do_bench_from_elem(b: &mut Bencher, src_len: usize) { let dst: Vec = repeat(5).take(src_len).collect(); assert_eq!(dst.len(), src_len); assert!(dst.iter().all(|x| *x == 5)); + dst }) } @@ -109,6 +113,7 @@ fn do_bench_from_slice(b: &mut Bencher, src_len: usize) { let dst = src.clone()[..].to_vec(); assert_eq!(dst.len(), src_len); assert!(dst.iter().enumerate().all(|(i, x)| i == *x)); + dst }); } @@ -141,6 +146,7 @@ fn do_bench_from_iter(b: &mut Bencher, src_len: usize) { let dst: Vec<_> = FromIterator::from_iter(src.clone()); assert_eq!(dst.len(), src_len); assert!(dst.iter().enumerate().all(|(i, x)| i == *x)); + dst }); } @@ -175,6 +181,7 @@ fn do_bench_extend(b: &mut Bencher, dst_len: usize, src_len: usize) { dst.extend(src.clone()); assert_eq!(dst.len(), dst_len + src_len); assert!(dst.iter().enumerate().all(|(i, x)| i == *x)); + dst }); } @@ -224,6 +231,7 @@ fn do_bench_push_all(b: &mut Bencher, dst_len: usize, src_len: usize) { dst.extend_from_slice(&src); assert_eq!(dst.len(), dst_len + src_len); assert!(dst.iter().enumerate().all(|(i, x)| i == *x)); + dst }); } @@ -273,6 +281,7 @@ fn do_bench_push_all_move(b: &mut Bencher, dst_len: usize, src_len: usize) { dst.extend(src.clone()); assert_eq!(dst.len(), dst_len + src_len); assert!(dst.iter().enumerate().all(|(i, x)| i == *x)); + dst }); } @@ -320,6 +329,7 @@ fn do_bench_clone(b: &mut Bencher, src_len: usize) { let dst = src.clone(); assert_eq!(dst.len(), src_len); assert!(dst.iter().enumerate().all(|(i, x)| i == *x)); + dst }); } @@ -354,10 +364,10 @@ fn do_bench_clone_from(b: &mut Bencher, times: usize, dst_len: usize, src_len: u for _ in 0..times { dst.clone_from(&src); - assert_eq!(dst.len(), src_len); assert!(dst.iter().enumerate().all(|(i, x)| dst_len + i == *x)); } + dst }); } From 646066cda8f59db1b60388f71e22bb3fb90a63eb Mon Sep 17 00:00:00 2001 From: The8472 Date: Wed, 20 Nov 2019 23:37:50 +0100 Subject: [PATCH 20/54] cyclic in-place reuse bench --- src/liballoc/benches/vec.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/liballoc/benches/vec.rs b/src/liballoc/benches/vec.rs index 97df15016cb28..301dc5fd009ed 100644 --- a/src/liballoc/benches/vec.rs +++ b/src/liballoc/benches/vec.rs @@ -531,6 +531,26 @@ bench_in_place![ bench_in_place_u128_i1_1000, u128, 1000, 1 ]; +#[bench] +fn bench_in_place_recycle(b: &mut test::Bencher) { + let mut data = vec![0; 1000]; + + b.iter(|| { + let tmp = std::mem::replace(&mut data, Vec::new()); + std::mem::replace( + &mut data, + black_box( + tmp.into_iter() + .enumerate() + .map(|(idx, e)| idx.wrapping_add(e)) + .fuse() + .peekable() + .collect::>(), + ), + ); + }); +} + #[bench] fn bench_chain_collect(b: &mut test::Bencher) { let data = black_box([0; LEN]); From 8749e46e358f7358d817ef30ed4ee9f6c0ab0ab8 Mon Sep 17 00:00:00 2001 From: The8472 Date: Thu, 21 Nov 2019 00:06:31 +0100 Subject: [PATCH 21/54] bench in-place collect of droppables --- src/liballoc/benches/vec.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/liballoc/benches/vec.rs b/src/liballoc/benches/vec.rs index 301dc5fd009ed..b22e8d06e3221 100644 --- a/src/liballoc/benches/vec.rs +++ b/src/liballoc/benches/vec.rs @@ -551,6 +551,21 @@ fn bench_in_place_recycle(b: &mut test::Bencher) { }); } +#[derive(Clone)] +struct Droppable(usize); + +impl Drop for Droppable { + fn drop(&mut self) { + black_box(self); + } +} + +#[bench] +fn bench_in_place_collect_droppable(b: &mut test::Bencher) { + let v: Vec = std::iter::repeat_with(|| Droppable(0)).take(1000).collect(); + b.iter(|| v.clone().into_iter().skip(100).collect::>()) +} + #[bench] fn bench_chain_collect(b: &mut test::Bencher) { let data = black_box([0; LEN]); From a9a2ebc8d4b7a8d4fda6a04ddf361080cf881308 Mon Sep 17 00:00:00 2001 From: The8472 Date: Thu, 21 Nov 2019 01:01:35 +0100 Subject: [PATCH 22/54] exercise more of the in-place pipeline in the bench --- src/liballoc/benches/vec.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/liballoc/benches/vec.rs b/src/liballoc/benches/vec.rs index b22e8d06e3221..88f30ee19a21e 100644 --- a/src/liballoc/benches/vec.rs +++ b/src/liballoc/benches/vec.rs @@ -563,7 +563,14 @@ impl Drop for Droppable { #[bench] fn bench_in_place_collect_droppable(b: &mut test::Bencher) { let v: Vec = std::iter::repeat_with(|| Droppable(0)).take(1000).collect(); - b.iter(|| v.clone().into_iter().skip(100).collect::>()) + b.iter(|| { + v.clone() + .into_iter() + .skip(100) + .enumerate() + .map(|(i, e)| Droppable(i ^ e.0)) + .collect::>() + }) } #[bench] From ce827cf12a89a1e7cbf4a3eae6f50802d83f238c Mon Sep 17 00:00:00 2001 From: The8472 Date: Thu, 21 Nov 2019 13:24:48 +0100 Subject: [PATCH 23/54] use From specializations on extend if extended Vec is empty this enables in-place iteration and allocation reuse in additional cases --- src/liballoc/vec.rs | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index bc79c22430e15..0c1f4fea37a37 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -1984,7 +1984,16 @@ impl<'a, T> IntoIterator for &'a mut Vec { impl Extend for Vec { #[inline] fn extend>(&mut self, iter: I) { - >::spec_extend(self, iter.into_iter()) + if self.capacity() > 0 { + >::spec_extend(self, iter.into_iter()) + } else { + // if self has no allocation then use the more powerful from_iter specializations + let other = SpecFrom::from_iter(iter.into_iter()); + // replace self, don't run drop since self was empty + unsafe { + ptr::write(self, other); + } + } } } @@ -2015,6 +2024,8 @@ where vector } }; + // must delegate to spec_extend() since extend() itself delegates + // to spec_from for empty Vecs as SpecExtend>::spec_extend(&mut vector, iterator); vector } @@ -2125,7 +2136,9 @@ impl SpecFrom> for Vec { } let mut vec = Vec::new(); - vec.extend(iterator); + // must delegate to spec_extend() since extend() itself delegates + // to spec_from for empty Vecs + vec.spec_extend(iterator); vec } } @@ -2370,7 +2383,16 @@ impl Vec { #[stable(feature = "extend_ref", since = "1.2.0")] impl<'a, T: 'a + Copy> Extend<&'a T> for Vec { fn extend>(&mut self, iter: I) { - self.spec_extend(iter.into_iter()) + if self.capacity() > 0 { + self.spec_extend(iter.into_iter()) + } else { + // if self has no allocation then use the more powerful from_iter specializations + let other = SpecFrom::from_iter(iter.into_iter()); + // replace self, don't run drop since self was empty + unsafe { + ptr::write(self, other); + } + } } } From 441f2d5d68c9e0acec6f78ab5f72a117a4b7e4c6 Mon Sep 17 00:00:00 2001 From: The8472 Date: Thu, 21 Nov 2019 13:40:49 +0100 Subject: [PATCH 24/54] restore SpecFrom> specialization by nesting specializations --- src/liballoc/vec.rs | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 0c1f4fea37a37..19f8f75c904c2 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2002,7 +2002,13 @@ trait SpecFrom { fn from_iter(iter: I) -> Self; } -impl SpecFrom for Vec +// Another specialization trait for Vec::from_iter +// necessary to manually prioritize overlapping specializations +trait SpecFromNested { + fn from_iter(iter: I) -> Self; +} + +impl SpecFromNested for Vec where I: Iterator, { @@ -2031,6 +2037,28 @@ where } } +impl SpecFromNested for Vec +where + I: TrustedLen, +{ + fn from_iter(iterator: I) -> Self { + let mut vector = Vec::new(); + // must delegate to spec_extend() since extend() itself delegates + // to spec_from for empty Vecs + vector.spec_extend(iterator); + vector + } +} + +impl SpecFrom for Vec +where + I: Iterator, +{ + default fn from_iter(iterator: I) -> Self { + SpecFromNested::from_iter(iterator) + } +} + struct InPlaceDrop { inner: *mut T, dst: *mut T, From c9fa7ce2068a5b8ebd59607ac5e25778cdadc125 Mon Sep 17 00:00:00 2001 From: The8472 Date: Thu, 21 Nov 2019 13:43:53 +0100 Subject: [PATCH 25/54] specialize creating a Vec from a slice iterator where T: Copy this was already implemented for Extend but not for FromIterator --- src/liballoc/vec.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 19f8f75c904c2..67079cff75fc2 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2194,6 +2194,20 @@ where } } +impl<'a, T: 'a> SpecFrom<&'a T, slice::Iter<'a, T>> for Vec +where + T: Copy, +{ + // reuses the extend specialization for T: Copy + fn from_iter(iterator: slice::Iter<'a, T>) -> Self { + let mut vec = Vec::new(); + // must delegate to spec_extend() since extend() itself delegates + // to spec_from for empty Vecs + vec.spec_extend(iterator); + vec + } +} + // Specialization trait used for Vec::extend trait SpecExtend { fn spec_extend(&mut self, iter: I); From a5243214e85da48e046742d3b7687a4ebdd2fcb9 Mon Sep 17 00:00:00 2001 From: The8472 Date: Thu, 21 Nov 2019 13:44:28 +0100 Subject: [PATCH 26/54] improve comments --- src/liballoc/vec.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 67079cff75fc2..fcb0c1c54e270 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2107,8 +2107,10 @@ where sink.did_panic = false; sink.dst } else { - // use try-fold since it vectorizes better, does not take ownership and lets us thread the - // write pointer through its innards + // use try-fold + // - it vectorizes better + // - unlike most internal iteration methods methods it only takes a &mut self + // - lets us thread the write pointer through its innards and get it back in the end iterator .try_fold::<_, _, Result<_, !>>(dst, move |mut dst, item| { unsafe { @@ -2126,7 +2128,7 @@ where let src = iterator.as_inner(); // check if SourceIter and InPlaceIterable contracts were upheld. - // but if they weren't we may not even make it to this point + // caveat: if they weren't we may not even make it to this point debug_assert_eq!(src_buf, src.buf.as_ptr()); debug_assert!(dst as *const _ <= src.ptr, "InPlaceIterable contract violation"); @@ -2171,10 +2173,9 @@ impl SpecFrom> for Vec { } } -// Further specialization potential once lattice specialization exists -// and https://github.com/rust-lang/rust/issues/62645 has been solved: -// This can be broadened to only require size and alignment equality between -// input and output Item types. +// Further specialization potential once +// https://github.com/rust-lang/rust/issues/62645 has been solved: +// T can be split into IN and OUT which only need to have the same size and alignment impl SpecFrom for Vec where I: Iterator + InPlaceIterable + SourceIter>, @@ -2290,6 +2291,8 @@ where } impl Vec { + // leaf method to which various SpecFrom/SpecExtend implementations delegate when + // they have no further optimizations to apply fn extend_desugared>(&mut self, mut iterator: I) { // This is the case for a general iterator. // From 14f88946e4d50e1537dedc63546ff3d3ff455f27 Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 23 Nov 2019 14:30:10 +0100 Subject: [PATCH 27/54] remove redundant code --- src/liballoc/vec.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index fcb0c1c54e270..16dc21282d9f9 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2281,12 +2281,7 @@ where { fn spec_extend(&mut self, iterator: slice::Iter<'a, T>) { let slice = iterator.as_slice(); - self.reserve(slice.len()); - unsafe { - let len = self.len(); - self.set_len(len + slice.len()); - self.get_unchecked_mut(len..).copy_from_slice(slice); - } + unsafe { self.append_elements(slice) }; } } From d077778f65da83b0ead71ae3d5289c769a3c6a36 Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 23 Nov 2019 14:32:20 +0100 Subject: [PATCH 28/54] fix some in-place-collect edge-cases - it's an allocation optimization, so don't attempt to do it on ZSTs - drop the tail of partially exhausted iters --- src/liballoc/vec.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 16dc21282d9f9..1a365813b12e6 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2086,6 +2086,12 @@ fn from_into_iter_source(mut iterator: I) -> Vec where I: Iterator + InPlaceIterable + SourceIter>, { + // This specialization only makes sense if we're juggling real allocations. + // Additionally some of the pointer arithmetic would panic on ZSTs. + if mem::size_of::() == 0 { + return SpecFromNested::from_iter(iterator); + } + let src_buf = iterator.as_inner().buf.as_ptr(); let src_end = iterator.as_inner().end; let dst = src_buf; @@ -2132,6 +2138,13 @@ where debug_assert_eq!(src_buf, src.buf.as_ptr()); debug_assert!(dst as *const _ <= src.ptr, "InPlaceIterable contract violation"); + if mem::needs_drop::() { + // drop tail if iterator was only partially exhaused + unsafe { + ptr::drop_in_place(src.as_mut_slice()); + } + } + let vec = unsafe { let len = dst.offset_from(src_buf) as usize; Vec::from_raw_parts(src.buf.as_ptr(), len, src.cap) From e207975af401f0ea0770a101f283b89d0ee795ee Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 23 Nov 2019 14:34:58 +0100 Subject: [PATCH 29/54] additional specializations tests --- src/liballoc/tests/vec.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/liballoc/tests/vec.rs b/src/liballoc/tests/vec.rs index 0f119a8a8e66f..e4f9adedbd96e 100644 --- a/src/liballoc/tests/vec.rs +++ b/src/liballoc/tests/vec.rs @@ -736,6 +736,28 @@ fn test_from_iter_specialization() { assert_eq!(srcptr, sinkptr); } +#[test] +fn test_from_iter_partially_drained_in_place_specialization() { + let src: Vec = vec![0usize; 10]; + let srcptr = src.as_ptr(); + let mut iter = src.into_iter(); + iter.next(); + iter.next(); + let sink = iter.collect::>(); + let sinkptr = sink.as_ptr(); + assert_eq!(srcptr, sinkptr); +} + +#[test] +fn test_extend_in_place_specialization() { + let src: Vec = vec![0usize; 1]; + let srcptr = src.as_ptr(); + let mut dst = Vec::new(); + dst.extend(src.into_iter()); + let dstptr = dst.as_ptr(); + assert_eq!(srcptr, dstptr); +} + #[test] fn test_from_iter_specialization_with_iterator_adapters() { fn assert_in_place_trait(_: &T) {}; From 96c049f1863abedea16cae8247c861784ba2d2c2 Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 23 Nov 2019 18:26:52 +0100 Subject: [PATCH 30/54] bench in-place zip --- src/liballoc/benches/vec.rs | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/liballoc/benches/vec.rs b/src/liballoc/benches/vec.rs index 88f30ee19a21e..d25fd8b4a6634 100644 --- a/src/liballoc/benches/vec.rs +++ b/src/liballoc/benches/vec.rs @@ -1,3 +1,4 @@ +use rand::prelude::*; use std::iter::{repeat, FromIterator}; use test::{black_box, Bencher}; @@ -551,6 +552,42 @@ fn bench_in_place_recycle(b: &mut test::Bencher) { }); } +#[bench] +fn bench_in_place_zip_recycle(b: &mut test::Bencher) { + let mut data = vec![0u8; 256]; + let mut rng = rand::thread_rng(); + let mut subst = (0..=255u8).collect::>(); + subst.shuffle(&mut rng); + + b.iter(|| { + let tmp = std::mem::replace(&mut data, Vec::new()); + let mangled = tmp + .into_iter() + .zip(subst.iter().copied()) + .enumerate() + .map(|(i, (d, s))| d.wrapping_add(i as u8) ^ s) + .collect::>(); + assert_eq!(mangled.len(), 256); + std::mem::replace(&mut data, black_box(mangled)); + }); +} + +#[bench] +fn bench_in_place_zip_iter_mut(b: &mut test::Bencher) { + let mut data = vec![0u8; 256]; + let mut rng = rand::thread_rng(); + let mut subst = (0..=255u8).collect::>(); + subst.shuffle(&mut rng); + + b.iter(|| { + data.iter_mut().enumerate().for_each(|(i, d)| { + *d = d.wrapping_add(i as u8) ^ subst[i]; + }); + }); + + black_box(data); +} + #[derive(Clone)] struct Droppable(usize); From 42208523d3e7d387d195521d1dd2d40eaffed9ff Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 23 Nov 2019 18:29:47 +0100 Subject: [PATCH 31/54] add in-place iteration for Zip this picks the left hand side as source since it might be more natural to consume that as IntoIter source --- src/libcore/iter/adapters/zip.rs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/libcore/iter/adapters/zip.rs b/src/libcore/iter/adapters/zip.rs index b13e12e2e8608..479bb665411c8 100644 --- a/src/libcore/iter/adapters/zip.rs +++ b/src/libcore/iter/adapters/zip.rs @@ -2,7 +2,10 @@ use crate::cmp; -use super::super::{DoubleEndedIterator, ExactSizeIterator, FusedIterator, Iterator, TrustedLen}; +use super::super::{ + DoubleEndedIterator, ExactSizeIterator, FusedIterator, InPlaceIterable, Iterator, SourceIter, + TrustedLen, +}; /// An iterator that iterates two other iterators simultaneously. /// @@ -289,6 +292,26 @@ where { } +// Arbitrarily selects the left side of the zip iteration as extractable "source" +// it would require negative trait bounds to be able to try both +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl SourceIter for Zip +where + A: SourceIter, + B: Iterator, + S: Iterator, +{ + type Source = S; + + #[inline] + fn as_inner(&mut self) -> &mut S { + SourceIter::as_inner(&mut self.a) + } +} + +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Zip {} + /// An iterator whose items are random-accessible efficiently /// /// # Safety From 366afd9a2fcab3dbc9adb8e07a3fa5c9fef85750 Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 23 Nov 2019 18:30:32 +0100 Subject: [PATCH 32/54] move free-standing method into trait impl --- src/liballoc/vec.rs | 154 +++++++++++++++++++++----------------------- 1 file changed, 75 insertions(+), 79 deletions(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 1a365813b12e6..0be9d6edd8761 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2082,83 +2082,6 @@ impl Drop for InPlaceDrop { } } -fn from_into_iter_source(mut iterator: I) -> Vec -where - I: Iterator + InPlaceIterable + SourceIter>, -{ - // This specialization only makes sense if we're juggling real allocations. - // Additionally some of the pointer arithmetic would panic on ZSTs. - if mem::size_of::() == 0 { - return SpecFromNested::from_iter(iterator); - } - - let src_buf = iterator.as_inner().buf.as_ptr(); - let src_end = iterator.as_inner().end; - let dst = src_buf; - - let dst = if mem::needs_drop::() { - // special-case drop handling since it prevents vectorization - let mut sink = InPlaceDrop { inner: src_buf, dst, did_panic: true }; - let _ = iterator.try_for_each::<_, Result<_, !>>(|item| { - unsafe { - debug_assert!( - sink.dst as *const _ <= src_end, - "InPlaceIterable contract violation" - ); - ptr::write(sink.dst, item); - sink.dst = sink.dst.add(1); - } - Ok(()) - }); - sink.did_panic = false; - sink.dst - } else { - // use try-fold - // - it vectorizes better - // - unlike most internal iteration methods methods it only takes a &mut self - // - lets us thread the write pointer through its innards and get it back in the end - iterator - .try_fold::<_, _, Result<_, !>>(dst, move |mut dst, item| { - unsafe { - // the InPlaceIterable contract cannot be verified precisely here since - // try_fold has an exclusive reference to the source pointer - // all we can do is check if it's still in range - debug_assert!(dst as *const _ <= src_end, "InPlaceIterable contract violation"); - ptr::write(dst, item); - dst = dst.add(1); - } - Ok(dst) - }) - .unwrap() - }; - - let src = iterator.as_inner(); - // check if SourceIter and InPlaceIterable contracts were upheld. - // caveat: if they weren't we may not even make it to this point - debug_assert_eq!(src_buf, src.buf.as_ptr()); - debug_assert!(dst as *const _ <= src.ptr, "InPlaceIterable contract violation"); - - if mem::needs_drop::() { - // drop tail if iterator was only partially exhaused - unsafe { - ptr::drop_in_place(src.as_mut_slice()); - } - } - - let vec = unsafe { - let len = dst.offset_from(src_buf) as usize; - Vec::from_raw_parts(src.buf.as_ptr(), len, src.cap) - }; - // prevent drop of the underlying storage by turning the IntoIter into - // the equivalent of Vec::new().into_iter() - src.cap = 0; - src.buf = unsafe { NonNull::new_unchecked(RawVec::NEW.ptr()) }; - src.ptr = src.buf.as_ptr(); - src.end = src.buf.as_ptr(); - - vec -} - impl SpecFrom> for Vec { fn from_iter(iterator: IntoIter) -> Self { // A common case is passing a vector into a function which immediately @@ -2193,8 +2116,81 @@ impl SpecFrom for Vec where I: Iterator + InPlaceIterable + SourceIter>, { - default fn from_iter(iterator: I) -> Self { - from_into_iter_source(iterator) + default fn from_iter(mut iterator: I) -> Self { + // This specialization only makes sense if we're juggling real allocations. + // Additionally some of the pointer arithmetic would panic on ZSTs. + if mem::size_of::() == 0 { + return SpecFromNested::from_iter(iterator); + } + + let src_buf = iterator.as_inner().buf.as_ptr(); + let src_end = iterator.as_inner().end; + let dst = src_buf; + + let dst = if mem::needs_drop::() { + // special-case drop handling since it prevents vectorization + let mut sink = InPlaceDrop { inner: src_buf, dst, did_panic: true }; + let _ = iterator.try_for_each::<_, Result<_, !>>(|item| { + unsafe { + debug_assert!( + sink.dst as *const _ <= src_end, + "InPlaceIterable contract violation" + ); + ptr::write(sink.dst, item); + sink.dst = sink.dst.add(1); + } + Ok(()) + }); + sink.did_panic = false; + sink.dst + } else { + // use try-fold + // - it vectorizes better + // - unlike most internal iteration methods methods it only takes a &mut self + // - lets us thread the write pointer through its innards and get it back in the end + iterator + .try_fold::<_, _, Result<_, !>>(dst, move |mut dst, item| { + unsafe { + // the InPlaceIterable contract cannot be verified precisely here since + // try_fold has an exclusive reference to the source pointer + // all we can do is check if it's still in range + debug_assert!( + dst as *const _ <= src_end, + "InPlaceIterable contract violation" + ); + ptr::write(dst, item); + dst = dst.add(1); + } + Ok(dst) + }) + .unwrap() + }; + + let src = iterator.as_inner(); + // check if SourceIter and InPlaceIterable contracts were upheld. + // caveat: if they weren't we may not even make it to this point + debug_assert_eq!(src_buf, src.buf.as_ptr()); + debug_assert!(dst as *const _ <= src.ptr, "InPlaceIterable contract violation"); + + if mem::needs_drop::() { + // drop tail if iterator was only partially exhaused + unsafe { + ptr::drop_in_place(src.as_mut_slice()); + } + } + + let vec = unsafe { + let len = dst.offset_from(src_buf) as usize; + Vec::from_raw_parts(src.buf.as_ptr(), len, src.cap) + }; + // prevent drop of the underlying storage by turning the IntoIter into + // the equivalent of Vec::new().into_iter() + src.cap = 0; + src.buf = unsafe { NonNull::new_unchecked(RawVec::NEW.ptr()) }; + src.ptr = src.buf.as_ptr(); + src.end = src.buf.as_ptr(); + + vec } } From 492a914533b3042aed7f0aba47e75b519efaf86f Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 23 Nov 2019 18:59:18 +0100 Subject: [PATCH 33/54] support in-place iteration for most adapters `Take` is not included since users probably call it with small constants and it doesn't make sense to hold onto huge allocations in that case --- src/libcore/iter/adapters/mod.rs | 114 +++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/src/libcore/iter/adapters/mod.rs b/src/libcore/iter/adapters/mod.rs index 633601629ff8e..35fd4ddea3a35 100644 --- a/src/libcore/iter/adapters/mod.rs +++ b/src/libcore/iter/adapters/mod.rs @@ -1093,6 +1093,22 @@ where #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for Filter where P: FnMut(&I::Item) -> bool {} +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl SourceIter for Filter where + P: FnMut(&I::Item) -> bool, + I: SourceIter +{ + type Source = S; + + #[inline] + fn as_inner(&mut self) -> &mut S { + SourceIter::as_inner(&mut self.iter) + } +} + +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Filter where P: FnMut(&I::Item) -> bool {} + /// An iterator that uses `f` to both filter and map elements from `iter`. /// /// This `struct` is created by the [`filter_map`] method on [`Iterator`]. See its @@ -1219,6 +1235,23 @@ where #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for FilterMap where F: FnMut(I::Item) -> Option {} +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl SourceIter for FilterMap where + F: FnMut(I::Item) -> Option, + I: SourceIter +{ + type Source = S; + + #[inline] + fn as_inner(&mut self) -> &mut S { + SourceIter::as_inner(&mut self.iter) + } +} + +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for FilterMap where F: FnMut(I::Item) -> Option {} + + /// An iterator that yields the current count and the element during iteration. /// /// This `struct` is created by the [`enumerate`] method on [`Iterator`]. See its @@ -1777,6 +1810,22 @@ where { } +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl SourceIter for SkipWhile where + P: FnMut(&I::Item) -> bool, + I: SourceIter +{ + type Source = S; + + #[inline] + fn as_inner(&mut self) -> &mut S { + SourceIter::as_inner(&mut self.iter) + } +} + +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for SkipWhile where F: FnMut(&I::Item) -> bool {} + /// An iterator that only accepts elements while `predicate` returns `true`. /// /// This `struct` is created by the [`take_while`] method on [`Iterator`]. See its @@ -1966,6 +2015,24 @@ where { } + +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl SourceIter for TakeWhile where + P: FnMut(&I::Item) -> bool, + I: SourceIter +{ + type Source = S; + + #[inline] + fn as_inner(&mut self) -> &mut S { + SourceIter::as_inner(&mut self.iter) + } +} + +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for TakeWhile where F: FnMut(&I::Item) -> bool {} + + /// An iterator that skips over `n` elements of `iter`. /// /// This `struct` is created by the [`skip`] method on [`Iterator`]. See its @@ -2259,6 +2326,19 @@ where } } +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl SourceIter for Take where I: SourceIter { + type Source = S; + + #[inline] + fn as_inner(&mut self) -> &mut S { + SourceIter::as_inner(&mut self.iter) + } +} + +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Take {} + #[stable(feature = "double_ended_take_iterator", since = "1.38.0")] impl DoubleEndedIterator for Take where @@ -2391,6 +2471,24 @@ where } } +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl SourceIter for Scan + where I: SourceIter, + F: FnMut(&mut St, I::Item) -> Option, +{ + type Source = S; + + #[inline] + fn as_inner(&mut self) -> &mut S { + SourceIter::as_inner(&mut self.iter) + } +} + +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Scan + where F: FnMut(&mut St, I::Item) -> Option, +{} + /// An iterator that yields `None` forever after the underlying iterator /// yields `None` once. /// @@ -2808,6 +2906,22 @@ where #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for Inspect where F: FnMut(&I::Item) {} +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl SourceIter for Inspect where + F: FnMut(&I::Item), + I: SourceIter +{ + type Source = S; + + #[inline] + fn as_inner(&mut self) -> &mut S { + SourceIter::as_inner(&mut self.iter) + } +} + +#[unstable(issue = "0", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Inspect where F: FnMut(&I::Item) {} + /// An iterator adapter that produces output as long as the underlying /// iterator produces `Result::Ok` values. /// From 0a8027e92e3e9aa977fb62a67e75bb57ea22e649 Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 23 Nov 2019 19:34:26 +0100 Subject: [PATCH 34/54] make tidy happy --- src/libcore/iter/adapters/mod.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libcore/iter/adapters/mod.rs b/src/libcore/iter/adapters/mod.rs index 35fd4ddea3a35..6fe8ef7878e68 100644 --- a/src/libcore/iter/adapters/mod.rs +++ b/src/libcore/iter/adapters/mod.rs @@ -1107,7 +1107,8 @@ unsafe impl SourceIter for Filter where } #[unstable(issue = "0", feature = "inplace_iteration")] -unsafe impl InPlaceIterable for Filter where P: FnMut(&I::Item) -> bool {} +unsafe impl InPlaceIterable for Filter + where P: FnMut(&I::Item) -> bool {} /// An iterator that uses `f` to both filter and map elements from `iter`. /// @@ -1249,7 +1250,8 @@ unsafe impl SourceIter for FilterMap where } #[unstable(issue = "0", feature = "inplace_iteration")] -unsafe impl InPlaceIterable for FilterMap where F: FnMut(I::Item) -> Option {} +unsafe impl InPlaceIterable for FilterMap + where F: FnMut(I::Item) -> Option {} /// An iterator that yields the current count and the element during iteration. @@ -1824,7 +1826,8 @@ unsafe impl SourceIter for SkipWhile where } #[unstable(issue = "0", feature = "inplace_iteration")] -unsafe impl InPlaceIterable for SkipWhile where F: FnMut(&I::Item) -> bool {} +unsafe impl InPlaceIterable for SkipWhile + where F: FnMut(&I::Item) -> bool {} /// An iterator that only accepts elements while `predicate` returns `true`. /// @@ -2030,7 +2033,8 @@ unsafe impl SourceIter for TakeWhile where } #[unstable(issue = "0", feature = "inplace_iteration")] -unsafe impl InPlaceIterable for TakeWhile where F: FnMut(&I::Item) -> bool {} +unsafe impl InPlaceIterable for TakeWhile + where F: FnMut(&I::Item) -> bool {} /// An iterator that skips over `n` elements of `iter`. From 1b4da4668e5af660c8e2a183ae1b005c8025430f Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 23 Nov 2019 20:59:17 +0100 Subject: [PATCH 35/54] remove unecessary feature flag --- src/liballoc/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/liballoc/lib.rs b/src/liballoc/lib.rs index 8959eebff8e17..4defefec07691 100644 --- a/src/liballoc/lib.rs +++ b/src/liballoc/lib.rs @@ -122,7 +122,6 @@ #![feature(try_trait)] #![feature(associated_type_bounds)] #![feature(inplace_iteration)] -#![feature(never_type)] #![feature(type_alias_impl_trait)] // Allow testing this library From 4bb786539745b22bb899fcefa97802dc3379f738 Mon Sep 17 00:00:00 2001 From: The8472 Date: Wed, 27 Nov 2019 22:19:06 +0100 Subject: [PATCH 36/54] include in-place .zip() in test --- src/liballoc/tests/vec.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/liballoc/tests/vec.rs b/src/liballoc/tests/vec.rs index e4f9adedbd96e..67cbee7ff5dd7 100644 --- a/src/liballoc/tests/vec.rs +++ b/src/liballoc/tests/vec.rs @@ -763,7 +763,14 @@ fn test_from_iter_specialization_with_iterator_adapters() { fn assert_in_place_trait(_: &T) {}; let src: Vec = vec![0usize; 65535]; let srcptr = src.as_ptr(); - let iter = src.into_iter().enumerate().map(|i| i.0 + i.1).peekable().skip(1); + let iter = src + .into_iter() + .enumerate() + .map(|i| i.0 + i.1) + .zip(std::iter::repeat(1usize)) + .map(|(a, b)| a + b) + .peekable() + .skip(1); assert_in_place_trait(&iter); let sink = iter.collect::>(); let sinkptr = sink.as_ptr(); From aa0a12b1affce1ad2d39d912eabbbd3f8eee6f83 Mon Sep 17 00:00:00 2001 From: The8472 Date: Wed, 27 Nov 2019 22:19:15 +0100 Subject: [PATCH 37/54] bench larger allocations --- src/liballoc/benches/vec.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/liballoc/benches/vec.rs b/src/liballoc/benches/vec.rs index d25fd8b4a6634..d269928f8021f 100644 --- a/src/liballoc/benches/vec.rs +++ b/src/liballoc/benches/vec.rs @@ -554,10 +554,10 @@ fn bench_in_place_recycle(b: &mut test::Bencher) { #[bench] fn bench_in_place_zip_recycle(b: &mut test::Bencher) { - let mut data = vec![0u8; 256]; + let mut data = vec![0u8; 1000]; let mut rng = rand::thread_rng(); - let mut subst = (0..=255u8).collect::>(); - subst.shuffle(&mut rng); + let mut subst = vec![0u8; 1000]; + rng.fill_bytes(&mut subst[..]); b.iter(|| { let tmp = std::mem::replace(&mut data, Vec::new()); @@ -567,7 +567,7 @@ fn bench_in_place_zip_recycle(b: &mut test::Bencher) { .enumerate() .map(|(i, (d, s))| d.wrapping_add(i as u8) ^ s) .collect::>(); - assert_eq!(mangled.len(), 256); + assert_eq!(mangled.len(), 1000); std::mem::replace(&mut data, black_box(mangled)); }); } @@ -576,8 +576,8 @@ fn bench_in_place_zip_recycle(b: &mut test::Bencher) { fn bench_in_place_zip_iter_mut(b: &mut test::Bencher) { let mut data = vec![0u8; 256]; let mut rng = rand::thread_rng(); - let mut subst = (0..=255u8).collect::>(); - subst.shuffle(&mut rng); + let mut subst = vec![0u8; 1000]; + rng.fill_bytes(&mut subst[..]); b.iter(|| { data.iter_mut().enumerate().for_each(|(i, d)| { From ca61c6bbf682ec1945b3cf1b9d0f197392fa27c3 Mon Sep 17 00:00:00 2001 From: The8472 Date: Fri, 29 Nov 2019 01:11:14 +0100 Subject: [PATCH 38/54] impl TrustedRandomAccess for vec::IntoIter --- src/liballoc/vec.rs | 20 +++++++++++++++++++- src/libcore/iter/adapters/mod.rs | 9 ++++++++- src/libcore/iter/adapters/zip.rs | 6 +++++- src/libcore/iter/mod.rs | 4 +++- src/libcore/slice/mod.rs | 17 +++++++++++------ src/libcore/str/mod.rs | 1 + 6 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 0be9d6edd8761..f284fb487aa46 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -61,7 +61,9 @@ use core::cmp::{self, Ordering}; use core::fmt; use core::hash::{self, Hash}; use core::intrinsics::{arith_offset, assume}; -use core::iter::{FromIterator, FusedIterator, InPlaceIterable, SourceIter, TrustedLen}; +use core::iter::{ + FromIterator, FusedIterator, InPlaceIterable, SourceIter, TrustedLen, TrustedRandomAccess, +}; use core::marker::PhantomData; use core::mem; use core::ops::Bound::{Excluded, Included, Unbounded}; @@ -2795,6 +2797,22 @@ impl FusedIterator for IntoIter {} #[unstable(feature = "trusted_len", issue = "37572")] unsafe impl TrustedLen for IntoIter {} +#[doc(hidden)] +#[unstable(issue = "0", feature = "std_internals")] +// T: Copy as approximation for !Drop since get_unchecked does not advance self.ptr +// and thus we can't implement drop-handling +unsafe impl TrustedRandomAccess for IntoIter +where + T: Copy, +{ + unsafe fn get_unchecked(&mut self, i: usize) -> Self::Item { + if mem::size_of::() == 0 { mem::zeroed() } else { ptr::read(self.ptr.add(i)) } + } + fn may_have_side_effect() -> bool { + false + } +} + #[stable(feature = "vec_into_iter_clone", since = "1.8.0")] impl Clone for IntoIter { fn clone(&self) -> IntoIter { diff --git a/src/libcore/iter/adapters/mod.rs b/src/libcore/iter/adapters/mod.rs index 6fe8ef7878e68..b752a6fafc970 100644 --- a/src/libcore/iter/adapters/mod.rs +++ b/src/libcore/iter/adapters/mod.rs @@ -16,8 +16,9 @@ mod zip; pub use self::chain::Chain; #[stable(feature = "rust1", since = "1.0.0")] pub use self::flatten::{FlatMap, Flatten}; -pub(crate) use self::zip::TrustedRandomAccess; pub use self::zip::Zip; +#[unstable(issue = "0", feature = "std_internals")] +pub use self::zip::TrustedRandomAccess; /// This trait provides transitive access to source-stages in an interator-adapter pipeline /// under the conditions that @@ -318,6 +319,7 @@ where } #[doc(hidden)] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl<'a, I, T: 'a> TrustedRandomAccess for Copied where I: TrustedRandomAccess, @@ -448,6 +450,7 @@ where } #[doc(hidden)] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl<'a, I, T: 'a> TrustedRandomAccess for Cloned where I: TrustedRandomAccess, @@ -464,6 +467,7 @@ where } #[doc(hidden)] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl<'a, I, T: 'a> TrustedRandomAccess for Cloned where I: TrustedRandomAccess, @@ -931,6 +935,7 @@ where } #[doc(hidden)] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl TrustedRandomAccess for Map where I: TrustedRandomAccess, @@ -1446,6 +1451,7 @@ where } #[doc(hidden)] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl TrustedRandomAccess for Enumerate where I: TrustedRandomAccess, @@ -2638,6 +2644,7 @@ where } } +#[unstable(issue = "0", feature = "std_internals")] unsafe impl TrustedRandomAccess for Fuse where I: TrustedRandomAccess, diff --git a/src/libcore/iter/adapters/zip.rs b/src/libcore/iter/adapters/zip.rs index 479bb665411c8..966b1adb0986c 100644 --- a/src/libcore/iter/adapters/zip.rs +++ b/src/libcore/iter/adapters/zip.rs @@ -262,6 +262,7 @@ where } #[doc(hidden)] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl TrustedRandomAccess for Zip where A: TrustedRandomAccess, @@ -322,7 +323,10 @@ unsafe impl InPlaceIterable for Zip {} /// .get_unchecked() must return distinct mutable references for distinct /// indices (if applicable), and must return a valid reference if index is in /// 0..self.len(). -pub(crate) unsafe trait TrustedRandomAccess: ExactSizeIterator { +#[unstable(issue = "0", feature = "std_internals")] +pub unsafe trait TrustedRandomAccess: ExactSizeIterator { + /// Returns item at offset `i` from the current position of the iterator. + /// It does not advance the iterator. unsafe fn get_unchecked(&mut self, i: usize) -> Self::Item; /// Returns `true` if getting an iterator element may have /// side effects. Remember to take inner iterators into account. diff --git a/src/libcore/iter/mod.rs b/src/libcore/iter/mod.rs index 674be45f76d95..f488f807e5c79 100644 --- a/src/libcore/iter/mod.rs +++ b/src/libcore/iter/mod.rs @@ -360,6 +360,8 @@ pub use self::adapters::MapWhile; pub use self::adapters::SourceIter; #[stable(feature = "iterator_step_by", since = "1.28.0")] pub use self::adapters::StepBy; +#[unstable(issue = "0", feature = "std_internals")] +pub use self::adapters::TrustedRandomAccess; #[stable(feature = "rust1", since = "1.0.0")] pub use self::adapters::{Chain, Cycle, Enumerate, Filter, FilterMap, Map, Rev, Zip}; #[stable(feature = "rust1", since = "1.0.0")] @@ -367,7 +369,7 @@ pub use self::adapters::{FlatMap, Peekable, Scan, Skip, SkipWhile, Take, TakeWhi #[stable(feature = "rust1", since = "1.0.0")] pub use self::adapters::{Fuse, Inspect}; -pub(crate) use self::adapters::{process_results, TrustedRandomAccess}; +pub(crate) use self::adapters::process_results; mod adapters; mod range; diff --git a/src/libcore/slice/mod.rs b/src/libcore/slice/mod.rs index 9b4d201573238..8ad75b9cd23a0 100644 --- a/src/libcore/slice/mod.rs +++ b/src/libcore/slice/mod.rs @@ -4210,6 +4210,7 @@ unsafe impl TrustedLen for Windows<'_, T> {} impl FusedIterator for Windows<'_, T> {} #[doc(hidden)] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for Windows<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a [T] { from_raw_parts(self.v.as_ptr().add(i), self.size) @@ -4349,6 +4350,7 @@ unsafe impl TrustedLen for Chunks<'_, T> {} impl FusedIterator for Chunks<'_, T> {} #[doc(hidden)] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for Chunks<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a [T] { let start = i * self.chunk_size; @@ -4491,6 +4493,7 @@ unsafe impl TrustedLen for ChunksMut<'_, T> {} impl FusedIterator for ChunksMut<'_, T> {} #[doc(hidden)] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for ChunksMut<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a mut [T] { let start = i * self.chunk_size; @@ -4631,7 +4634,7 @@ unsafe impl TrustedLen for ChunksExact<'_, T> {} impl FusedIterator for ChunksExact<'_, T> {} #[doc(hidden)] -#[stable(feature = "chunks_exact", since = "1.31.0")] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for ChunksExact<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a [T] { let start = i * self.chunk_size; @@ -4765,7 +4768,7 @@ unsafe impl TrustedLen for ChunksExactMut<'_, T> {} impl FusedIterator for ChunksExactMut<'_, T> {} #[doc(hidden)] -#[stable(feature = "chunks_exact", since = "1.31.0")] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for ChunksExactMut<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a mut [T] { let start = i * self.chunk_size; @@ -4908,7 +4911,7 @@ unsafe impl TrustedLen for RChunks<'_, T> {} impl FusedIterator for RChunks<'_, T> {} #[doc(hidden)] -#[stable(feature = "rchunks", since = "1.31.0")] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for RChunks<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a [T] { let end = self.v.len() - i * self.chunk_size; @@ -5053,7 +5056,7 @@ unsafe impl TrustedLen for RChunksMut<'_, T> {} impl FusedIterator for RChunksMut<'_, T> {} #[doc(hidden)] -#[stable(feature = "rchunks", since = "1.31.0")] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for RChunksMut<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a mut [T] { let end = self.v.len() - i * self.chunk_size; @@ -5197,7 +5200,7 @@ unsafe impl TrustedLen for RChunksExact<'_, T> {} impl FusedIterator for RChunksExact<'_, T> {} #[doc(hidden)] -#[stable(feature = "rchunks", since = "1.31.0")] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for RChunksExact<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a [T] { let end = self.v.len() - i * self.chunk_size; @@ -5336,7 +5339,7 @@ unsafe impl TrustedLen for RChunksExactMut<'_, T> {} impl FusedIterator for RChunksExactMut<'_, T> {} #[doc(hidden)] -#[stable(feature = "rchunks", since = "1.31.0")] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for RChunksExactMut<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a mut [T] { let end = self.v.len() - i * self.chunk_size; @@ -5683,6 +5686,7 @@ impl_marker_for!(BytewiseEquality, u8 i8 u16 i16 u32 i32 u64 i64 u128 i128 usize isize char bool); #[doc(hidden)] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for Iter<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a T { &*self.ptr.as_ptr().add(i) @@ -5693,6 +5697,7 @@ unsafe impl<'a, T> TrustedRandomAccess for Iter<'a, T> { } #[doc(hidden)] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for IterMut<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a mut T { &mut *self.ptr.as_ptr().add(i) diff --git a/src/libcore/str/mod.rs b/src/libcore/str/mod.rs index 5a7cddd4041d5..45e6ff2535878 100644 --- a/src/libcore/str/mod.rs +++ b/src/libcore/str/mod.rs @@ -863,6 +863,7 @@ impl FusedIterator for Bytes<'_> {} unsafe impl TrustedLen for Bytes<'_> {} #[doc(hidden)] +#[unstable(issue = "0", feature = "std_internals")] unsafe impl TrustedRandomAccess for Bytes<'_> { unsafe fn get_unchecked(&mut self, i: usize) -> u8 { self.0.get_unchecked(i) From abf9ff8fd2fa96a43f3e3aed21d3a1cc9f6e0c25 Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 30 Nov 2019 15:35:18 +0100 Subject: [PATCH 39/54] fix build issue due to stabilized feature --- src/liballoc/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/liballoc/lib.rs b/src/liballoc/lib.rs index 4defefec07691..642e7639d30f6 100644 --- a/src/liballoc/lib.rs +++ b/src/liballoc/lib.rs @@ -123,6 +123,7 @@ #![feature(associated_type_bounds)] #![feature(inplace_iteration)] #![feature(type_alias_impl_trait)] +#![cfg_attr(bootstrap, feature(never_type))] // Allow testing this library From 65e240daa2dae751d62c3b295f99dfe17b6f8d41 Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 30 Nov 2019 19:40:28 +0100 Subject: [PATCH 40/54] fix: bench didn't black_box its results --- src/liballoc/benches/vec.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/liballoc/benches/vec.rs b/src/liballoc/benches/vec.rs index d269928f8021f..540a9c4e7e1f6 100644 --- a/src/liballoc/benches/vec.rs +++ b/src/liballoc/benches/vec.rs @@ -693,9 +693,7 @@ fn bench_rev_1(b: &mut test::Bencher) { #[bench] fn bench_rev_2(b: &mut test::Bencher) { let data = black_box([0; LEN]); - b.iter(|| { - example_plain_slow(&data); - }); + b.iter(|| example_plain_slow(&data)); } #[bench] From 1114f833df11792bdfc98bb4198dd97fc3ecf4b9 Mon Sep 17 00:00:00 2001 From: The8472 Date: Fri, 20 Dec 2019 20:28:10 +0100 Subject: [PATCH 41/54] avoid exposing that binary heap's IntoIter is backed by vec::IntoIter, use a private trait instead --- src/liballoc/collections/binary_heap.rs | 12 +++++++++--- src/liballoc/lib.rs | 2 +- src/liballoc/vec.rs | 21 ++++++++++++++++----- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/liballoc/collections/binary_heap.rs b/src/liballoc/collections/binary_heap.rs index 6e380bfca9bd7..25aafbee67e9b 100644 --- a/src/liballoc/collections/binary_heap.rs +++ b/src/liballoc/collections/binary_heap.rs @@ -152,7 +152,7 @@ use core::ops::{Deref, DerefMut}; use core::ptr; use crate::slice; -use crate::vec::{self, Vec}; +use crate::vec::{self, Vec, AsIntoIter}; use super::SpecExtend; @@ -1147,17 +1147,23 @@ impl FusedIterator for IntoIter {} #[unstable(issue = "0", feature = "inplace_iteration")] unsafe impl SourceIter for IntoIter { - type Source = impl Iterator; + type Source = IntoIter; #[inline] fn as_inner(&mut self) -> &mut Self::Source { - &mut self.iter + self } } #[unstable(issue = "0", feature = "inplace_iteration")] unsafe impl InPlaceIterable for IntoIter {} +impl AsIntoIter for IntoIter { + fn as_into_iter(&mut self) -> &mut vec::IntoIter { + &mut self.iter + } +} + #[unstable(feature = "binary_heap_into_iter_sorted", issue = "59278")] #[derive(Clone, Debug)] pub struct IntoIterSorted { diff --git a/src/liballoc/lib.rs b/src/liballoc/lib.rs index 642e7639d30f6..c250b0185c17e 100644 --- a/src/liballoc/lib.rs +++ b/src/liballoc/lib.rs @@ -123,7 +123,7 @@ #![feature(associated_type_bounds)] #![feature(inplace_iteration)] #![feature(type_alias_impl_trait)] -#![cfg_attr(bootstrap, feature(never_type))] +#![feature(never_type)] // Allow testing this library diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index f284fb487aa46..f6d4263f408b1 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2116,7 +2116,7 @@ impl SpecFrom> for Vec { // T can be split into IN and OUT which only need to have the same size and alignment impl SpecFrom for Vec where - I: Iterator + InPlaceIterable + SourceIter>, + I: Iterator + InPlaceIterable + SourceIter>, { default fn from_iter(mut iterator: I) -> Self { // This specialization only makes sense if we're juggling real allocations. @@ -2125,8 +2125,8 @@ where return SpecFromNested::from_iter(iterator); } - let src_buf = iterator.as_inner().buf.as_ptr(); - let src_end = iterator.as_inner().end; + let src_buf = iterator.as_inner().as_into_iter().buf.as_ptr(); + let src_end = iterator.as_inner().as_into_iter().end; let dst = src_buf; let dst = if mem::needs_drop::() { @@ -2168,14 +2168,14 @@ where .unwrap() }; - let src = iterator.as_inner(); + let src = iterator.as_inner().as_into_iter(); // check if SourceIter and InPlaceIterable contracts were upheld. // caveat: if they weren't we may not even make it to this point debug_assert_eq!(src_buf, src.buf.as_ptr()); debug_assert!(dst as *const _ <= src.ptr, "InPlaceIterable contract violation"); if mem::needs_drop::() { - // drop tail if iterator was only partially exhaused + // drop tail if iterator was only partially exhausted unsafe { ptr::drop_in_place(src.as_mut_slice()); } @@ -2844,6 +2844,17 @@ unsafe impl SourceIter for IntoIter { } } +// internal helper trait for in-place iteration specialization. +pub(crate) trait AsIntoIter { + fn as_into_iter(&mut self) -> &mut IntoIter; +} + +impl AsIntoIter for IntoIter { + fn as_into_iter(&mut self) -> &mut IntoIter { + self + } +} + /// A draining iterator for `Vec`. /// /// This `struct` is created by the [`drain`] method on [`Vec`]. From ee2330e9f8a7cef79a096bf6687bb47d373d2a73 Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 18 Jan 2020 16:03:58 +0100 Subject: [PATCH 42/54] mark as_inner as unsafe and update comments --- src/liballoc/collections/binary_heap.rs | 2 +- src/liballoc/vec.rs | 12 +++-- src/libcore/iter/adapters/mod.rs | 60 +++++++++++++------------ src/libcore/iter/adapters/zip.rs | 9 +++- 4 files changed, 48 insertions(+), 35 deletions(-) diff --git a/src/liballoc/collections/binary_heap.rs b/src/liballoc/collections/binary_heap.rs index 25aafbee67e9b..eb7765407d348 100644 --- a/src/liballoc/collections/binary_heap.rs +++ b/src/liballoc/collections/binary_heap.rs @@ -1150,7 +1150,7 @@ unsafe impl SourceIter for IntoIter { type Source = IntoIter; #[inline] - fn as_inner(&mut self) -> &mut Self::Source { + unsafe fn as_inner(&mut self) -> &mut Self::Source { self } } diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index f6d4263f408b1..df878a831208b 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2061,6 +2061,8 @@ where } } +// A helper struct for in-place iteration that drops the destination slice of iteration. +// The source slice is dropped by IntoIter struct InPlaceDrop { inner: *mut T, dst: *mut T, @@ -2125,8 +2127,10 @@ where return SpecFromNested::from_iter(iterator); } - let src_buf = iterator.as_inner().as_into_iter().buf.as_ptr(); - let src_end = iterator.as_inner().as_into_iter().end; + let (src_buf, src_end) = { + let inner = unsafe { iterator.as_inner().as_into_iter() }; + (inner.buf.as_ptr(), inner.end) + }; let dst = src_buf; let dst = if mem::needs_drop::() { @@ -2168,7 +2172,7 @@ where .unwrap() }; - let src = iterator.as_inner().as_into_iter(); + let src = unsafe { iterator.as_inner().as_into_iter() }; // check if SourceIter and InPlaceIterable contracts were upheld. // caveat: if they weren't we may not even make it to this point debug_assert_eq!(src_buf, src.buf.as_ptr()); @@ -2839,7 +2843,7 @@ unsafe impl SourceIter for IntoIter { type Source = IntoIter; #[inline] - fn as_inner(&mut self) -> &mut Self::Source { + unsafe fn as_inner(&mut self) -> &mut Self::Source { self } } diff --git a/src/libcore/iter/adapters/mod.rs b/src/libcore/iter/adapters/mod.rs index b752a6fafc970..eda56262d8365 100644 --- a/src/libcore/iter/adapters/mod.rs +++ b/src/libcore/iter/adapters/mod.rs @@ -20,7 +20,7 @@ pub use self::zip::Zip; #[unstable(issue = "0", feature = "std_internals")] pub use self::zip::TrustedRandomAccess; -/// This trait provides transitive access to source-stages in an interator-adapter pipeline +/// This trait provides transitive access to source-stage in an interator-adapter pipeline /// under the conditions that /// * the iterator source `S` itself implements `SourceIter` /// * there is a delegating implementation of this trait for each adapter in the pipeline between @@ -47,7 +47,7 @@ pub use self::zip::TrustedRandomAccess; /// /// let mut iter = vec![9, 9, 9].into_iter().map(|i| i * i); /// let _ = iter.next(); -/// let mut remainder = std::mem::replace(iter.as_inner(), Vec::new().into_iter()); +/// let mut remainder = std::mem::replace(unsafe { iter.as_inner() }, Vec::new().into_iter()); /// println!("n = {} elements remaining", remainder.len()); /// ``` /// @@ -58,29 +58,33 @@ pub unsafe trait SourceIter { /// A source stage in an iterator pipeline. type Source: Iterator; - /// Extract the source of an iterator pipeline. + /// Retrieve the source of an iterator pipeline. /// - /// Callers may assume that calls to [`next()`] or any method taking `&self` - /// does no replace the referenced value. - /// But callers may replace the referenced values as long they in turn do not - /// expose it through a delegating implementation of this trait. - /// Which means that while adapters may not modify the reference they cannot - /// rely on it not being modified. + /// # Safety /// - /// Adapters must not rely on exclusive ownership or immutability of the source. - /// The lack of exclusive ownership also requires that adapters must uphold the source's - /// public API even when they have crate- or module-internal access. + /// Implementations of must return the same mutable reference for their lifetime, unless + /// replaced by a caller. + /// Callers may only replace the reference when they stopped iteration and drop the + /// iterator pipeline after extracting the source. + /// + /// This means iterator adapters can rely on the source not changing during + /// iteration but they cannot rely on it in their Drop implementations. + /// + /// Implementing this method means adapters relinquish private-only access to their + /// source and can only rely on guarantees made based on method receiver types. + /// The lack of restricted access also requires that adapters must uphold the source's + /// public API even when they have access to its internals. /// /// Callers in turn must expect the source to be in any state that is consistent with /// its public API since adapters sitting between it and the source have the same /// access. In particular an adapter may have consumed more elements than strictly necessary. /// - /// The overall goal of these requirements is to grant the consumer of a pipeline - /// access to the underlying storage of an iterator while restricting any statefulness - /// and side-effects of the pipeline stages from affecting or relying on that storage. + /// The overall goal of these requirements is to let the consumer of a pipeline use + /// * whatever remains in the source after iteration has stopped + /// * the memory that has become unused by advancing a consuming iterator /// /// [`next()`]: trait.Iterator.html#method.next - fn as_inner(&mut self) -> &mut Self::Source; + unsafe fn as_inner(&mut self) -> &mut Self::Source; } /// A double-ended iterator with the direction inverted. @@ -959,7 +963,7 @@ where type Source = S; #[inline] - fn as_inner(&mut self) -> &mut S { + unsafe fn as_inner(&mut self) -> &mut S { SourceIter::as_inner(&mut self.iter) } } @@ -1106,7 +1110,7 @@ unsafe impl SourceIter for Filter where type Source = S; #[inline] - fn as_inner(&mut self) -> &mut S { + unsafe fn as_inner(&mut self) -> &mut S { SourceIter::as_inner(&mut self.iter) } } @@ -1249,7 +1253,7 @@ unsafe impl SourceIter for FilterMap where type Source = S; #[inline] - fn as_inner(&mut self) -> &mut S { + unsafe fn as_inner(&mut self) -> &mut S { SourceIter::as_inner(&mut self.iter) } } @@ -1479,7 +1483,7 @@ where type Source = S; #[inline] - fn as_inner(&mut self) -> &mut S { + unsafe fn as_inner(&mut self) -> &mut S { SourceIter::as_inner(&mut self.iter) } } @@ -1709,7 +1713,7 @@ where type Source = S; #[inline] - fn as_inner(&mut self) -> &mut S { + unsafe fn as_inner(&mut self) -> &mut S { SourceIter::as_inner(&mut self.iter) } } @@ -1826,7 +1830,7 @@ unsafe impl SourceIter for SkipWhile where type Source = S; #[inline] - fn as_inner(&mut self) -> &mut S { + unsafe fn as_inner(&mut self) -> &mut S { SourceIter::as_inner(&mut self.iter) } } @@ -2033,7 +2037,7 @@ unsafe impl SourceIter for TakeWhile where type Source = S; #[inline] - fn as_inner(&mut self) -> &mut S { + unsafe fn as_inner(&mut self) -> &mut S { SourceIter::as_inner(&mut self.iter) } } @@ -2232,7 +2236,7 @@ where type Source = S; #[inline] - fn as_inner(&mut self) -> &mut S { + unsafe fn as_inner(&mut self) -> &mut S { SourceIter::as_inner(&mut self.iter) } } @@ -2341,7 +2345,7 @@ unsafe impl SourceIter for Take where I: SourceIter type Source = S; #[inline] - fn as_inner(&mut self) -> &mut S { + unsafe fn as_inner(&mut self) -> &mut S { SourceIter::as_inner(&mut self.iter) } } @@ -2489,7 +2493,7 @@ unsafe impl SourceIter for Scan type Source = S; #[inline] - fn as_inner(&mut self) -> &mut S { + unsafe fn as_inner(&mut self) -> &mut S { SourceIter::as_inner(&mut self.iter) } } @@ -2763,7 +2767,7 @@ where type Source = S; #[inline] - fn as_inner(&mut self) -> &mut S { + unsafe fn as_inner(&mut self) -> &mut S { SourceIter::as_inner(&mut self.iter) } } @@ -2925,7 +2929,7 @@ unsafe impl SourceIter for Inspect where type Source = S; #[inline] - fn as_inner(&mut self) -> &mut S { + unsafe fn as_inner(&mut self) -> &mut S { SourceIter::as_inner(&mut self.iter) } } diff --git a/src/libcore/iter/adapters/zip.rs b/src/libcore/iter/adapters/zip.rs index 966b1adb0986c..30a2402158665 100644 --- a/src/libcore/iter/adapters/zip.rs +++ b/src/libcore/iter/adapters/zip.rs @@ -305,13 +305,18 @@ where type Source = S; #[inline] - fn as_inner(&mut self) -> &mut S { + unsafe fn as_inner(&mut self) -> &mut S { SourceIter::as_inner(&mut self.a) } } #[unstable(issue = "0", feature = "inplace_iteration")] -unsafe impl InPlaceIterable for Zip {} +// Limited to Item: Copy since interaction between Zip's use of TrustedRandomAccess +// and Drop implementation of the source is unclear. +// +// An additional method returning the number of times the source has been logically advanced +// (without calling next()) would be needed to properly drop the remainder of the source. +unsafe impl InPlaceIterable for Zip where A::Item: Copy {} /// An iterator whose items are random-accessible efficiently /// From 8efa793def29504fd8f61c17414e92daa948001c Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 18 Jan 2020 16:13:28 +0100 Subject: [PATCH 43/54] replace drop flag with ManuallyDrop --- src/liballoc/vec.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index df878a831208b..75518a777612e 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2066,7 +2066,6 @@ where struct InPlaceDrop { inner: *mut T, dst: *mut T, - did_panic: bool, } impl InPlaceDrop { @@ -2079,9 +2078,7 @@ impl Drop for InPlaceDrop { #[inline] fn drop(&mut self) { unsafe { - if self.did_panic { - ptr::drop_in_place(slice::from_raw_parts_mut(self.inner, self.len()) as *mut _); - } + ptr::drop_in_place(slice::from_raw_parts_mut(self.inner, self.len()) as *mut _); } } } @@ -2135,7 +2132,7 @@ where let dst = if mem::needs_drop::() { // special-case drop handling since it prevents vectorization - let mut sink = InPlaceDrop { inner: src_buf, dst, did_panic: true }; + let mut sink = InPlaceDrop { inner: src_buf, dst }; let _ = iterator.try_for_each::<_, Result<_, !>>(|item| { unsafe { debug_assert!( @@ -2147,7 +2144,8 @@ where } Ok(()) }); - sink.did_panic = false; + // iteration succeeded, don't drop head + let sink = mem::ManuallyDrop::new(sink); sink.dst } else { // use try-fold From e300c1939653fe1c3d389a7c02cfb8de90efb4e7 Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 18 Jan 2020 22:20:04 +0100 Subject: [PATCH 44/54] pacify tidy --- src/liballoc/collections/binary_heap.rs | 6 +- src/liballoc/vec.rs | 6 +- src/libcore/iter/adapters/mod.rs | 134 +++++++++++++----------- src/libcore/iter/adapters/zip.rs | 8 +- src/libcore/iter/mod.rs | 7 +- src/libcore/iter/traits/marker.rs | 2 +- src/libcore/iter/traits/mod.rs | 2 +- src/libcore/slice/mod.rs | 22 ++-- src/libcore/str/mod.rs | 2 +- 9 files changed, 101 insertions(+), 88 deletions(-) diff --git a/src/liballoc/collections/binary_heap.rs b/src/liballoc/collections/binary_heap.rs index eb7765407d348..5e85bf2affece 100644 --- a/src/liballoc/collections/binary_heap.rs +++ b/src/liballoc/collections/binary_heap.rs @@ -152,7 +152,7 @@ use core::ops::{Deref, DerefMut}; use core::ptr; use crate::slice; -use crate::vec::{self, Vec, AsIntoIter}; +use crate::vec::{self, AsIntoIter, Vec}; use super::SpecExtend; @@ -1145,7 +1145,7 @@ impl ExactSizeIterator for IntoIter { #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for IntoIter {} -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl SourceIter for IntoIter { type Source = IntoIter; @@ -1155,7 +1155,7 @@ unsafe impl SourceIter for IntoIter { } } -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl InPlaceIterable for IntoIter {} impl AsIntoIter for IntoIter { diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 75518a777612e..76dc10c436a7b 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2800,7 +2800,7 @@ impl FusedIterator for IntoIter {} unsafe impl TrustedLen for IntoIter {} #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] // T: Copy as approximation for !Drop since get_unchecked does not advance self.ptr // and thus we can't implement drop-handling unsafe impl TrustedRandomAccess for IntoIter @@ -2833,10 +2833,10 @@ unsafe impl<#[may_dangle] T> Drop for IntoIter { } } -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl InPlaceIterable for IntoIter {} -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl SourceIter for IntoIter { type Source = IntoIter; diff --git a/src/libcore/iter/adapters/mod.rs b/src/libcore/iter/adapters/mod.rs index eda56262d8365..fdb98bcc2e657 100644 --- a/src/libcore/iter/adapters/mod.rs +++ b/src/libcore/iter/adapters/mod.rs @@ -16,9 +16,9 @@ mod zip; pub use self::chain::Chain; #[stable(feature = "rust1", since = "1.0.0")] pub use self::flatten::{FlatMap, Flatten}; -pub use self::zip::Zip; -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] pub use self::zip::TrustedRandomAccess; +pub use self::zip::Zip; /// This trait provides transitive access to source-stage in an interator-adapter pipeline /// under the conditions that @@ -53,7 +53,7 @@ pub use self::zip::TrustedRandomAccess; /// /// [`FromIterator`]: trait.FromIterator.html /// [`as_inner`]: #method.as_inner -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] pub unsafe trait SourceIter { /// A source stage in an iterator pipeline. type Source: Iterator; @@ -323,7 +323,7 @@ where } #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl<'a, I, T: 'a> TrustedRandomAccess for Copied where I: TrustedRandomAccess, @@ -454,7 +454,7 @@ where } #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl<'a, I, T: 'a> TrustedRandomAccess for Cloned where I: TrustedRandomAccess, @@ -471,7 +471,7 @@ where } #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl<'a, I, T: 'a> TrustedRandomAccess for Cloned where I: TrustedRandomAccess, @@ -939,7 +939,7 @@ where } #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl TrustedRandomAccess for Map where I: TrustedRandomAccess, @@ -954,7 +954,7 @@ where } } -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl SourceIter for Map where F: FnMut(I::Item) -> B, @@ -968,7 +968,7 @@ where } } -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl InPlaceIterable for Map where F: FnMut(I::Item) -> B {} /// An iterator that filters the elements of `iter` with `predicate`. @@ -1102,10 +1102,11 @@ where #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for Filter where P: FnMut(&I::Item) -> bool {} -#[unstable(issue = "0", feature = "inplace_iteration")] -unsafe impl SourceIter for Filter where +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for Filter +where P: FnMut(&I::Item) -> bool, - I: SourceIter + I: SourceIter, { type Source = S; @@ -1115,9 +1116,8 @@ unsafe impl SourceIter for Filter where } } -#[unstable(issue = "0", feature = "inplace_iteration")] -unsafe impl InPlaceIterable for Filter - where P: FnMut(&I::Item) -> bool {} +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Filter where P: FnMut(&I::Item) -> bool {} /// An iterator that uses `f` to both filter and map elements from `iter`. /// @@ -1245,10 +1245,11 @@ where #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for FilterMap where F: FnMut(I::Item) -> Option {} -#[unstable(issue = "0", feature = "inplace_iteration")] -unsafe impl SourceIter for FilterMap where +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for FilterMap +where F: FnMut(I::Item) -> Option, - I: SourceIter + I: SourceIter, { type Source = S; @@ -1258,10 +1259,11 @@ unsafe impl SourceIter for FilterMap where } } -#[unstable(issue = "0", feature = "inplace_iteration")] -unsafe impl InPlaceIterable for FilterMap - where F: FnMut(I::Item) -> Option {} - +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for FilterMap where + F: FnMut(I::Item) -> Option +{ +} /// An iterator that yields the current count and the element during iteration. /// @@ -1455,7 +1457,7 @@ where } #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl TrustedRandomAccess for Enumerate where I: TrustedRandomAccess, @@ -1475,7 +1477,7 @@ impl FusedIterator for Enumerate where I: FusedIterator {} #[unstable(feature = "trusted_len", issue = "37572")] unsafe impl TrustedLen for Enumerate where I: TrustedLen {} -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl SourceIter for Enumerate where I: SourceIter, @@ -1488,7 +1490,7 @@ where } } -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl InPlaceIterable for Enumerate {} /// An iterator with a `peek()` that returns an optional reference to the next @@ -1705,7 +1707,7 @@ impl Peekable { #[unstable(feature = "trusted_len", issue = "37572")] unsafe impl TrustedLen for Peekable where I: TrustedLen {} -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl SourceIter for Peekable where I: SourceIter, @@ -1718,7 +1720,7 @@ where } } -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl InPlaceIterable for Peekable {} /// An iterator that rejects elements while `predicate` returns `true`. @@ -1822,10 +1824,11 @@ where { } -#[unstable(issue = "0", feature = "inplace_iteration")] -unsafe impl SourceIter for SkipWhile where +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for SkipWhile +where P: FnMut(&I::Item) -> bool, - I: SourceIter + I: SourceIter, { type Source = S; @@ -1835,9 +1838,11 @@ unsafe impl SourceIter for SkipWhile where } } -#[unstable(issue = "0", feature = "inplace_iteration")] -unsafe impl InPlaceIterable for SkipWhile - where F: FnMut(&I::Item) -> bool {} +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for SkipWhile where + F: FnMut(&I::Item) -> bool +{ +} /// An iterator that only accepts elements while `predicate` returns `true`. /// @@ -2028,11 +2033,11 @@ where { } - -#[unstable(issue = "0", feature = "inplace_iteration")] -unsafe impl SourceIter for TakeWhile where +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for TakeWhile +where P: FnMut(&I::Item) -> bool, - I: SourceIter + I: SourceIter, { type Source = S; @@ -2042,10 +2047,11 @@ unsafe impl SourceIter for TakeWhile where } } -#[unstable(issue = "0", feature = "inplace_iteration")] -unsafe impl InPlaceIterable for TakeWhile - where F: FnMut(&I::Item) -> bool {} - +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for TakeWhile where + F: FnMut(&I::Item) -> bool +{ +} /// An iterator that skips over `n` elements of `iter`. /// @@ -2228,7 +2234,7 @@ where #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for Skip where I: FusedIterator {} -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl SourceIter for Skip where I: SourceIter, @@ -2241,7 +2247,7 @@ where } } -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl InPlaceIterable for Skip {} /// An iterator that only iterates over the first `n` iterations of `iter`. @@ -2340,8 +2346,11 @@ where } } -#[unstable(issue = "0", feature = "inplace_iteration")] -unsafe impl SourceIter for Take where I: SourceIter { +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for Take +where + I: SourceIter, +{ type Source = S; #[inline] @@ -2350,7 +2359,7 @@ unsafe impl SourceIter for Take where I: SourceIter } } -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl InPlaceIterable for Take {} #[stable(feature = "double_ended_take_iterator", since = "1.38.0")] @@ -2485,10 +2494,11 @@ where } } -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl SourceIter for Scan - where I: SourceIter, - F: FnMut(&mut St, I::Item) -> Option, +where + I: SourceIter, + F: FnMut(&mut St, I::Item) -> Option, { type Source = S; @@ -2498,10 +2508,11 @@ unsafe impl SourceIter for Scan } } -#[unstable(issue = "0", feature = "inplace_iteration")] -unsafe impl InPlaceIterable for Scan - where F: FnMut(&mut St, I::Item) -> Option, -{} +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Scan where + F: FnMut(&mut St, I::Item) -> Option +{ +} /// An iterator that yields `None` forever after the underlying iterator /// yields `None` once. @@ -2648,7 +2659,7 @@ where } } -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl TrustedRandomAccess for Fuse where I: TrustedRandomAccess, @@ -2759,7 +2770,7 @@ where } } -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl SourceIter for Fuse where I: SourceIter, @@ -2772,7 +2783,7 @@ where } } -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl InPlaceIterable for Fuse {} /// An iterator that calls a function with a reference to each element before @@ -2921,10 +2932,11 @@ where #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for Inspect where F: FnMut(&I::Item) {} -#[unstable(issue = "0", feature = "inplace_iteration")] -unsafe impl SourceIter for Inspect where +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for Inspect +where F: FnMut(&I::Item), - I: SourceIter + I: SourceIter, { type Source = S; @@ -2934,8 +2946,8 @@ unsafe impl SourceIter for Inspect where } } -#[unstable(issue = "0", feature = "inplace_iteration")] -unsafe impl InPlaceIterable for Inspect where F: FnMut(&I::Item) {} +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Inspect where F: FnMut(&I::Item) {} /// An iterator adapter that produces output as long as the underlying /// iterator produces `Result::Ok` values. diff --git a/src/libcore/iter/adapters/zip.rs b/src/libcore/iter/adapters/zip.rs index 30a2402158665..3be48b1abb356 100644 --- a/src/libcore/iter/adapters/zip.rs +++ b/src/libcore/iter/adapters/zip.rs @@ -262,7 +262,7 @@ where } #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl TrustedRandomAccess for Zip where A: TrustedRandomAccess, @@ -295,7 +295,7 @@ where // Arbitrarily selects the left side of the zip iteration as extractable "source" // it would require negative trait bounds to be able to try both -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl SourceIter for Zip where A: SourceIter, @@ -310,7 +310,7 @@ where } } -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] // Limited to Item: Copy since interaction between Zip's use of TrustedRandomAccess // and Drop implementation of the source is unclear. // @@ -328,7 +328,7 @@ unsafe impl InPlaceIterable for Zip where /// .get_unchecked() must return distinct mutable references for distinct /// indices (if applicable), and must return a valid reference if index is in /// 0..self.len(). -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] pub unsafe trait TrustedRandomAccess: ExactSizeIterator { /// Returns item at offset `i` from the current position of the iterator. /// It does not advance the iterator. diff --git a/src/libcore/iter/mod.rs b/src/libcore/iter/mod.rs index f488f807e5c79..e84d00b09f2ad 100644 --- a/src/libcore/iter/mod.rs +++ b/src/libcore/iter/mod.rs @@ -345,7 +345,7 @@ pub use self::traits::{DoubleEndedIterator, Extend, FromIterator, IntoIterator}; #[stable(feature = "rust1", since = "1.0.0")] pub use self::traits::{ExactSizeIterator, Product, Sum}; -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] pub use self::traits::InPlaceIterable; #[stable(feature = "iter_cloned", since = "1.1.0")] @@ -354,13 +354,14 @@ pub use self::adapters::Cloned; pub use self::adapters::Copied; #[stable(feature = "iterator_flatten", since = "1.29.0")] pub use self::adapters::Flatten; + #[unstable(feature = "iter_map_while", reason = "recently added", issue = "68537")] pub use self::adapters::MapWhile; -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] pub use self::adapters::SourceIter; #[stable(feature = "iterator_step_by", since = "1.28.0")] pub use self::adapters::StepBy; -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] pub use self::adapters::TrustedRandomAccess; #[stable(feature = "rust1", since = "1.0.0")] pub use self::adapters::{Chain, Cycle, Enumerate, Filter, FilterMap, Map, Rev, Zip}; diff --git a/src/libcore/iter/traits/marker.rs b/src/libcore/iter/traits/marker.rs index b27cfc0d2d7c6..e9c81aa029ba7 100644 --- a/src/libcore/iter/traits/marker.rs +++ b/src/libcore/iter/traits/marker.rs @@ -52,5 +52,5 @@ unsafe impl TrustedLen for &mut I {} /// In other words this trait indicates that an iterator pipeline can be collected in place. /// /// [`SourceIter`]: ../../std/iter/trait.SourceIter.html -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] pub unsafe trait InPlaceIterable: Iterator {} diff --git a/src/libcore/iter/traits/mod.rs b/src/libcore/iter/traits/mod.rs index 9ed2de7313df1..880f8d831fd92 100644 --- a/src/libcore/iter/traits/mod.rs +++ b/src/libcore/iter/traits/mod.rs @@ -11,7 +11,7 @@ pub use self::double_ended::DoubleEndedIterator; pub use self::exact_size::ExactSizeIterator; #[stable(feature = "rust1", since = "1.0.0")] pub use self::iterator::Iterator; -#[unstable(issue = "0", feature = "inplace_iteration")] +#[unstable(issue = "none", feature = "inplace_iteration")] pub use self::marker::InPlaceIterable; #[stable(feature = "rust1", since = "1.0.0")] pub use self::marker::{FusedIterator, TrustedLen}; diff --git a/src/libcore/slice/mod.rs b/src/libcore/slice/mod.rs index 8ad75b9cd23a0..7d29f9a742345 100644 --- a/src/libcore/slice/mod.rs +++ b/src/libcore/slice/mod.rs @@ -4210,7 +4210,7 @@ unsafe impl TrustedLen for Windows<'_, T> {} impl FusedIterator for Windows<'_, T> {} #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for Windows<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a [T] { from_raw_parts(self.v.as_ptr().add(i), self.size) @@ -4350,7 +4350,7 @@ unsafe impl TrustedLen for Chunks<'_, T> {} impl FusedIterator for Chunks<'_, T> {} #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for Chunks<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a [T] { let start = i * self.chunk_size; @@ -4493,7 +4493,7 @@ unsafe impl TrustedLen for ChunksMut<'_, T> {} impl FusedIterator for ChunksMut<'_, T> {} #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for ChunksMut<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a mut [T] { let start = i * self.chunk_size; @@ -4634,7 +4634,7 @@ unsafe impl TrustedLen for ChunksExact<'_, T> {} impl FusedIterator for ChunksExact<'_, T> {} #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for ChunksExact<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a [T] { let start = i * self.chunk_size; @@ -4768,7 +4768,7 @@ unsafe impl TrustedLen for ChunksExactMut<'_, T> {} impl FusedIterator for ChunksExactMut<'_, T> {} #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for ChunksExactMut<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a mut [T] { let start = i * self.chunk_size; @@ -4911,7 +4911,7 @@ unsafe impl TrustedLen for RChunks<'_, T> {} impl FusedIterator for RChunks<'_, T> {} #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for RChunks<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a [T] { let end = self.v.len() - i * self.chunk_size; @@ -5056,7 +5056,7 @@ unsafe impl TrustedLen for RChunksMut<'_, T> {} impl FusedIterator for RChunksMut<'_, T> {} #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for RChunksMut<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a mut [T] { let end = self.v.len() - i * self.chunk_size; @@ -5200,7 +5200,7 @@ unsafe impl TrustedLen for RChunksExact<'_, T> {} impl FusedIterator for RChunksExact<'_, T> {} #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for RChunksExact<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a [T] { let end = self.v.len() - i * self.chunk_size; @@ -5339,7 +5339,7 @@ unsafe impl TrustedLen for RChunksExactMut<'_, T> {} impl FusedIterator for RChunksExactMut<'_, T> {} #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for RChunksExactMut<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a mut [T] { let end = self.v.len() - i * self.chunk_size; @@ -5686,7 +5686,7 @@ impl_marker_for!(BytewiseEquality, u8 i8 u16 i16 u32 i32 u64 i64 u128 i128 usize isize char bool); #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for Iter<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a T { &*self.ptr.as_ptr().add(i) @@ -5697,7 +5697,7 @@ unsafe impl<'a, T> TrustedRandomAccess for Iter<'a, T> { } #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl<'a, T> TrustedRandomAccess for IterMut<'a, T> { unsafe fn get_unchecked(&mut self, i: usize) -> &'a mut T { &mut *self.ptr.as_ptr().add(i) diff --git a/src/libcore/str/mod.rs b/src/libcore/str/mod.rs index 45e6ff2535878..ececb1ef76e62 100644 --- a/src/libcore/str/mod.rs +++ b/src/libcore/str/mod.rs @@ -863,7 +863,7 @@ impl FusedIterator for Bytes<'_> {} unsafe impl TrustedLen for Bytes<'_> {} #[doc(hidden)] -#[unstable(issue = "0", feature = "std_internals")] +#[unstable(issue = "none", feature = "std_internals")] unsafe impl TrustedRandomAccess for Bytes<'_> { unsafe fn get_unchecked(&mut self, i: usize) -> u8 { self.0.get_unchecked(i) From bc7199f95e28d3ad9da76e7f171e905becba99d6 Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 25 Jan 2020 15:01:39 +0100 Subject: [PATCH 45/54] replace unsafe ptr::write with mem::replace, benchmarks show no difference --- src/liballoc/vec.rs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 76dc10c436a7b..19001116c8212 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -1990,11 +1990,8 @@ impl Extend for Vec { >::spec_extend(self, iter.into_iter()) } else { // if self has no allocation then use the more powerful from_iter specializations - let other = SpecFrom::from_iter(iter.into_iter()); - // replace self, don't run drop since self was empty - unsafe { - ptr::write(self, other); - } + // and overwrite self + mem::replace(self, SpecFrom::from_iter(iter.into_iter())); } } } @@ -2440,11 +2437,8 @@ impl<'a, T: 'a + Copy> Extend<&'a T> for Vec { self.spec_extend(iter.into_iter()) } else { // if self has no allocation then use the more powerful from_iter specializations - let other = SpecFrom::from_iter(iter.into_iter()); - // replace self, don't run drop since self was empty - unsafe { - ptr::write(self, other); - } + // and overwrite self + mem::replace(self, SpecFrom::from_iter(iter.into_iter())); } } } From c8081595ecae841584333f5251c34b4814baa431 Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 25 Jan 2020 15:02:13 +0100 Subject: [PATCH 46/54] move unsafety into method, not relevant to caller --- src/liballoc/vec.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 19001116c8212..156d605240fa6 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2066,8 +2066,8 @@ struct InPlaceDrop { } impl InPlaceDrop { - unsafe fn len(&self) -> usize { - self.dst.offset_from(self.inner) as usize + fn len(&self) -> usize { + unsafe { self.dst.offset_from(self.inner) as usize } } } From 2ad9a4b01799efcc2cc278bc7a605f71411138c9 Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 25 Jan 2020 17:55:47 +0100 Subject: [PATCH 47/54] test drops during in-place iteration --- src/liballoc/tests/vec.rs | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/liballoc/tests/vec.rs b/src/liballoc/tests/vec.rs index 67cbee7ff5dd7..986dec42eec62 100644 --- a/src/liballoc/tests/vec.rs +++ b/src/liballoc/tests/vec.rs @@ -2,6 +2,8 @@ use std::borrow::Cow; use std::collections::TryReserveError::*; use std::iter::InPlaceIterable; use std::mem::size_of; +use std::panic::AssertUnwindSafe; +use std::rc::Rc; use std::vec::{Drain, IntoIter}; use std::{isize, usize}; @@ -777,6 +779,45 @@ fn test_from_iter_specialization_with_iterator_adapters() { assert_eq!(srcptr, sinkptr); } +#[test] +fn test_from_iter_specialization_head_tail_drop() { + let drop_count: Vec<_> = (0..=2).map(|_| Rc::new(())).collect(); + let src: Vec<_> = drop_count.iter().cloned().collect(); + let srcptr = src.as_ptr(); + let iter = src.into_iter(); + let sink: Vec<_> = iter.skip(1).take(1).collect(); + let sinkptr = sink.as_ptr(); + assert_eq!(srcptr, sinkptr, "specialization was applied"); + assert_eq!(Rc::strong_count(&drop_count[0]), 1, "front was dropped"); + assert_eq!(Rc::strong_count(&drop_count[1]), 2, "one element was collected"); + assert_eq!(Rc::strong_count(&drop_count[2]), 1, "tail was dropped"); + assert_eq!(sink.len(), 1); +} + +#[test] +fn test_from_iter_specialization_panic_drop() { + let drop_count: Vec<_> = (0..=2).map(|_| Rc::new(())).collect(); + let src: Vec<_> = drop_count.iter().cloned().collect(); + let iter = src.into_iter(); + + let _ = std::panic::catch_unwind(AssertUnwindSafe(|| { + let _ = iter + .enumerate() + .filter_map(|(i, e)| { + if i == 1 { + std::panic!("aborting iteration"); + } + Some(e) + }) + .collect::>(); + })); + + assert!( + drop_count.iter().map(Rc::strong_count).all(|count| count == 1), + "all items were dropped once" + ); +} + #[test] fn test_cow_from() { let borrowed: &[_] = &["borrowed", "(slice)"]; From aaa16a017887ee76d96c574836a4b28a31c9a9cc Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 25 Jan 2020 17:58:34 +0100 Subject: [PATCH 48/54] remove redundant cast --- src/liballoc/vec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 156d605240fa6..39fdd4bf0f155 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2075,7 +2075,7 @@ impl Drop for InPlaceDrop { #[inline] fn drop(&mut self) { unsafe { - ptr::drop_in_place(slice::from_raw_parts_mut(self.inner, self.len()) as *mut _); + ptr::drop_in_place(slice::from_raw_parts_mut(self.inner, self.len())); } } } From 96cbc9d04c78c188ee1409d17f87d4ce670e2310 Mon Sep 17 00:00:00 2001 From: The8472 Date: Sat, 25 Jan 2020 20:08:46 +0100 Subject: [PATCH 49/54] add benchmark to cover in-place extend --- src/liballoc/benches/vec.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/liballoc/benches/vec.rs b/src/liballoc/benches/vec.rs index 540a9c4e7e1f6..1a6501d315fee 100644 --- a/src/liballoc/benches/vec.rs +++ b/src/liballoc/benches/vec.rs @@ -236,6 +236,20 @@ fn do_bench_push_all(b: &mut Bencher, dst_len: usize, src_len: usize) { }); } +#[bench] +fn bench_extend_recycle(b: &mut Bencher) { + let mut data = vec![0; 1000]; + + b.iter(|| { + let tmp = std::mem::replace(&mut data, Vec::new()); + let mut to_extend = black_box(Vec::new()); + to_extend.extend(tmp.into_iter()); + std::mem::replace(&mut data, black_box(to_extend)); + }); + + black_box(data); +} + #[bench] fn bench_push_all_0000_0000(b: &mut Bencher) { do_bench_push_all(b, 0, 0) From 6cf243ab728425461613793795c1f0cee1043d6a Mon Sep 17 00:00:00 2001 From: The8472 Date: Sun, 26 Jan 2020 20:50:05 +0100 Subject: [PATCH 50/54] extract IntoIter drop/forget used by specialization into separate methods --- src/liballoc/vec.rs | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 39fdd4bf0f155..aceaeac548009 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2121,9 +2121,9 @@ where return SpecFromNested::from_iter(iterator); } - let (src_buf, src_end) = { + let (src_buf, src_end, cap) = { let inner = unsafe { iterator.as_inner().as_into_iter() }; - (inner.buf.as_ptr(), inner.end) + (inner.buf.as_ptr(), inner.end, inner.cap) }; let dst = src_buf; @@ -2173,23 +2173,15 @@ where debug_assert_eq!(src_buf, src.buf.as_ptr()); debug_assert!(dst as *const _ <= src.ptr, "InPlaceIterable contract violation"); - if mem::needs_drop::() { - // drop tail if iterator was only partially exhausted - unsafe { - ptr::drop_in_place(src.as_mut_slice()); - } - } + // drop any remaining values at the tail of the source + src.drop_in_place(); + // but prevent drop of the allocation itself once IntoIter goes out of scope + src.forget_in_place(); let vec = unsafe { let len = dst.offset_from(src_buf) as usize; - Vec::from_raw_parts(src.buf.as_ptr(), len, src.cap) + Vec::from_raw_parts(src_buf, len, cap) }; - // prevent drop of the underlying storage by turning the IntoIter into - // the equivalent of Vec::new().into_iter() - src.cap = 0; - src.buf = unsafe { NonNull::new_unchecked(RawVec::NEW.ptr()) }; - src.ptr = src.buf.as_ptr(); - src.end = src.buf.as_ptr(); vec } @@ -2705,6 +2697,24 @@ impl IntoIter { pub fn as_mut_slice(&mut self) -> &mut [T] { unsafe { slice::from_raw_parts_mut(self.ptr as *mut T, self.len()) } } + + fn drop_in_place(&mut self) { + if mem::needs_drop::() { + unsafe { + ptr::drop_in_place(self.as_mut_slice()); + } + } + self.ptr = self.end; + } + + /// Relinquishes the backing allocation, equivalent to + /// `ptr::write(&mut self, Vec::new().into_iter())` + fn forget_in_place(&mut self) { + self.cap = 0; + self.buf = unsafe { NonNull::new_unchecked(RawVec::NEW.ptr()) }; + self.ptr = self.buf.as_ptr(); + self.end = self.buf.as_ptr(); + } } #[stable(feature = "rust1", since = "1.0.0")] From 089c9c87f4b3b77fff364757d0ad873f9c0918ab Mon Sep 17 00:00:00 2001 From: The8472 Date: Sun, 26 Jan 2020 20:51:57 +0100 Subject: [PATCH 51/54] work around compiler overhead around lambdas in generics by extracting them into free functions --- src/liballoc/vec.rs | 73 ++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index aceaeac548009..21f32642837c4 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2107,6 +2107,34 @@ impl SpecFrom> for Vec { } } +fn write_in_place(src_end: *const T) -> impl FnMut(*mut T, T) -> Result<*mut T, !> { + move |mut dst, item| { + unsafe { + // the InPlaceIterable contract cannot be verified precisely here since + // try_fold has an exclusive reference to the source pointer + // all we can do is check if it's still in range + debug_assert!(dst as *const _ <= src_end, "InPlaceIterable contract violation"); + ptr::write(dst, item); + dst = dst.add(1); + } + Ok(dst) + } +} + +fn write_in_place_with_drop( + src_end: *const T, +) -> impl FnMut(InPlaceDrop, T) -> Result, !> { + move |mut sink, item| { + unsafe { + // same caveat as above + debug_assert!(sink.dst as *const _ <= src_end, "InPlaceIterable contract violation"); + ptr::write(sink.dst, item); + sink.dst = sink.dst.add(1); + } + Ok(sink) + } +} + // Further specialization potential once // https://github.com/rust-lang/rust/issues/62645 has been solved: // T can be split into IN and OUT which only need to have the same size and alignment @@ -2125,46 +2153,23 @@ where let inner = unsafe { iterator.as_inner().as_into_iter() }; (inner.buf.as_ptr(), inner.end, inner.cap) }; - let dst = src_buf; + // use try-fold + // - it vectorizes better for some iterator adapters + // - unlike most internal iteration methods methods it only takes a &mut self + // - lets us thread the write pointer through its innards and get it back in the end let dst = if mem::needs_drop::() { - // special-case drop handling since it prevents vectorization - let mut sink = InPlaceDrop { inner: src_buf, dst }; - let _ = iterator.try_for_each::<_, Result<_, !>>(|item| { - unsafe { - debug_assert!( - sink.dst as *const _ <= src_end, - "InPlaceIterable contract violation" - ); - ptr::write(sink.dst, item); - sink.dst = sink.dst.add(1); - } - Ok(()) - }); + // special-case drop handling since it forces us to lug that extra field around which + // can inhibit optimizations + let sink = InPlaceDrop { inner: src_buf, dst: src_buf }; + let sink = iterator + .try_fold::<_, _, Result<_, !>>(sink, write_in_place_with_drop(src_end)) + .unwrap(); // iteration succeeded, don't drop head let sink = mem::ManuallyDrop::new(sink); sink.dst } else { - // use try-fold - // - it vectorizes better - // - unlike most internal iteration methods methods it only takes a &mut self - // - lets us thread the write pointer through its innards and get it back in the end - iterator - .try_fold::<_, _, Result<_, !>>(dst, move |mut dst, item| { - unsafe { - // the InPlaceIterable contract cannot be verified precisely here since - // try_fold has an exclusive reference to the source pointer - // all we can do is check if it's still in range - debug_assert!( - dst as *const _ <= src_end, - "InPlaceIterable contract violation" - ); - ptr::write(dst, item); - dst = dst.add(1); - } - Ok(dst) - }) - .unwrap() + iterator.try_fold::<_, _, Result<_, !>>(src_buf, write_in_place(src_end)).unwrap() }; let src = unsafe { iterator.as_inner().as_into_iter() }; From 0576929cdb01df85b868389667fdc2cef9b966fe Mon Sep 17 00:00:00 2001 From: The8472 Date: Sun, 26 Jan 2020 20:52:59 +0100 Subject: [PATCH 52/54] increase comment verbosity --- src/liballoc/vec.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 21f32642837c4..33676de236e78 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2058,8 +2058,8 @@ where } } -// A helper struct for in-place iteration that drops the destination slice of iteration. -// The source slice is dropped by IntoIter +// A helper struct for in-place iteration that drops the destination slice of iteration, +// i.e. the head. The source slice (the tail) is dropped by IntoIter. struct InPlaceDrop { inner: *mut T, dst: *mut T, From a724f9a91b89d62d7fca0f17bdd8dcdb6754101a Mon Sep 17 00:00:00 2001 From: The8472 Date: Tue, 28 Jan 2020 15:14:55 +0100 Subject: [PATCH 53/54] generalize in-place collect to types of same size and alignment --- src/liballoc/collections/binary_heap.rs | 6 ++-- src/liballoc/tests/vec.rs | 5 +-- src/liballoc/vec.rs | 46 +++++++++++++++---------- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/src/liballoc/collections/binary_heap.rs b/src/liballoc/collections/binary_heap.rs index 5e85bf2affece..6b6e03ea5fb3e 100644 --- a/src/liballoc/collections/binary_heap.rs +++ b/src/liballoc/collections/binary_heap.rs @@ -1158,8 +1158,10 @@ unsafe impl SourceIter for IntoIter { #[unstable(issue = "none", feature = "inplace_iteration")] unsafe impl InPlaceIterable for IntoIter {} -impl AsIntoIter for IntoIter { - fn as_into_iter(&mut self) -> &mut vec::IntoIter { +impl AsIntoIter for IntoIter { + type Item = I; + + fn as_into_iter(&mut self) -> &mut vec::IntoIter { &mut self.iter } } diff --git a/src/liballoc/tests/vec.rs b/src/liballoc/tests/vec.rs index 986dec42eec62..3c0b681ab0662 100644 --- a/src/liballoc/tests/vec.rs +++ b/src/liballoc/tests/vec.rs @@ -772,11 +772,12 @@ fn test_from_iter_specialization_with_iterator_adapters() { .zip(std::iter::repeat(1usize)) .map(|(a, b)| a + b) .peekable() - .skip(1); + .skip(1) + .map(|e| std::num::NonZeroUsize::new(e)); assert_in_place_trait(&iter); let sink = iter.collect::>(); let sinkptr = sink.as_ptr(); - assert_eq!(srcptr, sinkptr); + assert_eq!(srcptr, sinkptr as *const usize); } #[test] diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 33676de236e78..ab7bce1dbf29e 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -2135,23 +2135,28 @@ fn write_in_place_with_drop( } } -// Further specialization potential once -// https://github.com/rust-lang/rust/issues/62645 has been solved: -// T can be split into IN and OUT which only need to have the same size and alignment impl SpecFrom for Vec where - I: Iterator + InPlaceIterable + SourceIter>, + I: Iterator + InPlaceIterable + SourceIter, { default fn from_iter(mut iterator: I) -> Self { - // This specialization only makes sense if we're juggling real allocations. - // Additionally some of the pointer arithmetic would panic on ZSTs. - if mem::size_of::() == 0 { + // Additional requirements which cannot expressed via trait bounds. We rely on const eval + // instead: + // a) no ZSTs as there would be no allocation to reuse and pointer arithmetic would panic + // b) size match as required by Alloc contract + // c) alignments match as required by Alloc contract + if mem::size_of::() == 0 + || mem::size_of::() + != mem::size_of::<<::Source as AsIntoIter>::Item>() + || mem::align_of::() + != mem::align_of::<<::Source as AsIntoIter>::Item>() + { return SpecFromNested::from_iter(iterator); } - let (src_buf, src_end, cap) = { - let inner = unsafe { iterator.as_inner().as_into_iter() }; - (inner.buf.as_ptr(), inner.end, inner.cap) + let (src_buf, dst_buf, dst_end, cap) = unsafe { + let inner = iterator.as_inner().as_into_iter(); + (inner.buf.as_ptr(), inner.buf.as_ptr() as *mut T, inner.end as *const T, inner.cap) }; // use try-fold @@ -2161,15 +2166,15 @@ where let dst = if mem::needs_drop::() { // special-case drop handling since it forces us to lug that extra field around which // can inhibit optimizations - let sink = InPlaceDrop { inner: src_buf, dst: src_buf }; + let sink = InPlaceDrop { inner: dst_buf, dst: dst_buf }; let sink = iterator - .try_fold::<_, _, Result<_, !>>(sink, write_in_place_with_drop(src_end)) + .try_fold::<_, _, Result<_, !>>(sink, write_in_place_with_drop(dst_end)) .unwrap(); // iteration succeeded, don't drop head let sink = mem::ManuallyDrop::new(sink); sink.dst } else { - iterator.try_fold::<_, _, Result<_, !>>(src_buf, write_in_place(src_end)).unwrap() + iterator.try_fold::<_, _, Result<_, !>>(dst_buf, write_in_place(dst_end)).unwrap() }; let src = unsafe { iterator.as_inner().as_into_iter() }; @@ -2184,8 +2189,8 @@ where src.forget_in_place(); let vec = unsafe { - let len = dst.offset_from(src_buf) as usize; - Vec::from_raw_parts(src_buf, len, cap) + let len = dst.offset_from(dst_buf) as usize; + Vec::from_raw_parts(dst_buf, len, cap) }; vec @@ -2856,12 +2861,15 @@ unsafe impl SourceIter for IntoIter { } // internal helper trait for in-place iteration specialization. -pub(crate) trait AsIntoIter { - fn as_into_iter(&mut self) -> &mut IntoIter; +pub(crate) trait AsIntoIter { + type Item; + fn as_into_iter(&mut self) -> &mut IntoIter; } -impl AsIntoIter for IntoIter { - fn as_into_iter(&mut self) -> &mut IntoIter { +impl AsIntoIter for IntoIter { + type Item = T; + + fn as_into_iter(&mut self) -> &mut IntoIter { self } } From 2d051dbe6fdbb462f620ef5098a375ce554b786d Mon Sep 17 00:00:00 2001 From: The8472 Date: Sun, 2 Feb 2020 20:24:40 +0100 Subject: [PATCH 54/54] pacify tidy --- src/libcore/iter/adapters/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libcore/iter/adapters/mod.rs b/src/libcore/iter/adapters/mod.rs index fdb98bcc2e657..96105d83df265 100644 --- a/src/libcore/iter/adapters/mod.rs +++ b/src/libcore/iter/adapters/mod.rs @@ -1,3 +1,4 @@ +// ignore-tidy-filelength use crate::cmp; use crate::fmt; use crate::intrinsics;