Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@ include = [
"LICENSE.txt",
"NOTICE.txt",
]
edition = "2021"
rust-version = "1.84"
edition = "2024"
rust-version = "1.85"

[workspace.dependencies]
arrow = { version = "56.2.0", path = "./arrow", default-features = false }
Expand Down
5 changes: 4 additions & 1 deletion arrow-arith/src/arithmetic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,10 @@ mod tests {

// `multiply` overflows on this case.
let err = mul(&a, &b).unwrap_err();
assert_eq!(err.to_string(), "Arithmetic overflow: Overflow happened on: 123456789000000000000000000 * 10000000000000000000");
assert_eq!(
err.to_string(),
"Arithmetic overflow: Overflow happened on: 123456789000000000000000000 * 10000000000000000000"
);

// Avoid overflow by reducing the scale.
let result = multiply_fixed_point(&a, &b, 28).unwrap();
Expand Down
7 changes: 5 additions & 2 deletions arrow-arith/src/numeric.rs
Original file line number Diff line number Diff line change
Expand Up @@ -519,7 +519,7 @@ fn timestamp_op<T: TimestampOp>(
"Invalid timestamp arithmetic operation: {} {op} {}",
l.data_type(),
r.data_type()
)))
)));
}
};
Ok(Arc::new(array.with_timezone_opt(l.timezone())))
Expand Down Expand Up @@ -1263,7 +1263,10 @@ mod tests {
.with_precision_and_scale(37, 37)
.unwrap();
let err = mul(&a, &b).unwrap_err().to_string();
assert_eq!(err, "Invalid argument error: Output scale of Decimal128(3, 3) * Decimal128(37, 37) would exceed max scale of 38");
assert_eq!(
err,
"Invalid argument error: Output scale of Decimal128(3, 3) * Decimal128(37, 37) would exceed max scale of 38"
);

let a = Decimal128Array::from(vec![1])
.with_precision_and_scale(3, -2)
Expand Down
8 changes: 4 additions & 4 deletions arrow-array/src/arithmetic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -420,13 +420,13 @@ native_type_float_op!(
1.,
unsafe {
// Need to allow in clippy because
// current MSRV (Minimum Supported Rust Version) is `1.84.0` but this item is stable since `1.87.0`
// current MSRV (Minimum Supported Rust Version) is `1.85.0` but this item is stable since `1.87.0`
#[allow(unnecessary_transmutes)]
std::mem::transmute(-1_i32)
},
unsafe {
// Need to allow in clippy because
// current MSRV (Minimum Supported Rust Version) is `1.84.0` but this item is stable since `1.87.0`
// current MSRV (Minimum Supported Rust Version) is `1.85.0` but this item is stable since `1.87.0`
#[allow(unnecessary_transmutes)]
std::mem::transmute(i32::MAX)
}
Expand All @@ -437,13 +437,13 @@ native_type_float_op!(
1.,
unsafe {
// Need to allow in clippy because
// current MSRV (Minimum Supported Rust Version) is `1.84.0` but this item is stable since `1.87.0`
// current MSRV (Minimum Supported Rust Version) is `1.85.0` but this item is stable since `1.87.0`
#[allow(unnecessary_transmutes)]
std::mem::transmute(-1_i64)
},
unsafe {
// Need to allow in clippy because
// current MSRV (Minimum Supported Rust Version) is `1.84.0` but this item is stable since `1.87.0`
// current MSRV (Minimum Supported Rust Version) is `1.85.0` but this item is stable since `1.87.0`
#[allow(unnecessary_transmutes)]
std::mem::transmute(i64::MAX)
}
Expand Down
2 changes: 1 addition & 1 deletion arrow-array/src/array/binary_array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ impl<OffsetSize: OffsetSizeTrait> GenericBinaryArray<OffsetSize> {
&'a self,
indexes: impl Iterator<Item = Option<usize>> + 'a,
) -> impl Iterator<Item = Option<&'a [u8]>> {
indexes.map(|opt_index| opt_index.map(|index| self.value_unchecked(index)))
unsafe { indexes.map(|opt_index| opt_index.map(|index| self.value_unchecked(index))) }
}
}

Expand Down
6 changes: 3 additions & 3 deletions arrow-array/src/array/boolean_array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ impl BooleanArray {
/// # Safety
/// This doesn't check bounds, the caller must ensure that index < self.len()
pub unsafe fn value_unchecked(&self, i: usize) -> bool {
self.values.value_unchecked(i)
unsafe { self.values.value_unchecked(i) }
}

/// Returns the boolean value at index `i`.
Expand Down Expand Up @@ -222,7 +222,7 @@ impl BooleanArray {
&'a self,
indexes: impl Iterator<Item = Option<usize>> + 'a,
) -> impl Iterator<Item = Option<bool>> + 'a {
indexes.map(|opt_index| opt_index.map(|index| self.value_unchecked(index)))
unsafe { indexes.map(|opt_index| opt_index.map(|index| self.value_unchecked(index))) }
}

/// Create a [`BooleanArray`] by evaluating the operation for
Expand Down Expand Up @@ -355,7 +355,7 @@ impl ArrayAccessor for &BooleanArray {
}

unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
BooleanArray::value_unchecked(self, index)
unsafe { BooleanArray::value_unchecked(self, index) }
}
}

Expand Down
19 changes: 15 additions & 4 deletions arrow-array/src/array/byte_array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,7 @@ impl<T: ByteArrayType> GenericByteArray<T> {
/// # Safety
/// Caller is responsible for ensuring that the index is within the bounds of the array
pub unsafe fn value_unchecked(&self, i: usize) -> &T::Native {
unsafe {
let end = *self.value_offsets().get_unchecked(i + 1);
let start = *self.value_offsets().get_unchecked(i);

Expand All @@ -306,6 +307,7 @@ impl<T: ByteArrayType> GenericByteArray<T> {
// ArrayData is valid
T::Native::from_bytes_unchecked(b)
}
}

/// Returns the element at index `i`
///
Expand Down Expand Up @@ -509,7 +511,7 @@ impl<'a, T: ByteArrayType> ArrayAccessor for &'a GenericByteArray<T> {
}

unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
GenericByteArray::value_unchecked(self, index)
unsafe { GenericByteArray::value_unchecked(self, index) }
}
}

Expand Down Expand Up @@ -603,14 +605,23 @@ mod tests {
let nulls = NullBuffer::new_null(3);
let err =
StringArray::try_new(offsets.clone(), data.clone(), Some(nulls.clone())).unwrap_err();
assert_eq!(err.to_string(), "Invalid argument error: Incorrect length of null buffer for StringArray, expected 2 got 3");
assert_eq!(
err.to_string(),
"Invalid argument error: Incorrect length of null buffer for StringArray, expected 2 got 3"
);

let err = BinaryArray::try_new(offsets.clone(), data.clone(), Some(nulls)).unwrap_err();
assert_eq!(err.to_string(), "Invalid argument error: Incorrect length of null buffer for BinaryArray, expected 2 got 3");
assert_eq!(
err.to_string(),
"Invalid argument error: Incorrect length of null buffer for BinaryArray, expected 2 got 3"
);

let non_utf8_data = Buffer::from_slice_ref(b"he\xFFloworld");
let err = StringArray::try_new(offsets.clone(), non_utf8_data.clone(), None).unwrap_err();
assert_eq!(err.to_string(), "Invalid argument error: Encountered non UTF-8 data: invalid utf-8 sequence of 1 bytes from index 2");
assert_eq!(
err.to_string(),
"Invalid argument error: Encountered non UTF-8 data: invalid utf-8 sequence of 1 bytes from index 2"
);

BinaryArray::new(offsets, non_utf8_data, None);

Expand Down
21 changes: 16 additions & 5 deletions arrow-array/src/array/byte_view_array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
/// Caller is responsible for ensuring that the index is within the bounds
/// of the array
pub unsafe fn value_unchecked(&self, idx: usize) -> &T::Native {
unsafe {
let v = self.views.get_unchecked(idx);
let len = *v as u32;
let b = if len <= MAX_INLINE_VIEW_LEN {
Expand All @@ -336,6 +337,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
};
T::Native::from_bytes_unchecked(b)
}
}

/// Returns the first `len` bytes the inline value of the view.
///
Expand All @@ -344,9 +346,11 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
/// - The `len` must be the length of the inlined value. It should never be larger than [`MAX_INLINE_VIEW_LEN`].
#[inline(always)]
pub unsafe fn inline_value(view: &u128, len: usize) -> &[u8] {
unsafe {
debug_assert!(len <= MAX_INLINE_VIEW_LEN as usize);
std::slice::from_raw_parts((view as *const u128 as *const u8).wrapping_add(4), len)
}
}

/// Constructs a new iterator for iterating over the values of this array
pub fn iter(&self) -> ArrayIter<&Self> {
Expand Down Expand Up @@ -540,6 +544,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
/// into the bytes just appended at the end of `data_buf`.
#[inline(always)]
unsafe fn copy_view_to_buffer(&self, i: usize, data_buf: &mut Vec<u8>) -> u128 {
unsafe {
// SAFETY: `i < self.len()` ensures this is in‑bounds.
let raw_view = *self.views().get_unchecked(i);
let mut bv = ByteView::from(raw_view);
Expand All @@ -564,6 +569,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
bv.into()
}
}
}

/// Returns the total number of bytes used by all non inlined views in all
/// buffers.
Expand Down Expand Up @@ -624,6 +630,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
right: &GenericByteViewArray<T>,
right_idx: usize,
) -> Ordering {
unsafe {
let l_view = left.views().get_unchecked(left_idx);
let l_byte_view = ByteView::from(*l_view);

Expand All @@ -650,11 +657,12 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
}

// unfortunately, we need to compare the full data
let l_full_data: &[u8] = unsafe { left.value_unchecked(left_idx).as_ref() };
let r_full_data: &[u8] = unsafe { right.value_unchecked(right_idx).as_ref() };
let l_full_data: &[u8] = left.value_unchecked(left_idx).as_ref();
let r_full_data: &[u8] = right.value_unchecked(right_idx).as_ref();

l_full_data.cmp(r_full_data)
}
}

/// Builds a 128-bit composite key for an inline value:
///
Expand Down Expand Up @@ -853,7 +861,7 @@ impl<'a, T: ByteViewType + ?Sized> ArrayAccessor for &'a GenericByteViewArray<T>
}

unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
GenericByteViewArray::value_unchecked(self, index)
unsafe { GenericByteViewArray::value_unchecked(self, index) }
}
}

Expand Down Expand Up @@ -999,7 +1007,7 @@ impl BinaryViewArray {
/// # Safety
/// Caller is responsible for ensuring that items in array are utf8 data.
pub unsafe fn to_string_view_unchecked(self) -> StringViewArray {
StringViewArray::new_unchecked(self.views, self.buffers, self.nulls)
unsafe { StringViewArray::new_unchecked(self.views, self.buffers, self.nulls) }
}
}

Expand Down Expand Up @@ -1171,7 +1179,10 @@ mod tests {
builder.finish()
};
assert_eq!(array.value(0), "large payload over 12 bytes");
assert_eq!(array.value(1), "another large payload over 12 bytes that double than the first one, so that we can trigger the in_progress in builder re-created");
assert_eq!(
array.value(1),
"another large payload over 12 bytes that double than the first one, so that we can trigger the in_progress in builder re-created"
);
assert_eq!(2, array.buffers.len());
}

Expand Down
2 changes: 2 additions & 0 deletions arrow-array/src/array/dictionary_array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -947,6 +947,7 @@ where
}

unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
unsafe {
let val = self.dictionary.keys.value_unchecked(index);
let value_idx = val.as_usize();

Expand All @@ -958,6 +959,7 @@ where
}
}
}
}

/// A [`DictionaryArray`] with the key type erased
///
Expand Down
9 changes: 7 additions & 2 deletions arrow-array/src/array/fixed_size_binary_array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -168,13 +168,15 @@ impl FixedSizeBinaryArray {
/// Caller is responsible for ensuring that the index is within the bounds
/// of the array
pub unsafe fn value_unchecked(&self, i: usize) -> &[u8] {
unsafe {
let offset = i + self.offset();
let pos = self.value_offset_at(offset);
std::slice::from_raw_parts(
self.value_data.as_ptr().offset(pos as isize),
(self.value_offset_at(offset + 1) - pos) as usize,
)
}
}

/// Returns the offset for the element at index `i`.
///
Expand Down Expand Up @@ -654,7 +656,7 @@ impl<'a> ArrayAccessor for &'a FixedSizeBinaryArray {
}

unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
FixedSizeBinaryArray::value_unchecked(self, index)
unsafe { FixedSizeBinaryArray::value_unchecked(self, index) }
}
}

Expand Down Expand Up @@ -996,6 +998,9 @@ mod tests {

let nulls = NullBuffer::new_null(3);
let err = FixedSizeBinaryArray::try_new(2, buffer, Some(nulls)).unwrap_err();
assert_eq!(err.to_string(), "Invalid argument error: Incorrect length of null buffer for FixedSizeBinaryArray, expected 5 got 3");
assert_eq!(
err.to_string(),
"Invalid argument error: Incorrect length of null buffer for FixedSizeBinaryArray, expected 5 got 3"
);
}
}
19 changes: 15 additions & 4 deletions arrow-array/src/array/fixed_size_list_array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,9 @@ impl From<ArrayData> for FixedSizeListArray {
let value_length = match data.data_type() {
DataType::FixedSizeList(_, len) => *len,
data_type => {
panic!("FixedSizeListArray data should contain a FixedSizeList data type, got {data_type:?}")
panic!(
"FixedSizeListArray data should contain a FixedSizeList data type, got {data_type:?}"
)
}
};

Expand Down Expand Up @@ -685,19 +687,28 @@ mod tests {

let nulls = NullBuffer::new_null(2);
let err = FixedSizeListArray::try_new(field, 2, values.clone(), Some(nulls)).unwrap_err();
assert_eq!(err.to_string(), "Invalid argument error: Incorrect length of null buffer for FixedSizeListArray, expected 3 got 2");
assert_eq!(
err.to_string(),
"Invalid argument error: Incorrect length of null buffer for FixedSizeListArray, expected 3 got 2"
);

let field = Arc::new(Field::new_list_field(DataType::Int32, false));
let err = FixedSizeListArray::try_new(field.clone(), 2, values.clone(), None).unwrap_err();
assert_eq!(err.to_string(), "Invalid argument error: Found unmasked nulls for non-nullable FixedSizeListArray field \"item\"");
assert_eq!(
err.to_string(),
"Invalid argument error: Found unmasked nulls for non-nullable FixedSizeListArray field \"item\""
);

// Valid as nulls in child masked by parent
let nulls = NullBuffer::new(BooleanBuffer::new(Buffer::from([0b0000101]), 0, 3));
FixedSizeListArray::new(field, 2, values.clone(), Some(nulls));

let field = Arc::new(Field::new_list_field(DataType::Int64, true));
let err = FixedSizeListArray::try_new(field, 2, values, None).unwrap_err();
assert_eq!(err.to_string(), "Invalid argument error: FixedSizeListArray expected data type Int64 got Int32 for \"item\"");
assert_eq!(
err.to_string(),
"Invalid argument error: FixedSizeListArray expected data type Int64 got Int32 for \"item\""
);
}

#[test]
Expand Down
2 changes: 2 additions & 0 deletions arrow-array/src/array/list_array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -334,10 +334,12 @@ impl<OffsetSize: OffsetSizeTrait> GenericListArray<OffsetSize> {
/// # Safety
/// Caller must ensure that the index is within the array bounds
pub unsafe fn value_unchecked(&self, i: usize) -> ArrayRef {
unsafe {
let end = self.value_offsets().get_unchecked(i + 1).as_usize();
let start = self.value_offsets().get_unchecked(i).as_usize();
self.values.slice(start, end - start)
}
}

/// Returns ith value of this list array.
///
Expand Down
Loading
Loading