Skip to content

RFC: Add append!(::StructVector, iterator::Any) using Tables.isrowtable #117

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Feb 14, 2020
Merged

RFC: Add append!(::StructVector, iterator::Any) using Tables.isrowtable #117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ae6a52b
Implement append!(::StructVector, ::Any) using Tables.isrowtable
tkf Feb 12, 2020
3a53a20
Merge branch 'master' into append
tkf Feb 14, 2020
8f1ce3f
Directly append columns; add some tests
tkf Feb 14, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions src/tables.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,30 @@ Tables.columnaccess(::Type{<:StructVector}) = true
Tables.columns(s::StructVector) = fieldarrays(s)

Tables.schema(s::StructVector) = Tables.Schema(staticschema(eltype(s)))

function Base.append!(s::StructVector, rows)
if Tables.isrowtable(rows) && Tables.columnaccess(rows)
# Input `rows` is a container of rows _and_ satisfies column
# table interface. Thus, we can add the input column-by-column.
table = Tables.columns(rows)
isempty(_setdiff(propertynames(s), Tables.columnnames(rows))) ||
_invalid_columns_error(s, rows)
_foreach(propertynames(s)) do name
Copy link
Collaborator

@piever piever Feb 14, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Generally for this StructArrays uses foreachfield, but that assumes that also rows implements getproperty for column access, which in the latest Tables doesn't need to be the case. What I don't understand is: how is foldl better than foreach for tuples? Is it as efficient as the generated function approach?

Copy link
Collaborator

@piever piever Feb 14, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

By looking at the @code_llvm or @code_native for mapfoldl on tuples, it seems to generate the optimal thing, I'm just a bit confused as to where is the special method for tuples. Nevermind, I now see that it dispatches to Base.afoldl.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for merging this. Maybe you've already figured it out why, but the reason why I didn't use foreachfield there was that AFAICT foreachfield does not pass property names to the function.

append!(getproperty(s, name), Tables.getcolumn(table, name))
end
return s
else
# Otherwise, fallback to a generic implementation expecting
# that `rows` is an iterator:
return foldl(push!, rows; init = s)
end
end

@noinline function _invalid_columns_error(s, rows)
missingnames = setdiff!(collect(Tables.columnnames(rows)), propertynames(s))
throw(ArgumentError(string(
"Cannot append rows from `$(typeof(rows))` to `$(typeof(s))` due to ",
"missing column(s):\n",
join(missingnames, ", "),
)))
end
13 changes: 13 additions & 0 deletions src/utils.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,3 +134,16 @@ hasfields(::Type{<:NTuple{N, Any}}) where {N} = true
hasfields(::Type{<:NamedTuple{names}}) where {names} = true
hasfields(::Type{T}) where {T} = !isabstracttype(T)
hasfields(::Union) = false

_setdiff(a, b) = setdiff(a, b)

@inline _setdiff(::Tuple{}, ::Tuple{}) = ()
@inline _setdiff(::Tuple{}, ::Tuple) = ()
@inline _setdiff(a::Tuple, ::Tuple{}) = a
@inline _setdiff(a::Tuple, b::Tuple) = _setdiff(_exclude(a, b[1]), Base.tail(b))
@inline _exclude(a, b) = foldl((ys, x) -> x == b ? ys : (ys..., x), a; init = ())

# _foreach(f, xs) = foreach(f, xs)
_foreach(f, xs::Tuple) = foldl((_, x) -> (f(x); nothing), xs; init = nothing)
# Note `foreach` is not optimized for tuples yet.
# See: https://github.com/JuliaLang/julia/pull/31901
7 changes: 7 additions & 0 deletions test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,13 @@ end
@test Tables.rowaccess(typeof(s))
@test Tables.columnaccess(s)
@test Tables.columnaccess(typeof(s))
@test append!(StructArray([1im]), [(re = 111, im = 222)]) ==
StructArray([1im, 111 + 222im])
@test append!(StructArray([1im]), (x for x in [(re = 111, im = 222)])) ==
StructArray([1im, 111 + 222im])
# Testing integer column "names":
@test invoke(append!, Tuple{StructVector,Any}, StructArray(([0],)), StructArray(([1],))) ==
StructArray(([0, 1],))
end

struct S
Expand Down