Skip to content

Add Compiler Plugin support for statistics on GroupBy #1077

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Mar 11, 2025
Merged

Add Compiler Plugin support for statistics on GroupBy #1077

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
783b37d
Add GroupBySumOf functionality to groupBy operations
zaleslaw Feb 25, 2025
9e1587b
Add commented GroupBySum0 support with updated scenarios and tests
zaleslaw Feb 25, 2025
04518ef
Refactor GroupBy statistics functionality and tests.
zaleslaw Feb 26, 2025
20fddbf
Add support for GroupBy mean and median operations.
zaleslaw Feb 27, 2025
e3738c4
Add support for min and max functions
zaleslaw Feb 27, 2025
41c201c
Added support for std function
zaleslaw Feb 27, 2025
324850b
Merge branch 'master' into issue-1074
zaleslaw Mar 3, 2025
34eda2b
Updated support for sum/sumFor
zaleslaw Mar 4, 2025
c7ae742
Added support for all statistics but faced with limitation of FiR
zaleslaw Mar 4, 2025
370b9cc
Fixed comparable types for median
zaleslaw Mar 5, 2025
9e6ade6
Fixed for max/min
zaleslaw Mar 5, 2025
ff39939
Fixed for std/mean
zaleslaw Mar 5, 2025
00af85c
Merge branch 'master' into issue-1074
zaleslaw Mar 10, 2025
22d4fb1
added missed casts to median/percentile. Could result in Comparable<A…
Jolanrensen Mar 7, 2025
aa8cd9a
Refactor groupBy for enhanced type safety and comparability
zaleslaw Mar 10, 2025
9b29026
Refactor `GroupBy` aggregation classes and test handling.
zaleslaw Mar 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/max.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,9 +133,12 @@ public fun <T, C : Comparable<C>> DataFrame<T>.maxByOrNull(column: KProperty<C?>
// endregion

// region GroupBy

@Refine
@Interpretable("GroupByMax1")
public fun <T> Grouped<T>.max(): DataFrame<T> = maxFor(interComparableColumns())

@Refine
@Interpretable("GroupByMax0")
public fun <T, C : Comparable<C>> Grouped<T>.maxFor(columns: ColumnsForAggregateSelector<T, C?>): DataFrame<T> =
Aggregators.max.aggregateFor(this, columns)

Expand All @@ -149,6 +152,8 @@ public fun <T, C : Comparable<C>> Grouped<T>.maxFor(vararg columns: ColumnRefere
public fun <T, C : Comparable<C>> Grouped<T>.maxFor(vararg columns: KProperty<C?>): DataFrame<T> =
maxFor { columns.toColumnSet() }

@Refine
@Interpretable("GroupByMax0")
public fun <T, C : Comparable<C>> Grouped<T>.max(name: String? = null, columns: ColumnsSelector<T, C?>): DataFrame<T> =
Aggregators.max.aggregateAll(this, name, columns)

Expand Down
11 changes: 10 additions & 1 deletion core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/mean.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import org.jetbrains.kotlinx.dataframe.DataRow
import org.jetbrains.kotlinx.dataframe.RowExpression
import org.jetbrains.kotlinx.dataframe.aggregation.ColumnsForAggregateSelector
import org.jetbrains.kotlinx.dataframe.annotations.AccessApiOverload
import org.jetbrains.kotlinx.dataframe.annotations.Interpretable
import org.jetbrains.kotlinx.dataframe.annotations.Refine
import org.jetbrains.kotlinx.dataframe.columns.ColumnReference
import org.jetbrains.kotlinx.dataframe.columns.toColumnSet
import org.jetbrains.kotlinx.dataframe.columns.toColumnsSetOf
Expand Down Expand Up @@ -98,9 +100,12 @@ public inline fun <T, reified D : Number> DataFrame<T>.meanOf(
// endregion

// region GroupBy

@Refine
@Interpretable("GroupByMean1")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's call it GroupByMeanFor* to distinguish from GroupBy.mean

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, don't understand, GroupByMeanFor will be under mean/meanFor and GroupByMean0 will be under meanFor at that way.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suggest GroupByMeanDefault for GroupBy.mean(), GroupByMeanFor for GroupBy.meanFor and GroupByMean for mean

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, agree, will do some renaming/refactoring in next PR in that area

public fun <T> Grouped<T>.mean(skipNA: Boolean = skipNA_default): DataFrame<T> = meanFor(skipNA, numberColumns())

@Refine
@Interpretable("GroupByMean0")
public fun <T, C : Number> Grouped<T>.meanFor(
skipNA: Boolean = skipNA_default,
columns: ColumnsForAggregateSelector<T, C?>,
Expand All @@ -121,6 +126,8 @@ public fun <T, C : Number> Grouped<T>.meanFor(
skipNA: Boolean = skipNA_default,
): DataFrame<T> = meanFor(skipNA) { columns.toColumnSet() }

@Refine
@Interpretable("GroupByMean0")
public fun <T, C : Number> Grouped<T>.mean(
name: String? = null,
skipNA: Boolean = skipNA_default,
Expand All @@ -147,6 +154,8 @@ public fun <T, C : Number> Grouped<T>.mean(
skipNA: Boolean = skipNA_default,
): DataFrame<T> = mean(name, skipNA) { columns.toColumnSet() }

@Refine
@Interpretable("GroupByMeanOf")
public inline fun <T, reified R : Number> Grouped<T>.meanOf(
name: String? = null,
skipNA: Boolean = skipNA_default,
Expand Down
16 changes: 13 additions & 3 deletions core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/median.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import org.jetbrains.kotlinx.dataframe.DataRow
import org.jetbrains.kotlinx.dataframe.RowExpression
import org.jetbrains.kotlinx.dataframe.aggregation.ColumnsForAggregateSelector
import org.jetbrains.kotlinx.dataframe.annotations.AccessApiOverload
import org.jetbrains.kotlinx.dataframe.annotations.Interpretable
import org.jetbrains.kotlinx.dataframe.annotations.Refine
import org.jetbrains.kotlinx.dataframe.columns.ColumnReference
import org.jetbrains.kotlinx.dataframe.columns.toColumnSet
import org.jetbrains.kotlinx.dataframe.impl.aggregation.aggregators.Aggregators
Expand All @@ -16,6 +18,7 @@ import org.jetbrains.kotlinx.dataframe.impl.aggregation.interComparableColumns
import org.jetbrains.kotlinx.dataframe.impl.aggregation.modes.aggregateAll
import org.jetbrains.kotlinx.dataframe.impl.aggregation.modes.aggregateFor
import org.jetbrains.kotlinx.dataframe.impl.aggregation.modes.aggregateOf
import org.jetbrains.kotlinx.dataframe.impl.aggregation.modes.aggregateOfDelegated
import org.jetbrains.kotlinx.dataframe.impl.aggregation.modes.of
import org.jetbrains.kotlinx.dataframe.impl.columns.toComparableColumns
import org.jetbrains.kotlinx.dataframe.impl.suggestIfNull
Expand Down Expand Up @@ -103,9 +106,12 @@ public inline fun <T, reified R : Comparable<R>> DataFrame<T>.medianOf(
// endregion

// region GroupBy

@Refine
@Interpretable("GroupByMedian1")
public fun <T> Grouped<T>.median(): DataFrame<T> = medianFor(interComparableColumns())

@Refine
@Interpretable("GroupByMedian0")
public fun <T, C : Comparable<C>> Grouped<T>.medianFor(columns: ColumnsForAggregateSelector<T, C?>): DataFrame<T> =
Aggregators.median.aggregateFor(this, columns)

Expand All @@ -119,6 +125,8 @@ public fun <T, C : Comparable<C>> Grouped<T>.medianFor(vararg columns: ColumnRef
public fun <T, C : Comparable<C>> Grouped<T>.medianFor(vararg columns: KProperty<C?>): DataFrame<T> =
medianFor { columns.toColumnSet() }

@Refine
@Interpretable("GroupByMedian0")
public fun <T, C : Comparable<C>> Grouped<T>.median(
name: String? = null,
columns: ColumnsSelector<T, C?>,
Expand All @@ -137,10 +145,12 @@ public fun <T, C : Comparable<C>> Grouped<T>.median(
public fun <T, C : Comparable<C>> Grouped<T>.median(vararg columns: KProperty<C?>, name: String? = null): DataFrame<T> =
median(name) { columns.toColumnSet() }

@Refine
@Interpretable("GroupByMedianOf")
public inline fun <T, reified R : Comparable<R>> Grouped<T>.medianOf(
name: String? = null,
crossinline expression: RowExpression<T, R?>,
): DataFrame<T> = Aggregators.median.aggregateOf(this, name, expression)
): DataFrame<T> = Aggregators.median.cast<R?>().aggregateOf(this, name, expression)

// endregion

Expand Down Expand Up @@ -227,6 +237,6 @@ public fun <T, C : Comparable<C>> PivotGroupBy<T>.median(vararg columns: KProper

public inline fun <T, reified R : Comparable<R>> PivotGroupBy<T>.medianOf(
crossinline expression: RowExpression<T, R?>,
): DataFrame<T> = Aggregators.median.aggregateOf(this, expression)
): DataFrame<T> = Aggregators.median.cast<R?>().aggregateOf(this, expression)

// endregion
7 changes: 6 additions & 1 deletion core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/min.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,9 +133,12 @@ public fun <T, C : Comparable<C>> DataFrame<T>.minByOrNull(column: KProperty<C?>
// endregion

// region GroupBy

@Refine
@Interpretable("GroupByMin1")
public fun <T> Grouped<T>.min(): DataFrame<T> = minFor(interComparableColumns())

@Refine
@Interpretable("GroupByMin0")
public fun <T, C : Comparable<C>> Grouped<T>.minFor(columns: ColumnsForAggregateSelector<T, C?>): DataFrame<T> =
Aggregators.min.aggregateFor(this, columns)

Expand All @@ -149,6 +152,8 @@ public fun <T, C : Comparable<C>> Grouped<T>.minFor(vararg columns: ColumnRefere
public fun <T, C : Comparable<C>> Grouped<T>.minFor(vararg columns: KProperty<C?>): DataFrame<T> =
minFor { columns.toColumnSet() }

@Refine
@Interpretable("GroupByMin0")
public fun <T, C : Comparable<C>> Grouped<T>.min(name: String? = null, columns: ColumnsSelector<T, C?>): DataFrame<T> =
Aggregators.min.aggregateAll(this, name, columns)

Expand Down
4 changes: 2 additions & 2 deletions core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/percentile.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ public inline fun <T, reified R : Comparable<R>> Grouped<T>.percentileOf(
percentile: Double,
name: String? = null,
crossinline expression: RowExpression<T, R?>,
): DataFrame<T> = Aggregators.percentile(percentile).aggregateOf(this, name, expression)
): DataFrame<T> = Aggregators.percentile(percentile).cast<R?>().aggregateOf(this, name, expression)

// endregion

Expand Down Expand Up @@ -289,6 +289,6 @@ public fun <T, C : Comparable<C>> PivotGroupBy<T>.percentile(
public inline fun <T, reified R : Comparable<R>> PivotGroupBy<T>.percentileOf(
percentile: Double,
crossinline expression: RowExpression<T, R?>,
): DataFrame<T> = Aggregators.percentile(percentile).aggregateOf(this, expression)
): DataFrame<T> = Aggregators.percentile(percentile).cast<R?>().aggregateOf(this, expression)

// endregion
13 changes: 12 additions & 1 deletion core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/std.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import org.jetbrains.kotlinx.dataframe.DataRow
import org.jetbrains.kotlinx.dataframe.RowExpression
import org.jetbrains.kotlinx.dataframe.aggregation.ColumnsForAggregateSelector
import org.jetbrains.kotlinx.dataframe.annotations.AccessApiOverload
import org.jetbrains.kotlinx.dataframe.annotations.Interpretable
import org.jetbrains.kotlinx.dataframe.annotations.Refine
import org.jetbrains.kotlinx.dataframe.columns.ColumnReference
import org.jetbrains.kotlinx.dataframe.columns.toColumnSet
import org.jetbrains.kotlinx.dataframe.columns.toColumnsSetOf
Expand Down Expand Up @@ -102,10 +104,13 @@ public inline fun <T, reified R : Number> DataFrame<T>.stdOf(
// endregion

// region GroupBy

@Refine
@Interpretable("GroupByStd1")
public fun <T> Grouped<T>.std(skipNA: Boolean = skipNA_default, ddof: Int = ddof_default): DataFrame<T> =
stdFor(skipNA, ddof, numberColumns())

@Refine
@Interpretable("GroupByStd0")
public fun <T> Grouped<T>.stdFor(
skipNA: Boolean = skipNA_default,
ddof: Int = ddof_default,
Expand All @@ -118,6 +123,7 @@ public fun <T> Grouped<T>.stdFor(
ddof: Int = ddof_default,
): DataFrame<T> = stdFor(skipNA, ddof) { columns.toColumnsSetOf() }

@AccessApiOverload
public fun <T, C : Number> Grouped<T>.stdFor(
vararg columns: ColumnReference<C?>,
skipNA: Boolean = skipNA_default,
Expand All @@ -131,13 +137,16 @@ public fun <T, C : Number> Grouped<T>.stdFor(
ddof: Int = ddof_default,
): DataFrame<T> = stdFor(skipNA, ddof) { columns.toColumnSet() }

@Refine
@Interpretable("GroupByStd0")
public fun <T> Grouped<T>.std(
name: String? = null,
skipNA: Boolean = skipNA_default,
ddof: Int = ddof_default,
columns: ColumnsSelector<T, Number?>,
): DataFrame<T> = Aggregators.std(skipNA, ddof).aggregateAll(this, name, columns)

@AccessApiOverload
public fun <T> Grouped<T>.std(
vararg columns: ColumnReference<Number?>,
name: String? = null,
Expand All @@ -160,6 +169,8 @@ public fun <T> Grouped<T>.std(
ddof: Int = ddof_default,
): DataFrame<T> = std(name, skipNA, ddof) { columns.toColumnSet() }

@Refine
@Interpretable("GroupByStdOf")
public inline fun <T, reified R : Number> Grouped<T>.stdOf(
name: String? = null,
skipNA: Boolean = skipNA_default,
Expand Down
11 changes: 10 additions & 1 deletion core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/sum.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import org.jetbrains.kotlinx.dataframe.DataRow
import org.jetbrains.kotlinx.dataframe.RowExpression
import org.jetbrains.kotlinx.dataframe.aggregation.ColumnsForAggregateSelector
import org.jetbrains.kotlinx.dataframe.annotations.AccessApiOverload
import org.jetbrains.kotlinx.dataframe.annotations.Interpretable
import org.jetbrains.kotlinx.dataframe.annotations.Refine
import org.jetbrains.kotlinx.dataframe.columns.ColumnReference
import org.jetbrains.kotlinx.dataframe.columns.toColumnSet
import org.jetbrains.kotlinx.dataframe.columns.toColumnsSetOf
Expand Down Expand Up @@ -89,9 +91,12 @@ public inline fun <T, reified C : Number?> DataFrame<T>.sumOf(crossinline expres
// endregion

// region GroupBy

@Refine
@Interpretable("GroupBySum1")
public fun <T> Grouped<T>.sum(): DataFrame<T> = sumFor(numberColumns())

@Refine
@Interpretable("GroupBySum0")
public fun <T, C : Number> Grouped<T>.sumFor(columns: ColumnsForAggregateSelector<T, C?>): DataFrame<T> =
Aggregators.sum.aggregateFor(this, columns)

Expand All @@ -105,6 +110,8 @@ public fun <T, C : Number> Grouped<T>.sumFor(vararg columns: ColumnReference<C?>
public fun <T, C : Number> Grouped<T>.sumFor(vararg columns: KProperty<C?>): DataFrame<T> =
sumFor { columns.toColumnSet() }

@Refine
@Interpretable("GroupBySum0")
public fun <T, C : Number> Grouped<T>.sum(name: String? = null, columns: ColumnsSelector<T, C?>): DataFrame<T> =
Aggregators.sum.aggregateAll(this, name, columns)

Expand All @@ -119,6 +126,8 @@ public fun <T, C : Number> Grouped<T>.sum(vararg columns: ColumnReference<C?>, n
public fun <T, C : Number> Grouped<T>.sum(vararg columns: KProperty<C?>, name: String? = null): DataFrame<T> =
sum(name) { columns.toColumnSet() }

@Refine
@Interpretable("GroupBySumOf")
public inline fun <T, reified R : Number> Grouped<T>.sumOf(
resultName: String? = null,
crossinline expression: RowExpression<T, R?>,
Expand Down
Loading