Merge pull request #10 from bitfield/freq

Add Freq() filter

Author: bitfield

README.md

@@ -334,7 +334,45 @@ fmt.Println(output)
 `First()` reads its input and passes on the first N lines of it (like Unix [`head`](examples/head/main.go)):
 
 ```go
-Stdin().First(10).Stdout()
+script.Stdin().First(10).Stdout()
+```
+
+### Freq
+
+`Freq()` counts the frequencies of input lines, and outputs only the unique lines in the input, each prefixed with a count of its frequency, in descending order of frequency (that is, most frequent lines first). Lines with the same frequency will be sorted alphabetically. For example, given this input:
+
+```
+banana
+apple
+orange
+apple
+banana
+```
+
+and a program like:
+
+```go
+script.Stdin().Freq().Stdout()
+```
+
+the output will be:
+
+```
+2 apple
+2 banana
+1 orange
+```
+
+This is a common pattern in shell scripts to find the most frequently-occurring lines in a file:
+
+```sh
+sort testdata/freq.input.txt |uniq -c |sort -rn
+```
+
+`Freq()`'s behaviour is like the combination of Unix `sort`, `uniq -c`, and `sort -rn` used here. You can use `Freq()` in combination with `First()` to get, for example, the ten most common lines in a file:
+
+```go
+script.Stdin().Freq().First(10).Stdout()
 ```
 
 ### Join

filters.go

@@ -3,10 +3,12 @@ package script
 import (
 	"bufio"
 	"bytes"
+	"fmt"
 	"io"
 	"os"
 	"os/exec"
 	"regexp"
+	"sort"
 	"strings"
 )
 
@@ -157,3 +159,38 @@ func (p *Pipe) First(lines int) *Pipe {
 	}
 	return Echo(output.String())
 }
+
+// Freq reads from the pipe, and returns a new pipe containing only unique lines
+// from the input, prefixed with a frequency count, in descending numerical
+// order (most frequent lines first). Lines with equal frequency will be sorted
+// alphabetically. If there is an error reading the pipe, the pipe's error
+// status is also set.
+func (p *Pipe) Freq() *Pipe {
+	if p == nil || p.Error() != nil {
+		return p
+	}
+	var freq = map[string]int{}
+	p.EachLine(func(line string, out *strings.Builder) {
+		freq[line]++
+	})
+	type frequency struct {
+		line  string
+		count int
+	}
+	var freqs = make([]frequency, 0, len(freq))
+	for line, count := range freq {
+		freqs = append(freqs, frequency{line, count})
+	}
+	sort.Slice(freqs, func(i, j int) bool {
+		if freqs[i].count == freqs[j].count {
+			return freqs[i].line < freqs[j].line
+		}
+		return freqs[i].count > freqs[j].count
+	})
+	var output strings.Builder
+	for _, item := range freqs {
+		output.WriteString(fmt.Sprintf("%d %s", item.count, item.line))
+		output.WriteRune('\n')
+	}
+	return Echo(output.String())
+}

filters_test.go

@@ -255,3 +255,18 @@ func TestFirst(t *testing.T) {
 		t.Errorf("want %q, got %q", want, got)
 	}
 }
+
+func TestFreq(t *testing.T) {
+	t.Parallel()
+	want, err := ioutil.ReadFile("testdata/freq.golden.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := File("testdata/freq.input.txt").Freq().Bytes()
+	if err != nil {
+		t.Error(err)
+	}
+	if !bytes.Equal(got, want) {
+		t.Errorf("want %q, got %q", want, got)
+	}
+}

pipes_test.go

@@ -102,6 +102,8 @@ func doMethodsOnPipe(t *testing.T, p *Pipe, kind string) {
 	p.Read([]byte{})
 	action = "First()"
 	p.First(1)
+	action = "Freq()"
+	p.Freq()
 }
 
 func TestNilPipes(t *testing.T) {

testdata/freq.golden.txt

@@ -0,0 +1,4 @@
+4 apple
+4 banana
+2 orange
+1 kumquat

testdata/freq.input.txt

@@ -0,0 +1,11 @@
+apple
+orange
+banana
+banana
+apple
+orange
+kumquat
+apple
+apple
+banana
+banana