Add Column() filter and Visitors example

.gitignore

@@ -2,3 +2,6 @@ examples/cat/cat
 examples/grep/grep
 examples/cat2/cat2
 examples/echo/echo
+examples/head/head
+examples/visitors/visitors
+.vscode/settings.json

examples/visitors/access.log

@@ -0,0 +1,25 @@
+212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 2028 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
+212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 162544 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
+212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 9419 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
+212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 2058 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
+212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 343743 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
+212.205.21.11 - - [30/Jun/2019:17:06:16 +0000] "GET / HTTP/1.1" 200 1150 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
+212.205.21.11 - - [30/Jun/2019:17:06:16 +0000] "GET / HTTP/1.1" 200 2946 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
+176.182.2.191 - - [30/Jun/2019:17:06:17 +0000] "GET / HTTP/1.1" 200 13278 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
+176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 29474 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
+176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 29349 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
+176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 48271 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
+176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 1380 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
+176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 2028 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
+176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 91819 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
+176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 305667 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
+176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 13194 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
+176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 12935 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
+176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 14598 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
+176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 22458 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
+176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 15737 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
+176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 404 17679 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
+176.182.2.191 - - [30/Jun/2019:17:06:23 +0000] "GET / HTTP/1.1" 200 5995 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
+190.253.121.1 - - [30/Jun/2019:17:06:23 +0000] "GET / HTTP/1.1" 200 8809 "-" "Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-J415FN Build/PPR1.180610.011) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/9.2 Chrome/67.0.3396.87 Mobile Safari/537.36"
+176.182.2.191 - - [30/Jun/2019:17:06:24 +0000] "GET / HTTP/1.1" 200 162544 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
+90.53.111.17 - - [30/Jun/2019:17:06:24 +0000] "GET / HTTP/1.1" 200 - "https://example.com/ "Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-J415FN Build/PPR1.180610.011) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/9.2 Chrome/67.0.3396.87 Mobile Safari/537.36"

examples/visitors/go.mod

@@ -0,0 +1,5 @@
+module visitors
+
+go 1.12
+
+require github.com/bitfield/script v0.9.0

examples/visitors/main.go

@@ -0,0 +1,24 @@
+/*
+This program reads an Apache logfile in Common Log Format, like this:
+
+212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 2028 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
+
+It extracts the first column of each line (the visitor IP address), counts the
+frequency of each unique IP address in the log, and outputs the 10 most frequent
+visitors in the log. Example output:
+
+16 176.182.2.191
+7 212.205.21.11
+1 190.253.121.1
+1 90.53.111.17
+
+*/
+package main
+
+import (
+	"github.com/bitfield/script"
+)
+
+func main() {
+	script.Stdin().Column(1).Freq().First(10).Stdout()
+}

filters.go

@@ -9,6 +9,7 @@ import (
 	"os/exec"
 	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 )
 
@@ -19,7 +20,7 @@ func (p *Pipe) Match(s string) *Pipe {
 	return p.EachLine(func(line string, out *strings.Builder) {
 		if strings.Contains(line, s) {
 			out.WriteString(line)
-			out.WriteByte('\n')
+			out.WriteRune('\n')
 		}
 	})
 }
@@ -31,7 +32,7 @@ func (p *Pipe) MatchRegexp(re *regexp.Regexp) *Pipe {
 	return p.EachLine(func(line string, out *strings.Builder) {
 		if re.MatchString(line) {
 			out.WriteString(line)
-			out.WriteByte('\n')
+			out.WriteRune('\n')
 		}
 	})
 }
@@ -43,7 +44,7 @@ func (p *Pipe) Reject(s string) *Pipe {
 	return p.EachLine(func(line string, out *strings.Builder) {
 		if !strings.Contains(line, s) {
 			out.WriteString(line)
-			out.WriteByte('\n')
+			out.WriteRune('\n')
 		}
 	})
 }
@@ -55,7 +56,7 @@ func (p *Pipe) RejectRegexp(re *regexp.Regexp) *Pipe {
 	return p.EachLine(func(line string, out *strings.Builder) {
 		if !re.MatchString(line) {
 			out.WriteString(line)
-			out.WriteByte('\n')
+			out.WriteRune('\n')
 		}
 	})
 }
@@ -153,6 +154,7 @@ func (p *Pipe) First(lines int) *Pipe {
 		output.WriteString(scanner.Text())
 		output.WriteRune('\n')
 	}
+	p.Close()
 	err := scanner.Err()
 	if err != nil {
 		p.SetError(err)
@@ -178,19 +180,39 @@ func (p *Pipe) Freq() *Pipe {
 		count int
 	}
 	var freqs = make([]frequency, 0, len(freq))
+	var maxCount int
 	for line, count := range freq {
 		freqs = append(freqs, frequency{line, count})
+		if count > maxCount {
+			maxCount = count
+		}
 	}
 	sort.Slice(freqs, func(i, j int) bool {
 		if freqs[i].count == freqs[j].count {
 			return freqs[i].line < freqs[j].line
 		}
 		return freqs[i].count > freqs[j].count
 	})
+	fieldWidth := len(strconv.Itoa(maxCount))
 	var output strings.Builder
 	for _, item := range freqs {
-		output.WriteString(fmt.Sprintf("%d %s", item.count, item.line))
+		output.WriteString(fmt.Sprintf("%*d %s", fieldWidth, item.count, item.line))
 		output.WriteRune('\n')
 	}
 	return Echo(output.String())
 }
+
+// Column reads from the pipe, and returns a new pipe containing only the Nth
+// column of each line in the input, where '1' means the first column, and
+// columns are delimited by whitespace. Specifically, whatever Unicode defines
+// as whitespace ('WSpace=yes'). If there is an error reading the pipe, the
+// pipe's error status is also set.
+func (p *Pipe) Column(col int) *Pipe {
+	return p.EachLine(func(line string, out *strings.Builder) {
+		columns := strings.Fields(line)
+		if col <= len(columns) {
+			out.WriteString(columns[col-1])
+			out.WriteRune('\n')
+		}
+	})
+}

filters_test.go

@@ -226,23 +226,30 @@ func TestFirst(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	got, err := File("testdata/first.input.txt").First(10).Bytes()
+	input := File("testdata/first.input.txt")
+	got, err := input.First(10).Bytes()
 	if err != nil {
 		t.Error(err)
 	}
 	if !bytes.Equal(got, want) {
 		t.Errorf("want %q, got %q", want, got)
 	}
-	// First(0) should return zero lines
-	zero := File("testdata/first.input.txt").First(0)
-	gotZero, err := zero.CountLines()
+	_, err = ioutil.ReadAll(input.Reader)
+	if err == nil {
+		t.Error("input not closed after reading")
+	}
+	input = File("testdata/first.input.txt")
+	gotZero, err := input.First(0).CountLines()
 	if err != nil {
 		t.Fatal(err)
 	}
 	if gotZero != 0 {
 		t.Errorf("want 0 lines, got %d lines", gotZero)
 	}
-	// First(N) where the input has less than N lines, should just return the input.
+	_, err = ioutil.ReadAll(input.Reader)
+	if err == nil {
+		t.Error("input not closed after reading")
+	}
 	want, err = File("testdata/first.input.txt").Bytes()
 	if err != nil {
 		t.Fatal(err)
@@ -270,3 +277,18 @@ func TestFreq(t *testing.T) {
 		t.Errorf("want %q, got %q", want, got)
 	}
 }
+
+func TestColumn(t *testing.T) {
+	t.Parallel()
+	want, err := ioutil.ReadFile("testdata/column.golden.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := File("testdata/column.input.txt").Column(3).Bytes()
+	if err != nil {
+		t.Error(err)
+	}
+	if !bytes.Equal(got, want) {
+		t.Errorf("want %q, got %q", want, got)
+	}
+}

pipes_test.go

@@ -104,6 +104,8 @@ func doMethodsOnPipe(t *testing.T, p *Pipe, kind string) {
 	p.First(1)
 	action = "Freq()"
 	p.Freq()
+	action = "Column()"
+	p.Column(2)
 }
 
 func TestNilPipes(t *testing.T) {

testdata/column.golden.txt

@@ -0,0 +1,8 @@
+Ss+
+R+
+Ss
+Ss+
+Ss+
+Ss+
+Ss+
+Ss

testdata/column.input.txt

@@ -0,0 +1,9 @@
+60916 s003  Ss+    0:00.51 /bin/bash -l
+ 6653 s004  R+     0:00.01 ps ax
+bogus line
+80159 s004  Ss     0:00.56 /bin/bash -l
+60942 s006  Ss+    0:00.53 /bin/bash -l
+60943 s007  Ss+    0:00.51 /bin/bash -l
+60977 s009  Ss+  	0:00.52 /bin/bash -l
+	60978 s010  Ss+    0:00.53 /bin/bash -l
+61356 s011	Ss     0:00.54 /bin/bash -l

testdata/freq.golden.txt

@@ -1,4 +1,4 @@
-4 apple
-4 banana
-2 orange
-1 kumquat
+10 apple
+ 4 banana
+ 2 orange
+ 1 kumquat

testdata/freq.input.txt

@@ -8,4 +8,10 @@ kumquat
 apple
 apple
 banana
-banana
+banana
+apple
+apple
+apple
+apple
+apple
+apple