Skip to content

Commit aa865a3

Browse files
bitfieldbitfield
authored
Merge pull request #11 from bitfield/column
Add Column() filter and Visitors example
2 parents f8d212a + 77c46ad commit aa865a3

12 files changed

+360
-64
lines changed

‎.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,6 @@ examples/cat/cat
22
examples/grep/grep
33
examples/cat2/cat2
44
examples/echo/echo
5+
examples/head/head
6+
examples/visitors/visitors
7+
.vscode/settings.json

‎README.md

Lines changed: 219 additions & 49 deletions
Large diffs are not rendered by default.

‎examples/visitors/access.log

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 2028 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
2+
212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 162544 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
3+
212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 9419 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
4+
212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 2058 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
5+
212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 343743 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
6+
212.205.21.11 - - [30/Jun/2019:17:06:16 +0000] "GET / HTTP/1.1" 200 1150 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
7+
212.205.21.11 - - [30/Jun/2019:17:06:16 +0000] "GET / HTTP/1.1" 200 2946 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
8+
176.182.2.191 - - [30/Jun/2019:17:06:17 +0000] "GET / HTTP/1.1" 200 13278 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
9+
176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 29474 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
10+
176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 29349 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
11+
176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 48271 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
12+
176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 1380 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
13+
176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 2028 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
14+
176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 91819 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
15+
176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 305667 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
16+
176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 13194 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
17+
176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 12935 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
18+
176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 14598 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
19+
176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 22458 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
20+
176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 15737 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
21+
176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 404 17679 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
22+
176.182.2.191 - - [30/Jun/2019:17:06:23 +0000] "GET / HTTP/1.1" 200 5995 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
23+
190.253.121.1 - - [30/Jun/2019:17:06:23 +0000] "GET / HTTP/1.1" 200 8809 "-" "Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-J415FN Build/PPR1.180610.011) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/9.2 Chrome/67.0.3396.87 Mobile Safari/537.36"
24+
176.182.2.191 - - [30/Jun/2019:17:06:24 +0000] "GET / HTTP/1.1" 200 162544 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
25+
90.53.111.17 - - [30/Jun/2019:17:06:24 +0000] "GET / HTTP/1.1" 200 - "https://example.com/ "Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-J415FN Build/PPR1.180610.011) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/9.2 Chrome/67.0.3396.87 Mobile Safari/537.36"

‎examples/visitors/go.mod

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
module visitors
2+
3+
go 1.12
4+
5+
require github.com/bitfield/script v0.9.0

‎examples/visitors/main.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
This program reads an Apache logfile in Common Log Format, like this:
3+
4+
212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 2028 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
5+
6+
It extracts the first column of each line (the visitor IP address), counts the
7+
frequency of each unique IP address in the log, and outputs the 10 most frequent
8+
visitors in the log. Example output:
9+
10+
16 176.182.2.191
11+
7 212.205.21.11
12+
1 190.253.121.1
13+
1 90.53.111.17
14+
15+
*/
16+
package main
17+
18+
import (
19+
"github.com/bitfield/script"
20+
)
21+
22+
func main() {
23+
script.Stdin().Column(1).Freq().First(10).Stdout()
24+
}

‎filters.go

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"os/exec"
1010
"regexp"
1111
"sort"
12+
"strconv"
1213
"strings"
1314
)
1415

@@ -19,7 +20,7 @@ func (p *Pipe) Match(s string) *Pipe {
1920
return p.EachLine(func(line string, out *strings.Builder) {
2021
if strings.Contains(line, s) {
2122
out.WriteString(line)
22-
out.WriteByte('\n')
23+
out.WriteRune('\n')
2324
}
2425
})
2526
}
@@ -31,7 +32,7 @@ func (p *Pipe) MatchRegexp(re *regexp.Regexp) *Pipe {
3132
return p.EachLine(func(line string, out *strings.Builder) {
3233
if re.MatchString(line) {
3334
out.WriteString(line)
34-
out.WriteByte('\n')
35+
out.WriteRune('\n')
3536
}
3637
})
3738
}
@@ -43,7 +44,7 @@ func (p *Pipe) Reject(s string) *Pipe {
4344
return p.EachLine(func(line string, out *strings.Builder) {
4445
if !strings.Contains(line, s) {
4546
out.WriteString(line)
46-
out.WriteByte('\n')
47+
out.WriteRune('\n')
4748
}
4849
})
4950
}
@@ -55,7 +56,7 @@ func (p *Pipe) RejectRegexp(re *regexp.Regexp) *Pipe {
5556
return p.EachLine(func(line string, out *strings.Builder) {
5657
if !re.MatchString(line) {
5758
out.WriteString(line)
58-
out.WriteByte('\n')
59+
out.WriteRune('\n')
5960
}
6061
})
6162
}
@@ -153,6 +154,7 @@ func (p *Pipe) First(lines int) *Pipe {
153154
output.WriteString(scanner.Text())
154155
output.WriteRune('\n')
155156
}
157+
p.Close()
156158
err := scanner.Err()
157159
if err != nil {
158160
p.SetError(err)
@@ -178,19 +180,39 @@ func (p *Pipe) Freq() *Pipe {
178180
count int
179181
}
180182
var freqs = make([]frequency, 0, len(freq))
183+
var maxCount int
181184
for line, count := range freq {
182185
freqs = append(freqs, frequency{line, count})
186+
if count > maxCount {
187+
maxCount = count
188+
}
183189
}
184190
sort.Slice(freqs, func(i, j int) bool {
185191
if freqs[i].count == freqs[j].count {
186192
return freqs[i].line < freqs[j].line
187193
}
188194
return freqs[i].count > freqs[j].count
189195
})
196+
fieldWidth := len(strconv.Itoa(maxCount))
190197
var output strings.Builder
191198
for _, item := range freqs {
192-
output.WriteString(fmt.Sprintf("%d %s", item.count, item.line))
199+
output.WriteString(fmt.Sprintf("%*d %s", fieldWidth, item.count, item.line))
193200
output.WriteRune('\n')
194201
}
195202
return Echo(output.String())
196203
}
204+
205+
// Column reads from the pipe, and returns a new pipe containing only the Nth
206+
// column of each line in the input, where '1' means the first column, and
207+
// columns are delimited by whitespace. Specifically, whatever Unicode defines
208+
// as whitespace ('WSpace=yes'). If there is an error reading the pipe, the
209+
// pipe's error status is also set.
210+
func (p *Pipe) Column(col int) *Pipe {
211+
return p.EachLine(func(line string, out *strings.Builder) {
212+
columns := strings.Fields(line)
213+
if col <= len(columns) {
214+
out.WriteString(columns[col-1])
215+
out.WriteRune('\n')
216+
}
217+
})
218+
}

‎filters_test.go

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -226,23 +226,30 @@ func TestFirst(t *testing.T) {
226226
if err != nil {
227227
t.Fatal(err)
228228
}
229-
got, err := File("testdata/first.input.txt").First(10).Bytes()
229+
input := File("testdata/first.input.txt")
230+
got, err := input.First(10).Bytes()
230231
if err != nil {
231232
t.Error(err)
232233
}
233234
if !bytes.Equal(got, want) {
234235
t.Errorf("want %q, got %q", want, got)
235236
}
236-
// First(0) should return zero lines
237-
zero := File("testdata/first.input.txt").First(0)
238-
gotZero, err := zero.CountLines()
237+
_, err = ioutil.ReadAll(input.Reader)
238+
if err == nil {
239+
t.Error("input not closed after reading")
240+
}
241+
input = File("testdata/first.input.txt")
242+
gotZero, err := input.First(0).CountLines()
239243
if err != nil {
240244
t.Fatal(err)
241245
}
242246
if gotZero != 0 {
243247
t.Errorf("want 0 lines, got %d lines", gotZero)
244248
}
245-
// First(N) where the input has less than N lines, should just return the input.
249+
_, err = ioutil.ReadAll(input.Reader)
250+
if err == nil {
251+
t.Error("input not closed after reading")
252+
}
246253
want, err = File("testdata/first.input.txt").Bytes()
247254
if err != nil {
248255
t.Fatal(err)
@@ -270,3 +277,18 @@ func TestFreq(t *testing.T) {
270277
t.Errorf("want %q, got %q", want, got)
271278
}
272279
}
280+
281+
func TestColumn(t *testing.T) {
282+
t.Parallel()
283+
want, err := ioutil.ReadFile("testdata/column.golden.txt")
284+
if err != nil {
285+
t.Fatal(err)
286+
}
287+
got, err := File("testdata/column.input.txt").Column(3).Bytes()
288+
if err != nil {
289+
t.Error(err)
290+
}
291+
if !bytes.Equal(got, want) {
292+
t.Errorf("want %q, got %q", want, got)
293+
}
294+
}

‎pipes_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,8 @@ func doMethodsOnPipe(t *testing.T, p *Pipe, kind string) {
104104
p.First(1)
105105
action = "Freq()"
106106
p.Freq()
107+
action = "Column()"
108+
p.Column(2)
107109
}
108110

109111
func TestNilPipes(t *testing.T) {

‎testdata/column.golden.txt

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
Ss+
2+
R+
3+
Ss
4+
Ss+
5+
Ss+
6+
Ss+
7+
Ss+
8+
Ss

‎testdata/column.input.txt

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
60916 s003 Ss+ 0:00.51 /bin/bash -l
2+
6653 s004 R+ 0:00.01 ps ax
3+
bogus line
4+
80159 s004 Ss 0:00.56 /bin/bash -l
5+
60942 s006 Ss+ 0:00.53 /bin/bash -l
6+
60943 s007 Ss+ 0:00.51 /bin/bash -l
7+
60977 s009 Ss+ 0:00.52 /bin/bash -l
8+
60978 s010 Ss+ 0:00.53 /bin/bash -l
9+
61356 s011 Ss 0:00.54 /bin/bash -l

‎testdata/freq.golden.txt

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
4 apple
2-
4 banana
3-
2 orange
4-
1 kumquat
1+
10 apple
2+
4 banana
3+
2 orange
4+
1 kumquat

‎testdata/freq.input.txt

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,10 @@ kumquat
88
apple
99
apple
1010
banana
11-
banana
11+
banana
12+
apple
13+
apple
14+
apple
15+
apple
16+
apple
17+
apple

0 commit comments

Comments
 (0)