Skip to content

Commit cdaa02d

Browse files
author
Fede Vilensky
committed
Refactor spanner.go and spanner_test.go to utilize memefish, instead of spansql, for SQL statement parsing and clean up expected outputs in tests.
1 parent 2788339 commit cdaa02d

File tree

4 files changed

+48
-42
lines changed

4 files changed

+48
-42
lines changed

database/spanner/spanner.go

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,14 @@ import (
1313

1414
"cloud.google.com/go/spanner"
1515
sdb "cloud.google.com/go/spanner/admin/database/apiv1"
16-
"cloud.google.com/go/spanner/spansql"
17-
18-
"github.com/golang-migrate/migrate/v4"
19-
"github.com/golang-migrate/migrate/v4/database"
20-
2116
adminpb "cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
17+
"github.com/cloudspannerecosystem/memefish"
2218
"github.com/hashicorp/go-multierror"
2319
uatomic "go.uber.org/atomic"
2420
"google.golang.org/api/iterator"
21+
22+
"github.com/golang-migrate/migrate/v4"
23+
"github.com/golang-migrate/migrate/v4/database"
2524
)
2625

2726
func init() {
@@ -51,7 +50,7 @@ var (
5150
type Config struct {
5251
MigrationsTable string
5352
DatabaseName string
54-
// Whether to parse the migration DDL with spansql before
53+
// Whether to parse the migration DDL with memefish before
5554
// running them towards Spanner.
5655
// Parsing outputs clean DDL statements such as reformatted
5756
// and void of comments.
@@ -343,13 +342,13 @@ func (s *Spanner) ensureVersionTable() (err error) {
343342
func cleanStatements(migration []byte) ([]string, error) {
344343
// The Spanner GCP backend does not yet support comments for the UpdateDatabaseDdl RPC
345344
// (see https://issuetracker.google.com/issues/159730604) we use
346-
// spansql to parse the DDL and output valid stamements without comments
347-
ddl, err := spansql.ParseDDL("", string(migration))
345+
// memefish to parse the SQL statements and output valid stamements without comments
346+
astStmts, err := memefish.ParseStatements("", string(migration))
348347
if err != nil {
349348
return nil, err
350349
}
351-
stmts := make([]string, 0, len(ddl.List))
352-
for _, stmt := range ddl.List {
350+
stmts := make([]string, 0, len(astStmts))
351+
for _, stmt := range astStmts {
353352
stmts = append(stmts, stmt.SQL())
354353
}
355354
return stmts, nil

database/spanner/spanner_test.go

Lines changed: 27 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,13 @@ import (
55
"os"
66
"testing"
77

8-
"github.com/golang-migrate/migrate/v4"
9-
10-
dt "github.com/golang-migrate/migrate/v4/database/testing"
11-
_ "github.com/golang-migrate/migrate/v4/source/file"
12-
138
"cloud.google.com/go/spanner/spannertest"
149
"github.com/stretchr/testify/assert"
1510
"github.com/stretchr/testify/require"
11+
12+
"github.com/golang-migrate/migrate/v4"
13+
dt "github.com/golang-migrate/migrate/v4/database/testing"
14+
_ "github.com/golang-migrate/migrate/v4/source/file"
1615
)
1716

1817
// withSpannerEmulator is not thread-safe and cannot be used with parallel tests since it sets the emulator
@@ -75,75 +74,75 @@ func TestCleanStatements(t *testing.T) {
7574
{
7675
name: "single statement, single line, no semicolon, no comment",
7776
multiStatement: "CREATE TABLE table_name (id STRING(255) NOT NULL) PRIMARY KEY (id)",
78-
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"},
77+
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL\n) PRIMARY KEY (id)"},
7978
},
8079
{
8180
name: "single statement, multi line, no semicolon, no comment",
8281
multiStatement: `CREATE TABLE table_name (
83-
id STRING(255) NOT NULL,
82+
id STRING(255) NOT NULL
8483
) PRIMARY KEY (id)`,
85-
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"},
84+
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL\n) PRIMARY KEY (id)"},
8685
},
8786
{
8887
name: "single statement, single line, with semicolon, no comment",
8988
multiStatement: "CREATE TABLE table_name (id STRING(255) NOT NULL) PRIMARY KEY (id);",
90-
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"},
89+
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL\n) PRIMARY KEY (id)"},
9190
},
9291
{
9392
name: "single statement, multi line, with semicolon, no comment",
9493
multiStatement: `CREATE TABLE table_name (
95-
id STRING(255) NOT NULL,
94+
id STRING(255) NOT NULL
9695
) PRIMARY KEY (id);`,
97-
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"},
96+
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL\n) PRIMARY KEY (id)"},
9897
},
9998
{
10099
name: "multi statement, with trailing semicolon. no comment",
101100
// From https://github.com/mattes/migrate/pull/281
102101
multiStatement: `CREATE TABLE table_name (
103-
id STRING(255) NOT NULL,
104-
) PRIMARY KEY(id);
102+
id STRING(255) NOT NULL
103+
) PRIMARY KEY (id);
105104
106105
CREATE INDEX table_name_id_idx ON table_name (id);`,
107106
expected: []string{`CREATE TABLE table_name (
108-
id STRING(255) NOT NULL,
109-
) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
107+
id STRING(255) NOT NULL
108+
) PRIMARY KEY (id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
110109
},
111110
{
112111
name: "multi statement, no trailing semicolon, no comment",
113112
// From https://github.com/mattes/migrate/pull/281
114113
multiStatement: `CREATE TABLE table_name (
115-
id STRING(255) NOT NULL,
116-
) PRIMARY KEY(id);
114+
id STRING(255) NOT NULL
115+
) PRIMARY KEY (id);
117116
118117
CREATE INDEX table_name_id_idx ON table_name (id)`,
119118
expected: []string{`CREATE TABLE table_name (
120-
id STRING(255) NOT NULL,
121-
) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
119+
id STRING(255) NOT NULL
120+
) PRIMARY KEY (id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
122121
},
123122
{
124123
name: "multi statement, no trailing semicolon, standalone comment",
125124
// From https://github.com/mattes/migrate/pull/281
126125
multiStatement: `CREATE TABLE table_name (
127126
-- standalone comment
128-
id STRING(255) NOT NULL,
129-
) PRIMARY KEY(id);
127+
id STRING(255) NOT NULL
128+
) PRIMARY KEY (id);
130129
131130
CREATE INDEX table_name_id_idx ON table_name (id)`,
132131
expected: []string{`CREATE TABLE table_name (
133-
id STRING(255) NOT NULL,
134-
) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
132+
id STRING(255) NOT NULL
133+
) PRIMARY KEY (id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
135134
},
136135
{
137136
name: "multi statement, no trailing semicolon, inline comment",
138137
// From https://github.com/mattes/migrate/pull/281
139138
multiStatement: `CREATE TABLE table_name (
140139
id STRING(255) NOT NULL, -- inline comment
141-
) PRIMARY KEY(id);
140+
) PRIMARY KEY (id);
142141
143142
CREATE INDEX table_name_id_idx ON table_name (id)`,
144143
expected: []string{`CREATE TABLE table_name (
145-
id STRING(255) NOT NULL,
146-
) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
144+
id STRING(255) NOT NULL
145+
) PRIMARY KEY (id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
147146
},
148147
{
149148
name: "alter table with SET OPTIONS",
@@ -155,9 +154,9 @@ func TestCleanStatements(t *testing.T) {
155154
name: "column with NUMERIC type",
156155
multiStatement: `CREATE TABLE table_name (
157156
id STRING(255) NOT NULL,
158-
sum NUMERIC,
157+
sum NUMERIC
159158
) PRIMARY KEY (id)`,
160-
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n sum NUMERIC,\n) PRIMARY KEY(id)"},
159+
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n sum NUMERIC\n) PRIMARY KEY (id)"},
161160
},
162161
}
163162

go.mod

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ require (
99
github.com/ClickHouse/clickhouse-go v1.4.3
1010
github.com/aws/aws-sdk-go v1.49.6
1111
github.com/cenkalti/backoff/v4 v4.1.2
12+
github.com/cloudspannerecosystem/memefish v0.6.1
1213
github.com/cockroachdb/cockroach-go/v2 v2.1.1
1314
github.com/dhui/dktest v0.4.5
1415
github.com/docker/docker v27.2.0+incompatible
@@ -147,7 +148,7 @@ require (
147148
github.com/klauspost/asmfmt v1.3.2 // indirect
148149
github.com/klauspost/compress v1.15.11 // indirect
149150
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
150-
github.com/mattn/go-colorable v0.1.6 // indirect
151+
github.com/mattn/go-colorable v0.1.13 // indirect
151152
github.com/mattn/go-isatty v0.0.16 // indirect
152153
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
153154
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect

go.sum

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCU
6565
github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
6666
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU=
6767
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
68+
github.com/MakeNowJust/heredoc/v2 v2.0.1 h1:rlCHh70XXXv7toz95ajQWOWQnN4WNLt0TdpZYIR/J6A=
69+
github.com/MakeNowJust/heredoc/v2 v2.0.1/go.mod h1:6/2Abh5s+hc3g9nbWLe9ObDIOhaRrqsyY9MWy+4JdRM=
6870
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
6971
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
7072
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
@@ -131,6 +133,8 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
131133
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
132134
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg=
133135
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
136+
github.com/cloudspannerecosystem/memefish v0.6.1 h1:EJNZNq0E2vrYGBlu/xBs6jN7a5eq9ovF/wK+5Mo1iks=
137+
github.com/cloudspannerecosystem/memefish v0.6.1/go.mod h1:mVw0xBxy0yOgm990BuR0+nqP8J+yBAAf7N/2uL69rBU=
134138
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
135139
github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 h1:DBmgJDC9dTfkVyGgipamEh2BpGYxScCH1TOF1LL1cXc=
136140
github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM=
@@ -272,8 +276,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
272276
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
273277
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
274278
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
275-
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
276-
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
279+
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
280+
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
277281
github.com/google/go-github/v39 v39.2.0 h1:rNNM311XtPOz5rDdsJXAp2o8F67X9FnROXTvto3aSnQ=
278282
github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE=
279283
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@@ -412,6 +416,8 @@ github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQ
412416
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
413417
github.com/k0kubun/pp v2.3.0+incompatible h1:EKhKbi34VQDWJtq+zpsKSEhkHHs9w2P8Izbq8IhLVSo=
414418
github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=
419+
github.com/k0kubun/pp/v3 v3.4.1 h1:1WdFZDRRqe8UsR61N/2RoOZ3ziTEqgTPVqKrHeb779Y=
420+
github.com/k0kubun/pp/v3 v3.4.1/go.mod h1:+SiNiqKnBfw1Nkj82Lh5bIeKQOAkPy6Xw9CAZUZ8npI=
415421
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
416422
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
417423
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
@@ -452,8 +458,9 @@ github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQ
452458
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
453459
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
454460
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
455-
github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE=
456461
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
462+
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
463+
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
457464
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
458465
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
459466
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=

0 commit comments

Comments
 (0)