Merge pull request #560 from dhui/re-add-iofs-src-driver

Re-add the io/fs source driver

Author: dhui

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15-alpine3.12 AS builder
+FROM golang:1.16-alpine3.13 AS builder
 ARG VERSION
 
 RUN apk add --no-cache git gcc musl-dev make
@@ -15,7 +15,7 @@ COPY . ./
 
 RUN make build-docker
 
-FROM alpine:3.12
+FROM alpine:3.13
 
 RUN apk add --no-cache ca-certificates
 

go.mod

@@ -64,4 +64,4 @@ require (
 	modernc.org/zappy v1.0.0 // indirect
 )
 
-go 1.13
+go 1.16

source/file/file.go

@@ -1,31 +1,22 @@
 package file
 
 import (
-	"net/http"
 	nurl "net/url"
 	"os"
 	"path/filepath"
 
 	"github.com/golang-migrate/migrate/v4/source"
-	"github.com/golang-migrate/migrate/v4/source/httpfs"
 )
 
 func init() {
 	source.Register("file", &File{})
 }
 
-type File struct {
-	httpfs.PartialDriver
-	url  string
-	path string
-}
-
-func (f *File) Open(url string) (source.Driver, error) {
+func parseURL(url string) (string, error) {
 	u, err := nurl.Parse(url)
 	if err != nil {
-		return nil, err
+		return "", err
 	}
-
 	// concat host and path to restore full path
 	// host might be `.`
 	p := u.Opaque
@@ -37,25 +28,17 @@ func (f *File) Open(url string) (source.Driver, error) {
 		// default to current directory if no path
 		wd, err := os.Getwd()
 		if err != nil {
-			return nil, err
+			return "", err
 		}
 		p = wd
 
 	} else if p[0:1] == "." || p[0:1] != "/" {
 		// make path absolute if relative
 		abs, err := filepath.Abs(p)
 		if err != nil {
-			return nil, err
+			return "", err
 		}
 		p = abs
 	}
-
-	nf := &File{
-		url:  url,
-		path: p,
-	}
-	if err := nf.Init(http.Dir(p), ""); err != nil {
-		return nil, err
-	}
-	return nf, nil
+	return p, nil
 }

source/file/file_go115.go

@@ -0,0 +1,32 @@
+// +build !go1.16
+
+package file
+
+import (
+	"net/http"
+
+	"github.com/golang-migrate/migrate/v4/source"
+	"github.com/golang-migrate/migrate/v4/source/httpfs"
+)
+
+type File struct {
+	httpfs.PartialDriver
+	url  string
+	path string
+}
+
+func (f *File) Open(url string) (source.Driver, error) {
+	p, err := parseURL(url)
+	if err != nil {
+		return nil, err
+	}
+
+	nf := &File{
+		url:  url,
+		path: p,
+	}
+	if err := nf.Init(http.Dir(p), ""); err != nil {
+		return nil, err
+	}
+	return nf, nil
+}

source/file/file_go116.go

@@ -0,0 +1,31 @@
+// +build go1.16
+
+package file
+
+import (
+	"os"
+
+	"github.com/golang-migrate/migrate/v4/source"
+	"github.com/golang-migrate/migrate/v4/source/iofs"
+)
+
+type File struct {
+	iofs.PartialDriver
+	url  string
+	path string
+}
+
+func (f *File) Open(url string) (source.Driver, error) {
+	p, err := parseURL(url)
+	if err != nil {
+		return nil, err
+	}
+	nf := &File{
+		url:  url,
+		path: p,
+	}
+	if err := nf.Init(os.DirFS(p), "."); err != nil {
+		return nil, err
+	}
+	return nf, nil
+}

source/iofs/README.md

@@ -0,0 +1,3 @@
+# iofs
+
+https://pkg.go.dev/github.com/golang-migrate/migrate/v4/source/iofs

source/iofs/doc.go

@@ -0,0 +1,10 @@
+/*
+Package iofs provides the Go 1.16+ io/fs#FS driver.
+
+It can accept various file systems (like embed.FS, archive/zip#Reader) implementing io/fs#FS.
+
+This driver cannot be used with Go versions 1.15 and below.
+
+Also, Opening with a URL scheme is not supported.
+*/
+package iofs

source/iofs/example_test.go

@@ -0,0 +1,31 @@
+// +build go1.16
+
+package iofs_test
+
+import (
+	"embed"
+	"log"
+
+	"github.com/golang-migrate/migrate/v4"
+	_ "github.com/golang-migrate/migrate/v4/database/postgres"
+	"github.com/golang-migrate/migrate/v4/source/iofs"
+)
+
+//go:embed testdata/migrations/*.sql
+var fs embed.FS
+
+func Example() {
+	d, err := iofs.New(fs, "testdata/migrations")
+	if err != nil {
+		log.Fatal(err)
+	}
+	m, err := migrate.NewWithSourceInstance("iofs", d, "postgres://postgres@localhost/postgres?sslmode=disable")
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = m.Up()
+	if err != nil {
+		// ...
+	}
+	// ...
+}

source/iofs/iofs.go

@@ -0,0 +1,175 @@
+// +build go1.16
+
+package iofs
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"path"
+	"strconv"
+
+	"github.com/golang-migrate/migrate/v4/source"
+)
+
+type driver struct {
+	PartialDriver
+}
+
+// New returns a new Driver from io/fs#FS and a relative path.
+func New(fsys fs.FS, path string) (source.Driver, error) {
+	var i driver
+	if err := i.Init(fsys, path); err != nil {
+		return nil, fmt.Errorf("failed to init driver with path %s: %w", path, err)
+	}
+	return &i, nil
+}
+
+// Open is part of source.Driver interface implementation.
+// Open cannot be called on the iofs passthrough driver.
+func (d *driver) Open(url string) (source.Driver, error) {
+	return nil, errors.New("Open() cannot be called on the iofs passthrough driver")
+}
+
+// PartialDriver is a helper service for creating new source drivers working with
+// io/fs.FS instances. It implements all source.Driver interface methods
+// except for Open(). New driver could embed this struct and add missing Open()
+// method.
+//
+// To prepare PartialDriver for use Init() function.
+type PartialDriver struct {
+	migrations *source.Migrations
+	fsys       fs.FS
+	path       string
+}
+
+// Init prepares not initialized IoFS instance to read migrations from a
+// io/fs#FS instance and a relative path.
+func (d *PartialDriver) Init(fsys fs.FS, path string) error {
+	entries, err := fs.ReadDir(fsys, path)
+	if err != nil {
+		return err
+	}
+
+	ms := source.NewMigrations()
+	for _, e := range entries {
+		if e.IsDir() {
+			continue
+		}
+		m, err := source.DefaultParse(e.Name())
+		if err != nil {
+			continue
+		}
+		file, err := e.Info()
+		if err != nil {
+			return err
+		}
+		if !ms.Append(m) {
+			return source.ErrDuplicateMigration{
+				Migration: *m,
+				FileInfo:  file,
+			}
+		}
+	}
+
+	d.fsys = fsys
+	d.path = path
+	d.migrations = ms
+	return nil
+}
+
+// Close is part of source.Driver interface implementation.
+// Closes the file system if possible.
+func (d *PartialDriver) Close() error {
+	c, ok := d.fsys.(io.Closer)
+	if !ok {
+		return nil
+	}
+	return c.Close()
+}
+
+// First is part of source.Driver interface implementation.
+func (d *PartialDriver) First() (version uint, err error) {
+	if version, ok := d.migrations.First(); ok {
+		return version, nil
+	}
+	return 0, &fs.PathError{
+		Op:   "first",
+		Path: d.path,
+		Err:  fs.ErrNotExist,
+	}
+}
+
+// Prev is part of source.Driver interface implementation.
+func (d *PartialDriver) Prev(version uint) (prevVersion uint, err error) {
+	if version, ok := d.migrations.Prev(version); ok {
+		return version, nil
+	}
+	return 0, &fs.PathError{
+		Op:   "prev for version " + strconv.FormatUint(uint64(version), 10),
+		Path: d.path,
+		Err:  fs.ErrNotExist,
+	}
+}
+
+// Next is part of source.Driver interface implementation.
+func (d *PartialDriver) Next(version uint) (nextVersion uint, err error) {
+	if version, ok := d.migrations.Next(version); ok {
+		return version, nil
+	}
+	return 0, &fs.PathError{
+		Op:   "next for version " + strconv.FormatUint(uint64(version), 10),
+		Path: d.path,
+		Err:  fs.ErrNotExist,
+	}
+}
+
+// ReadUp is part of source.Driver interface implementation.
+func (d *PartialDriver) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) {
+	if m, ok := d.migrations.Up(version); ok {
+		body, err := d.open(path.Join(d.path, m.Raw))
+		if err != nil {
+			return nil, "", err
+		}
+		return body, m.Identifier, nil
+	}
+	return nil, "", &fs.PathError{
+		Op:   "read up for version " + strconv.FormatUint(uint64(version), 10),
+		Path: d.path,
+		Err:  fs.ErrNotExist,
+	}
+}
+
+// ReadDown is part of source.Driver interface implementation.
+func (d *PartialDriver) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) {
+	if m, ok := d.migrations.Down(version); ok {
+		body, err := d.open(path.Join(d.path, m.Raw))
+		if err != nil {
+			return nil, "", err
+		}
+		return body, m.Identifier, nil
+	}
+	return nil, "", &fs.PathError{
+		Op:   "read down for version " + strconv.FormatUint(uint64(version), 10),
+		Path: d.path,
+		Err:  fs.ErrNotExist,
+	}
+}
+
+func (d *PartialDriver) open(path string) (fs.File, error) {
+	f, err := d.fsys.Open(path)
+	if err == nil {
+		return f, nil
+	}
+	// Some non-standard file systems may return errors that don't include the path, that
+	// makes debugging harder.
+	if !errors.As(err, new(*fs.PathError)) {
+		err = &fs.PathError{
+			Op:   "open",
+			Path: path,
+			Err:  err,
+		}
+	}
+	return nil, err
+}

source/iofs/iofs_test.go

@@ -0,0 +1,20 @@
+// +build go1.16
+
+package iofs_test
+
+import (
+	"testing"
+
+	"github.com/golang-migrate/migrate/v4/source/iofs"
+	st "github.com/golang-migrate/migrate/v4/source/testing"
+)
+
+func Test(t *testing.T) {
+	// reuse the embed.FS set in example_test.go
+	d, err := iofs.New(fs, "testdata/migrations")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st.Test(t, d)
+}

source/iofs/testdata/migrations/1_foobar.down.sql

@@ -0,0 +1 @@
+1 down

source/iofs/testdata/migrations/1_foobar.up.sql

@@ -0,0 +1 @@
+1 up

source/iofs/testdata/migrations/3_foobar.up.sql

@@ -0,0 +1 @@
+3 up