Skip to content
This repository was archived by the owner on Feb 7, 2024. It is now read-only.

fix: lazy load version, build correct multipart #305

Merged
merged 1 commit into from
Aug 22, 2023
Merged

fix: lazy load version, build correct multipart #305

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions add.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,11 @@ func CidVersion(version int) AddOpts {
func (s *Shell) Add(r io.Reader, options ...AddOpts) (string, error) {
fr := files.NewReaderFile(r)
slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", fr)})
fileReader := files.NewMultiFileReader(slf, true)

fileReader, err := s.newMultiFileReader(slf)
if err != nil {
return "", err
}

var out object
rb := s.Request("add")
Expand All @@ -90,7 +94,11 @@ func (s *Shell) AddWithOpts(r io.Reader, pin bool, rawLeaves bool) (string, erro
func (s *Shell) AddLink(target string) (string, error) {
link := files.NewLinkFile(target, nil)
slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", link)})
reader := files.NewMultiFileReader(slf, true)

reader, err := s.newMultiFileReader(slf)
if err != nil {
return "", err
}

var out object
return out.Hash, s.Request("add").Body(reader).Exec(context.Background(), &out)
Expand All @@ -108,7 +116,11 @@ func (s *Shell) AddDir(dir string, options ...AddOpts) (string, error) {
return "", err
}
slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry(filepath.Base(dir), sf)})
reader := files.NewMultiFileReader(slf, true)

reader, err := s.newMultiFileReader(slf)
if err != nil {
return "", err
}

rb := s.Request("add").Option("recursive", true)
for _, option := range options {
Expand Down
10 changes: 4 additions & 6 deletions dag.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func (s *Shell) DagPutWithOpts(data interface{}, opts ...options.DagPutOption) (
return "", err
}

fileReader, err := dagToFilesReader(data)
fileReader, err := s.dagToFilesReader(data)
if err != nil {
return "", err
}
Expand Down Expand Up @@ -80,7 +80,7 @@ func (s *Shell) DagImportWithOpts(data interface{}, opts ...options.DagImportOpt
return nil, err
}

fileReader, err := dagToFilesReader(data)
fileReader, err := s.dagToFilesReader(data)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -130,7 +130,7 @@ func (s *Shell) DagImportWithOpts(data interface{}, opts ...options.DagImportOpt
return &out, err
}

func dagToFilesReader(data interface{}) (*files.MultiFileReader, error) {
func (s *Shell) dagToFilesReader(data interface{}) (*files.MultiFileReader, error) {
var r io.Reader
switch data := data.(type) {
case *files.MultiFileReader:
Expand All @@ -147,7 +147,5 @@ func dagToFilesReader(data interface{}) (*files.MultiFileReader, error) {

fr := files.NewReaderFile(r)
slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", fr)})
fileReader := files.NewMultiFileReader(slf, true)

return fileReader, nil
return s.newMultiFileReader(slf)
}
13 changes: 7 additions & 6 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,33 +3,34 @@ go 1.19
module github.com/ipfs/go-ipfs-api

require (
github.com/blang/semver/v4 v4.0.0
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927
github.com/ipfs/boxo v0.8.0
github.com/ipfs/boxo v0.12.0
github.com/libp2p/go-libp2p v0.26.3
github.com/mitchellh/go-homedir v1.1.0
github.com/multiformats/go-multiaddr v0.8.0
github.com/multiformats/go-multibase v0.1.1
github.com/multiformats/go-multibase v0.2.0
)

require (
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
github.com/ipfs/go-cid v0.4.0 // indirect
github.com/ipfs/go-cid v0.4.1 // indirect
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
github.com/minio/sha256-simd v1.0.0 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base32 v0.1.0 // indirect
github.com/multiformats/go-base36 v0.2.0 // indirect
github.com/multiformats/go-multicodec v0.8.1 // indirect
github.com/multiformats/go-multihash v0.2.1 // indirect
github.com/multiformats/go-multicodec v0.9.0 // indirect
github.com/multiformats/go-multihash v0.2.3 // indirect
github.com/multiformats/go-multistream v0.4.1 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
golang.org/x/crypto v0.6.0 // indirect
golang.org/x/sys v0.6.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
google.golang.org/protobuf v1.30.0 // indirect
lukechampine.com/blake3 v1.1.7 // indirect
)
28 changes: 15 additions & 13 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764=
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg=
Expand All @@ -11,10 +13,10 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2U
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/ipfs/boxo v0.8.0 h1:UdjAJmHzQHo/j3g3b1bAcAXCj/GM6iTwvSlBDvPBNBs=
github.com/ipfs/boxo v0.8.0/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA=
github.com/ipfs/go-cid v0.4.0 h1:a4pdZq0sx6ZSxbCizebnKiMCx/xI/aBBFlB73IgH4rA=
github.com/ipfs/go-cid v0.4.0/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk=
github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ=
github.com/ipfs/boxo v0.12.0/go.mod h1:xAnfiU6PtxWCnRqu7dcXQ10bB5/kvI1kXRotuGqGBhg=
github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s=
github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk=
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU=
Expand All @@ -37,20 +39,20 @@ github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9
github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4=
github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU=
github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs=
github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI=
github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8=
github.com/multiformats/go-multicodec v0.8.1 h1:ycepHwavHafh3grIbR1jIXnKCsFm0fqsfEOsJ8NtKE8=
github.com/multiformats/go-multicodec v0.8.1/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k=
github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108=
github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc=
github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g=
github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk=
github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg=
github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k=
github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U=
github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM=
github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo=
github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q=
github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8=
github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand All @@ -59,8 +61,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0=
lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=
5 changes: 4 additions & 1 deletion key.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,10 @@ func (keyImportOpt) AllowAnyKeyType(allow bool) KeyImportOpt {
func (s *Shell) KeyImport(ctx context.Context, name string, key io.Reader, options ...KeyImportOpt) error {
fr := files.NewReaderFile(key)
slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", fr)})
fileReader := files.NewMultiFileReader(slf, true)
fileReader, err := s.newMultiFileReader(slf)
if err != nil {
return err
}

rb := s.Request("key/import", name)
for _, opt := range options {
Expand Down
5 changes: 4 additions & 1 deletion mfs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,10 @@ func (s *Shell) FilesStat(ctx context.Context, path string, options ...FilesOpt)
func (s *Shell) FilesWrite(ctx context.Context, path string, data io.Reader, options ...FilesOpt) error {
fr := files.NewReaderFile(data)
slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", fr)})
fileReader := files.NewMultiFileReader(slf, true)
fileReader, err := s.newMultiFileReader(slf)
if err != nil {
return err
}

rb := s.Request("files/write", path)
for _, opt := range options {
Expand Down
51 changes: 46 additions & 5 deletions shell.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,10 @@ import (
"os"
"path"
"strings"
"sync"
"time"

"github.com/blang/semver/v4"
files "github.com/ipfs/boxo/files"
tar "github.com/ipfs/boxo/tar"
homedir "github.com/mitchellh/go-homedir"
Expand All @@ -35,6 +37,9 @@ const (
type Shell struct {
url string
httpcli gohttp.Client

versionOnce sync.Once
version *semver.Version
}

func NewLocalShell() *Shell {
Expand Down Expand Up @@ -121,6 +126,39 @@ func NewShellWithClient(url string, client *gohttp.Client) *Shell {
return &sh
}

// encodedAbsolutePathVersion is the version from which the absolute path header in
// multipart requests is %-encoded. Before this version, its sent raw.
var encodedAbsolutePathVersion = semver.MustParse("0.23.0-dev")

func (s *Shell) loadRemoteVersion() error {
if s.version == nil {
version, _, err := s.Version()
if err != nil {
return err
}

remoteVersion, err := semver.New(version)
if err != nil {
return err
}

s.versionOnce.Do(func() {
s.version = remoteVersion
})
}

return nil
}
Comment on lines +133 to +151
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm pretty sure the .Do should englobe all of loadRemoteVersion's body.
Because this code will concurrently request the version number but save it only once, this leads to a data race between your s.version == nil and s.version = remoteVersion.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll fix in separate PR.


func (s *Shell) newMultiFileReader(dir files.Directory) (*files.MultiFileReader, error) {
err := s.loadRemoteVersion()
if err != nil {
return nil, err
}

return files.NewMultiFileReader(dir, true, s.version.LT(encodedAbsolutePathVersion)), nil
}

func (s *Shell) SetTimeout(d time.Duration) {
s.httpcli.Timeout = d
}
Expand Down Expand Up @@ -368,7 +406,7 @@ func (s *Shell) PatchData(root string, set bool, data interface{}) (string, erro

fr := files.NewReaderFile(read)
slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", fr)})
fileReader := files.NewMultiFileReader(slf, true)
fileReader := files.NewMultiFileReader(slf, true, s.version.LT(encodedAbsolutePathVersion))

var out object
return out.Hash, s.Request("object/patch/"+cmd, root).
Expand Down Expand Up @@ -470,7 +508,7 @@ func (s *Shell) BlockPut(block []byte, format, mhtype string, mhlen int) (string

fr := files.NewBytesFile(block)
slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", fr)})
fileReader := files.NewMultiFileReader(slf, true)
fileReader := files.NewMultiFileReader(slf, true, s.version.LT(encodedAbsolutePathVersion))

return out.Key, s.Request("block/put").
Option("mhtype", mhtype).
Expand Down Expand Up @@ -507,7 +545,7 @@ func (s *Shell) ObjectPut(obj *IpfsObject) (string, error) {

fr := files.NewReaderFile(&data)
slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", fr)})
fileReader := files.NewMultiFileReader(slf, true)
fileReader := files.NewMultiFileReader(slf, true, s.version.LT(encodedAbsolutePathVersion))

var out object
return out.Hash, s.Request("object/put").
Expand All @@ -530,10 +568,13 @@ func (s *Shell) PubSubSubscribe(topic string) (*PubSubSubscription, error) {
}

func (s *Shell) PubSubPublish(topic, data string) (err error) {

fr := files.NewReaderFile(bytes.NewReader([]byte(data)))
slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", fr)})
fileReader := files.NewMultiFileReader(slf, true)

fileReader, err := s.newMultiFileReader(slf)
if err != nil {
return err
}

encoder, _ := mbase.EncoderByName("base64url")
resp, err := s.Request("pubsub/pub", encoder.Encode([]byte(topic))).
Expand Down
5 changes: 4 additions & 1 deletion shell_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -496,8 +496,11 @@ func TestDagImportMultipleCARs(t *testing.T) {
files.FileEntry("", files.NewReaderFile(bytes.NewReader(carFile2))),
})

fileReader, err := s.newMultiFileReader(slf)
is.Nil(err)

dagImported, err := s.DagImportWithOpts(
files.NewMultiFileReader(slf, true),
fileReader,
options.Dag.Stats(true),
options.Dag.Silent(false),
)
Expand Down