@@ -37,25 +37,41 @@ const (
37
37
HashMurmur3 uint64 = 0x22
38
38
)
39
39
40
- func (ds * Shard ) isValueNode () bool {
40
+ // Hash function declared as global variable only for testing purposes.
41
+ // FIXME: We shoul have a cleaner way to replace this during tests.
42
+ var HAMTHashFunction = murmur3Hash
43
+
44
+ func (ds * Shard ) IsValueNode () bool {
41
45
return ds .key != "" && ds .val != nil
42
46
}
43
47
44
48
// A Shard represents the HAMT. It should be initialized with NewShard().
45
49
type Shard struct {
46
50
childer * childer
47
51
48
- tableSize int
52
+ // Entries per node (number of possible childs indexed by the partial key).
53
+ tableSize int
54
+ // Bits needed to encode child indexes (log2 of number of entries). This is
55
+ // the number of bits taken from the hash key on each level of the tree.
49
56
tableSizeLg2 int
50
57
51
58
builder cid.Builder
52
59
hashFunc uint64
53
60
61
+ // String format with number of zeros that will be present in the hexadecimal
62
+ // encoding of the child index to always reach the fixed maxpadlen chars.
63
+ // Example: maxpadlen = 4 => prefixPadStr: "%04X" (print number in hexadecimal
64
+ // format padding with zeros to always reach 4 characters).
54
65
prefixPadStr string
55
- maxpadlen int
66
+ // Length in chars of string that encodes child indexes. We encode indexes
67
+ // as hexadecimal strings to this is log4 of number of entries.
68
+ maxpadlen int
56
69
57
70
dserv ipld.DAGService
58
71
72
+ // FIXME: Remove. We don't actually store "value nodes". This confusing
73
+ // abstraction just removes the maxpadlen from the link names to extract
74
+ // the actual value link the trie is storing.
59
75
// leaf node
60
76
key string
61
77
val * ipld.Link
@@ -68,6 +84,7 @@ func NewShard(dserv ipld.DAGService, size int) (*Shard, error) {
68
84
return nil , err
69
85
}
70
86
87
+ // FIXME: Make this at least a static configuration for testing.
71
88
ds .hashFunc = HashMurmur3
72
89
return ds , nil
73
90
}
@@ -211,7 +228,7 @@ func (ds *Shard) Set(ctx context.Context, name string, nd ipld.Node) error {
211
228
// name key in this Shard or its children. It also returns the previous link
212
229
// under that name key (if any).
213
230
func (ds * Shard ) SetAndPrevious (ctx context.Context , name string , node ipld.Node ) (* ipld.Link , error ) {
214
- hv := & hashBits { b : hash ([] byte ( name ))}
231
+ hv := newHashBits ( name )
215
232
err := ds .dserv .Add (ctx , node )
216
233
if err != nil {
217
234
return nil , err
@@ -221,6 +238,9 @@ func (ds *Shard) SetAndPrevious(ctx context.Context, name string, node ipld.Node
221
238
if err != nil {
222
239
return nil , err
223
240
}
241
+
242
+ // FIXME: We don't need to set the name here, it will get overwritten.
243
+ // This is confusing, confirm and remove this line.
224
244
lnk .Name = ds .linkNamePrefix (0 ) + name
225
245
226
246
return ds .setValue (ctx , hv , name , lnk )
@@ -236,13 +256,13 @@ func (ds *Shard) Remove(ctx context.Context, name string) error {
236
256
// RemoveAndPrevious is similar to the public Remove but also returns the
237
257
// old removed link (if it exists).
238
258
func (ds * Shard ) RemoveAndPrevious (ctx context.Context , name string ) (* ipld.Link , error ) {
239
- hv := & hashBits { b : hash ([] byte ( name ))}
259
+ hv := newHashBits ( name )
240
260
return ds .setValue (ctx , hv , name , nil )
241
261
}
242
262
243
263
// Find searches for a child node by 'name' within this hamt
244
264
func (ds * Shard ) Find (ctx context.Context , name string ) (* ipld.Link , error ) {
245
- hv := & hashBits { b : hash ([] byte ( name ))}
265
+ hv := newHashBits ( name )
246
266
247
267
var out * ipld.Link
248
268
err := ds .getValue (ctx , hv , name , func (sv * Shard ) error {
@@ -276,7 +296,7 @@ func (ds *Shard) childLinkType(lnk *ipld.Link) (linkType, error) {
276
296
277
297
// Link returns a merklelink to this shard node
278
298
func (ds * Shard ) Link () (* ipld.Link , error ) {
279
- if ds .isValueNode () {
299
+ if ds .IsValueNode () {
280
300
return ds .val , nil
281
301
}
282
302
@@ -305,7 +325,7 @@ func (ds *Shard) getValue(ctx context.Context, hv *hashBits, key string, cb func
305
325
return err
306
326
}
307
327
308
- if child .isValueNode () {
328
+ if child .IsValueNode () {
309
329
if child .key == key {
310
330
return cb (child )
311
331
}
@@ -332,6 +352,21 @@ func (ds *Shard) EnumLinks(ctx context.Context) ([]*ipld.Link, error) {
332
352
return links , nil
333
353
}
334
354
355
+ // FIXME: Check which functions do we need to actually expose.
356
+ func (ds * Shard ) EnumAll (ctx context.Context ) ([]* ipld.Link , error ) {
357
+ var links []* ipld.Link
358
+
359
+ linkResults := ds .EnumAllAsync (ctx )
360
+
361
+ for linkResult := range linkResults {
362
+ if linkResult .Err != nil {
363
+ return links , linkResult .Err
364
+ }
365
+ links = append (links , linkResult .Link )
366
+ }
367
+ return links , nil
368
+ }
369
+
335
370
// ForEachLink walks the Shard and calls the given function.
336
371
func (ds * Shard ) ForEachLink (ctx context.Context , f func (* ipld.Link ) error ) error {
337
372
return ds .walkTrie (ctx , func (sv * Shard ) error {
@@ -345,6 +380,31 @@ func (ds *Shard) ForEachLink(ctx context.Context, f func(*ipld.Link) error) erro
345
380
// EnumLinksAsync returns a channel which will receive Links in the directory
346
381
// as they are enumerated, where order is not guaranteed
347
382
func (ds * Shard ) EnumLinksAsync (ctx context.Context ) <- chan format.LinkResult {
383
+ linkResults := make (chan format.LinkResult )
384
+ ctx , cancel := context .WithCancel (ctx )
385
+ go func () {
386
+ defer close (linkResults )
387
+ defer cancel ()
388
+ getLinks := makeAsyncTrieGetLinks (ds .dserv , linkResults )
389
+ cset := cid .NewSet ()
390
+ rootNode , err := ds .Node ()
391
+ if err != nil {
392
+ emitResult (ctx , linkResults , format.LinkResult {Link : nil , Err : err })
393
+ return
394
+ }
395
+ // FIXME: Make concurrency an option for testing.
396
+ //err := dag.Walk(ctx, getLinks, ds.cid, cset.Visit, dag.Concurrent())
397
+ err = dag .Walk (ctx , getLinks , rootNode .Cid (), cset .Visit )
398
+ if err != nil {
399
+ emitResult (ctx , linkResults , format.LinkResult {Link : nil , Err : err })
400
+ }
401
+ }()
402
+ return linkResults
403
+ }
404
+
405
+ // EnumLinksAsync returns a channel which will receive Links in the directory
406
+ // as they are enumerated, where order is not guaranteed
407
+ func (ds * Shard ) EnumAllAsync (ctx context.Context ) <- chan format.LinkResult {
348
408
linkResults := make (chan format.LinkResult )
349
409
ctx , cancel := context .WithCancel (ctx )
350
410
go func () {
@@ -405,6 +465,39 @@ func makeAsyncTrieGetLinks(dagService ipld.DAGService, linkResults chan<- format
405
465
}
406
466
}
407
467
468
+ //// same as makeAsyncTrieGetLinks but return all
469
+ //// FIXME: Check how to abstract this.
470
+ //func makeAsyncTrieGetAll(dagService ipld.DAGService, linkResults chan<- format.LinkResult) dag.GetLinks {
471
+ //
472
+ // return func(ctx context.Context, currentCid cid.Cid) ([]*ipld.Link, error) {
473
+ // node, err := dagService.Get(ctx, currentCid)
474
+ // if err != nil {
475
+ // return nil, err
476
+ // }
477
+ // directoryShard, err := NewHamtFromDag(dagService, node)
478
+ // if err != nil {
479
+ // return nil, err
480
+ // }
481
+ //
482
+ // childShards := make([]*ipld.Link, 0, directoryShard.childer.length())
483
+ // links := directoryShard.childer.links
484
+ // for idx := range directoryShard.childer.children {
485
+ // lnk := links[idx]
486
+ // // We don't care about the link type (shard or value), just count
487
+ // // *all* nodes in this HAMT.
488
+ // emitResult(ctx, linkResults, format.LinkResult{Link: lnk, Err: nil})
489
+ // lnkLinkType, err := directoryShard.childLinkType(lnk)
490
+ // if err != nil {
491
+ // return nil, err
492
+ // }
493
+ // if lnkLinkType == shardLink {
494
+ // childShards = append(childShards, lnk)
495
+ // }
496
+ // }
497
+ // return childShards, nil
498
+ // }
499
+ //}
500
+
408
501
func emitResult (ctx context.Context , linkResults chan <- format.LinkResult , r format.LinkResult ) {
409
502
// make sure that context cancel is processed first
410
503
// the reason is due to the concurrency of EnumerateChildrenAsync
@@ -423,7 +516,7 @@ func emitResult(ctx context.Context, linkResults chan<- format.LinkResult, r for
423
516
424
517
func (ds * Shard ) walkTrie (ctx context.Context , cb func (* Shard ) error ) error {
425
518
return ds .childer .each (ctx , func (s * Shard ) error {
426
- if s .isValueNode () {
519
+ if s .IsValueNode () {
427
520
if err := cb (s ); err != nil {
428
521
return err
429
522
}
@@ -455,7 +548,7 @@ func (ds *Shard) setValue(ctx context.Context, hv *hashBits, key string, value *
455
548
return
456
549
}
457
550
458
- if child .isValueNode () {
551
+ if child .IsValueNode () {
459
552
// Leaf node. This is the base case of this recursive function.
460
553
if child .key == key {
461
554
// We are in the correct shard (tree level) so we modify this child
@@ -489,10 +582,7 @@ func (ds *Shard) setValue(ctx context.Context, hv *hashBits, key string, value *
489
582
return nil , err
490
583
}
491
584
child .builder = ds .builder
492
- chhv := & hashBits {
493
- b : hash ([]byte (grandChild .key )),
494
- consumed : hv .consumed ,
495
- }
585
+ chhv := newConsumedHashBits (grandChild .key , hv .consumed )
496
586
497
587
// We explicitly ignore the oldValue returned by the next two insertions
498
588
// (which will be nil) to highlight there is no overwrite here: they are
@@ -536,7 +626,7 @@ func (ds *Shard) setValue(ctx context.Context, hv *hashBits, key string, value *
536
626
// Have we loaded the child? Prefer that.
537
627
schild := child .childer .child (0 )
538
628
if schild != nil {
539
- if schild .isValueNode () {
629
+ if schild .IsValueNode () {
540
630
ds .childer .set (schild , i )
541
631
}
542
632
return
0 commit comments