diff --git a/cmd/boostd/pieces.go b/cmd/boostd/pieces.go index 46f47bf6e..9de6abd00 100644 --- a/cmd/boostd/pieces.go +++ b/cmd/boostd/pieces.go @@ -14,7 +14,7 @@ import ( var piecesCmd = &cli.Command{ Name: "pieces", - Usage: "interact with the piecestore", + Usage: "Interact with the Piece Store", Description: "The piecestore is a database that tracks and manages data that is made available to the retrieval market", Subcommands: []*cli.Command{ piecesListPiecesCmd, @@ -26,7 +26,7 @@ var piecesCmd = &cli.Command{ var piecesListPiecesCmd = &cli.Command{ Name: "list-pieces", - Usage: "list registered pieces", + Usage: "List registered pieces", Action: func(cctx *cli.Context) error { nodeApi, closer, err := bcli.GetBoostAPI(cctx) if err != nil { @@ -49,7 +49,7 @@ var piecesListPiecesCmd = &cli.Command{ var piecesListCidInfosCmd = &cli.Command{ Name: "list-cids", - Usage: "list registered payload CIDs", + Usage: "List registered payload CIDs", Flags: []cli.Flag{ &cli.BoolFlag{ Name: "verbose", @@ -123,7 +123,7 @@ var piecesListCidInfosCmd = &cli.Command{ var piecesInfoCmd = &cli.Command{ Name: "piece-info", - Usage: "get registered information for a given piece CID", + Usage: "Get registered information for a given piece CID", Action: func(cctx *cli.Context) error { if !cctx.Args().Present() { return lcli.ShowHelp(cctx, fmt.Errorf("must specify piece cid")) @@ -158,7 +158,7 @@ var piecesInfoCmd = &cli.Command{ var piecesCidInfoCmd = &cli.Command{ Name: "cid-info", - Usage: "get registered information for a given payload CID", + Usage: "Get registered information for a given payload CID", Action: func(cctx *cli.Context) error { if !cctx.Args().Present() { return lcli.ShowHelp(cctx, fmt.Errorf("must specify payload cid")) diff --git a/cmd/boostx/utils_cmd.go b/cmd/boostx/utils_cmd.go index 2239c1f19..f460912b1 100644 --- a/cmd/boostx/utils_cmd.go +++ b/cmd/boostx/utils_cmd.go @@ -1,7 +1,6 @@ package main import ( - "bufio" "context" "fmt" "io" @@ -213,16 +212,6 @@ var commpCmd = &cli.Command{ } defer rdr.Close() //nolint:errcheck - // check that the data is a car file; if it's not, retrieval won't work - _, err = car.ReadHeader(bufio.NewReader(rdr)) - if err != nil { - return fmt.Errorf("not a car file: %w", err) - } - - if _, err := rdr.Seek(0, io.SeekStart); err != nil { - return fmt.Errorf("seek to start: %w", err) - } - w := &writer.Writer{} _, err = io.CopyBuffer(w, rdr, make([]byte, writer.CommPBuf)) if err != nil { diff --git a/db/deals.go b/db/deals.go index 26edd3f17..9e4c388bc 100644 --- a/db/deals.go +++ b/db/deals.go @@ -212,6 +212,14 @@ func (d *DealsDB) ByPublishCID(ctx context.Context, publishCid string) ([]*types return deals, nil } +func (d *DealsDB) ByPieceCID(ctx context.Context, pieceCid cid.Cid) ([]*types.ProviderDealState, error) { + return d.list(ctx, 0, 0, "PieceCID=?", pieceCid.String()) +} + +func (d *DealsDB) ByRootPayloadCID(ctx context.Context, payloadCid cid.Cid) ([]*types.ProviderDealState, error) { + return d.list(ctx, 0, 0, "DealDataRoot=?", payloadCid.String()) +} + func (d *DealsDB) BySignedProposalCID(ctx context.Context, proposalCid cid.Cid) (*types.ProviderDealState, error) { qry := "SELECT " + dealFieldsStr + " FROM Deals WHERE SignedProposalCID=?" row := d.db.QueryRowContext(ctx, qry, proposalCid.String()) diff --git a/gql/resolver.go b/gql/resolver.go index b8ceed8c8..4884fda73 100644 --- a/gql/resolver.go +++ b/gql/resolver.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "errors" "fmt" - "github.com/filecoin-project/boost/db" "github.com/filecoin-project/boost/fundmanager" gqltypes "github.com/filecoin-project/boost/gql/types" @@ -16,6 +15,9 @@ import ( "github.com/filecoin-project/boost/storagemarket/types" "github.com/filecoin-project/boost/storagemarket/types/dealcheckpoints" "github.com/filecoin-project/boost/transport" + "github.com/filecoin-project/dagstore" + "github.com/filecoin-project/go-fil-markets/piecestore" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" lotus_storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/api/v1api" "github.com/filecoin-project/lotus/markets/storageadapter" @@ -49,12 +51,15 @@ type resolver struct { provider *storagemarket.Provider legacyProv lotus_storagemarket.StorageProvider legacyDT lotus_dtypes.ProviderDataTransfer + ps piecestore.PieceStore + sa retrievalmarket.SectorAccessor + dagst dagstore.Interface publisher *storageadapter.DealPublisher spApi sealingpipeline.API fullNode v1api.FullNode } -func NewResolver(cfg *config.Boost, r lotus_repo.LockedRepo, h host.Host, dealsDB *db.DealsDB, logsDB *db.LogsDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, spApi sealingpipeline.API, provider *storagemarket.Provider, legacyProv lotus_storagemarket.StorageProvider, legacyDT lotus_dtypes.ProviderDataTransfer, publisher *storageadapter.DealPublisher, fullNode v1api.FullNode) *resolver { +func NewResolver(cfg *config.Boost, r lotus_repo.LockedRepo, h host.Host, dealsDB *db.DealsDB, logsDB *db.LogsDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, spApi sealingpipeline.API, provider *storagemarket.Provider, legacyProv lotus_storagemarket.StorageProvider, legacyDT lotus_dtypes.ProviderDataTransfer, ps piecestore.PieceStore, sa retrievalmarket.SectorAccessor, dagst dagstore.Interface, publisher *storageadapter.DealPublisher, fullNode v1api.FullNode) *resolver { return &resolver{ cfg: cfg, repo: r, @@ -68,6 +73,9 @@ func NewResolver(cfg *config.Boost, r lotus_repo.LockedRepo, h host.Host, dealsD provider: provider, legacyProv: legacyProv, legacyDT: legacyDT, + ps: ps, + sa: sa, + dagst: dagst, publisher: publisher, spApi: spApi, fullNode: fullNode, diff --git a/gql/resolver_piece.go b/gql/resolver_piece.go new file mode 100644 index 000000000..cf507a8c0 --- /dev/null +++ b/gql/resolver_piece.go @@ -0,0 +1,310 @@ +package gql + +import ( + "context" + "errors" + "fmt" + + gqltypes "github.com/filecoin-project/boost/gql/types" + "github.com/filecoin-project/boost/storagemarket/types/dealcheckpoints" + "github.com/filecoin-project/dagstore" + "github.com/filecoin-project/dagstore/shard" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v8/market" + "github.com/graph-gophers/graphql-go" + "github.com/ipfs/go-cid" +) + +type IndexStatus string + +const ( + IndexStatusUnknown IndexStatus = "" + IndexStatusNotFound IndexStatus = "NotFound" + IndexStatusRegistered IndexStatus = "Registered" + IndexStatusComplete IndexStatus = "Complete" + IndexStatusFailed IndexStatus = "Failed" +) + +type pieceDealResolver struct { + Deal *basicDealResolver + Sector *sectorResolver + SealStatus *sealStatus +} + +type sealStatus struct { + IsUnsealed bool + Error string +} + +type pieceInfoDeal struct { + ChainDealID gqltypes.Uint64 + Sector *sectorResolver + SealStatus *sealStatus +} + +type indexStatus struct { + Status string + Error string +} + +type pieceResolver struct { + PieceCid string + IndexStatus *indexStatus + Deals []*pieceDealResolver + PieceInfoDeals []*pieceInfoDeal +} + +func (r *resolver) PiecesWithPayloadCid(ctx context.Context, args struct{ PayloadCid string }) ([]string, error) { + payloadCid, err := cid.Parse(args.PayloadCid) + if err != nil { + return nil, fmt.Errorf("%s is not a valid payload cid", args.PayloadCid) + } + + var pieceCidSet = make(map[string]struct{}) + + // Get boost deals by piece Cid + boostDeals, err := r.dealsDB.ByRootPayloadCID(ctx, payloadCid) + if err != nil { + return nil, err + } + for _, dl := range boostDeals { + pieceCidSet[dl.ClientDealProposal.Proposal.PieceCID.String()] = struct{}{} + } + + // Get legacy markets deals by payload Cid + // TODO: add method to markets to filter deals by payload CID + allLegacyDeals, err := r.legacyProv.ListLocalDeals() + if err != nil { + return nil, err + } + for _, dl := range allLegacyDeals { + if dl.Ref.Root == payloadCid { + pieceCidSet[dl.ClientDealProposal.Proposal.PieceCID.String()] = struct{}{} + } + } + + pieceCids := make([]string, 0, len(pieceCidSet)) + for pieceCid := range pieceCidSet { + pieceCids = append(pieceCids, pieceCid) + } + return pieceCids, nil +} + +func (r *resolver) PieceStatus(ctx context.Context, args struct{ PieceCid string }) (*pieceResolver, error) { + pieceCid, err := cid.Parse(args.PieceCid) + if err != nil { + return nil, fmt.Errorf("%s is not a valid piece cid", args.PieceCid) + } + + // Get sector info from PieceStore + pieceInfo, err := r.ps.GetPieceInfo(pieceCid) + if err != nil && !errors.Is(err, retrievalmarket.ErrNotFound) { + return nil, err + } + + // Get boost deals by piece Cid + boostDeals, err := r.dealsDB.ByPieceCID(ctx, pieceCid) + if err != nil { + return nil, err + } + + // Get legacy markets deals by piece Cid + // TODO: add method to markets to filter deals by piece CID + allLegacyDeals, err := r.legacyProv.ListLocalDeals() + if err != nil { + return nil, err + } + var legacyDeals []storagemarket.MinerDeal + for _, dl := range allLegacyDeals { + if dl.Ref.PieceCid != nil && *dl.Ref.PieceCid == pieceCid { + legacyDeals = append(legacyDeals, dl) + } + } + + // Convert piece info deals to graphQL format + var pids []*pieceInfoDeal + for _, dl := range pieceInfo.Deals { + // Check the sealing status of each deal + errMsg := "" + isUnsealed, err := r.sa.IsUnsealed(ctx, dl.SectorID, dl.Offset.Unpadded(), dl.Length.Unpadded()) + if err != nil { + errMsg = err.Error() + } + + pids = append(pids, &pieceInfoDeal{ + SealStatus: &sealStatus{ + IsUnsealed: isUnsealed, + Error: errMsg, + }, + ChainDealID: gqltypes.Uint64(dl.DealID), + Sector: §orResolver{ + ID: gqltypes.Uint64(dl.SectorID), + Offset: gqltypes.Uint64(dl.Offset), + Length: gqltypes.Uint64(dl.Length), + }, + }) + } + + // Convert boost deals to graphQL format + deals := make([]*pieceDealResolver, 0, len(boostDeals)+len(legacyDeals)) + for _, dl := range boostDeals { + bd := propToBasicDeal(dl.ClientDealProposal.Proposal) + bd.IsLegacy = false + bd.ID = graphql.ID(dl.DealUuid.String()) + bd.CreatedAt = graphql.Time{Time: dl.CreatedAt} + bd.ClientPeerID = dl.ClientPeerID.String() + bd.DealDataRoot = dl.DealDataRoot.String() + bd.PublishCid = cidToString(dl.PublishCID) + bd.Transfer = dealTransfer{ + Type: dl.Transfer.Type, + Size: gqltypes.Uint64(dl.Transfer.Size), + } + bd.Message = dl.Checkpoint.String() + + // Only check the unseal state if the deal has already been added to a piece + st := &sealStatus{IsUnsealed: false} + if dl.Checkpoint >= dealcheckpoints.AddedPiece { + isUnsealed, err := r.sa.IsUnsealed(ctx, dl.SectorID, dl.Offset.Unpadded(), dl.Length.Unpadded()) + if err != nil { + st.Error = err.Error() + } + st.IsUnsealed = isUnsealed + } + + deals = append(deals, &pieceDealResolver{ + Deal: &bd, + SealStatus: st, + Sector: §orResolver{ + ID: gqltypes.Uint64(dl.SectorID), + Offset: gqltypes.Uint64(dl.Offset), + Length: gqltypes.Uint64(dl.Length), + }, + }) + } + + // Convert legacy deals to graphQL format + for _, dl := range legacyDeals { + bd := propToBasicDeal(dl.Proposal) + bd.IsLegacy = true + bd.ID = graphql.ID(dl.ProposalCid.String()) + bd.CreatedAt = graphql.Time{Time: dl.CreationTime.Time()} + bd.ClientPeerID = dl.Client.String() + bd.DealDataRoot = dl.Ref.Root.String() + bd.PublishCid = cidToString(dl.PublishCid) + bd.Transfer = dealTransfer{ + Type: "graphsync", + Size: gqltypes.Uint64(dl.Ref.RawBlockSize), + } + bd.Message = dl.Message + + // For legacy deals the sector information is stored in the piece store + sector := r.getLegacyDealSector(ctx, pids, dl.DealID) + + st := &sealStatus{IsUnsealed: false} + if sector == nil { + sector = §orResolver{ID: gqltypes.Uint64(dl.SectorNumber)} + } else { + secID := abi.SectorNumber(sector.ID) + offset := abi.PaddedPieceSize(sector.Offset).Unpadded() + len := abi.PaddedPieceSize(sector.Length).Unpadded() + isUnsealed, err := r.sa.IsUnsealed(ctx, secID, offset, len) + st = &sealStatus{IsUnsealed: isUnsealed} + if err != nil { + st.Error = err.Error() + } + } + + deals = append(deals, &pieceDealResolver{ + Deal: &bd, + Sector: sector, + SealStatus: st, + }) + } + + // Get the state of the piece in the DAG store + idxStatus, err := r.getIndexStatus(ctx, pieceCid, deals) + if err != nil { + return nil, err + } + + return &pieceResolver{ + PieceCid: args.PieceCid, + IndexStatus: idxStatus, + PieceInfoDeals: pids, + Deals: deals, + }, nil +} + +func (r *resolver) getIndexStatus(ctx context.Context, pieceCid cid.Cid, deals []*pieceDealResolver) (*indexStatus, error) { + si, err := r.dagst.GetShardInfo(shard.KeyFromCID(pieceCid)) + if err != nil && !errors.Is(err, dagstore.ErrShardUnknown) { + return nil, err + } + idxst := IndexStatusUnknown + idxerr := "" + switch { + case err != nil && errors.Is(err, dagstore.ErrShardUnknown): + idxst = IndexStatusNotFound + case si.ShardState == dagstore.ShardStateNew, si.ShardState == dagstore.ShardStateInitializing: + idxst = IndexStatusRegistered + case si.ShardState == dagstore.ShardStateAvailable, si.ShardState == dagstore.ShardStateServing: + idxst = IndexStatusComplete + case si.ShardState == dagstore.ShardStateErrored, si.ShardState == dagstore.ShardStateRecovering: + idxst = IndexStatusFailed + if si.Error != nil { + idxerr = si.Error.Error() + } + } + + // Try retrieving the piece payload cid as a means to check if the + // payload cid => piece cid index has been created correctly + if idxst == IndexStatusComplete && len(deals) > 0 { + cidstr := deals[0].Deal.DealDataRoot + c, err := cid.Parse(cidstr) + if err != nil { + // This should never happen, but check just in case + return nil, fmt.Errorf("parsing retrieved deal data root cid %s: %w", cidstr, err) + } + ks, err := r.dagst.ShardsContainingMultihash(ctx, c.Hash()) + if err != nil || len(ks) == 0 { + idxst = IndexStatusFailed + idxerr = fmt.Sprintf("unable to resolve piece's root payload cid %s to piece cid", cidstr) + } + } + + return &indexStatus{Status: string(idxst), Error: idxerr}, nil +} + +func cidToString(c *cid.Cid) string { + cstr := "" + if c != nil { + cstr = c.String() + } + return cstr +} + +func propToBasicDeal(prop market.DealProposal) basicDealResolver { + return basicDealResolver{ + ClientAddress: prop.Client.String(), + ProviderAddress: prop.Provider.String(), + PieceCid: prop.PieceCID.String(), + PieceSize: gqltypes.Uint64(prop.PieceSize), + ProviderCollateral: gqltypes.Uint64(prop.ProviderCollateral.Uint64()), + StartEpoch: gqltypes.Uint64(prop.StartEpoch), + EndEpoch: gqltypes.Uint64(prop.EndEpoch), + } +} + +// Get the index status of the deal with the given on-chain ID from the piece info data +func (r *resolver) getLegacyDealSector(ctx context.Context, pids []*pieceInfoDeal, chainDealId abi.DealID) *sectorResolver { + for _, pid := range pids { + if abi.DealID(pid.ChainDealID) != chainDealId { + continue + } + + return pid.Sector + } + return nil +} diff --git a/gql/schema.graphql b/gql/schema.graphql index 146c57d65..2be280db6 100644 --- a/gql/schema.graphql +++ b/gql/schema.graphql @@ -140,6 +140,35 @@ type DealLog { Subsystem: String! } +type IndexStatus { + Status: String! + Error: String! +} + +type SealStatus { + IsUnsealed: Boolean! + Error: String! +} + +type PieceDeal { + SealStatus: SealStatus! + Deal: DealBasic! + Sector: Sector! +} + +type PieceInfoDeal { + ChainDealID: Uint64! + Sector: Sector! + SealStatus: SealStatus! +} + +type PieceStatus { + PieceCid: String! + IndexStatus: IndexStatus! + Deals: [PieceDeal]! + PieceInfoDeals: [PieceInfoDeal]! +} + type ProposalLog { DealUUID: ID! Accepted: Boolean! @@ -321,6 +350,12 @@ type RootQuery { """Get the number of accepted and rejected deal proposal logs""" proposalLogsCount: ProposalLogsCount! + """Get information about a piece from the piece store, DAG store and database""" + pieceStatus(pieceCid: String!): PieceStatus! + + """Get the pieces that contain a particular payload CID""" + piecesWithPayloadCid(payloadCid: String!): [String!]! + """Get storage space usage""" storage: Storage! diff --git a/node/builder.go b/node/builder.go index 94e7e12d3..b58a18973 100644 --- a/node/builder.go +++ b/node/builder.go @@ -22,6 +22,7 @@ import ( "github.com/filecoin-project/boost/storagemanager" "github.com/filecoin-project/boost/storagemarket" "github.com/filecoin-project/boost/storagemarket/dealfilter" + "github.com/filecoin-project/dagstore" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" @@ -35,7 +36,7 @@ import ( "github.com/filecoin-project/lotus/journal/alerting" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" - "github.com/filecoin-project/lotus/markets/dagstore" + mktsdagstore "github.com/filecoin-project/lotus/markets/dagstore" lotus_dealfilter "github.com/filecoin-project/lotus/markets/dealfilter" "github.com/filecoin-project/lotus/markets/idxprov" "github.com/filecoin-project/lotus/markets/retrievaladapter" @@ -500,12 +501,13 @@ func ConfigBoost(cfg *config.Boost) Option { })), // DAG Store - Override(new(dagstore.MinerAPI), lotus_modules.NewMinerAPI(cfg.DAGStore)), + Override(new(mktsdagstore.MinerAPI), lotus_modules.NewMinerAPI(cfg.DAGStore)), Override(DAGStoreKey, lotus_modules.DAGStore(cfg.DAGStore)), + Override(new(dagstore.Interface), From(new(*dagstore.DAGStore))), // Lotus Markets (retrieval) - Override(new(dagstore.SectorAccessor), sectoraccessor.NewSectorAccessor), - Override(new(retrievalmarket.SectorAccessor), From(new(dagstore.SectorAccessor))), + Override(new(mktsdagstore.SectorAccessor), sectoraccessor.NewSectorAccessor), + Override(new(retrievalmarket.SectorAccessor), From(new(mktsdagstore.SectorAccessor))), Override(new(retrievalmarket.RetrievalProviderNode), retrievaladapter.NewRetrievalProviderNode), Override(new(rmnet.RetrievalMarketNetwork), lotus_modules.RetrievalNetwork), Override(new(retrievalmarket.RetrievalProvider), lotus_modules.RetrievalProvider), diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index b2bfb563e..0747c805c 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -21,6 +21,7 @@ import ( "github.com/filecoin-project/boost/storagemarket/logs" "github.com/filecoin-project/boost/storagemarket/lp2pimpl" "github.com/filecoin-project/boost/transport/httptransport" + "github.com/filecoin-project/dagstore" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/shared" @@ -30,7 +31,7 @@ import ( ctypes "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/sigs" - "github.com/filecoin-project/lotus/markets/dagstore" + mktsdagstore "github.com/filecoin-project/lotus/markets/dagstore" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/node/modules" lotus_dtypes "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -390,12 +391,12 @@ func NewChainDealManager(a v1api.FullNode) *storagemarket.ChainDealManager { func NewStorageMarketProvider(provAddr address.Address, cfg *config.Boost) func(lc fx.Lifecycle, h host.Host, a v1api.FullNode, sqldb *sql.DB, dealsDB *db.DealsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, dp *storageadapter.DealPublisher, secb *sectorblocks.SectorBlocks, sps sealingpipeline.API, df dtypes.StorageDealFilter, logsSqlDB *LogSqlDB, logsDB *db.LogsDB, - dagst *dagstore.Wrapper, ps lotus_dtypes.ProviderPieceStore, ip *indexprovider.Wrapper, lp lotus_storagemarket.StorageProvider, + dagst *mktsdagstore.Wrapper, ps lotus_dtypes.ProviderPieceStore, ip *indexprovider.Wrapper, lp lotus_storagemarket.StorageProvider, cdm *storagemarket.ChainDealManager) (*storagemarket.Provider, error) { return func(lc fx.Lifecycle, h host.Host, a v1api.FullNode, sqldb *sql.DB, dealsDB *db.DealsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, dp *storageadapter.DealPublisher, secb *sectorblocks.SectorBlocks, sps sealingpipeline.API, df dtypes.StorageDealFilter, logsSqlDB *LogSqlDB, logsDB *db.LogsDB, - dagst *dagstore.Wrapper, ps lotus_dtypes.ProviderPieceStore, ip *indexprovider.Wrapper, + dagst *mktsdagstore.Wrapper, ps lotus_dtypes.ProviderPieceStore, ip *indexprovider.Wrapper, lp lotus_storagemarket.StorageProvider, cdm *storagemarket.ChainDealManager) (*storagemarket.Provider, error) { prvCfg := storagemarket.Config{ @@ -413,12 +414,13 @@ func NewStorageMarketProvider(provAddr address.Address, cfg *config.Boost) func( } } -func NewGraphqlServer(cfg *config.Boost) func(lc fx.Lifecycle, r repo.LockedRepo, h host.Host, prov *storagemarket.Provider, dealsDB *db.DealsDB, logsDB *db.LogsDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, publisher *storageadapter.DealPublisher, spApi sealingpipeline.API, legacyProv lotus_storagemarket.StorageProvider, legacyDT lotus_dtypes.ProviderDataTransfer, fullNode v1api.FullNode) *gql.Server { +func NewGraphqlServer(cfg *config.Boost) func(lc fx.Lifecycle, r repo.LockedRepo, h host.Host, prov *storagemarket.Provider, dealsDB *db.DealsDB, logsDB *db.LogsDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, publisher *storageadapter.DealPublisher, spApi sealingpipeline.API, legacyProv lotus_storagemarket.StorageProvider, legacyDT lotus_dtypes.ProviderDataTransfer, ps lotus_dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, dagst dagstore.Interface, fullNode v1api.FullNode) *gql.Server { return func(lc fx.Lifecycle, r repo.LockedRepo, h host.Host, prov *storagemarket.Provider, dealsDB *db.DealsDB, logsDB *db.LogsDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, publisher *storageadapter.DealPublisher, spApi sealingpipeline.API, - legacyProv lotus_storagemarket.StorageProvider, legacyDT lotus_dtypes.ProviderDataTransfer, fullNode v1api.FullNode) *gql.Server { + legacyProv lotus_storagemarket.StorageProvider, legacyDT lotus_dtypes.ProviderDataTransfer, + ps lotus_dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, dagst dagstore.Interface, fullNode v1api.FullNode) *gql.Server { - resolver := gql.NewResolver(cfg, r, h, dealsDB, logsDB, plDB, fundsDB, fundMgr, storageMgr, spApi, prov, legacyProv, legacyDT, publisher, fullNode) + resolver := gql.NewResolver(cfg, r, h, dealsDB, logsDB, plDB, fundsDB, fundMgr, storageMgr, spApi, prov, legacyProv, legacyDT, ps, sa, dagst, publisher, fullNode) server := gql.NewServer(resolver) lc.Append(fx.Hook{ diff --git a/react/src/App.js b/react/src/App.js index a35540d3a..c5af7eb69 100644 --- a/react/src/App.js +++ b/react/src/App.js @@ -18,6 +18,7 @@ import {LegacyDealDetail} from "./LegacyDealDetail" import {SettingsPage} from "./Settings"; import {Banner} from "./Banner"; import {ProposalLogsPage} from "./ProposalLogs"; +import {InspectPage} from "./Inspect"; function App(props) { return ( @@ -48,6 +49,8 @@ function App(props) { } /> } /> } /> + } /> + } /> } /> diff --git a/react/src/Components.js b/react/src/Components.js index 0cd159a82..3dae107fb 100644 --- a/react/src/Components.js +++ b/react/src/Components.js @@ -11,7 +11,8 @@ export function PageContainer(props) { } export function ShortDealLink(props) { - return + const linkBase = props.isLegacy ? "/legacy-deals/" : "/deals/" + return } diff --git a/react/src/DealDetail.js b/react/src/DealDetail.js index 301e2104c..b61b4f3a4 100644 --- a/react/src/DealDetail.js +++ b/react/src/DealDetail.js @@ -3,7 +3,7 @@ import React, {useEffect, useState} from "react"; import {useMutation, useQuery, useSubscription} from "@apollo/react-hooks"; import {DealCancelMutation, DealFailPausedMutation, DealRetryPausedMutation, DealSubscription, EpochQuery} from "./gql"; -import {useNavigate, useParams} from "react-router-dom"; +import {useNavigate, useParams, Link} from "react-router-dom"; import {dateFormat} from "./util-date"; import moment from "moment"; import {addCommas, humanFIL, humanFileSize} from "./util"; @@ -143,7 +143,7 @@ export function DealDetail(props) { Deal Data Root CID - {deal.DealDataRoot} + {deal.DealDataRoot} Verified @@ -151,7 +151,7 @@ export function DealDetail(props) { Piece CID - {deal.PieceCid} + {deal.PieceCid} Piece Size diff --git a/react/src/DealPublish.js b/react/src/DealPublish.js index 3cc8d7570..f0f523e8e 100644 --- a/react/src/DealPublish.js +++ b/react/src/DealPublish.js @@ -93,13 +93,7 @@ function DealsTable(props) { {moment(deal.CreatedAt).fromNow()} - {deal.IsLegacy ? ( - - - - ) : ( - - )} + {humanFileSize(deal.Transfer.Size)} {humanFileSize(deal.PieceSize)} diff --git a/react/src/Deals.js b/react/src/Deals.js index 98ab86fcc..59d66e02c 100644 --- a/react/src/Deals.js +++ b/react/src/Deals.js @@ -195,7 +195,7 @@ export function StorageDealsMenuItem(props) {

Storage Deals

{data ? ( - +
{data.dealsCount} deal{data.dealsCount === 1 ? '' : 's'}
diff --git a/react/src/Inspect.css b/react/src/Inspect.css new file mode 100644 index 000000000..d213d6f82 --- /dev/null +++ b/react/src/Inspect.css @@ -0,0 +1,92 @@ +.inspect table { + font-size: 1em; + width: 100%; +} + +.inspect td, .inspect th { + padding: 0.5em 1em; + font-weight: normal; +} + +.inspect th { + white-space: nowrap; + text-align: left; + opacity: 0.6; +} + +.inspect .search { + position: absolute; + right: 8em; + top: 2em; +} + +.inspect .search input { + background-image: url("./bootstrap-icons/icons/search.svg"); + background-repeat: no-repeat; + background-position: left 0.5em center; + padding-left: 2em; + padding-right: 2em; +} + +.inspect .search .clear-text { + position: absolute; + right: 1em; + top: 1.1em; + cursor: pointer; +} + +.piece-detail h3 { + margin-top: 2em; +} + +.piece-detail .title { + font-size: 1.25em; + color: #a61e4d; + margin: 2em 0.8em 1.5em 0.8em; +} + +.piece-detail { + padding: 1em 0 5em 0; +} + +.piece-detail .title { + color: inherit; + font-weight: normal; + margin: 0.8em; +} + +.piece-detail p { + padding: 0.5em 1em; +} + +.piece-detail table { + font-size: 1em; +} + +.piece-detail table.deals tr td:first-child { + width: 8em; +} +.piece-detail table.deals tr td:last-child { + line-break: anywhere; +} + +.piece-detail table th { + text-align: left; +} + +.piece-detail table.deals .param { + color: #777777; +} + +.piece-detail td, .piece-detail th { + padding: 0.5em 1em; +} + +.inspect .title { + font-size: 1.25em; + margin: 2em 0.8em 1em 0.8em; +} + +.inspect .payload-cid { + padding: 0.5em 1em; +} diff --git a/react/src/Inspect.js b/react/src/Inspect.js new file mode 100644 index 000000000..efa79e429 --- /dev/null +++ b/react/src/Inspect.js @@ -0,0 +1,211 @@ +import {useQuery} from "@apollo/react-hooks"; +import { + PieceStatusQuery, PiecesWithPayloadCidQuery +} from "./gql"; +import moment from "moment"; +import {DebounceInput} from 'react-debounce-input'; +import React, {useState} from "react"; +import {PageContainer, ShortDealLink} from "./Components"; +import {Link, useParams} from "react-router-dom"; +import {dateFormat} from "./util-date"; +import xImg from './bootstrap-icons/icons/x-lg.svg' +import inspectImg from './bootstrap-icons/icons/wrench.svg' +import './Inspect.css' + +export function InspectMenuItem(props) { + return ( + + +

Inspect

+ + ) +} + +export function InspectPage(props) { + return + + +} + +function InspectContent(props) { + const params = useParams() + const [searchQuery, setSearchQuery] = useState(params.query) + const handleSearchQueryChange = (event) => { + setSearchQuery(event.target.value) + } + const clearSearchBox = () => { + setSearchQuery('') + } + + // Look up pieces by payload + const payloadRes = useQuery(PiecesWithPayloadCidQuery, { + variables: { + payloadCid: searchQuery + }, + // Don't do this query if the search query is empty + skip: !searchQuery + }) + + // If the request for a payload CID has completed + var pieceCid = null + var pieceCids = [] + if (payloadRes && payloadRes.data) { + pieceCids = payloadRes.data.piecesWithPayloadCid + if (pieceCids.length === 0) { + // If there were no results for the lookup by payload CID, use the search + // query for a lookup by piece CID + pieceCid = searchQuery + } else if (pieceCids.length === 1) { + // If there was exactly one result for the lookup by payload CID, use + // the piece CID for the lookup by piece CID + pieceCid = pieceCids[0] + } + } + + // Lookup a piece by piece CID + const pieceRes = useQuery(PieceStatusQuery, { + variables: { + pieceCid: pieceCid, + }, + // Don't do this query if there is no piece CID yet + skip: !pieceCid + }) + + if ((pieceRes || {}).loading || payloadRes.loading) { + return
Loading ...
+ } + + var errorMsg = "" + if ((pieceRes || {}).error || payloadRes.error) { + errorMsg = ((pieceRes || {}).error ? pieceRes.error.message : payloadRes.error.message) + } + + const pieceStatus = ((pieceRes || {}).data || {}).pieceStatus + const showPayload = pieceCids.length > 1 + const showInstructions = !errorMsg && !pieceStatus && !showPayload + return
+ + { errorMsg ?
Error: {errorMsg}
: null} + { pieceStatus ? : null } + { showPayload ? : null } + { showInstructions ?

Enter piece CID or payload CID into the search box

: null } +
+} + +function PiecesWithPayload({payloadCid, pieceCids, setSearchQuery}) { + return
+
Pieces with payload CID {payloadCid}:
+ {pieceCids.map(pc => ( +
+ setSearchQuery(pc)} to={"/inspect/"+pc}>{pc} +
+ ))} +
+} + +function PieceStatus({pieceCid, pieceStatus}) { + if (!pieceStatus) { + return
No piece found with piece CID {pieceCid}
+ } + + const rootCid = pieceStatus.Deals.length ? pieceStatus.Deals[0].Deal.DealDataRoot : null + + return
+
+ + + + + + + {rootCid ? ( + + + + + ) : null} + + + + + +
Piece CID{pieceCid}
Data Root CID{rootCid}
Index Status{pieceStatus.IndexStatus.Status}
+ +

Piece Store

+ {pieceStatus.PieceInfoDeals.length ? ( + + + + + + + + + + {pieceStatus.PieceInfoDeals.map(deal => ( + + + + + + + + ))} + +
Chain Deal IDSector NumberPiece OffsetPiece LengthUnsealed
{deal.ChainDealID+''}{deal.Sector.ID+''}{deal.Sector.Offset+''}{deal.Sector.Length+''}
+ ) : ( +

No data found in piece store for piece CID {pieceCid}

+ )} + +

Deals

+ {pieceStatus.PieceInfoDeals.length ? ( + + + + + + + + + + + + {pieceStatus.Deals.map(deal => ( + + + + + + + + + + ))} + +
CreatedAtDeal IDLegacy DealSector NumberPiece OffsetPiece LengthUnsealed
{moment(deal.Deal.CreatedAt).format(dateFormat)}{deal.Deal.IsLegacy ? 'Yes' : 'No'}{deal.Sector.ID+''}{deal.Sector.Offset+''}{deal.Sector.Length+''}
+ ) : ( +

No deals found with piece CID {pieceCid}

+ )} +
+
+} + +function SealStatus({status}) { + if (status.Error) { + return status.Error + } + return status.IsUnsealed ? 'Yes' : 'No' +} + +function SearchBox(props) { + return
+ + { props.value ? clear : null } +
+} diff --git a/react/src/LegacyDealDetail.js b/react/src/LegacyDealDetail.js index 16ef9a3e0..45739258b 100644 --- a/react/src/LegacyDealDetail.js +++ b/react/src/LegacyDealDetail.js @@ -5,7 +5,7 @@ import {useNavigate} from "react-router-dom"; import {dateFormat} from "./util-date"; import moment from "moment"; import {humanFIL, addCommas, humanFileSize} from "./util"; -import {useParams} from "react-router-dom"; +import {useParams, Link} from "react-router-dom"; import './DealDetail.css' import closeImg from './bootstrap-icons/icons/x-circle.svg' @@ -79,7 +79,7 @@ export function LegacyDealDetail(props) { Deal Data Root CID - {deal.DealDataRoot} + {deal.DealDataRoot} CAR File Path @@ -87,7 +87,11 @@ export function LegacyDealDetail(props) { Piece CID - {deal.PieceCid} + {deal.PieceCid} + + + Verified + {deal.IsVerified ? 'Yes' : 'No'} Piece Size diff --git a/react/src/Menu.js b/react/src/Menu.js index a458a173f..6416fc4c2 100644 --- a/react/src/Menu.js +++ b/react/src/Menu.js @@ -8,6 +8,7 @@ import {FundsMenuItem} from "./Funds"; import gridImg from './bootstrap-icons/icons/grid-3x3-gap.svg' import './Menu.css' import {SettingsMenuItem} from "./Settings"; +import {InspectMenuItem} from "./Inspect"; import {ProposalLogsMenuItem} from "./ProposalLogs"; export function Menu(props) { @@ -24,6 +25,7 @@ export function Menu(props) { +

Message Pool

diff --git a/react/src/gql.js b/react/src/gql.js index 8e0dd3f89..999ee87b6 100644 --- a/react/src/gql.js +++ b/react/src/gql.js @@ -300,6 +300,53 @@ const LegacyDealQuery = gql` } `; +const PiecesWithPayloadCidQuery = gql` + query AppPiecesWithPayloadCidQuery($payloadCid: String!) { + piecesWithPayloadCid(payloadCid: $payloadCid) + } +`; + +const PieceStatusQuery = gql` + query AppPieceStatusQuery($pieceCid: String!) { + pieceStatus(pieceCid: $pieceCid) { + PieceCid + IndexStatus { + Status + Error + } + Deals { + SealStatus { + IsUnsealed + Error + } + Deal { + ID + IsLegacy + CreatedAt + DealDataRoot + } + Sector { + ID + Offset + Length + } + } + PieceInfoDeals { + ChainDealID + Sector { + ID + Offset + Length + } + SealStatus { + IsUnsealed + Error + } + } + } + } +`; + const StorageQuery = gql` query AppStorageQuery { storage { @@ -515,6 +562,8 @@ export { NewDealsSubscription, ProposalLogsListQuery, ProposalLogsCountQuery, + PiecesWithPayloadCidQuery, + PieceStatusQuery, StorageQuery, LegacyStorageQuery, FundsQuery,